Merge pull request #240 from sasjs/allanbowe/error-the-function-md-239

fix: avoiding use of md5() in sysfunc().  Closes #239

.all-contributorsrc

@@ -126,6 +126,15 @@
       "contributions": [
         "code"
       ]
+    },
+    {
+      "login": "eltociear",
+      "name": "Ikko Ashimine",
+      "avatar_url": "https://avatars.githubusercontent.com/u/22633385?v=4",
+      "profile": "https://bandism.net/",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.npmignore

@@ -6,7 +6,6 @@ sasjs/
 .github/
 .git-hooks/
 .vscode/
-main.dox
 make_singlefile.sh
 *.md
 .all-contributorsrc

README.md

@@ -18,7 +18,7 @@
 [dependency-url]:https://github.com/sasjs/core/blob/main/package.json
 
 
-Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
+Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/.github/CONTRIBUTING.md) are welcome.
 
 You can download and compile them all in just two lines of SAS code:
 
@@ -224,12 +224,13 @@ The following repositories are also worth checking out:
 * [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
 * [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
 * [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)
+* [rogerjdeangelis](https://github.com/rogerjdeangelis)
 * [scottbass/sas](https://github.com/scottbass/SAS)
 * [yabwon/sas_packages](https://github.com/yabwon/SAS_PACKAGES)
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-11-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-12-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
 
@@ -251,6 +252,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <td align="center"><a href="https://github.com/VladislavParhomchik"><img src="https://avatars.githubusercontent.com/u/83717836?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vladislav Parhomchik</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=VladislavParhomchik" title="Tests">⚠️</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3AVladislavParhomchik" title="Reviewed Pull Requests">👀</a></td>
     <td align="center"><a href="https://github.com/vznesh"><img src="https://avatars.githubusercontent.com/u/28916792?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Vignesh T.</b></sub></a><br /><a href="https://github.com/sasjs/core/issues?q=author%3Avznesh" title="Bug reports">🐛</a></td>
     <td align="center"><a href="https://github.com/yabwon"><img src="https://avatars.githubusercontent.com/u/9314894?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Bart Jablonski</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=yabwon" title="Code">💻</a></td>
+    <td align="center"><a href="https://bandism.net/"><img src="https://avatars.githubusercontent.com/u/22633385?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Ikko Ashimine</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=eltociear" title="Code">💻</a></td>
   </tr>
 </table>
 

all.sas

@@ -369,6 +369,48 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
 %mend mf_existvarlist;
 
 /** @endcond *//**
+  @file
+  @brief Returns E8601DT26.6 if compatible else DATETIME19.3
+  @details From our experience in [Data Controller for SAS]
+  (https://datacontroller.io) deployments, the E8601DT26.6 datetime format has
+  the widest support when it comes to pass-through SQL queries.
+
+  However, it is not supported in WPS or early versions of SAS 9 (M3 and below)
+  when used as a datetime literal, eg:
+
+      data _null_;
+        demo="%sysfunc(datetime(),E8601DT26.6)"dt;
+        demo=;
+      run;
+
+  This macro will therefore return DATEITME19.3 as an alternative format
+  based on the runtime environment so that it can be used in such cases, eg:
+
+      data _null_;
+        demo="%sysfunc(datetime(),%mf_fmtdttm())"dt;
+        demo=;
+      run;
+
+  <h4> Related Macros </h4>
+  @li mf_fmtdttm.test.sas
+
+  @author Allan Bowe
+**/
+
+%macro mf_fmtdttm(
+)/*/STORE SOURCE*/;
+
+%if "&sysver"="9.2" or "&sysver"="9.3"
+  or ("&sysver"="9.4" and "%substr(&SYSVLONG,9,1)" le "3")
+  or "%substr(&sysver,1,1)"="4"
+  or "%substr(&sysver,1,1)"="5"
+%then %do;DATETIME19.3%end;
+%else %do;E8601DT26.6%end;
+
+%mend mf_fmtdttm;
+
+
+/**
   @file
   @brief Returns the appLoc from the _program variable
   @details When working with SASjs apps, web services / tests / jobs are always
@@ -994,12 +1036,12 @@ or %index(&pgm,/tests/testteardown)
   @brief Returns an unused libref
   @details Use as follows:
 
-    libname mclib0 (work);
-    libname mclib1 (work);
-    libname mclib2 (work);
+      libname mclib0 (work);
+      libname mclib1 (work);
+      libname mclib2 (work);
 
-    %let libref=%mf_getuniquelibref();
-    %put &=libref;
+      %let libref=%mf_getuniquelibref();
+      %put &=libref;
 
   which returns:
 
@@ -1148,33 +1190,50 @@ or %index(&pgm,/tests/testteardown)
   @file
   @brief Returns number of variables in a dataset
   @details Useful to identify those renagade datasets that have no columns!
+  Can also be used to count for numeric, or character columns
 
-        %put Number of Variables=%mf_getvarcount(sashelp.class);
+      %put Number of Variables=%mf_getvarcount(sashelp.class);
+      %put Character Variables=%mf_getvarcount(sashelp.class,typefilter=C);
+      %put Numeric Variables = %mf_getvarcount(sashelp.class,typefilter=N);
 
   returns:
   > Number of Variables=4
 
-  @param libds Two part dataset (or view) reference.
+
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] typefilter= (A) Filter for certain types of column.  Valid values:
+    @li A Count All columns
+    @li C Count Character columns only
+    @li N Count Numeric columns only
 
   @version 9.2
   @author Allan Bowe
 
 **/
 
-%macro mf_getvarcount(libds
+%macro mf_getvarcount(libds,typefilter=A
 )/*/STORE SOURCE*/;
-  %local dsid nvars rc ;
+  %local dsid nvars rc outcnt x;
   %let dsid=%sysfunc(open(&libds));
   %let nvars=.;
+  %let outcnt=0;
+  %let typefilter=%upcase(&typefilter);
   %if &dsid %then %do;
     %let nvars=%sysfunc(attrn(&dsid,NVARS));
+    %if &typefilter=A %then %let outcnt=&nvars;
+    %else %if &nvars>0 %then %do x=1 %to &nvars;
+      /* increment based on variable type */
+      %if %sysfunc(vartype(&dsid,&x))=&typefilter %then %do;
+        %let outcnt=%eval(&outcnt+1);
+      %end;
+    %end;
     %let rc=%sysfunc(close(&dsid));
   %end;
   %else %do;
     %put unable to open &libds (rc=&dsid);
     %let rc=%sysfunc(close(&dsid));
   %end;
-  &nvars
+  &outcnt
 %mend mf_getvarcount;/**
   @file
   @brief Returns the format of a variable
@@ -2203,7 +2262,12 @@ Usage:
     or "&SYSPROCESSNAME "="Compute Server "
     or &mode=INCLUDE
   %then %do;
-    options obs=max replace nosyntaxcheck mprint;
+    options obs=max replace mprint;
+    %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5"
+    %then %do;
+      options nosyntaxcheck;
+    %end;
+
     %if &mode=INCLUDE %then %do;
       %if %sysfunc(exist(&errds))=1 %then %do;
         data _null_;
@@ -3940,10 +4004,12 @@ proc sql;
 %end;
 
 %if &libds=0 %then %do;
+  proc sql;
   describe table &syslast;
   drop table &syslast;
 %end;
-%mend mp_coretable;/**
+%mend mp_coretable;
+/**
   @file mp_createconstraints.sas
   @brief Creates constraints
   @details Takes the output from mp_getconstraints.sas as input
@@ -4718,7 +4784,7 @@ drop table &out_ds;
 
   Usage:
 
-      %mp_ds2cards(base_ds=sashelp.class
+      %mp_ds2cards(sashelp.class
         , tgt_ds=work.class
         , cards_file= "C:\temp\class.sas"
         , showlog=NO
@@ -4730,7 +4796,7 @@ drop table &out_ds;
     - explicity setting a unix LF
     - constraints / indexes etc
 
-  @param [in] base_ds= Should be two level - eg work.blah.  This is the table
+  @param [in] base_ds Should be two level - eg work.blah.  This is the table
                   that is converted to a cards file.
   @param [in] tgt_ds= Table that the generated cards file would create.
     Optional - if omitted, will be same as BASE_DS.
@@ -5828,7 +5894,36 @@ data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $4032;
+  length reason_cd $4032 vtype $1 vnum dsid 8;
+
+  /* quick check to ensure column exists */
+  if upcase(VARIABLE_NM) not in
+  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
+  then do;
+    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
+    putlog REASON_CD= VARIABLE_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
+    output;
+    return;
+  end;
+
+  /* need to open the dataset to get the column type */
+  dsid=open("&targetds","i");
+  if dsid>0 then do;
+    vnum=varnum(dsid,VARIABLE_NM);
+    if vnum<1 then do;
+      /* should not happen as was also tested for above */
+      REASON_CD=cats("Variable (",VARIABLE_NM,") not found in &targetds");
+      putlog REASON_CD= dsid=;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
+      output;
+      return;
+    end;
+    /* now we can get the type */
+    else vtype=vartype(dsid,vnum);
+  end;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -5852,17 +5947,8 @@ data &outds;
     call symputx('nobs',_n_,'l');
     output;
   end;
-  if upcase(VARIABLE_NM) not in
-  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
-  then do;
-    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
-    putlog REASON_CD= VARIABLE_NM=;
-    call symputx('reason_cd',reason_cd,'l');
-    call symputx('nobs',_n_,'l');
-    output;
-  end;
   if OPERATOR_NM not in
-  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
+  ('=','>','<','<=','>=','NE','GE','LE','BETWEEN','IN','NOT IN','CONTAINS')
   then do;
     REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
@@ -5871,6 +5957,18 @@ data &outds;
     output;
   end;
 
+  /* special missing logic */
+  if vtype='N'
+    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
+    and cats(upcase(raw_value)) in (
+      '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
+      '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
+    )
+  then do;
+    /* valid numeric - exit data step loop */
+    return;
+  end;
+
   /* special logic */
   if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
   else if OPERATOR_NM in ('IN','NOT IN') then do;
@@ -6024,6 +6122,9 @@ filename &outref temp;
   run;
 %end;
 %else %do;
+  proc sort data=&inds;
+    by SUBGROUP_ID;
+  run;
   data _null_;
     file &outref lrecl=32800;
     set &inds end=last;
@@ -6506,6 +6607,22 @@ drop table &dropds;
 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
 
+/**
+  * Cater for environments where sashelp.vcncolu is not available
+  */
+%if %sysfunc(exist(sashelp.vcncolu,view))=0 %then %do;
+  proc sql;
+  create table &outds(
+      libref char(8)
+    ,TABLE_NAME char(32)
+    ,constraint_type char(8)     label='Constraint Type'
+    ,constraint_name char(32)     label='Constraint Name'
+    ,column_name char(32)     label='Column'
+    ,constraint_order num
+  );
+  %return;
+%end;
+
 /**
   * Neither dictionary tables nor sashelp provides a constraint order column,
   * however they DO arrive in the correct order.  So, create the col.
@@ -7458,6 +7575,7 @@ create table &outsummary as
   <h4> SAS Macros </h4>
   @li mcf_length.sas
   @li mf_getuniquename.sas
+  @li mf_getvarcount.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_getvarformat.sas
@@ -7477,7 +7595,7 @@ create table &outsummary as
   ,outds=work.mp_getmaxvarlengths
 )/*/STORE SOURCE*/;
 
-%local vars prefix x var fmt;
+%local vars prefix x var fmt srcds;
 %let vars=%mf_getvarlist(libds=&libds);
 %let prefix=%substr(%mf_getuniquename(),1,25);
 %let num2char=%upcase(&num2char);
@@ -7487,6 +7605,24 @@ create table &outsummary as
   %mcf_length(wrap=YES, insert_cmplib=YES)
 %end;
 
+%if &num2char=NO
+  and ("%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5")
+  and %mf_getvarcount(&libds,typefilter=N) gt 0
+%then %do;
+  /* custom functions not supported in summary operations */
+  %let srcds=%mf_getuniquename();
+  data &srcds/view=&srcds;
+    set &libds;
+  %do x=1 %to %sysfunc(countw(&vars,%str( )));
+    %let var=%scan(&vars,&x);
+    %if %mf_getvartype(&libds,&var)=N %then %do;
+      &prefix.&x=mcf_length(&var);
+    %end;
+  %end;
+  run;
+%end;
+%else %let srcds=&libds;
+
 proc sql;
 create table &outds (rename=(
     %do x=1 %to %sysfunc(countw(&vars,%str( )));
@@ -7511,10 +7647,15 @@ create table &outds (rename=(
         %end;
       %end;
       %else %do;
-        max(mcf_length(&var)) as &prefix.&x
+        %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do;
+          max(&prefix.&x) as &prefix.&x
+        %end;
+        %else %do;
+          max(mcf_length(&var)) as &prefix.&x
+        %end;
       %end;
     %end;
-  from &libds;
+  from &srcds;
 
   proc transpose data=&outds
     out=&outds(rename=(_name_=NAME COL1=MAXLEN));
@@ -8244,7 +8385,7 @@ run;
 %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
   data &outds;
     length hashkey $32;
-    retain hashkey "%sysfunc(md5(%str(&salt)),$hex32.)";
+    hashkey=put(md5("&salt"),$hex32.);
     output;
     stop;
   run;
@@ -8255,9 +8396,14 @@ run;
   %put %str(ERR)OR: Dataset &libds is not a dataset;
 %end;
 %else %do;
-  data &outds(rename=(&keyvar=hashkey) keep=&keyvar)/nonote2err;
+  data &outds(rename=(&keyvar=hashkey) keep=&keyvar)
+  %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+    /nonote2err
+  %end;
+  ;
     length &prevkeyvar &keyvar $32;
-    retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
+    retain &prevkeyvar;
+    if _n_=1 then &prevkeyvar=put(md5("&salt"),$hex32.);
     set &libds end=&lastvar;
     /* hash should include previous row */
     &keyvar=%mp_md5(
@@ -8443,7 +8589,7 @@ options
   validvarname=V7         /* avoid special characters etc in variable names */
   varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
   varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
-%if %substr(&sysver,1,1) ne 4 %then %do;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
   noautocorrect           /* disallow misspelled procedure names            */
   dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
 %end;
@@ -9213,6 +9359,7 @@ options ibufsize=&ibufsize;
   @param [in] loop_secs= (1) Seconds to wait between each lock attempt
 
   <h4> SAS Macros </h4>
+  @li mf_fmtdttm.sas
   @li mp_abort.sas
   @li mp_lockfilecheck.sas
   @li mf_getuser.sas
@@ -9302,7 +9449,7 @@ run;
       LOCK_LIB ="&lib";
       LOCK_DS="&ds";
       LOCK_STATUS_CD='LOCKED';
-      LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt;
+      LOCK_START_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt;
       LOCK_USER_NM="&user";
       LOCK_PID="&sysjobid";
       LOCK_REF="&ref";
@@ -9322,7 +9469,7 @@ run;
     proc sql;
     update &ctl_ds
       set LOCK_STATUS_CD='LOCKED'
-        , LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_START_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
         , LOCK_USER_NM="&user"
         , LOCK_PID="&sysjobid"
         , LOCK_REF="&ref"
@@ -9397,7 +9544,7 @@ run;
     proc sql;
     update &ctl_ds
       set LOCK_STATUS_CD='UNLOCKED'
-        , LOCK_END_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
         , LOCK_USER_NM="&user"
         , LOCK_PID="&sysjobid"
         , LOCK_REF="&ref"
@@ -10186,6 +10333,7 @@ run;
 
   <h4> SAS Macros </h4>
   @li mf_existvar.sas
+  @li mf_fmtdttm.sas
   @li mf_getquotedstr.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
@@ -10345,12 +10493,12 @@ quit;
     set keytable="&base_libds"
       ,keycolumn="&retained_key"
       ,max_key=%eval(&maxkey+&newkey_cnt)
-      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt;
+      ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt;
   %end;
   %else %do;
   update &maxkeytable
     set max_key=%eval(&maxkey+&newkey_cnt)
-      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt
+      ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt
     where keytable="&base_libds";
   %end;
   %mp_lockanytable(UNLOCK
@@ -11546,7 +11694,7 @@ select distinct tgtvar_nm into: missvars separated by ' '
 data &ds1;
   set &dslist indsname=&inds_auto;
   &hashkey=put(md5(catx('|',%mf_getquotedstr(&key,quote=N))),$hex32.);
-  &inds_keep=&inds_auto;
+  &inds_keep=upcase(&inds_auto);
 proc sort;
   by &inds_keep &hashkey;
 run;
@@ -11581,8 +11729,8 @@ data &ds4;
   tgtvar_nm=upcase(tgtvar_nm);
   if tgtvar_nm in (%upcase(&vlist));
 
-  if &inds_auto="&ds2" then tgtvar_type='N';
-  else if &inds_auto="&ds3" then tgtvar_type='C';
+  if upcase(&inds_auto)="&ds2" then tgtvar_type='N';
+  else if upcase(&inds_auto)="&ds3" then tgtvar_type='C';
   else do;
     putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
     call symputx('syscc',98);
@@ -12934,11 +13082,21 @@ ods package close;
       oldval_num num format=best32. label='Old (numeric) value',
       newval_num num format=best32. label='New (numeric) value',
       oldval_char char(32765) label='Old (character) value',
-      newval_char char(32765) label='New (character) value',
-    constraint pk_mpe_audit
-      primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
+      newval_char char(32765) label='New (character) value'
   );
 
+  %local lib;
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create
+      pk_mpe_audit=(load_ref libref dsn key_hash tgtvar_nm)
+      /nomiss unique;
+  quit;
+
 %mend mddl_dc_difftable;/**
   @file
   @brief Filtertable DDL
@@ -12950,21 +13108,34 @@ ods package close;
 
 %macro mddl_dc_filterdetail(libds=WORK.FILTER_DETAIL);
 
+%local nn lib;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  %let nn=not null;
+%end;
+%else %let nn=;
+
   proc sql;
   create table &libds(
-      filter_hash char(32) not null,
-      filter_line num not null,
-      group_logic char(3) not null,
-      subgroup_logic char(3) not null,
-      subgroup_id num not null,
-      variable_nm varchar(32) not null,
-      operator_nm varchar(12) not null,
-      raw_value varchar(4000) not null,
-      processed_dttm num not null format=E8601DT26.6,
-    constraint pk_mpe_filteranytable
-      primary key(filter_hash,filter_line)
+      filter_hash char(32) &nn,
+      filter_line num &nn,
+      group_logic char(3) &nn,
+      subgroup_logic char(3) &nn,
+      subgroup_id num &nn,
+      variable_nm varchar(32) &nn,
+      operator_nm varchar(12) &nn,
+      raw_value varchar(4000) &nn,
+      processed_dttm num &nn format=E8601DT26.6
   );
 
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create pk_mpe_filterdetail=(filter_hash filter_line)/nomiss unique;
+  quit;
+
 %mend mddl_dc_filterdetail;/**
   @file
   @brief Filtersummary DDL
@@ -12976,16 +13147,29 @@ ods package close;
 
 %macro mddl_dc_filtersummary(libds=WORK.FILTER_SUMMARY);
 
+%local nn lib;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  %let nn=not null;
+%end;
+%else %let nn=;
+
   proc sql;
   create table &libds(
-      filter_rk num not null,
-      filter_hash char(32) not null,
-      filter_table char(41) not null,
-      processed_dttm num not null format=E8601DT26.6,
-    constraint pk_mpe_filteranytable
-      primary key(filter_rk)
+      filter_rk num &nn,
+      filter_hash char(32) &nn,
+      filter_table char(41) &nn,
+      processed_dttm num &nn format=E8601DT26.6
   );
 
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create filter_rk /nomiss unique;
+  quit;
+
 %mend mddl_dc_filtersummary;/**
   @file
   @brief Locktable DDL
@@ -12997,19 +13181,35 @@ ods package close;
 
 %macro mddl_dc_locktable(libds=WORK.LOCKTABLE);
 
+%local nn lib;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  %let nn=not null;
+%end;
+%else %let nn=;
+
   proc sql;
   create table &libds(
       lock_lib char(8),
       lock_ds char(32),
-      lock_status_cd char(10) not null,
-      lock_user_nm char(100) not null ,
+      lock_status_cd char(10) &nn,
+      lock_user_nm char(100) &nn ,
       lock_ref char(200),
       lock_pid char(10),
       lock_start_dttm num format=E8601DT26.6,
-      lock_end_dttm num format=E8601DT26.6,
-    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds)
+      lock_end_dttm num format=E8601DT26.6
   );
 
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create
+      pk_mp_lockanytable=(lock_lib lock_ds)
+      /nomiss unique;
+  quit;
+
 %mend mddl_dc_locktable;/**
   @file
   @brief Maxkeytable DDL
@@ -13029,9 +13229,18 @@ ods package close;
       max_key num label=
         'Integer representing current max RK or SK value in the KEYTABLE',
       processed_dttm num format=E8601DT26.6
-        label='Datetime this value was last updated',
-    constraint pk_mpe_maxkeyvalues
-        primary key(keytable));
+        label='Datetime this value was last updated'
+  );
+
+  %local lib;
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create keytable /nomiss unique;
+  quit;
 
 %mend mddl_dc_maxkeytable;/**
   @file
@@ -13049,8 +13258,8 @@ proc sql;
 create table &libds(
     FMTNAME char(32)      label='Format name'
     /*
-      to accomodate larger START values, mp_loadformat.sas will need the
-      SQL dependency removed (proc sql needs to accomodate 3 index values in
+      to accommodate larger START values, mp_loadformat.sas will need the
+      SQL dependency removed (proc sql needs to accommodate 3 index values in
       a 32767 ibufsize limit)
     */
     ,START char(10000)    label='Starting value for format'
@@ -13075,7 +13284,8 @@ create table &libds(
     ,LANGUAGE char(8)     label='Language for date strings'
 );
 
-%mend mddl_sas_cntlout;/**
+%mend mddl_sas_cntlout;
+/**
   @file mm_adduser2group.sas
   @brief Adds a user to a group
   @details Adds a user to a metadata group.  The macro first checks whether the
@@ -19229,8 +19439,8 @@ data _null_;
   file &fname1 lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put "Content-Type: multipart/form-data; boundary=&boundary";
-  put "Authorization: Bearer " _infile_;
+  if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary";
+  put _infile_;
 run;
 
 %if &mdebug=1 %then %do;
@@ -19360,9 +19570,11 @@ data _null_;
   file &fref1 lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put "Authorization: Bearer " _infile_;
-  put "Content-Type: application/json";
-  put "accept: application/json";
+  if _n_=1 then do;
+    put "Content-Type: application/json";
+    put "accept: application/json";
+  end;
+  put _infile_;
 run;
 
 %if &mdebug=1 %then %do;
@@ -19807,6 +20019,11 @@ data _null_;
   put '%end; ';
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
+  put '  %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do; ';
+  put '    /* functions in formats unsupported */ ';
+  put '    %put &sysmacroname: forcing missing back to NULL as feature not supported; ';
+  put '    %let missing=NULL; ';
+  put '  %end; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
   put '    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
   put '  ) ';
@@ -19982,7 +20199,7 @@ data _null_;
   file &headref lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put "Authorization: Bearer " _infile_;
+  put _infile_;
 run;
 
 proc http method='DELETE' headerin=&headref
@@ -20031,7 +20248,7 @@ data _null_;
   file &headref lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put "Authorization: Bearer " _infile_;
+  put _infile_;
 run;
 
 proc http method='GET' out=&binaryfref headerin=&headref
@@ -20102,8 +20319,8 @@ data _null_;
   file &fref0 lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put "Authorization: Bearer " _infile_;
-  put "accept: application/json";
+  if _n_=1 then put "accept: application/json";
+  put _infile_;
 run;
 
 %if &mdebug=1 %then %do;
@@ -20290,8 +20507,8 @@ data _null_;
   file &authref lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put 'Authorization: Bearer ' _infile_;
-  put "Content-Type: multipart/form-data; boundary=&boundary";
+  if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary";
+  put _infile_;
 run;
 
 %if &mdebug=1 %then %do;
@@ -20603,6 +20820,11 @@ run;
 %end;
 
 %else %if &action=ARR or &action=OBJ %then %do;
+  %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do;
+    /* functions in formats unsupported */
+    %put &sysmacroname: forcing missing back to NULL as feature not supported;
+    %let missing=NULL;
+  %end;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
     ,engine=DATASTEP,missing=&missing,showmeta=&showmeta
   )

base/mf_fmtdttm.sas

@@ -0,0 +1,42 @@
+/**
+  @file
+  @brief Returns E8601DT26.6 if compatible else DATETIME19.3
+  @details From our experience in [Data Controller for SAS]
+  (https://datacontroller.io) deployments, the E8601DT26.6 datetime format has
+  the widest support when it comes to pass-through SQL queries.
+
+  However, it is not supported in WPS or early versions of SAS 9 (M3 and below)
+  when used as a datetime literal, eg:
+
+      data _null_;
+        demo="%sysfunc(datetime(),E8601DT26.6)"dt;
+        demo=;
+      run;
+
+  This macro will therefore return DATEITME19.3 as an alternative format
+  based on the runtime environment so that it can be used in such cases, eg:
+
+      data _null_;
+        demo="%sysfunc(datetime(),%mf_fmtdttm())"dt;
+        demo=;
+      run;
+
+  <h4> Related Macros </h4>
+  @li mf_fmtdttm.test.sas
+
+  @author Allan Bowe
+**/
+
+%macro mf_fmtdttm(
+)/*/STORE SOURCE*/;
+
+%if "&sysver"="9.2" or "&sysver"="9.3"
+  or ("&sysver"="9.4" and "%substr(&SYSVLONG,9,1)" le "3")
+  or "%substr(&sysver,1,1)"="4"
+  or "%substr(&sysver,1,1)"="5"
+%then %do;DATETIME19.3%end;
+%else %do;E8601DT26.6%end;
+
+%mend mf_fmtdttm;
+
+

base/mf_getuniquelibref.sas

@@ -3,12 +3,12 @@
   @brief Returns an unused libref
   @details Use as follows:
 
-    libname mclib0 (work);
-    libname mclib1 (work);
-    libname mclib2 (work);
+      libname mclib0 (work);
+      libname mclib1 (work);
+      libname mclib2 (work);
 
-    %let libref=%mf_getuniquelibref();
-    %put &=libref;
+      %let libref=%mf_getuniquelibref();
+      %put &=libref;
 
   which returns:
 

base/mf_getvarcount.sas

@@ -2,31 +2,48 @@
   @file
   @brief Returns number of variables in a dataset
   @details Useful to identify those renagade datasets that have no columns!
+  Can also be used to count for numeric, or character columns
 
-        %put Number of Variables=%mf_getvarcount(sashelp.class);
+      %put Number of Variables=%mf_getvarcount(sashelp.class);
+      %put Character Variables=%mf_getvarcount(sashelp.class,typefilter=C);
+      %put Numeric Variables = %mf_getvarcount(sashelp.class,typefilter=N);
 
   returns:
   > Number of Variables=4
 
-  @param libds Two part dataset (or view) reference.
+
+  @param [in] libds Two part dataset (or view) reference.
+  @param [in] typefilter= (A) Filter for certain types of column.  Valid values:
+    @li A Count All columns
+    @li C Count Character columns only
+    @li N Count Numeric columns only
 
   @version 9.2
   @author Allan Bowe
 
 **/
 
-%macro mf_getvarcount(libds
+%macro mf_getvarcount(libds,typefilter=A
 )/*/STORE SOURCE*/;
-  %local dsid nvars rc ;
+  %local dsid nvars rc outcnt x;
   %let dsid=%sysfunc(open(&libds));
   %let nvars=.;
+  %let outcnt=0;
+  %let typefilter=%upcase(&typefilter);
   %if &dsid %then %do;
     %let nvars=%sysfunc(attrn(&dsid,NVARS));
+    %if &typefilter=A %then %let outcnt=&nvars;
+    %else %if &nvars>0 %then %do x=1 %to &nvars;
+      /* increment based on variable type */
+      %if %sysfunc(vartype(&dsid,&x))=&typefilter %then %do;
+        %let outcnt=%eval(&outcnt+1);
+      %end;
+    %end;
     %let rc=%sysfunc(close(&dsid));
   %end;
   %else %do;
     %put unable to open &libds (rc=&dsid);
     %let rc=%sysfunc(close(&dsid));
   %end;
-  &nvars
+  &outcnt
 %mend mf_getvarcount;

base/mp_abort.sas

@@ -89,7 +89,12 @@
     or "&SYSPROCESSNAME "="Compute Server "
     or &mode=INCLUDE
   %then %do;
-    options obs=max replace nosyntaxcheck mprint;
+    options obs=max replace mprint;
+    %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5"
+    %then %do;
+      options nosyntaxcheck;
+    %end;
+
     %if &mode=INCLUDE %then %do;
       %if %sysfunc(exist(&errds))=1 %then %do;
         data _null_;

base/mp_coretable.sas

@@ -65,7 +65,8 @@ proc sql;
 %end;
 
 %if &libds=0 %then %do;
+  proc sql;
   describe table &syslast;
   drop table &syslast;
 %end;
-%mend mp_coretable;
+%mend mp_coretable;

base/mp_ds2cards.sas

@@ -11,7 +11,7 @@
 
   Usage:
 
-      %mp_ds2cards(base_ds=sashelp.class
+      %mp_ds2cards(sashelp.class
         , tgt_ds=work.class
         , cards_file= "C:\temp\class.sas"
         , showlog=NO
@@ -23,7 +23,7 @@
     - explicity setting a unix LF
     - constraints / indexes etc
 
-  @param [in] base_ds= Should be two level - eg work.blah.  This is the table
+  @param [in] base_ds Should be two level - eg work.blah.  This is the table
                   that is converted to a cards file.
   @param [in] tgt_ds= Table that the generated cards file would create.
     Optional - if omitted, will be same as BASE_DS.

base/mp_filtercheck.sas

@@ -92,7 +92,36 @@ data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $4032;
+  length reason_cd $4032 vtype $1 vnum dsid 8;
+
+  /* quick check to ensure column exists */
+  if upcase(VARIABLE_NM) not in
+  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
+  then do;
+    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
+    putlog REASON_CD= VARIABLE_NM=;
+    call symputx('reason_cd',reason_cd,'l');
+    call symputx('nobs',_n_,'l');
+    output;
+    return;
+  end;
+
+  /* need to open the dataset to get the column type */
+  dsid=open("&targetds","i");
+  if dsid>0 then do;
+    vnum=varnum(dsid,VARIABLE_NM);
+    if vnum<1 then do;
+      /* should not happen as was also tested for above */
+      REASON_CD=cats("Variable (",VARIABLE_NM,") not found in &targetds");
+      putlog REASON_CD= dsid=;
+      call symputx('reason_cd',reason_cd,'l');
+      call symputx('nobs',_n_,'l');
+      output;
+      return;
+    end;
+    /* now we can get the type */
+    else vtype=vartype(dsid,vnum);
+  end;
 
   /* closed list checks */
   if GROUP_LOGIC not in ('AND','OR') then do;
@@ -116,17 +145,8 @@ data &outds;
     call symputx('nobs',_n_,'l');
     output;
   end;
-  if upcase(VARIABLE_NM) not in
-  (%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
-  then do;
-    REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
-    putlog REASON_CD= VARIABLE_NM=;
-    call symputx('reason_cd',reason_cd,'l');
-    call symputx('nobs',_n_,'l');
-    output;
-  end;
   if OPERATOR_NM not in
-  ('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS','GE','LE')
+  ('=','>','<','<=','>=','NE','GE','LE','BETWEEN','IN','NOT IN','CONTAINS')
   then do;
     REASON_CD='Invalid OPERATOR_NM: '!!cats(OPERATOR_NM);
     putlog REASON_CD= OPERATOR_NM=;
@@ -135,6 +155,18 @@ data &outds;
     output;
   end;
 
+  /* special missing logic */
+  if vtype='N'
+    and OPERATOR_NM in ('=','>','<','<=','>=','NE','GE','LE')
+    and cats(upcase(raw_value)) in (
+      '.','.A','.B','.C','.D','.E','.F','.G','.H','.I','.J','.K','.L','.M','.N'
+      '.N','.O','.P','.Q','.R','.S','.T','.U','.V','.W','.X','.Y','.Z','._'
+    )
+  then do;
+    /* valid numeric - exit data step loop */
+    return;
+  end;
+
   /* special logic */
   if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
   else if OPERATOR_NM in ('IN','NOT IN') then do;

base/mp_filtergenerate.sas

@@ -84,6 +84,9 @@ filename &outref temp;
   run;
 %end;
 %else %do;
+  proc sort data=&inds;
+    by SUBGROUP_ID;
+  run;
   data _null_;
     file &outref lrecl=32800;
     set &inds end=last;

base/mp_getconstraints.sas

@@ -40,6 +40,22 @@
 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
 
+/**
+  * Cater for environments where sashelp.vcncolu is not available
+  */
+%if %sysfunc(exist(sashelp.vcncolu,view))=0 %then %do;
+  proc sql;
+  create table &outds(
+      libref char(8)
+    ,TABLE_NAME char(32)
+    ,constraint_type char(8)     label='Constraint Type'
+    ,constraint_name char(32)     label='Constraint Name'
+    ,column_name char(32)     label='Column'
+    ,constraint_order num
+  );
+  %return;
+%end;
+
 /**
   * Neither dictionary tables nor sashelp provides a constraint order column,
   * however they DO arrive in the correct order.  So, create the col.

base/mp_getmaxvarlengths.sas

@@ -41,6 +41,7 @@
   <h4> SAS Macros </h4>
   @li mcf_length.sas
   @li mf_getuniquename.sas
+  @li mf_getvarcount.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
   @li mf_getvarformat.sas
@@ -60,7 +61,7 @@
   ,outds=work.mp_getmaxvarlengths
 )/*/STORE SOURCE*/;
 
-%local vars prefix x var fmt;
+%local vars prefix x var fmt srcds;
 %let vars=%mf_getvarlist(libds=&libds);
 %let prefix=%substr(%mf_getuniquename(),1,25);
 %let num2char=%upcase(&num2char);
@@ -70,6 +71,24 @@
   %mcf_length(wrap=YES, insert_cmplib=YES)
 %end;
 
+%if &num2char=NO
+  and ("%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5")
+  and %mf_getvarcount(&libds,typefilter=N) gt 0
+%then %do;
+  /* custom functions not supported in summary operations */
+  %let srcds=%mf_getuniquename();
+  data &srcds/view=&srcds;
+    set &libds;
+  %do x=1 %to %sysfunc(countw(&vars,%str( )));
+    %let var=%scan(&vars,&x);
+    %if %mf_getvartype(&libds,&var)=N %then %do;
+      &prefix.&x=mcf_length(&var);
+    %end;
+  %end;
+  run;
+%end;
+%else %let srcds=&libds;
+
 proc sql;
 create table &outds (rename=(
     %do x=1 %to %sysfunc(countw(&vars,%str( )));
@@ -94,10 +113,15 @@ create table &outds (rename=(
         %end;
       %end;
       %else %do;
-        max(mcf_length(&var)) as &prefix.&x
+        %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do;
+          max(&prefix.&x) as &prefix.&x
+        %end;
+        %else %do;
+          max(mcf_length(&var)) as &prefix.&x
+        %end;
       %end;
     %end;
-  from &libds;
+  from &srcds;
 
   proc transpose data=&outds
     out=&outds(rename=(_name_=NAME COL1=MAXLEN));

base/mp_hashdataset.sas

@@ -58,7 +58,7 @@
 %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
   data &outds;
     length hashkey $32;
-    retain hashkey "%sysfunc(md5(%str(&salt)),$hex32.)";
+    hashkey=put(md5("&salt"),$hex32.);
     output;
     stop;
   run;
@@ -69,9 +69,14 @@
   %put %str(ERR)OR: Dataset &libds is not a dataset;
 %end;
 %else %do;
-  data &outds(rename=(&keyvar=hashkey) keep=&keyvar)/nonote2err;
+  data &outds(rename=(&keyvar=hashkey) keep=&keyvar)
+  %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+    /nonote2err
+  %end;
+  ;
     length &prevkeyvar &keyvar $32;
-    retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
+    retain &prevkeyvar;
+    if _n_=1 then &prevkeyvar=put(md5("&salt"),$hex32.);
     set &libds end=&lastvar;
     /* hash should include previous row */
     &keyvar=%mp_md5(

base/mp_init.sas

@@ -67,7 +67,7 @@ options
   validvarname=V7         /* avoid special characters etc in variable names */
   varinitchk=%str(ERR)OR  /* avoid data mistakes from variable name typos   */
   varlenchk=%str(ERR)OR   /* fail hard if truncation (data loss) can result */
-%if %substr(&sysver,1,1) ne 4 %then %do;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
   noautocorrect           /* disallow misspelled procedure names            */
   dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
 %end;

base/mp_lockanytable.sas

@@ -22,6 +22,7 @@
   @param [in] loop_secs= (1) Seconds to wait between each lock attempt
 
   <h4> SAS Macros </h4>
+  @li mf_fmtdttm.sas
   @li mp_abort.sas
   @li mp_lockfilecheck.sas
   @li mf_getuser.sas
@@ -111,7 +112,7 @@ run;
       LOCK_LIB ="&lib";
       LOCK_DS="&ds";
       LOCK_STATUS_CD='LOCKED';
-      LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt;
+      LOCK_START_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt;
       LOCK_USER_NM="&user";
       LOCK_PID="&sysjobid";
       LOCK_REF="&ref";
@@ -131,7 +132,7 @@ run;
     proc sql;
     update &ctl_ds
       set LOCK_STATUS_CD='LOCKED'
-        , LOCK_START_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_START_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
         , LOCK_USER_NM="&user"
         , LOCK_PID="&sysjobid"
         , LOCK_REF="&ref"
@@ -206,7 +207,7 @@ run;
     proc sql;
     update &ctl_ds
       set LOCK_STATUS_CD='UNLOCKED'
-        , LOCK_END_DTTM="%sysfunc(datetime(),E8601DT26.6)"dt
+        , LOCK_END_DTTM="%sysfunc(datetime(),%mf_fmtdttm())"dt
         , LOCK_USER_NM="&user"
         , LOCK_PID="&sysjobid"
         , LOCK_REF="&ref"

base/mp_retainedkey.sas

@@ -58,6 +58,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_existvar.sas
+  @li mf_fmtdttm.sas
   @li mf_getquotedstr.sas
   @li mf_getuniquename.sas
   @li mf_nobs.sas
@@ -217,12 +218,12 @@ quit;
     set keytable="&base_libds"
       ,keycolumn="&retained_key"
       ,max_key=%eval(&maxkey+&newkey_cnt)
-      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt;
+      ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt;
   %end;
   %else %do;
   update &maxkeytable
     set max_key=%eval(&maxkey+&newkey_cnt)
-      ,processed_dttm="%sysfunc(datetime(),E8601DT26.6)"dt
+      ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt
     where keytable="&base_libds";
   %end;
   %mp_lockanytable(UNLOCK

base/mp_storediffs.sas

@@ -125,7 +125,7 @@
 data &ds1;
   set &dslist indsname=&inds_auto;
   &hashkey=put(md5(catx('|',%mf_getquotedstr(&key,quote=N))),$hex32.);
-  &inds_keep=&inds_auto;
+  &inds_keep=upcase(&inds_auto);
 proc sort;
   by &inds_keep &hashkey;
 run;
@@ -160,8 +160,8 @@ data &ds4;
   tgtvar_nm=upcase(tgtvar_nm);
   if tgtvar_nm in (%upcase(&vlist));
 
-  if &inds_auto="&ds2" then tgtvar_type='N';
-  else if &inds_auto="&ds3" then tgtvar_type='C';
+  if upcase(&inds_auto)="&ds2" then tgtvar_type='N';
+  else if upcase(&inds_auto)="&ds3" then tgtvar_type='C';
   else do;
     putlog "%str(ERR)OR: unidentified vartype input!" &inds_auto;
     call symputx('syscc',98);

ddl/mddl_dc_difftable.sas

@@ -26,9 +26,19 @@
       oldval_num num format=best32. label='Old (numeric) value',
       newval_num num format=best32. label='New (numeric) value',
       oldval_char char(32765) label='Old (character) value',
-      newval_char char(32765) label='New (character) value',
-    constraint pk_mpe_audit
-      primary key(load_ref,libref,dsn,key_hash,tgtvar_nm)
+      newval_char char(32765) label='New (character) value'
   );
 
+  %local lib;
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create
+      pk_mpe_audit=(load_ref libref dsn key_hash tgtvar_nm)
+      /nomiss unique;
+  quit;
+
 %mend mddl_dc_difftable;

ddl/mddl_dc_filterdetail.sas

@@ -9,19 +9,32 @@
 
 %macro mddl_dc_filterdetail(libds=WORK.FILTER_DETAIL);
 
+%local nn lib;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  %let nn=not null;
+%end;
+%else %let nn=;
+
   proc sql;
   create table &libds(
-      filter_hash char(32) not null,
-      filter_line num not null,
-      group_logic char(3) not null,
-      subgroup_logic char(3) not null,
-      subgroup_id num not null,
-      variable_nm varchar(32) not null,
-      operator_nm varchar(12) not null,
-      raw_value varchar(4000) not null,
-      processed_dttm num not null format=E8601DT26.6,
-    constraint pk_mpe_filteranytable
-      primary key(filter_hash,filter_line)
+      filter_hash char(32) &nn,
+      filter_line num &nn,
+      group_logic char(3) &nn,
+      subgroup_logic char(3) &nn,
+      subgroup_id num &nn,
+      variable_nm varchar(32) &nn,
+      operator_nm varchar(12) &nn,
+      raw_value varchar(4000) &nn,
+      processed_dttm num &nn format=E8601DT26.6
   );
 
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create pk_mpe_filterdetail=(filter_hash filter_line)/nomiss unique;
+  quit;
+
 %mend mddl_dc_filterdetail;

ddl/mddl_dc_filtersummary.sas

@@ -9,14 +9,27 @@
 
 %macro mddl_dc_filtersummary(libds=WORK.FILTER_SUMMARY);
 
+%local nn lib;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  %let nn=not null;
+%end;
+%else %let nn=;
+
   proc sql;
   create table &libds(
-      filter_rk num not null,
-      filter_hash char(32) not null,
-      filter_table char(41) not null,
-      processed_dttm num not null format=E8601DT26.6,
-    constraint pk_mpe_filteranytable
-      primary key(filter_rk)
+      filter_rk num &nn,
+      filter_hash char(32) &nn,
+      filter_table char(41) &nn,
+      processed_dttm num &nn format=E8601DT26.6
   );
 
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create filter_rk /nomiss unique;
+  quit;
+
 %mend mddl_dc_filtersummary;

ddl/mddl_dc_locktable.sas

@@ -9,17 +9,33 @@
 
 %macro mddl_dc_locktable(libds=WORK.LOCKTABLE);
 
+%local nn lib;
+%if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
+  %let nn=not null;
+%end;
+%else %let nn=;
+
   proc sql;
   create table &libds(
       lock_lib char(8),
       lock_ds char(32),
-      lock_status_cd char(10) not null,
-      lock_user_nm char(100) not null ,
+      lock_status_cd char(10) &nn,
+      lock_user_nm char(100) &nn ,
       lock_ref char(200),
       lock_pid char(10),
       lock_start_dttm num format=E8601DT26.6,
-      lock_end_dttm num format=E8601DT26.6,
-    constraint pk_mp_lockanytable primary key(lock_lib,lock_ds)
+      lock_end_dttm num format=E8601DT26.6
   );
 
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create
+      pk_mp_lockanytable=(lock_lib lock_ds)
+      /nomiss unique;
+  quit;
+
 %mend mddl_dc_locktable;

ddl/mddl_dc_maxkeytable.sas

@@ -17,8 +17,17 @@
       max_key num label=
         'Integer representing current max RK or SK value in the KEYTABLE',
       processed_dttm num format=E8601DT26.6
-        label='Datetime this value was last updated',
-    constraint pk_mpe_maxkeyvalues
-        primary key(keytable));
+        label='Datetime this value was last updated'
+  );
+
+  %local lib;
+  %let libds=%upcase(&libds);
+  %if %index(&libds,.)=0 %then %let lib=WORK;
+  %else %let lib=%scan(&libds,1,.);
+
+  proc datasets lib=&lib noprint;
+    modify %scan(&libds,-1,.);
+    index create keytable /nomiss unique;
+  quit;
 
 %mend mddl_dc_maxkeytable;

ddl/mddl_sas_cntlout.sas

@@ -14,8 +14,8 @@ proc sql;
 create table &libds(
     FMTNAME char(32)      label='Format name'
     /*
-      to accomodate larger START values, mp_loadformat.sas will need the
-      SQL dependency removed (proc sql needs to accomodate 3 index values in
+      to accommodate larger START values, mp_loadformat.sas will need the
+      SQL dependency removed (proc sql needs to accommodate 3 index values in
       a 32767 ibufsize limit)
     */
     ,START char(10000)    label='Starting value for format'
@@ -40,4 +40,4 @@ create table &libds(
     ,LANGUAGE char(8)     label='Language for date strings'
 );
 
-%mend mddl_sas_cntlout;
+%mend mddl_sas_cntlout;

main.dox

@@ -92,4 +92,15 @@
         * Auto-generated from the plain source `.lua` files in the same directory
         * Prefixes: _ml_
 
+*/
+
+/*! \dir ddl
+ *  \brief Data Definition Language files
+ *  \details Provides templates for commonly used tables in sasjs/core.
+   Attributes:
+
+        * OS independent
+        * No X command
+        * Prefixes: _mddl_
+
  */

sasjs/sasjsconfig.json

@@ -43,7 +43,7 @@
     },
     {
       "name": "sas9",
-      "serverUrl": "https://sas.analytium.co.uk:8343",
+      "serverUrl": "",
       "serverType": "SAS9",
       "httpsAgentOptions": {
         "allowInsecureRequests": false
@@ -78,7 +78,7 @@
     },
     {
       "name": "docsonly",
-      "serverType": "SAS9",
+      "serverType": "SASJS",
       "appLoc": "dummy",
       "macroFolders": [
         "meta",
@@ -105,4 +105,4 @@
       "contextName": "SAS Job Execution compute context"
     }
   ]
-}
+}

server/ms_createfile.sas

@@ -68,8 +68,8 @@ data _null_;
   file &fname1 lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put "Content-Type: multipart/form-data; boundary=&boundary";
-  put "Authorization: Bearer " _infile_;
+  if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary";
+  put _infile_;
 run;
 
 %if &mdebug=1 %then %do;

server/ms_createuser.sas

@@ -87,9 +87,11 @@ data _null_;
   file &fref1 lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put "Authorization: Bearer " _infile_;
-  put "Content-Type: application/json";
-  put "accept: application/json";
+  if _n_=1 then do;
+    put "Content-Type: application/json";
+    put "accept: application/json";
+  end;
+  put _infile_;
 run;
 
 %if &mdebug=1 %then %do;

server/ms_createwebservice.sas

@@ -390,6 +390,11 @@ data _null_;
   put '%end; ';
   put ' ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
+  put '  %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do; ';
+  put '    /* functions in formats unsupported */ ';
+  put '    %put &sysmacroname: forcing missing back to NULL as feature not supported; ';
+  put '    %let missing=NULL; ';
+  put '  %end; ';
   put '  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref ';
   put '    ,engine=DATASTEP,missing=&missing,showmeta=&showmeta ';
   put '  ) ';

server/ms_deletefile.sas

@@ -33,7 +33,7 @@ data _null_;
   file &headref lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put "Authorization: Bearer " _infile_;
+  put _infile_;
 run;
 
 proc http method='DELETE' headerin=&headref

server/ms_getfile.sas

@@ -34,7 +34,7 @@ data _null_;
   file &headref lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put "Authorization: Bearer " _infile_;
+  put _infile_;
 run;
 
 proc http method='GET' out=&binaryfref headerin=&headref

server/ms_getusers.sas

@@ -56,8 +56,8 @@ data _null_;
   file &fref0 lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put "Authorization: Bearer " _infile_;
-  put "accept: application/json";
+  if _n_=1 then put "accept: application/json";
+  put _infile_;
 run;
 
 %if &mdebug=1 %then %do;

server/ms_runstp.sas

@@ -137,8 +137,8 @@ data _null_;
   file &authref lrecl=1000;
   infile "&_sasjs_tokenfile" lrecl=1000;
   input;
-  put 'Authorization: Bearer ' _infile_;
-  put "Content-Type: multipart/form-data; boundary=&boundary";
+  if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary";
+  put _infile_;
 run;
 
 %if &mdebug=1 %then %do;

server/ms_webout.sas

@@ -100,6 +100,11 @@
 %end;
 
 %else %if &action=ARR or &action=OBJ %then %do;
+  %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then %do;
+    /* functions in formats unsupported */
+    %put &sysmacroname: forcing missing back to NULL as feature not supported;
+    %let missing=NULL;
+  %end;
   %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
     ,engine=DATASTEP,missing=&missing,showmeta=&showmeta
   )

tests/crossplatform/mf_fmtdttm.test.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Testing mf_fmtdttm macro
+
+  <h4> SAS Macros </h4>
+  @li mf_fmtdttm.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%global test1;
+
+%mp_assertscope(SNAPSHOT)
+%let test1=%mf_fmtdttm();
+%mp_assertscope(COMPARE,ignorelist=test1)
+
+%mp_assert(
+  iftrue=("&test1"="DATETIME19.3" or "&test1"="E8601DT26.6"),
+  desc=Basic test,
+  outds=work.test_results
+)

tests/crossplatform/mf_getvarcount.test.sas

@@ -0,0 +1,46 @@
+/**
+  @file
+  @brief Testing mf_getvarlist macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getvarcount.sas
+  @li mp_assertscope.sas
+
+**/
+
+data work.all work.nums(keep=num1 num2) work.chars(keep=char1 char2);
+  length num1 num2 8 char1 char2 char3 $4;
+  call missing (of _all_);
+  output;
+run;
+
+%mp_assertscope(SNAPSHOT)
+%put scope check:%mf_getvarcount(work.all);
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(%mf_getvarcount(work.all)=5),
+  desc=%str(Checking for mixed vars),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_getvarcount(work.all,typefilter=C)=3),
+  desc=%str(Checking for char in mixed vars),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_getvarcount(work.all,typefilter=N)=2),
+  desc=%str(Checking for num in mixed vars),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_getvarcount(work.nums,typefilter=c)=0),
+  desc=%str(Checking for char in num vars),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_getvarcount(work.chars,typefilter=N)=0),
+  desc=%str(Checking for num in char vars),
+  outds=work.test_results
+)
+

tests/crossplatform/mp_filtercheck.test.sas

@@ -6,11 +6,39 @@
   @li mp_filtercheck.sas
   @li mp_assertdsobs.sas
   @li mp_assert.sas
+  @li mp_assertscope.sas
 
 **/
 
+/* set up test data */
+data work.class ;
+length name $8 sex $1 age height weight 8;
+infile cards dsd;
+input Name:$char. Sex :$char. Age Height Weight;
+datalines4;
+Alfred,M,14,69,112.5
+Alice,F,13,56.5,84
+Barbara,F,13,65.3,98
+Carol,F,14,62.8,102.5
+Henry,M,14,63.5,102.5
+James,M,12,57.3,83
+Jane,F,12,59.8,84.5
+Janet,F,15,62.5,112.5
+Jeffrey,M,13,62.5,84
+John,M,12,59,99.5
+Joyce,F,11,51.3,50.5
+Judy,F,14,64.3,90
+Louise,F,12,56.3,77
+Mary,F,15,66.5,112
+Philip,M,16,72,150
+Robert,M,12,64.8,128
+Ronald,M,15,67,133
+Thomas,M,11,57.5,85
+William,M,15,66.5,112
+;;;;
+run;
 
-/* valid filter */
+/* valid filter conditions */
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
@@ -21,14 +49,19 @@ AND,AND,1,SEX,<=,"'M'"
 AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
 AND,OR,2,Weight,>=,77.7
 AND,OR,2,Weight,NE,77.7
+AND,AND,1,age,=,.A
+AND,AND,1,height,<,.B
 ;;;;
 run;
 
+%mp_assertscope(SNAPSHOT)
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )
+%mp_assertscope(COMPARE)
+
 %let syscc=0;
 %mp_assertdsobs(work.badrecords,
   desc=Valid filter query,
@@ -49,7 +82,7 @@ AND,OR,2,Weight,>=,7
 ;;;;
 run;
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )
@@ -71,7 +104,7 @@ AND,OR,2,Name,NOT IN,"(''''Jane','Alfred')"
 run;
 
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )
@@ -94,7 +127,7 @@ AND,OR,2,Weight,>=,7
 run;
 
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )
@@ -115,7 +148,7 @@ AND,AND,1,age,=,;;%abort
 ;;;;
 run;
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )
@@ -137,7 +170,7 @@ AND,AND,1,age,=,0
 run;
 %let syscc=0;
 %mp_filtercheck(work.inds,
-  targetds=sashelp.class,
+  targetds=work.class,
   outds=work.badrecords,
   abort=NO
 )

tests/crossplatform/mp_filtergenerate.test.sas

@@ -11,6 +11,34 @@
 
 options source2;
 
+/* set up test data */
+data work.class ;
+length name $8 sex $1 age height weight 8;
+infile cards dsd;
+input Name:$char. Sex :$char. Age Height Weight;
+datalines4;
+Alfred,M,14,69,112.5
+Alice,F,13,56.5,84
+Barbara,F,13,65.3,98
+Carol,F,14,62.8,102.5
+Henry,M,14,63.5,102.5
+James,M,12,57.3,83
+Jane,F,12,59.8,84.5
+Janet,F,15,62.5,112.5
+Jeffrey,M,13,62.5,84
+John,M,12,59,99.5
+Joyce,F,11,51.3,50.5
+Judy,F,14,64.3,90
+Louise,F,12,56.3,77
+Mary,F,15,66.5,112
+Philip,M,16,72,150
+Robert,M,12,64.8,128
+Ronald,M,15,67,133
+Thomas,M,11,57.5,85
+William,M,15,66.5,112
+;;;;
+run;
+
 /* valid filter */
 data work.inds;
   infile datalines4 dsd;
@@ -23,10 +51,10 @@ AND,OR,2,Name,NOT IN,"('Jane','Janet')"
 AND,OR,2,Weight,>=,84.6
 ;;;;
 run;
-%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtercheck(work.inds,targetds=work.class)
 %mp_filtergenerate(work.inds,outref=myfilter)
 data work.test;
-  set sashelp.class;
+  set work.class;
   where %inc myfilter;;
 run;
 %mp_assertdsobs(work.test,
@@ -43,10 +71,10 @@ data work.inds;
 datalines4;
 ;;;;
 run;
-%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtercheck(work.inds,targetds=work.class)
 %mp_filtergenerate(work.inds,outref=myfilter)
 data work.test;
-  set sashelp.class;
+  set work.class;
   where %inc myfilter;;
 run;
 %mp_assertdsobs(work.test,
@@ -64,10 +92,10 @@ datalines4;
 AND,OR,2,Name,IN,"('Jane','Janet')"
 ;;;;
 run;
-%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtercheck(work.inds,targetds=work.class)
 %mp_filtergenerate(work.inds,outref=myfilter)
 data work.test;
-  set sashelp.class;
+  set work.class;
   where %inc myfilter;;
 run;
 %mp_assertdsobs(work.test,
@@ -86,10 +114,10 @@ OR,OR,2,Name,IN,"('Jane','Janet')"
 OR,OR,3,Name,IN,"('James')"
 ;;;;
 run;
-%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtercheck(work.inds,targetds=work.class)
 %mp_filtergenerate(work.inds,outref=myfilter)
 data work.test;
-  set sashelp.class;
+  set work.class;
   where %inc myfilter;;
 run;
 %mp_assertdsobs(work.test,
@@ -108,10 +136,10 @@ AND,OR,2,Name,IN,"('Jane','Janet')"
 AND,OR,3,Name,IN,"('James')"
 ;;;;
 run;
-%mp_filtercheck(work.inds,targetds=sashelp.class)
+%mp_filtercheck(work.inds,targetds=work.class)
 %mp_filtergenerate(work.inds,outref=myfilter)
 data work.test;
-  set sashelp.class;
+  set work.class;
   where %inc myfilter;;
 run;
 %mp_assertdsobs(work.test,

tests/crossplatform/mp_filterstore.test.1.sas

@@ -10,6 +10,34 @@
 
 **/
 
+/* set up test data */
+data work.class ;
+length name $8 sex $1 age height weight 8;
+infile cards dsd;
+input Name:$char. Sex :$char. Age Height Weight;
+datalines4;
+Alfred,M,14,69,112.5
+Alice,F,13,56.5,84
+Barbara,F,13,65.3,98
+Carol,F,14,62.8,102.5
+Henry,M,14,63.5,102.5
+James,M,12,57.3,83
+Jane,F,12,59.8,84.5
+Janet,F,15,62.5,112.5
+Jeffrey,M,13,62.5,84
+John,M,12,59,99.5
+Joyce,F,11,51.3,50.5
+Judy,F,14,64.3,90
+Louise,F,12,56.3,77
+Mary,F,15,66.5,112
+Philip,M,16,72,150
+Robert,M,12,64.8,128
+Ronald,M,15,67,133
+Thomas,M,11,57.5,85
+William,M,15,66.5,112
+;;;;
+run;
+
 libname permlib (work);
 
 %mp_coretable(LOCKTABLE,libds=permlib.locktable)
@@ -31,7 +59,7 @@ AND,OR,2,Weight,NE,77.7
 ;;;;
 run;
 
-%mp_filterstore(libds=sashelp.class,
+%mp_filterstore(libds=work.class,
   queryds=work.inds,
   filter_summary=permlib.filtsum,
   filter_detail=permlib.filtdet,
@@ -60,7 +88,7 @@ select max(filter_rk) into: test1 from work.result;
 )
 
 /* Test 2 - load same table again and ensure we get the same RK */
-%mp_filterstore(libds=sashelp.class,
+%mp_filterstore(libds=work.class,
   queryds=work.inds,
   filter_summary=permlib.filtsum,
   filter_detail=permlib.filtdet,

tests/crossplatform/mp_filtervalidate.test.sas

@@ -7,9 +7,36 @@
   @li mp_filtervalidate.sas
   @li mp_assertdsobs.sas
   @li mp_assert.sas
+  @li mp_assertscope.sas
 
 **/
 
+data work.class ;
+length name $8 sex $1 age height weight 8;
+infile cards dsd;
+input Name:$char. Sex :$char. Age Height Weight;
+datalines4;
+Alfred,M,14,69,112.5
+Alice,F,13,56.5,84
+Barbara,F,13,65.3,98
+Carol,F,14,62.8,102.5
+Henry,M,14,63.5,102.5
+James,M,12,57.3,83
+Jane,F,12,59.8,84.5
+Janet,F,15,62.5,112.5
+Jeffrey,M,13,62.5,84
+John,M,12,59,99.5
+Joyce,F,11,51.3,50.5
+Judy,F,14,64.3,90
+Louise,F,12,56.3,77
+Mary,F,15,66.5,112
+Philip,M,16,72,150
+Robert,M,12,64.8,128
+Ronald,M,15,67,133
+Thomas,M,11,57.5,85
+William,M,15,66.5,112
+;;;;
+run;
 
 /* valid filter */
 data work.inds;
@@ -21,10 +48,16 @@ AND,AND,1,AGE,>,5
 AND,AND,1,SEX,NE,"'M'"
 AND,OR,2,Name,NOT IN,"('Jane','Janet')"
 AND,OR,2,Weight,>=,84.6
+AND,AND,3,age,=,.a
+AND,AND,4,weight,NE,._
 ;;;;
 run;
 %mp_filtergenerate(work.inds,outref=myfilter)
-%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+
+%mp_assertscope(SNAPSHOT)
+%mp_filtervalidate(myfilter,work.class,outds=work.results,abort=NO)
+%mp_assertscope(COMPARE)
+
 %mp_assertdsobs(work.results,
   desc=Valid filter,
   test=EMPTY,
@@ -40,7 +73,7 @@ datalines4;
 ;;;;
 run;
 %mp_filtergenerate(work.inds,outref=myfilter)
-%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_filtervalidate(myfilter,work.class,outds=work.results,abort=NO)
 %mp_assertdsobs(work.results,
   desc=Empty filter,
   test=EMPTY,
@@ -59,7 +92,7 @@ AND,AND,1,SEX,NE,2
 ;;;;
 run;
 %mp_filtergenerate(work.inds,outref=myfilter)
-%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_filtervalidate(myfilter,work.class,outds=work.results,abort=NO)
 %let syscc=0;
 %let test3=0;
 data _null_;
@@ -84,7 +117,7 @@ AND,AND,1,age,NE,"'M'"
 ;;;;
 run;
 %mp_filtergenerate(work.inds,outref=myfilter)
-%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%mp_filtervalidate(myfilter,work.class,outds=work.results,abort=NO)
 %let syscc=0;
 %let test4=0;
 data _null_;

tests/crossplatform/mp_getconstraints.test.sas

@@ -6,24 +6,52 @@
   @li mf_nobs.sas
   @li mp_getconstraints.sas
   @li mp_assert.sas
+  @li mp_assertscope.sas
 
 **/
 
-proc sql;
-create table work.example(
-  TX_FROM float format=datetime19.,
-  DD_TYPE char(16),
-  DD_SOURCE char(2048),
-  DD_SHORTDESC char(256),
-  constraint pk primary key(tx_from, dd_type,dd_source),
-  constraint unq unique(tx_from, dd_type),
-  constraint nnn not null(DD_SHORTDESC)
-);
 
-%mp_getconstraints(lib=work,ds=example,outds=work.constraints)
+%macro conditional();
 
-%mp_assert(
-  iftrue=(%mf_nobs(work.constraints)=6),
-  desc=Output table work.constraints created with correct number of records,
-  outds=work.test_results
-)
+%if %sysfunc(exist(sashelp.vcncolu,view))=1 %then %do;
+  proc sql;
+  create table work.example(
+    TX_FROM float format=datetime19.,
+    DD_TYPE char(16),
+    DD_SOURCE char(2048),
+    DD_SHORTDESC char(256),
+    constraint pk primary key(tx_from, dd_type,dd_source),
+    constraint unq unique(tx_from, dd_type),
+    constraint nnn not null(DD_SHORTDESC)
+  );
+  %mp_assertscope(SNAPSHOT)
+  %mp_getconstraints(lib=work,ds=example,outds=work.constraints)
+  %mp_assertscope(COMPARE)
+
+  %mp_assert(
+    iftrue=(%mf_nobs(work.constraints)=6),
+    desc=Output table work.constraints created with correct number of records,
+    outds=work.test_results
+  )
+%end;
+%else %do;
+  proc sql;
+  create table work.example(
+    TX_FROM float format=datetime19.,
+    DD_TYPE char(16),
+    DD_SOURCE char(2048),
+    DD_SHORTDESC char(256)
+  );
+  %mp_assertscope(SNAPSHOT)
+  %mp_getconstraints(lib=work,ds=example,outds=work.constraints)
+  %mp_assertscope(COMPARE)
+
+  %mp_assert(
+    iftrue=(%mf_nobs(work.constraints)=0),
+    desc=Empty table created as constraints not supported,
+    outds=work.test_results
+  )
+%end;
+%mend conditional;
+
+%conditional()

tests/crossplatform/mp_getmaxvarlengths.test.sas

@@ -10,10 +10,48 @@
 
 **/
 
+data work.class ;
+attrib
+Name                             length= $8
+Sex                              length= $1
+Age                              length= 8
+Height                           length= 8
+Weight                           length= 8
+;
+infile cards dsd;
+input
+  Name                             :$char.
+  Sex                              :$char.
+  Age
+  Height
+  Weight
+;
+datalines4;
+Alfred,M,14,69,112.5
+Alice,F,13,56.5,84
+Barbara,F,13,65.3,98
+Carol,F,14,62.8,102.5
+Henry,M,14,63.5,102.5
+James,M,12,57.3,83
+Jane,F,12,59.8,84.5
+Janet,F,15,62.5,112.5
+Jeffrey,M,13,62.5,84
+John,M,12,59,99.5
+Joyce,F,11,51.3,50.5
+Judy,F,14,64.3,90
+Louise,F,12,56.3,77
+Mary,F,15,66.5,112
+Philip,M,16,72,150
+Robert,M,12,64.8,128
+Ronald,M,15,67,133
+Thomas,M,11,57.5,85
+William,M,15,66.5,112
+;;;;
+run;
 
 /* regular usage */
 %mp_assertscope(SNAPSHOT)
-%mp_getmaxvarlengths(sashelp.class,outds=work.myds)
+%mp_getmaxvarlengths(work.class,outds=work.myds)
 %mp_assertscope(COMPARE,desc=checking scope leakage on mp_getmaxvarlengths)
 %mp_assert(
   iftrue=(&syscc=0),