fix: removing old email address

385 support nldatm in mp getcols

.github/vpn/config.ovpn

@@ -3,10 +3,12 @@ client
 tls-client
 dev tun
 # this will connect with whatever proto DNS tells us (https://community.openvpn.net/openvpn/ticket/934)
-proto tcp
-remote vpn.4gl.io 7494
+proto udp
+remote vpn.4gl.io 7194
 resolv-retry infinite
-cipher AES-256-CBC
+# this will fallback from udp6 to udp4 as well
+connect-timeout 5
+data-ciphers AES-256-CBC:AES-256-GCM
 auth SHA256
 script-security 2
 keepalive 10 120

.github/workflows/run-tests.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
@@ -34,6 +34,10 @@ jobs:
           USER_KEY: ${{ secrets.USER_KEY }}
           TLS_KEY: ${{ secrets.TLS_KEY }}
 
+      - name: Chmod VPN files
+        run: |
+          chmod 600 .github/vpn/ca.crt .github/vpn/user.crt .github/vpn/user.key .github/vpn/tls.key
+
       - name: Install Open VPN
         run: |
           sudo apt install apt-transport-https
@@ -42,8 +46,13 @@ jobs:
           sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
           sudo apt update
           sudo apt install openvpn3=17~betaUb22042+jammy
+
       - name: Start Open VPN 3
         run: openvpn3 session-start --config .github/vpn/config.ovpn
+
+      - name: Fetch SASJS server
+        run: curl ${{ secrets.SASJS_SERVER_URL }}/SASjsApi/info
+
       - name: Install Doxygen
         run: sudo apt-get install doxygen
 

all.sas

@@ -4861,7 +4861,7 @@ data;run;
 data &out_ds(compress=no
     keep=file_or_folder filepath filename ext msg directory level
   );
-  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
+  length directory filepath $2000 fref fref2 $8 file_or_folder $6 filename $255
     ext $20 msg $200 foption $16;
   if _n_=1 then call missing(of _all_);
   retain level &level;
@@ -7157,10 +7157,11 @@ data &outds(keep=name type length varnum format label ddtype fmtname);
     else if formatd=0 then format=cats(fmtname,formatl,'.');
     else format=cats(fmtname,formatl,'.',formatd);
     type='N';
-    if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
+    if format=:'DATETIME' or format=:'E8601DT' or format=:'NLDATM'
+    then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
       or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
-      or format=:'MONYY'
+      or format=:'MONYY' or format=:'NLDATE'
     then ddtype='DATE';
     else if format=:'TIME' then ddtype='TIME';
     else ddtype='NUMERIC';
@@ -9675,6 +9676,7 @@ options
 %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
   noautocorrect           /* disallow misspelled procedure names            */
   dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
+                          /* turn off with dsoptions=nonote2err             */
 %end;
 ;
 
@@ -12475,6 +12477,7 @@ run;
   @li mp_coretable.sas
   @li mp_stackdiffs.test.sas
   @li mp_storediffs.sas
+  @li mp_stripdiffs.sas
 
   @todo The current approach assumes that a variable called KEY_HASH is not on
     the base table.  This part will need to be refactored (eg using
@@ -12933,6 +12936,7 @@ select distinct tgtvar_nm into: missvars separated by ' '
   <h4> Related Macros </h4>
   @li mp_stackdiffs.sas
   @li mp_storediffs.test.sas
+  @li mp_stripdiffs.sas
 
   @version 9.2
   @author Allan Bowe
@@ -13052,7 +13056,7 @@ data &ds4;
 run;
 
 %if "&loadref"="0" %then %let loadref=%sysfunc(uuidgen());
-%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime());
+%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime(),8.6);
 %let libds=%upcase(&libds);
 
 /* join orig vals for modified & deleted */
@@ -13382,6 +13386,260 @@ run;
 
 %mend mp_streamfile;
 /**
+  @file
+  @brief Generates a stage dataset to revert diffs tracked in an audit table
+  @details A big benefit of tracking data changes in an audit table is that
+    those changes can be subsequently reverted if necessary!
+
+    This macro prepares a staging dataset containing those differences - eg for:
+
+    @li deleted rows - these are re-inserted
+    @li changed rows - differences are reverted
+    @li added rows - marked with `_____DELETE__THIS__RECORD_____="YES"`
+
+    These changes are NOT applied to the base table - a staging dataset is
+    simply prepared for an ETL process to action.  In Data Controller, this
+    dataset is used directly as an input to the APPROVE process (so that the
+    reversion diffs can be reviewed prior to being applied).
+
+
+  @param [in] libds Base library.dataset (will not be modified).  The library
+    must be assigned.
+  @param [in] loadref Unique identifier for the version to be reverted.  This
+    change, plus ALL SUBSEQUENT CHANGES, will be reverted in the output table.
+  @param [in] difftable The dataset containing the diffs.  Definition available
+    in mddl_dc_difftable.sas
+  @param [in] filtervar= (0) If provided, the contents of this macro variable
+    will be applied as an additional filter against &libds
+  @param [out] outds= (work.mp_stripdiffs) Output table containing the diffs.
+    Has the same format as the base datset, plus a
+    `_____DELETE__THIS__RECORD_____` variable.
+  @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
+  @li mf_islibds.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_abort.sas
+
+  <h4> Related Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mp_stackdiffs.sas
+  @li mp_storediffs.sas
+  @li mp_stripdiffs.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mp_stripdiffs(libds
+  ,loadref
+  ,difftable
+  ,filtervar=0
+  ,outds=work.mp_stripdiffs
+  ,mdebug=0
+)/*/STORE SOURCE*/;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+%let libds=%upcase(&libds);
+
+/* safety checks */
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc on entry.  Clean session required!)
+)
+%let libds=%upcase(&libds);
+%mp_abort(iftrue= (%mf_islibds(&libds)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid library.dataset reference - %superq(libds))
+)
+
+/* set up unique and temporary vars */
+%local ds1 ds2 ds3 ds4 ds5 fref1 filterstr;
+%let fref1=%mf_getuniquefileref();
+%if &filtervar ne 0 %then %let filterstr=%superq(&filtervar);
+%else %let filterstr=%str(1=1);
+
+/* get timestamp of the diff to be reverted */
+%local ts;
+proc sql noprint;
+select put(processed_dttm,datetime19.6) into: ts
+  from &difftable where load_ref="&loadref";
+%mp_abort(iftrue= (&sqlobs=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Load ref %superq(loadref) not found!)
+)
+
+/* extract diffs for this base table from this timestamp onwards */
+%let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_diffs));
+create table &ds1 (drop=libref dsn) as
+  select * from &difftable
+  where upcase(cats(libref))="%scan(&libds,1,.)"
+  and upcase(cats(dsn))="%scan(&libds,2,.)"
+  and processed_dttm ge "&ts"dt
+  order by processed_dttm desc, key_hash, is_pk;
+
+/* extract key values only */
+%let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
+%local keyhash processed;
+%let keyhash=%upcase(%mf_getuniquename(prefix=mpsdvar_keyhash));
+%let processed=%upcase(%mf_getuniquename(prefix=mpsdvar_processed));
+create table &ds2 as
+  select key_hash as &keyhash,
+    tgtvar_nm,
+    tgtvar_type,
+    coalescec(oldval_char,newval_char) as charval,
+    coalesce(oldval_num, newval_num) as numval,
+    processed_dttm as &processed
+  from &ds1
+  where is_pk=1
+  order by &keyhash, &processed;
+
+/* grab pk values */
+%local pk;
+select distinct upcase(tgtvar_nm) into: pk separated by ' ' from &ds2;
+
+%let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
+proc transpose data=&ds2(where=(tgtvar_type='C'))
+    out=&ds3(drop=_name_);
+  by &keyhash &processed;
+  id TGTVAR_NM;
+  var charval;
+run;
+
+%let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
+proc transpose data=&ds2(where=(tgtvar_type='N'))
+    out=&ds4(drop=_name_);
+  by &keyhash &processed;
+  id TGTVAR_NM;
+  var numval;
+run;
+/* shorten the lengths */
+%mp_ds2squeeze(&ds3,outds=&ds3)
+%mp_ds2squeeze(&ds4,outds=&ds4)
+
+/* now merge to get all key values and de-dup */
+%let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
+data &ds5;
+  length &keyhash $32 &processed 8;
+  merge &ds3 &ds4;
+  by &keyhash &processed;
+  if not missing(&keyhash);
+run;
+proc sort data=&ds5 nodupkey;
+  by &pk;
+run;
+
+/* join to base table for preliminary stage DS */
+proc sql;
+create table &outds as select "No " as _____DELETE__THIS__RECORD_____
+  %do x=1 %to %sysfunc(countw(&pk,%str( )));
+    ,a.%scan(&pk,&x,%str( ))
+  %end;
+  %local notpkcols;
+  %let notpkcols=%upcase(%mf_getvarlist(&libds));
+  %let notpkcols=%mf_wordsinstr1butnotstr2(str1=&notpkcols,str2=&pk);
+  %do x=1 %to %sysfunc(countw(&notpkcols,%str( )));
+    ,b.%scan(&notpkcols,&x,%str( ))
+  %end;
+  from &ds5 a
+  left join &libds (where=(&filterstr)) b
+  on 1=1
+%do x=1 %to %sysfunc(countw(&pk,%str( )));
+  and a.%scan(&pk,&x,%str( ))=b.%scan(&pk,&x,%str( ))
+%end;
+;
+
+/* create SAS code to apply to stage_ds */
+data _null_;
+  set &ds1;
+  file &fref1 lrecl=33000;
+  length charval $32767;
+  if _n_=1 then put 'proc sql noprint;';
+  by descending processed_dttm key_hash is_pk;
+  if move_type='M' then do;
+    if first.key_hash then do;
+      put "update &outds set " @@;
+    end;
+    if IS_PK=0 then do;
+      put "  " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+      if not last.is_pk then put ',';
+    end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+  end;
+  else if move_type='A' then do;
+    if first.key_hash then do;
+      put "update &outds set _____DELETE__THIS__RECORD_____='Yes' where 1=1 "@@;
+    end;
+    /* gating if - as only need PK now */
+    if is_pk=1;
+    put '  AND ' tgtvar_nm '=' @@;
+    cnt=count(newval_char,'"');
+    charval=quote(trim(substr(newval_char,1,32765-cnt)));
+    if tgtvar_type='C' then put charval @@;
+    else put newval_num @@;
+  end;
+  else if move_type='D' then do;
+    if first.key_hash then do;
+      put "update &outds set _____DELETE__THIS__RECORD_____='No' " @@;
+    end;
+    if IS_PK=0 then do;
+      put "  ," tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+  end;
+  if last.key_hash then put ';';
+run;
+
+/* apply the modification statements */
+%inc &fref1/source2 lrecl=33000;
+
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &ds1, &ds2, &ds3, &ds4, &ds5;
+  file &fref1 clear;
+%end;
+%else %do;
+  data _null_;
+    infile &fref1;
+    input;
+    if _n_=1 then putlog "Contents of SQL adjustments";
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_stripdiffs;
+/** @endcond *//**
   @file
   @brief Runs arbitrary code for a specified amount of time
   @details Executes a series of procs and data steps to enable performance
@@ -14518,7 +14776,8 @@ filename __us2grp clear;
 
 %local cur_engine;
 %let cur_engine=%mf_getengine(&libref);
-%if &cur_engine ne META and &cur_engine ne %then %do;
+%if &cur_engine ne META and &cur_engine ne and %length(&open_passthrough)=0
+%then %do;
   %put NOTE:  &libref already has a direct (&cur_engine) libname connection;
   %return;
 %end;
@@ -17428,22 +17687,25 @@ run;
 %mend mm_getcols;/**
   @file mm_getdetails.sas
   @brief extracts metadata attributes and associations for a particular uri
-
   @param [in] uri the metadata object for which to return
     attributes / associations
+  @param [in] sortoptions= Enables sorting of the output datasets, for example,
+    `SORTSEQ=LINGUISTIC`
   @param [out] outattrs= (work.attributes)
     The dataset to create that contains the list of attributes
   @param [out] outassocs= (work.associations)
     The dataset to contain the list of associations
 
-  @version 9.2
-  @author Allan Bowe
+  <h4> Related Files </h4>
+  @li mm_getobjects.sas
+  @li mm_gettypes.sas
 
 **/
 
 %macro mm_getdetails(uri
   ,outattrs=work.attributes
   ,outassocs=work.associations
+  ,sortoptions=
 )/*/STORE SOURCE*/;
 
 data &outassocs;
@@ -17468,7 +17730,7 @@ data &outassocs;
     n1+1;
   end;
 run;
-proc sort;
+proc sort &sortoptions;
   by assoc name;
 run;
 
@@ -17488,7 +17750,7 @@ data &outattrs;
     n1+1;
   end;
 run;
-proc sort;
+proc sort &sortoptions;
   by type name;
 run;
 
@@ -17973,10 +18235,11 @@ run;
 
   @param [in] user= the metadata user to return groups for.  Leave blank for all
     groups.
-  @param [in] repo= the metadata repository that contains the user/group
-    information
-  @param [in] mDebug= set to 1 to show debug messages in the log
-  @param [out] outds= the dataset to create that contains the list of groups
+  @param [in] repo= (foundation) the metadata repository that contains the
+    user/group information
+  @param [in] mDebug= (0) set to 1 to show debug messages in the log
+  @param [out] outds= (work.mm_getgroups) The dataset to create that contains
+    the list of groups
 
   @returns outds  dataset containing all groups in a column named "metagroup"
     - groupuri
@@ -22856,6 +23119,236 @@ run;
 
 %mend ms_testservice;
 /**
+  @file
+  @brief Triggers a SASjs Server STP using the /SASjsApi/stp/trigger endpoint
+  @details Triggers the STP and returns the sessionId
+
+  Example:
+
+      %ms_triggerstp(/some/stored/program
+        ,debug=131
+        ,outds=work.myresults
+      )
+
+  @param [in] pgm The full path to the Stored Program in SASjs Drive (_program
+    parameter)
+  @param [in] debug= (131) The value to supply to the _debug URL parameter
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] inputparams=(_null_) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$10000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+  @param [in] inputfiles= (_null_) A dataset containing fileref/name/filename in
+    the following format:
+    |fileref:$8|name:$32|filename:$256|
+    |---|---|--|
+    |someref|some_name|some_filename.xls|
+    |fref2|another_file|zyx_v2.csv|
+  @param [in] expiresaftermins= (15) The number of minutes to retain the session
+    folder after the session ends.
+
+  @param [out] outds= (work.ms_triggerstp) Set to the name of a dataset to
+  contain the sessionId. If this dataset already exists, and contains the
+  sessionId, it will be appended to.
+  Format:
+    |sessionId:$36|
+    |---|
+    |20241028074744-54132-1730101664824|
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+  @li mp_dropmembers.sas
+  @li mf_nobs.sas
+
+**/
+
+%macro ms_triggerstp(pgm
+    ,debug=131
+    ,inputparams=_null_
+    ,inputfiles=_null_
+    ,expiresAfterMins=15
+    ,outds=work.ms_triggerstp
+    ,mdebug=0
+  );
+  %local dbg mainref authref boundary libref triggered_sid;
+  %let mainref=%mf_getuniquefileref();
+  %let authref=%mf_getuniquefileref();
+  %let boundary=%mf_getuniquename();
+  %if &inputparams=0 %then %let inputparams=_null_;
+
+  %if &mdebug=1 %then %do;
+    %put &sysmacroname entry vars:;
+    %put _local_;
+  %end;
+  %else %let dbg=*;
+
+
+  %mp_abort(iftrue=("&pgm"="")
+    ,mac=&sysmacroname
+    ,msg=%str(Program not provided)
+  )
+  %mp_abort(iftrue=("&outds"="")
+    ,mac=&sysmacroname
+    ,msg=%str(Output dataset not provided)
+  )
+
+  /* avoid sending bom marker to API */
+  %local optval;
+  %let optval=%sysfunc(getoption(bomfile));
+  options nobomfile;
+
+  /* Add params to the content */
+  data _null_;
+    file &mainref termstr=crlf lrecl=32767 mod;
+    length line $1000 name $32 value $32767;
+    if _n_=1 then call missing(of _all_);
+    set &inputparams;
+    put "--&boundary";
+    line=cats('Content-Disposition: form-data; name="',name,'"');
+    put line;
+    put ;
+    put value;
+  run;
+
+  /* parse input file list */
+  %local webcount;
+  %let webcount=0;
+  data _null_;
+    set &inputfiles end=last;
+    length fileref $8 name $32 filename $256;
+    call symputx(cats('webref',_n_),fileref,'l');
+    call symputx(cats('webname',_n_),name,'l');
+    call symputx(cats('webfilename',_n_),filename,'l');
+    if last then do;
+      call symputx('webcount',_n_);
+      call missing(of _all_);
+    end;
+  run;
+
+  /* write out the input files to the content */
+  %local i;
+  %do i=1 %to &webcount;
+    data _null_;
+      file &mainref termstr=crlf lrecl=32767 mod;
+      infile &&webref&i lrecl=32767;
+      if _n_ = 1 then do;
+        length line $32767;
+        line=cats(
+          'Content-Disposition: form-data; name="'
+          ,"&&webname&i"
+          ,'"; filename="'
+          ,"&&webfilename&i"
+          ,'"'
+        );
+        put "--&boundary";
+        put line;
+        put "Content-Type: text/plain";
+        put ;
+      end;
+      input;
+      put _infile_; /* add the actual file to be sent */
+    run;
+  %end;
+
+  /* Add footer to the content */
+  data _null_;
+    file &mainref termstr=crlf mod;
+    put / "--&boundary--";
+  run;
+
+  data _null_;
+    file &authref lrecl=1000;
+    infile "&_sasjs_tokenfile" lrecl=1000;
+    input;
+    if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary";
+    put _infile_;
+  run;
+
+  %if &mdebug=1 %then %do;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** authref=&authref content *****";
+      infile &authref;
+      input;
+      put _infile_;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** mainref=&mainref content *****";
+      infile &mainref;
+      input;
+      put _infile_;
+    run;
+  %end;
+
+  %local resp_path outref;
+  %let resp_path=%sysfunc(pathname(work))/%mf_getuniquename();
+  %let outref=%mf_getuniquefileref();
+  filename &outref "&resp_path" lrecl=32767;
+
+  /* prepare request*/
+  proc http method='POST' headerin=&authref in=&mainref out=&outref
+    url="&_sasjs_apiserverurl/SASjsApi/stp/trigger?%trim(
+      )_program=&pgm%str(&)_debug=131%str(&)expiresAfterMins=&expiresaftermins";
+  %if &mdebug=1 %then %do;
+    debug level=2;
+  %end;
+  run;
+
+  %if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  or &mdebug=1
+  %then %do;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** outref=&outref content *****";
+      infile &outref;
+      input;
+      putlog _infile_;
+    run;
+  %end;
+  %mp_abort(
+    iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200
+        and &SYS_PROCHTTP_STATUS_CODE ne 201)
+    ,mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+
+  /* reset options */
+  options &optval;
+
+  %let libref=%mf_getuniquelibref();
+  libname &libref JSON fileref=&outref;
+  %let triggered_sid=%mf_getuniquename(prefix=triggered_sid_);
+
+  data work.&triggered_sid (keep=sessionid);
+    set &libref..root;
+
+    %if &mdebug=1 %then %do;
+      putlog (_all_)(=);
+    %end;
+  run;
+
+  %if %mf_nobs(work.&triggered_sid)>0 %then %do;
+    proc append base=&outds data=work.&triggered_sid;
+    run;
+  %end;
+
+  %if &mdebug=1 %then %do;
+    %put &sysmacroname exit vars:;
+    %put _local_;
+  %end;
+  %else %do;
+    /* clear refs */
+    filename &authref;
+    filename &mainref;
+    filename &outref;
+    libname &libref clear;
+    /* and remove temp dataset */
+    %mp_dropmembers(&triggered_sid,libref=work);
+  %end;
+
+%mend ms_triggerstp;/**
   @file
   @brief Send data to/from sasjs/server
   @details This macro should be added to the start of each web service,
@@ -29532,6 +30025,103 @@ Usage:
 
 %mend mx_getcode;
 /**
+  @file
+  @brief Fetches all groups or the groups for a particular member
+  @details  When building applications that run on multiple flavours of SAS, it
+  is convenient to use a single macro (like this one) to fetch the groups
+  regardless of the flavour of SAS being used
+
+  The alternative would be to compile a generic macro in target-specific
+  folders (SASVIYA, SAS9 and SASJS).  This avoids compiling unnecessary macros
+  at the expense of a more complex sasjsconfig.json setup.
+
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] user= (0) Provide the username on which to filter
+  @param [in] uid= (0) Provide the userid on which to filter
+  @param [in] repo= (foundation) SAS9 only, choose the metadata repo to query
+  @param [in] access_token_var= (ACCESS_TOKEN) VIYA only.
+    The global macro variable to contain the access token
+  @param [in] grant_type= (sas_services) VIYA only.
+    Valid values are "password" or "authorization_code" (unquoted).
+  @param [out] outds= (work.mx_getgroups) This output dataset will contain the
+    list of groups. Format:
+|GROUPNAME:$32.|GROUPDESC:$256.|GROUPURI:best.|
+|---|---|---|
+|`SomeGroup `|`A group `|`1`|
+|`Another Group`|`this is a different group`|`2`|
+|`admin`|`Administrators `|`3`|
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mm_getgroups.sas
+  @li ms_getgroups.sas
+  @li mv_getgroups.sas
+  @li mv_getusergroups.sas
+
+**/
+
+%macro mx_getgroups(
+  mdebug=0,
+  user=0,
+  uid=0,
+  repo=foundation,
+  access_token_var=ACCESS_TOKEN,
+  grant_type=sas_services,
+  outds=work.mx_getgroups
+)/*/STORE SOURCE*/;
+%local platform name shortloc;
+%let platform=%mf_getplatform();
+
+%if &platform=SASJS %then %do;
+  %ms_getgroups(
+    user=&user,
+    uid=&uid,
+    outds=&outds,
+    mdebug=&mdebug
+  )
+  data &outds;
+    length groupuri groupname $32 groupdesc $128 ;
+    set &outds;
+    keep groupuri groupname groupdesc;
+    groupuri=cats(groupid);
+    groupname=name;
+    groupdesc=description;
+  run;
+  proc sort; by groupname; run;
+%end;
+%else %if &platform=SAS9 or &platform=SASMETA %then %do;
+  %if &user=0 %then %let user=;
+  %mm_getGroups(
+    user=&user
+    ,outds=&outds
+    ,repo=&repo
+    ,mDebug=&mdebug
+  )
+  proc sort data=&outds; by groupname; run;
+%end;
+%else %if &platform=SASVIYA %then %do;
+  %if &user=0 %then %do;
+    %mv_getgroups(access_token_var=&access_token_var
+      ,grant_type=&grant_type
+      ,outds=&outds
+    )
+  %end;
+  %else %do;
+    %mv_getusergroups(&user
+      ,outds=&outds
+      ,access_token_var=&access_token_var
+      ,grant_type=&grant_type
+    )
+  %end;
+  proc sort
+      data=&outds(rename=(id=groupuri name=groupname description=groupdesc))
+      out=&outds (keep=groupuri groupname groupdesc);
+    by groupname;
+  run;
+%end;
+
+%mend mx_getgroups;/**
   @file
   @brief Will execute a SASjs web service on SAS 9, Viya or SASjs Server
   @details Prepares the input files and retrieves the resulting datasets from

base/mp_dirlist.sas

@@ -85,7 +85,7 @@ data;run;
 data &out_ds(compress=no
     keep=file_or_folder filepath filename ext msg directory level
   );
-  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
+  length directory filepath $2000 fref fref2 $8 file_or_folder $6 filename $255
     ext $20 msg $200 foption $16;
   if _n_=1 then call missing(of _all_);
   retain level &level;

base/mp_getcols.sas

@@ -53,10 +53,11 @@ data &outds(keep=name type length varnum format label ddtype fmtname);
     else if formatd=0 then format=cats(fmtname,formatl,'.');
     else format=cats(fmtname,formatl,'.',formatd);
     type='N';
-    if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
+    if format=:'DATETIME' or format=:'E8601DT' or format=:'NLDATM'
+    then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
       or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
-      or format=:'MONYY'
+      or format=:'MONYY' or format=:'NLDATE'
     then ddtype='DATE';
     else if format=:'TIME' then ddtype='TIME';
     else ddtype='NUMERIC';

base/mp_init.sas

@@ -70,6 +70,7 @@ options
 %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
   noautocorrect           /* disallow misspelled procedure names            */
   dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
+                          /* turn off with dsoptions=nonote2err             */
 %end;
 ;
 

base/mp_stackdiffs.sas

@@ -197,6 +197,7 @@
   @li mp_coretable.sas
   @li mp_stackdiffs.test.sas
   @li mp_storediffs.sas
+  @li mp_stripdiffs.sas
 
   @todo The current approach assumes that a variable called KEY_HASH is not on
     the base table.  This part will need to be refactored (eg using

base/mp_storediffs.sas

@@ -64,6 +64,7 @@
   <h4> Related Macros </h4>
   @li mp_stackdiffs.sas
   @li mp_storediffs.test.sas
+  @li mp_stripdiffs.sas
 
   @version 9.2
   @author Allan Bowe
@@ -183,7 +184,7 @@ data &ds4;
 run;
 
 %if "&loadref"="0" %then %let loadref=%sysfunc(uuidgen());
-%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime());
+%if &processed_dttm=0 %then %let processed_dttm=%sysfunc(datetime(),8.6);
 %let libds=%upcase(&libds);
 
 /* join orig vals for modified & deleted */

base/mp_stripdiffs.sas

@@ -0,0 +1,255 @@
+/**
+  @file
+  @brief Generates a stage dataset to revert diffs tracked in an audit table
+  @details A big benefit of tracking data changes in an audit table is that
+    those changes can be subsequently reverted if necessary!
+
+    This macro prepares a staging dataset containing those differences - eg for:
+
+    @li deleted rows - these are re-inserted
+    @li changed rows - differences are reverted
+    @li added rows - marked with `_____DELETE__THIS__RECORD_____="YES"`
+
+    These changes are NOT applied to the base table - a staging dataset is
+    simply prepared for an ETL process to action.  In Data Controller, this
+    dataset is used directly as an input to the APPROVE process (so that the
+    reversion diffs can be reviewed prior to being applied).
+
+
+  @param [in] libds Base library.dataset (will not be modified).  The library
+    must be assigned.
+  @param [in] loadref Unique identifier for the version to be reverted.  This
+    change, plus ALL SUBSEQUENT CHANGES, will be reverted in the output table.
+  @param [in] difftable The dataset containing the diffs.  Definition available
+    in mddl_dc_difftable.sas
+  @param [in] filtervar= (0) If provided, the contents of this macro variable
+    will be applied as an additional filter against &libds
+  @param [out] outds= (work.mp_stripdiffs) Output table containing the diffs.
+    Has the same format as the base datset, plus a
+    `_____DELETE__THIS__RECORD_____` variable.
+  @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
+  @li mf_islibds.sas
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_abort.sas
+
+  <h4> Related Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mp_stackdiffs.sas
+  @li mp_storediffs.sas
+  @li mp_stripdiffs.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mp_stripdiffs(libds
+  ,loadref
+  ,difftable
+  ,filtervar=0
+  ,outds=work.mp_stripdiffs
+  ,mdebug=0
+)/*/STORE SOURCE*/;
+%local dbg;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+%let libds=%upcase(&libds);
+
+/* safety checks */
+%mp_abort(iftrue= (&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc on entry.  Clean session required!)
+)
+%let libds=%upcase(&libds);
+%mp_abort(iftrue= (%mf_islibds(&libds)=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid library.dataset reference - %superq(libds))
+)
+
+/* set up unique and temporary vars */
+%local ds1 ds2 ds3 ds4 ds5 fref1 filterstr;
+%let fref1=%mf_getuniquefileref();
+%if &filtervar ne 0 %then %let filterstr=%superq(&filtervar);
+%else %let filterstr=%str(1=1);
+
+/* get timestamp of the diff to be reverted */
+%local ts;
+proc sql noprint;
+select put(processed_dttm,datetime19.6) into: ts
+  from &difftable where load_ref="&loadref";
+%mp_abort(iftrue= (&sqlobs=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Load ref %superq(loadref) not found!)
+)
+
+/* extract diffs for this base table from this timestamp onwards */
+%let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_diffs));
+create table &ds1 (drop=libref dsn) as
+  select * from &difftable
+  where upcase(cats(libref))="%scan(&libds,1,.)"
+  and upcase(cats(dsn))="%scan(&libds,2,.)"
+  and processed_dttm ge "&ts"dt
+  order by processed_dttm desc, key_hash, is_pk;
+
+/* extract key values only */
+%let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
+%local keyhash processed;
+%let keyhash=%upcase(%mf_getuniquename(prefix=mpsdvar_keyhash));
+%let processed=%upcase(%mf_getuniquename(prefix=mpsdvar_processed));
+create table &ds2 as
+  select key_hash as &keyhash,
+    tgtvar_nm,
+    tgtvar_type,
+    coalescec(oldval_char,newval_char) as charval,
+    coalesce(oldval_num, newval_num) as numval,
+    processed_dttm as &processed
+  from &ds1
+  where is_pk=1
+  order by &keyhash, &processed;
+
+/* grab pk values */
+%local pk;
+select distinct upcase(tgtvar_nm) into: pk separated by ' ' from &ds2;
+
+%let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
+proc transpose data=&ds2(where=(tgtvar_type='C'))
+    out=&ds3(drop=_name_);
+  by &keyhash &processed;
+  id TGTVAR_NM;
+  var charval;
+run;
+
+%let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
+proc transpose data=&ds2(where=(tgtvar_type='N'))
+    out=&ds4(drop=_name_);
+  by &keyhash &processed;
+  id TGTVAR_NM;
+  var numval;
+run;
+/* shorten the lengths */
+%mp_ds2squeeze(&ds3,outds=&ds3)
+%mp_ds2squeeze(&ds4,outds=&ds4)
+
+/* now merge to get all key values and de-dup */
+%let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
+data &ds5;
+  length &keyhash $32 &processed 8;
+  merge &ds3 &ds4;
+  by &keyhash &processed;
+  if not missing(&keyhash);
+run;
+proc sort data=&ds5 nodupkey;
+  by &pk;
+run;
+
+/* join to base table for preliminary stage DS */
+proc sql;
+create table &outds as select "No " as _____DELETE__THIS__RECORD_____
+  %do x=1 %to %sysfunc(countw(&pk,%str( )));
+    ,a.%scan(&pk,&x,%str( ))
+  %end;
+  %local notpkcols;
+  %let notpkcols=%upcase(%mf_getvarlist(&libds));
+  %let notpkcols=%mf_wordsinstr1butnotstr2(str1=&notpkcols,str2=&pk);
+  %do x=1 %to %sysfunc(countw(&notpkcols,%str( )));
+    ,b.%scan(&notpkcols,&x,%str( ))
+  %end;
+  from &ds5 a
+  left join &libds (where=(&filterstr)) b
+  on 1=1
+%do x=1 %to %sysfunc(countw(&pk,%str( )));
+  and a.%scan(&pk,&x,%str( ))=b.%scan(&pk,&x,%str( ))
+%end;
+;
+
+/* create SAS code to apply to stage_ds */
+data _null_;
+  set &ds1;
+  file &fref1 lrecl=33000;
+  length charval $32767;
+  if _n_=1 then put 'proc sql noprint;';
+  by descending processed_dttm key_hash is_pk;
+  if move_type='M' then do;
+    if first.key_hash then do;
+      put "update &outds set " @@;
+    end;
+    if IS_PK=0 then do;
+      put "  " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+      if not last.is_pk then put ',';
+    end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+  end;
+  else if move_type='A' then do;
+    if first.key_hash then do;
+      put "update &outds set _____DELETE__THIS__RECORD_____='Yes' where 1=1 "@@;
+    end;
+    /* gating if - as only need PK now */
+    if is_pk=1;
+    put '  AND ' tgtvar_nm '=' @@;
+    cnt=count(newval_char,'"');
+    charval=quote(trim(substr(newval_char,1,32765-cnt)));
+    if tgtvar_type='C' then put charval @@;
+    else put newval_num @@;
+  end;
+  else if move_type='D' then do;
+    if first.key_hash then do;
+      put "update &outds set _____DELETE__THIS__RECORD_____='No' " @@;
+    end;
+    if IS_PK=0 then do;
+      put "  ," tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+  end;
+  if last.key_hash then put ';';
+run;
+
+/* apply the modification statements */
+%inc &fref1/source2 lrecl=33000;
+
+%if &mdebug=0 %then %do;
+  proc sql;
+  drop table &ds1, &ds2, &ds3, &ds4, &ds5;
+  file &fref1 clear;
+%end;
+%else %do;
+  data _null_;
+    infile &fref1;
+    input;
+    if _n_=1 then putlog "Contents of SQL adjustments";
+    putlog _infile_;
+  run;
+%end;
+
+%mend mp_stripdiffs;
+/** @endcond */

fcmp/mcf_getfmttype.sas

@@ -80,8 +80,8 @@ function mcf_getfmttype(fmtnm $) $8;
     /* apply lookups */
     if cats(fmt) in ('DATETIME','B8601DN','B8601DN','B8601DT','B8601DT'
       ,'B8601DZ','B8601DZ','DATEAMPM','DTDATE','DTMONYY','DTWKDATX','DTYEAR'
-      ,'DTYYQC','E8601DN','E8601DN','E8601DT','E8601DT','E8601DZ','E8601DZ')
-      then return('DATETIME');
+      ,'DTYYQC','E8601DN','E8601DN','E8601DT','E8601DT','E8601DZ','E8601DZ'
+      ,'NLDATM') then return('DATETIME');
     else if fmt in ('DATE','YYMMDD','B8601DA','B8601DA','DAY','DDMMYY'
       ,'DDMMYYB','DDMMYYC','DDMMYYD','DDMMYYN','DDMMYYP','DDMMYYS','DDMMYYx'
       ,'DOWNAME','E8601DA','E8601DA','JULDAY','JULIAN','MMDDYY','MMDDYYB'
@@ -92,7 +92,7 @@ function mcf_getfmttype(fmtnm $) $8;
       ,'YYMMD','YYMMDDB','YYMMDDC','YYMMDDD','YYMMDDN','YYMMDDP','YYMMDDS'
       ,'YYMMDDx','YYMMN','YYMMP','YYMMS','YYMMx','YYMON','YYQ','YYQC','YYQD'
       ,'YYQN','YYQP','YYQR','YYQRC','YYQRD','YYQRN','YYQRP','YYQRS','YYQRx'
-      ,'YYQS','YYQx','YYQZ') then return('DATE');
+      ,'YYQS','YYQx','YYQZ','NLDATE') then return('DATE');
     else if fmt in ('TIME','B8601LZ','B8601LZ','B8601TM','B8601TM','B8601TZ'
       ,'B8601TZ','E8601LZ','E8601LZ','E8601TM','E8601TM','E8601TZ','E8601TZ'
       ,'HHMM','HOUR','MMSS','TIMEAMPM','TOD') then return('TIME');

meta/mm_assigndirectlib.sas

@@ -54,7 +54,8 @@
 
 %local cur_engine;
 %let cur_engine=%mf_getengine(&libref);
-%if &cur_engine ne META and &cur_engine ne %then %do;
+%if &cur_engine ne META and &cur_engine ne and %length(&open_passthrough)=0
+%then %do;
   %put NOTE:  &libref already has a direct (&cur_engine) libname connection;
   %return;
 %end;

meta/mm_getdetails.sas

@@ -1,22 +1,25 @@
 /**
   @file mm_getdetails.sas
   @brief extracts metadata attributes and associations for a particular uri
-
   @param [in] uri the metadata object for which to return
     attributes / associations
+  @param [in] sortoptions= Enables sorting of the output datasets, for example,
+    `SORTSEQ=LINGUISTIC`
   @param [out] outattrs= (work.attributes)
     The dataset to create that contains the list of attributes
   @param [out] outassocs= (work.associations)
     The dataset to contain the list of associations
 
-  @version 9.2
-  @author Allan Bowe
+  <h4> Related Files </h4>
+  @li mm_getobjects.sas
+  @li mm_gettypes.sas
 
 **/
 
 %macro mm_getdetails(uri
   ,outattrs=work.attributes
   ,outassocs=work.associations
+  ,sortoptions=
 )/*/STORE SOURCE*/;
 
 data &outassocs;
@@ -41,7 +44,7 @@ data &outassocs;
     n1+1;
   end;
 run;
-proc sort;
+proc sort &sortoptions;
   by assoc name;
 run;
 
@@ -61,7 +64,7 @@ data &outattrs;
     n1+1;
   end;
 run;
-proc sort;
+proc sort &sortoptions;
   by type name;
 run;
 

meta/mm_getgroups.sas

@@ -11,10 +11,11 @@
 
   @param [in] user= the metadata user to return groups for.  Leave blank for all
     groups.
-  @param [in] repo= the metadata repository that contains the user/group
-    information
-  @param [in] mDebug= set to 1 to show debug messages in the log
-  @param [out] outds= the dataset to create that contains the list of groups
+  @param [in] repo= (foundation) the metadata repository that contains the
+    user/group information
+  @param [in] mDebug= (0) set to 1 to show debug messages in the log
+  @param [out] outds= (work.mm_getgroups) The dataset to create that contains
+    the list of groups
 
   @returns outds  dataset containing all groups in a column named "metagroup"
     - groupuri

package.json

@@ -7,7 +7,7 @@
     "Viya",
     "SASjs"
   ],
-  "author": "Allan Bowe <support@macropeople.com>",
+  "author": "Allan Bowe",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/sasjs/core.git"
@@ -26,13 +26,10 @@
   "homepage": "https://core.sasjs.io",
   "main": "index.js",
   "scripts": {
-    "build": "sasjs cbd -t server",
-    "docs": "sasjs doc -t docsonly && ./sasjs/utils/build.sh",
-    "test": "sasjs test -t server",
-    "lint": "sasjs lint",
+    "build": "npx @sasjs/cli cbd -t server",
+    "docs": "npx @sasjs/cli doc -t docsonly && ./sasjs/utils/build.sh",
+    "test": "npx @sasjs/cli test -t server",
+    "lint": "npx @sasjs/cli lint",
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
-  },
-  "devDependencies": {
-    "@sasjs/cli": "^4.4.1"
   }
-}
+}

sasjs/utils/build.sh

@@ -1,22 +1,27 @@
 #!/bin/bash
 ####################################################################
-# PROJECT: Macro Core Docs Build                                   #
-# To execute, use the npm command (npm run docs)                   #
+# PROJECT: SASjs Core Docs Build
+# To execute, use the npm command (npm run docs)
+# Target repo will have github action to create sitemap
+# https://github.com/marketplace/actions/generate-sitemap
 ####################################################################
 
 # refresh github pages site
 rm -rf sasjsbuild/docsite
 git clone git@github.com:sasjs/core.github.io.git sasjsbuild/docsite
-rm -rf sasjsbuild/docsite/*
-mv sasjsbuild/docs/* sasjsbuild/docsite/
+rm -rf sasjsbuild/docsite/*.html
+rm -rf sasjsbuild/docsite/*.js
+rm -rf sasjsbuild/docsite/*.png
+rm -rf sasjsbuild/docsite/*.dot
+rm -rf sasjsbuild/docsite/*.css
+rm -rf sasjsbuild/docsite/*.svg
+rm -rf search
+cp -R sasjsbuild/docs/* sasjsbuild/docsite/
 cd sasjsbuild/docsite/
+git config user.name sasjs
 echo 'core.sasjs.io' > CNAME
 git add .
 git commit -m "build.sh build on $(date +%F:%H:%M:%S)"
 git push
-npx sitemap-generator-cli https://core.sasjs.io
-git add .
-git commit -m "adding sitemap"
-git push
 
 echo "check it out:  https://sasjs.github.io/core.github.io/files.html"

sasjs/utils/create_sas_package.sas

@@ -8,6 +8,12 @@
 
     Requires the server to have SSH keys.
 
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mp_gitadd.sas
+  @li mp_gitreleaseinfo.sas
+  @li mp_gitstatus.sas
+
   <h4> SAS Macros </h4>
   @li mp_gitadd.sas
   @li mp_gitreleaseinfo.sas
@@ -124,11 +130,15 @@ data members(compress=char);
   keep name name2 path;
 run;
 
+proc sort data=members;
+  by name name2;
+run;
+
 %let temp_options = %sysfunc(getoption(source)) %sysfunc(getoption(notes));
 options nosource nonotes;
 data _null_;
   set members;
-  by name notsorted;
+  by name;
 
   ord + first.name;
 

server/ms_triggerstp.sas

@@ -0,0 +1,231 @@
+/**
+  @file
+  @brief Triggers a SASjs Server STP using the /SASjsApi/stp/trigger endpoint
+  @details Triggers the STP and returns the sessionId
+
+  Example:
+
+      %ms_triggerstp(/some/stored/program
+        ,debug=131
+        ,outds=work.myresults
+      )
+
+  @param [in] pgm The full path to the Stored Program in SASjs Drive (_program
+    parameter)
+  @param [in] debug= (131) The value to supply to the _debug URL parameter
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] inputparams=(_null_) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$10000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+  @param [in] inputfiles= (_null_) A dataset containing fileref/name/filename in
+    the following format:
+    |fileref:$8|name:$32|filename:$256|
+    |---|---|--|
+    |someref|some_name|some_filename.xls|
+    |fref2|another_file|zyx_v2.csv|
+  @param [in] expiresaftermins= (15) The number of minutes to retain the session
+    folder after the session ends.
+
+  @param [out] outds= (work.ms_triggerstp) Set to the name of a dataset to
+  contain the sessionId. If this dataset already exists, and contains the
+  sessionId, it will be appended to.
+  Format:
+    |sessionId:$36|
+    |---|
+    |20241028074744-54132-1730101664824|
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+  @li mp_dropmembers.sas
+  @li mf_nobs.sas
+
+**/
+
+%macro ms_triggerstp(pgm
+    ,debug=131
+    ,inputparams=_null_
+    ,inputfiles=_null_
+    ,expiresAfterMins=15
+    ,outds=work.ms_triggerstp
+    ,mdebug=0
+  );
+  %local dbg mainref authref boundary libref triggered_sid;
+  %let mainref=%mf_getuniquefileref();
+  %let authref=%mf_getuniquefileref();
+  %let boundary=%mf_getuniquename();
+  %if &inputparams=0 %then %let inputparams=_null_;
+
+  %if &mdebug=1 %then %do;
+    %put &sysmacroname entry vars:;
+    %put _local_;
+  %end;
+  %else %let dbg=*;
+
+
+  %mp_abort(iftrue=("&pgm"="")
+    ,mac=&sysmacroname
+    ,msg=%str(Program not provided)
+  )
+  %mp_abort(iftrue=("&outds"="")
+    ,mac=&sysmacroname
+    ,msg=%str(Output dataset not provided)
+  )
+
+  /* avoid sending bom marker to API */
+  %local optval;
+  %let optval=%sysfunc(getoption(bomfile));
+  options nobomfile;
+
+  /* Add params to the content */
+  data _null_;
+    file &mainref termstr=crlf lrecl=32767 mod;
+    length line $1000 name $32 value $32767;
+    if _n_=1 then call missing(of _all_);
+    set &inputparams;
+    put "--&boundary";
+    line=cats('Content-Disposition: form-data; name="',name,'"');
+    put line;
+    put ;
+    put value;
+  run;
+
+  /* parse input file list */
+  %local webcount;
+  %let webcount=0;
+  data _null_;
+    set &inputfiles end=last;
+    length fileref $8 name $32 filename $256;
+    call symputx(cats('webref',_n_),fileref,'l');
+    call symputx(cats('webname',_n_),name,'l');
+    call symputx(cats('webfilename',_n_),filename,'l');
+    if last then do;
+      call symputx('webcount',_n_);
+      call missing(of _all_);
+    end;
+  run;
+
+  /* write out the input files to the content */
+  %local i;
+  %do i=1 %to &webcount;
+    data _null_;
+      file &mainref termstr=crlf lrecl=32767 mod;
+      infile &&webref&i lrecl=32767;
+      if _n_ = 1 then do;
+        length line $32767;
+        line=cats(
+          'Content-Disposition: form-data; name="'
+          ,"&&webname&i"
+          ,'"; filename="'
+          ,"&&webfilename&i"
+          ,'"'
+        );
+        put "--&boundary";
+        put line;
+        put "Content-Type: text/plain";
+        put ;
+      end;
+      input;
+      put _infile_; /* add the actual file to be sent */
+    run;
+  %end;
+
+  /* Add footer to the content */
+  data _null_;
+    file &mainref termstr=crlf mod;
+    put / "--&boundary--";
+  run;
+
+  data _null_;
+    file &authref lrecl=1000;
+    infile "&_sasjs_tokenfile" lrecl=1000;
+    input;
+    if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary";
+    put _infile_;
+  run;
+
+  %if &mdebug=1 %then %do;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** authref=&authref content *****";
+      infile &authref;
+      input;
+      put _infile_;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** mainref=&mainref content *****";
+      infile &mainref;
+      input;
+      put _infile_;
+    run;
+  %end;
+
+  %local resp_path outref;
+  %let resp_path=%sysfunc(pathname(work))/%mf_getuniquename();
+  %let outref=%mf_getuniquefileref();
+  filename &outref "&resp_path" lrecl=32767;
+
+  /* prepare request*/
+  proc http method='POST' headerin=&authref in=&mainref out=&outref
+    url="&_sasjs_apiserverurl/SASjsApi/stp/trigger?%trim(
+      )_program=&pgm%str(&)_debug=131%str(&)expiresAfterMins=&expiresaftermins";
+  %if &mdebug=1 %then %do;
+    debug level=2;
+  %end;
+  run;
+
+  %if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  or &mdebug=1
+  %then %do;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** outref=&outref content *****";
+      infile &outref;
+      input;
+      putlog _infile_;
+    run;
+  %end;
+  %mp_abort(
+    iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200
+        and &SYS_PROCHTTP_STATUS_CODE ne 201)
+    ,mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+
+  /* reset options */
+  options &optval;
+
+  %let libref=%mf_getuniquelibref();
+  libname &libref JSON fileref=&outref;
+  %let triggered_sid=%mf_getuniquename(prefix=triggered_sid_);
+
+  data work.&triggered_sid (keep=sessionid);
+    set &libref..root;
+
+    %if &mdebug=1 %then %do;
+      putlog (_all_)(=);
+    %end;
+  run;
+
+  %if %mf_nobs(work.&triggered_sid)>0 %then %do;
+    proc append base=&outds data=work.&triggered_sid;
+    run;
+  %end;
+
+  %if &mdebug=1 %then %do;
+    %put &sysmacroname exit vars:;
+    %put _local_;
+  %end;
+  %else %do;
+    /* clear refs */
+    filename &authref;
+    filename &mainref;
+    filename &outref;
+    libname &libref clear;
+    /* and remove temp dataset */
+    %mp_dropmembers(&triggered_sid,libref=work);
+  %end;
+
+%mend ms_triggerstp;

tests/base/mp_stripdiffs.test.sas

@@ -0,0 +1,106 @@
+/**
+  @file
+  @brief Testing mp_stripdiffs.sas macro
+  @details
+
+  <h4> SAS Macros </h4>
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mp_ds2md.sas
+  @li mp_stripdiffs.sas
+
+**/
+
+/* make an adjustable base dataset */
+/* use a composite key also (name weight) */
+libname libby (work);
+data libby.class;
+  set sashelp.class;
+run;
+
+/* first, store some diffs */
+data work.orig work.deleted work.changed work.appended;
+  set libby.class;
+  if _n_=1 then do;
+    call symputx('delname',name);
+    output work.orig work.deleted;
+  end;
+  else if _n_=2 then do;
+    output work.orig;
+    call symputx('modname',name);
+    call symputx('modval',age);
+    age=99;
+    output work.changed;
+  end;
+  else do;
+    name='Newbie';
+    output work.appended;
+    stop;
+  end;
+run;
+%mp_storediffs(libby.class,work.orig,NAME WEIGHT
+  ,delds=work.deleted
+  ,modds=work.changed
+  ,appds=work.appended
+  ,outds=work.audit
+  ,loadref=UPLOAD1
+  ,mdebug=0
+)
+%mp_ds2md(work.audit)
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking preparation case,
+  outds=work.test_results
+)
+
+/* apply the changes */
+proc sql;
+delete from libby.class where name in ("&delname","&modname");
+proc append base=libby.class data=work.appended;
+proc append base=libby.class data=work.changed;
+run;
+
+/* now, prepare the revert dataset */
+%mp_assertscope(SNAPSHOT)
+%mp_stripdiffs(libby.class
+  ,UPLOAD1
+  ,work.audit
+  ,outds=work.mp_stripdiffs
+  ,mdebug=1
+)
+%mp_ds2md(work.mp_stripdiffs)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking error condition,
+  outds=work.test_results
+)
+
+%let delpass=0;
+%let modpass=0;
+%let addpass=0;
+data _null_;
+  set work.mp_stripdiffs;
+  if upcase(_____DELETE__THIS__RECORD_____)='NO' and name="&delname"
+  then call symputx('delpass',1);
+  if name="&modname" and age=&modval then call symputx('modpass',1);
+  if upcase(_____DELETE__THIS__RECORD_____)='YES' and name="Newbie"
+  then call symputx('addpass',1);
+run;
+
+%mp_assert(
+  iftrue=(&delpass=1),
+  desc=Ensuring deleted record is back in the dataset,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&modpass=1),
+  desc=Ensuring modified record now has old value,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&addpass=1),
+  desc=Ensuring added record is now marked for deletion,
+  outds=work.test_results
+)

tests/serveronly/ms_triggerstp.test.sas

@@ -0,0 +1,90 @@
+/**
+  @file
+  @brief Testing ms_triggerstp.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li ms_createfile.sas
+  @li ms_triggerstp.sas
+  @li mf_existds.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertcols.sas
+  @li mf_getvartype.sas
+  @li ms_deletefile.sas
+
+**/
+
+/* first, create multiple STPs to run */
+filename stpcode1 temp;
+data _null_;
+  file stpcode1;
+  put '%put hello world;';
+  put '%put _all_;';
+  put 'data _null_; file _webout1; put "triggerstp test 1";run;';
+run;
+filename stpcode2 temp;
+data _null_;
+  file stpcode2;
+  put '%put Lorem Ipsum;';
+  put '%put _all_;';
+  put 'data _null_; file _webout2; put "triggerstp test 2";run;';
+run;
+options mprint;
+%let fname1=%mf_getuniquename();
+%let fname2=%mf_getuniquename();
+
+%ms_createfile(/sasjs/tests/&fname1..sas
+  ,inref=stpcode1
+  ,mdebug=1
+)
+%ms_createfile(/sasjs/tests/&fname2..sas
+  ,inref=stpcode2
+)
+
+%mp_assertscope(SNAPSHOT)
+  %ms_triggerstp(/sasjs/tests/&fname1
+    ,debug=131
+    ,outds=work.mySessions
+  )
+  %ms_triggerstp(/sasjs/tests/&fname2
+    ,outds=work.mySessions
+  )
+%mp_assertscope(COMPARE
+                ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN)
+
+%mp_assert(iftrue=%str(%mf_existds(work.mySessions)=1)
+          ,desc=Testing output exists
+          ,outds=work.test_results
+)
+
+%mp_assertdsobs(work.mySessions,
+  test=EQUALS 2,
+  desc=Testing observations,
+  outds=work.test_results
+)
+%mp_assertcols(work.mySessions,
+  cols=sessionid,
+  test=ALL,
+  desc=Testing column exists,
+  outds=work.test_results
+)
+
+data _null_;
+  retain contentCheck 1;
+  set work.mySessions end=last;
+  if missing(sessionID) then contentCheck = 0;
+  if last then do;
+    call symputx("contentCheck",contentCheck,"l");
+  end;
+run;
+%let typeCheck = %mf_getvartype(work.mySessions,sessionid);
+
+%mp_assert(iftrue=%str(&typeCheck = C and &contentCheck = 1)
+          ,desc=Testing type and content of output
+          ,outds=work.test_results
+)
+
+%ms_deletefile(/sasjs/tests/&fname1..sas)
+%ms_deletefile(/sasjs/tests/&fname2..sas)

tests/x-platform/mx_getgroups.test.sas

@@ -0,0 +1,41 @@
+/**
+  @file
+  @brief Testing mx_getgroups.test.sas macro
+
+  Be sure to run <code>%let mcTestAppLoc=/Public/temp/macrocore;</code> when
+  running in Studio
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mf_getuser.sas
+  @li mp_assert.sas
+  @li mx_getgroups.sas
+
+**/
+
+
+%mx_getgroups(outds=work.test1)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.test1)>0),
+  desc=groups were found,
+  outds=work.test_results
+)
+%mp_assertcols(work.test1,
+  cols=groupuri groupname groupdesc,
+  test=ALL,
+  desc=check all columns exist
+)
+
+%mx_getgroups(outds=work.test2,user=%mf_getuser())
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.test2)>0),
+  desc=groups for current user were found,
+  outds=work.test_results
+)
+%mp_assertcols(work.test2,
+  cols=groupuri groupname groupdesc,
+  test=ALL,
+  desc=check all columns exist
+)

xplatform/mx_getgroups.sas

@@ -0,0 +1,98 @@
+/**
+  @file
+  @brief Fetches all groups or the groups for a particular member
+  @details  When building applications that run on multiple flavours of SAS, it
+  is convenient to use a single macro (like this one) to fetch the groups
+  regardless of the flavour of SAS being used
+
+  The alternative would be to compile a generic macro in target-specific
+  folders (SASVIYA, SAS9 and SASJS).  This avoids compiling unnecessary macros
+  at the expense of a more complex sasjsconfig.json setup.
+
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] user= (0) Provide the username on which to filter
+  @param [in] uid= (0) Provide the userid on which to filter
+  @param [in] repo= (foundation) SAS9 only, choose the metadata repo to query
+  @param [in] access_token_var= (ACCESS_TOKEN) VIYA only.
+    The global macro variable to contain the access token
+  @param [in] grant_type= (sas_services) VIYA only.
+    Valid values are "password" or "authorization_code" (unquoted).
+  @param [out] outds= (work.mx_getgroups) This output dataset will contain the
+    list of groups. Format:
+|GROUPNAME:$32.|GROUPDESC:$256.|GROUPURI:best.|
+|---|---|---|
+|`SomeGroup `|`A group `|`1`|
+|`Another Group`|`this is a different group`|`2`|
+|`admin`|`Administrators `|`3`|
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mm_getgroups.sas
+  @li ms_getgroups.sas
+  @li mv_getgroups.sas
+  @li mv_getusergroups.sas
+
+**/
+
+%macro mx_getgroups(
+  mdebug=0,
+  user=0,
+  uid=0,
+  repo=foundation,
+  access_token_var=ACCESS_TOKEN,
+  grant_type=sas_services,
+  outds=work.mx_getgroups
+)/*/STORE SOURCE*/;
+%local platform name shortloc;
+%let platform=%mf_getplatform();
+
+%if &platform=SASJS %then %do;
+  %ms_getgroups(
+    user=&user,
+    uid=&uid,
+    outds=&outds,
+    mdebug=&mdebug
+  )
+  data &outds;
+    length groupuri groupname $32 groupdesc $128 ;
+    set &outds;
+    keep groupuri groupname groupdesc;
+    groupuri=cats(groupid);
+    groupname=name;
+    groupdesc=description;
+  run;
+  proc sort; by groupname; run;
+%end;
+%else %if &platform=SAS9 or &platform=SASMETA %then %do;
+  %if &user=0 %then %let user=;
+  %mm_getGroups(
+    user=&user
+    ,outds=&outds
+    ,repo=&repo
+    ,mDebug=&mdebug
+  )
+  proc sort data=&outds; by groupname; run;
+%end;
+%else %if &platform=SASVIYA %then %do;
+  %if &user=0 %then %do;
+    %mv_getgroups(access_token_var=&access_token_var
+      ,grant_type=&grant_type
+      ,outds=&outds
+    )
+  %end;
+  %else %do;
+    %mv_getusergroups(&user
+      ,outds=&outds
+      ,access_token_var=&access_token_var
+      ,grant_type=&grant_type
+    )
+  %end;
+  proc sort
+      data=&outds(rename=(id=groupuri name=groupname description=groupdesc))
+      out=&outds (keep=groupuri groupname groupdesc);
+    by groupname;
+  run;
+%end;
+
+%mend mx_getgroups;