feat: adding NLDATE and NLDATM to mcf_getfmttype

385 support nldatm in mp getcols

.github/vpn/config.ovpn

@@ -3,10 +3,12 @@ client
 tls-client
 dev tun
 # this will connect with whatever proto DNS tells us (https://community.openvpn.net/openvpn/ticket/934)
-proto tcp
-remote vpn.4gl.io 7494
+proto udp
+remote vpn.4gl.io 7194
 resolv-retry infinite
-cipher AES-256-CBC
+# this will fallback from udp6 to udp4 as well
+connect-timeout 5
+data-ciphers AES-256-CBC:AES-256-GCM
 auth SHA256
 script-security 2
 keepalive 10 120

.github/workflows/run-tests.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     strategy:
       matrix:
@@ -34,6 +34,10 @@ jobs:
           USER_KEY: ${{ secrets.USER_KEY }}
           TLS_KEY: ${{ secrets.TLS_KEY }}
 
+      - name: Chmod VPN files
+        run: |
+          chmod 600 .github/vpn/ca.crt .github/vpn/user.crt .github/vpn/user.key .github/vpn/tls.key
+
       - name: Install Open VPN
         run: |
           sudo apt install apt-transport-https
@@ -42,8 +46,13 @@ jobs:
           sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
           sudo apt update
           sudo apt install openvpn3=17~betaUb22042+jammy
+
       - name: Start Open VPN 3
         run: openvpn3 session-start --config .github/vpn/config.ovpn
+
+      - name: Fetch SASJS server
+        run: curl ${{ secrets.SASJS_SERVER_URL }}/SASjsApi/info
+
       - name: Install Doxygen
         run: sudo apt-get install doxygen
 

all.sas

@@ -4861,7 +4861,7 @@ data;run;
 data &out_ds(compress=no
     keep=file_or_folder filepath filename ext msg directory level
   );
-  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
+  length directory filepath $2000 fref fref2 $8 file_or_folder $6 filename $255
     ext $20 msg $200 foption $16;
   if _n_=1 then call missing(of _all_);
   retain level &level;
@@ -7157,10 +7157,11 @@ data &outds(keep=name type length varnum format label ddtype fmtname);
     else if formatd=0 then format=cats(fmtname,formatl,'.');
     else format=cats(fmtname,formatl,'.',formatd);
     type='N';
-    if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
+    if format=:'DATETIME' or format=:'E8601DT' or format=:'NLDATM'
+    then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
       or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
-      or format=:'MONYY'
+      or format=:'MONYY' or format=:'NLDATE'
     then ddtype='DATE';
     else if format=:'TIME' then ddtype='TIME';
     else ddtype='NUMERIC';
@@ -9675,6 +9676,7 @@ options
 %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
   noautocorrect           /* disallow misspelled procedure names            */
   dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
+                          /* turn off with dsoptions=nonote2err             */
 %end;
 ;
 
@@ -13407,6 +13409,8 @@ run;
     change, plus ALL SUBSEQUENT CHANGES, will be reverted in the output table.
   @param [in] difftable The dataset containing the diffs.  Definition available
     in mddl_dc_difftable.sas
+  @param [in] filtervar= (0) If provided, the contents of this macro variable
+    will be applied as an additional filter against &libds
   @param [out] outds= (work.mp_stripdiffs) Output table containing the diffs.
     Has the same format as the base datset, plus a
     `_____DELETE__THIS__RECORD_____` variable.
@@ -13415,7 +13419,9 @@ run;
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
   @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
   @li mf_islibds.sas
+  @li mf_wordsinstr1butnotstr2.sas
   @li mp_abort.sas
 
   <h4> Related Macros </h4>
@@ -13432,6 +13438,7 @@ run;
 %macro mp_stripdiffs(libds
   ,loadref
   ,difftable
+  ,filtervar=0
   ,outds=work.mp_stripdiffs
   ,mdebug=0
 )/*/STORE SOURCE*/;
@@ -13455,11 +13462,11 @@ run;
   ,msg=%str(Invalid library.dataset reference - %superq(libds))
 )
 
-
-
 /* set up unique and temporary vars */
-%local ds1 ds2 ds3 ds4 ds5 fref1;
+%local ds1 ds2 ds3 ds4 ds5 fref1 filterstr;
 %let fref1=%mf_getuniquefileref();
+%if &filtervar ne 0 %then %let filterstr=%superq(&filtervar);
+%else %let filterstr=%str(1=1);
 
 /* get timestamp of the diff to be reverted */
 %local ts;
@@ -13482,30 +13489,28 @@ create table &ds1 (drop=libref dsn) as
 
 /* extract key values only */
 %let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
+%local keyhash processed;
+%let keyhash=%upcase(%mf_getuniquename(prefix=mpsdvar_keyhash));
+%let processed=%upcase(%mf_getuniquename(prefix=mpsdvar_processed));
 create table &ds2 as
-  select key_hash,
+  select key_hash as &keyhash,
     tgtvar_nm,
     tgtvar_type,
     coalescec(oldval_char,newval_char) as charval,
     coalesce(oldval_num, newval_num) as numval,
-    processed_dttm
+    processed_dttm as &processed
   from &ds1
   where is_pk=1
-  order by key_hash, processed_dttm;
+  order by &keyhash, &processed;
 
 /* grab pk values */
 %local pk;
-data _null_;
-  set &ds2;
-  by key_hash;
-  call symputx('pk',catx(' ',symget('pk'),tgtvar_nm),'l');
-  if last.key_hash then stop;
-run;
+select distinct upcase(tgtvar_nm) into: pk separated by ' ' from &ds2;
 
 %let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
 proc transpose data=&ds2(where=(tgtvar_type='C'))
     out=&ds3(drop=_name_);
-  by KEY_HASH PROCESSED_DTTM;
+  by &keyhash &processed;
   id TGTVAR_NM;
   var charval;
 run;
@@ -13513,7 +13518,7 @@ run;
 %let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
 proc transpose data=&ds2(where=(tgtvar_type='N'))
     out=&ds4(drop=_name_);
-  by KEY_HASH PROCESSED_DTTM;
+  by &keyhash &processed;
   id TGTVAR_NM;
   var numval;
 run;
@@ -13521,20 +13526,32 @@ run;
 %mp_ds2squeeze(&ds3,outds=&ds3)
 %mp_ds2squeeze(&ds4,outds=&ds4)
 
+/* now merge to get all key values and de-dup */
 %let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
 data &ds5;
-  length key_hash $32 processed_dttm 8;
+  length &keyhash $32 &processed 8;
   merge &ds3 &ds4;
-  by key_hash;
-  if not missing(key_hash);
+  by &keyhash &processed;
+  if not missing(&keyhash);
+run;
+proc sort data=&ds5 nodupkey;
+  by &pk;
 run;
 
 /* join to base table for preliminary stage DS */
 proc sql;
-create table &outds as select "No " as _____DELETE__THIS__RECORD_____,
-    b.*
+create table &outds as select "No " as _____DELETE__THIS__RECORD_____
+  %do x=1 %to %sysfunc(countw(&pk,%str( )));
+    ,a.%scan(&pk,&x,%str( ))
+  %end;
+  %local notpkcols;
+  %let notpkcols=%upcase(%mf_getvarlist(&libds));
+  %let notpkcols=%mf_wordsinstr1butnotstr2(str1=&notpkcols,str2=&pk);
+  %do x=1 %to %sysfunc(countw(&notpkcols,%str( )));
+    ,b.%scan(&notpkcols,&x,%str( ))
+  %end;
   from &ds5 a
-  inner join &libds b
+  left join &libds (where=(&filterstr)) b
   on 1=1
 %do x=1 %to %sysfunc(countw(&pk,%str( )));
   and a.%scan(&pk,&x,%str( ))=b.%scan(&pk,&x,%str( ))
@@ -13583,13 +13600,23 @@ data _null_;
   end;
   else if move_type='D' then do;
     if first.key_hash then do;
-      put "insert into &outds set _____DELETE__THIS__RECORD_____='No' " @@;
+      put "update &outds set _____DELETE__THIS__RECORD_____='No' " @@;
+    end;
+    if IS_PK=0 then do;
+      put "  ," tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
     end;
-    put "  ," tgtvar_nm '=' @@;
-    cnt=count(oldval_char,'"');
-    charval=quote(trim(substr(oldval_char,1,32765-cnt)));
-    if tgtvar_type='C' then put charval @@;
-    else put oldval_num @@;
   end;
   if last.key_hash then put ';';
 run;
@@ -14749,7 +14776,8 @@ filename __us2grp clear;
 
 %local cur_engine;
 %let cur_engine=%mf_getengine(&libref);
-%if &cur_engine ne META and &cur_engine ne %then %do;
+%if &cur_engine ne META and &cur_engine ne and %length(&open_passthrough)=0
+%then %do;
   %put NOTE:  &libref already has a direct (&cur_engine) libname connection;
   %return;
 %end;
@@ -17659,22 +17687,25 @@ run;
 %mend mm_getcols;/**
   @file mm_getdetails.sas
   @brief extracts metadata attributes and associations for a particular uri
-
   @param [in] uri the metadata object for which to return
     attributes / associations
+  @param [in] sortoptions= Enables sorting of the output datasets, for example,
+    `SORTSEQ=LINGUISTIC`
   @param [out] outattrs= (work.attributes)
     The dataset to create that contains the list of attributes
   @param [out] outassocs= (work.associations)
     The dataset to contain the list of associations
 
-  @version 9.2
-  @author Allan Bowe
+  <h4> Related Files </h4>
+  @li mm_getobjects.sas
+  @li mm_gettypes.sas
 
 **/
 
 %macro mm_getdetails(uri
   ,outattrs=work.attributes
   ,outassocs=work.associations
+  ,sortoptions=
 )/*/STORE SOURCE*/;
 
 data &outassocs;
@@ -17699,7 +17730,7 @@ data &outassocs;
     n1+1;
   end;
 run;
-proc sort;
+proc sort &sortoptions;
   by assoc name;
 run;
 
@@ -17719,7 +17750,7 @@ data &outattrs;
     n1+1;
   end;
 run;
-proc sort;
+proc sort &sortoptions;
   by type name;
 run;
 
@@ -23088,6 +23119,236 @@ run;
 
 %mend ms_testservice;
 /**
+  @file
+  @brief Triggers a SASjs Server STP using the /SASjsApi/stp/trigger endpoint
+  @details Triggers the STP and returns the sessionId
+
+  Example:
+
+      %ms_triggerstp(/some/stored/program
+        ,debug=131
+        ,outds=work.myresults
+      )
+
+  @param [in] pgm The full path to the Stored Program in SASjs Drive (_program
+    parameter)
+  @param [in] debug= (131) The value to supply to the _debug URL parameter
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] inputparams=(_null_) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$10000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+  @param [in] inputfiles= (_null_) A dataset containing fileref/name/filename in
+    the following format:
+    |fileref:$8|name:$32|filename:$256|
+    |---|---|--|
+    |someref|some_name|some_filename.xls|
+    |fref2|another_file|zyx_v2.csv|
+  @param [in] expiresaftermins= (15) The number of minutes to retain the session
+    folder after the session ends.
+
+  @param [out] outds= (work.ms_triggerstp) Set to the name of a dataset to
+  contain the sessionId. If this dataset already exists, and contains the
+  sessionId, it will be appended to.
+  Format:
+    |sessionId:$36|
+    |---|
+    |20241028074744-54132-1730101664824|
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+  @li mp_dropmembers.sas
+  @li mf_nobs.sas
+
+**/
+
+%macro ms_triggerstp(pgm
+    ,debug=131
+    ,inputparams=_null_
+    ,inputfiles=_null_
+    ,expiresAfterMins=15
+    ,outds=work.ms_triggerstp
+    ,mdebug=0
+  );
+  %local dbg mainref authref boundary libref triggered_sid;
+  %let mainref=%mf_getuniquefileref();
+  %let authref=%mf_getuniquefileref();
+  %let boundary=%mf_getuniquename();
+  %if &inputparams=0 %then %let inputparams=_null_;
+
+  %if &mdebug=1 %then %do;
+    %put &sysmacroname entry vars:;
+    %put _local_;
+  %end;
+  %else %let dbg=*;
+
+
+  %mp_abort(iftrue=("&pgm"="")
+    ,mac=&sysmacroname
+    ,msg=%str(Program not provided)
+  )
+  %mp_abort(iftrue=("&outds"="")
+    ,mac=&sysmacroname
+    ,msg=%str(Output dataset not provided)
+  )
+
+  /* avoid sending bom marker to API */
+  %local optval;
+  %let optval=%sysfunc(getoption(bomfile));
+  options nobomfile;
+
+  /* Add params to the content */
+  data _null_;
+    file &mainref termstr=crlf lrecl=32767 mod;
+    length line $1000 name $32 value $32767;
+    if _n_=1 then call missing(of _all_);
+    set &inputparams;
+    put "--&boundary";
+    line=cats('Content-Disposition: form-data; name="',name,'"');
+    put line;
+    put ;
+    put value;
+  run;
+
+  /* parse input file list */
+  %local webcount;
+  %let webcount=0;
+  data _null_;
+    set &inputfiles end=last;
+    length fileref $8 name $32 filename $256;
+    call symputx(cats('webref',_n_),fileref,'l');
+    call symputx(cats('webname',_n_),name,'l');
+    call symputx(cats('webfilename',_n_),filename,'l');
+    if last then do;
+      call symputx('webcount',_n_);
+      call missing(of _all_);
+    end;
+  run;
+
+  /* write out the input files to the content */
+  %local i;
+  %do i=1 %to &webcount;
+    data _null_;
+      file &mainref termstr=crlf lrecl=32767 mod;
+      infile &&webref&i lrecl=32767;
+      if _n_ = 1 then do;
+        length line $32767;
+        line=cats(
+          'Content-Disposition: form-data; name="'
+          ,"&&webname&i"
+          ,'"; filename="'
+          ,"&&webfilename&i"
+          ,'"'
+        );
+        put "--&boundary";
+        put line;
+        put "Content-Type: text/plain";
+        put ;
+      end;
+      input;
+      put _infile_; /* add the actual file to be sent */
+    run;
+  %end;
+
+  /* Add footer to the content */
+  data _null_;
+    file &mainref termstr=crlf mod;
+    put / "--&boundary--";
+  run;
+
+  data _null_;
+    file &authref lrecl=1000;
+    infile "&_sasjs_tokenfile" lrecl=1000;
+    input;
+    if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary";
+    put _infile_;
+  run;
+
+  %if &mdebug=1 %then %do;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** authref=&authref content *****";
+      infile &authref;
+      input;
+      put _infile_;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** mainref=&mainref content *****";
+      infile &mainref;
+      input;
+      put _infile_;
+    run;
+  %end;
+
+  %local resp_path outref;
+  %let resp_path=%sysfunc(pathname(work))/%mf_getuniquename();
+  %let outref=%mf_getuniquefileref();
+  filename &outref "&resp_path" lrecl=32767;
+
+  /* prepare request*/
+  proc http method='POST' headerin=&authref in=&mainref out=&outref
+    url="&_sasjs_apiserverurl/SASjsApi/stp/trigger?%trim(
+      )_program=&pgm%str(&)_debug=131%str(&)expiresAfterMins=&expiresaftermins";
+  %if &mdebug=1 %then %do;
+    debug level=2;
+  %end;
+  run;
+
+  %if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  or &mdebug=1
+  %then %do;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** outref=&outref content *****";
+      infile &outref;
+      input;
+      putlog _infile_;
+    run;
+  %end;
+  %mp_abort(
+    iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200
+        and &SYS_PROCHTTP_STATUS_CODE ne 201)
+    ,mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+
+  /* reset options */
+  options &optval;
+
+  %let libref=%mf_getuniquelibref();
+  libname &libref JSON fileref=&outref;
+  %let triggered_sid=%mf_getuniquename(prefix=triggered_sid_);
+
+  data work.&triggered_sid (keep=sessionid);
+    set &libref..root;
+
+    %if &mdebug=1 %then %do;
+      putlog (_all_)(=);
+    %end;
+  run;
+
+  %if %mf_nobs(work.&triggered_sid)>0 %then %do;
+    proc append base=&outds data=work.&triggered_sid;
+    run;
+  %end;
+
+  %if &mdebug=1 %then %do;
+    %put &sysmacroname exit vars:;
+    %put _local_;
+  %end;
+  %else %do;
+    /* clear refs */
+    filename &authref;
+    filename &mainref;
+    filename &outref;
+    libname &libref clear;
+    /* and remove temp dataset */
+    %mp_dropmembers(&triggered_sid,libref=work);
+  %end;
+
+%mend ms_triggerstp;/**
   @file
   @brief Send data to/from sasjs/server
   @details This macro should be added to the start of each web service,

base/mp_dirlist.sas

@@ -85,7 +85,7 @@ data;run;
 data &out_ds(compress=no
     keep=file_or_folder filepath filename ext msg directory level
   );
-  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
+  length directory filepath $2000 fref fref2 $8 file_or_folder $6 filename $255
     ext $20 msg $200 foption $16;
   if _n_=1 then call missing(of _all_);
   retain level &level;

base/mp_getcols.sas

@@ -53,10 +53,11 @@ data &outds(keep=name type length varnum format label ddtype fmtname);
     else if formatd=0 then format=cats(fmtname,formatl,'.');
     else format=cats(fmtname,formatl,'.',formatd);
     type='N';
-    if format=:'DATETIME' or format=:'E8601DT' then ddtype='DATETIME';
+    if format=:'DATETIME' or format=:'E8601DT' or format=:'NLDATM'
+    then ddtype='DATETIME';
     else if format=:'DATE' or format=:'DDMMYY' or format=:'MMDDYY'
       or format=:'YYMMDD' or format=:'E8601DA' or format=:'B8601DA'
-      or format=:'MONYY'
+      or format=:'MONYY' or format=:'NLDATE'
     then ddtype='DATE';
     else if format=:'TIME' then ddtype='TIME';
     else ddtype='NUMERIC';

base/mp_init.sas

@@ -70,6 +70,7 @@ options
 %if "%substr(&sysver,1,1)" ne "4" and "%substr(&sysver,1,1)" ne "5" %then %do;
   noautocorrect           /* disallow misspelled procedure names            */
   dsoptions=note2err      /* undocumented - convert bad NOTEs to ERRs       */
+                          /* turn off with dsoptions=nonote2err             */
 %end;
 ;
 

base/mp_stripdiffs.sas

@@ -22,6 +22,8 @@
     change, plus ALL SUBSEQUENT CHANGES, will be reverted in the output table.
   @param [in] difftable The dataset containing the diffs.  Definition available
     in mddl_dc_difftable.sas
+  @param [in] filtervar= (0) If provided, the contents of this macro variable
+    will be applied as an additional filter against &libds
   @param [out] outds= (work.mp_stripdiffs) Output table containing the diffs.
     Has the same format as the base datset, plus a
     `_____DELETE__THIS__RECORD_____` variable.
@@ -30,7 +32,9 @@
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
   @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
   @li mf_islibds.sas
+  @li mf_wordsinstr1butnotstr2.sas
   @li mp_abort.sas
 
   <h4> Related Macros </h4>
@@ -47,6 +51,7 @@
 %macro mp_stripdiffs(libds
   ,loadref
   ,difftable
+  ,filtervar=0
   ,outds=work.mp_stripdiffs
   ,mdebug=0
 )/*/STORE SOURCE*/;
@@ -70,11 +75,11 @@
   ,msg=%str(Invalid library.dataset reference - %superq(libds))
 )
 
-
-
 /* set up unique and temporary vars */
-%local ds1 ds2 ds3 ds4 ds5 fref1;
+%local ds1 ds2 ds3 ds4 ds5 fref1 filterstr;
 %let fref1=%mf_getuniquefileref();
+%if &filtervar ne 0 %then %let filterstr=%superq(&filtervar);
+%else %let filterstr=%str(1=1);
 
 /* get timestamp of the diff to be reverted */
 %local ts;
@@ -97,30 +102,28 @@ create table &ds1 (drop=libref dsn) as
 
 /* extract key values only */
 %let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
+%local keyhash processed;
+%let keyhash=%upcase(%mf_getuniquename(prefix=mpsdvar_keyhash));
+%let processed=%upcase(%mf_getuniquename(prefix=mpsdvar_processed));
 create table &ds2 as
-  select key_hash,
+  select key_hash as &keyhash,
     tgtvar_nm,
     tgtvar_type,
     coalescec(oldval_char,newval_char) as charval,
     coalesce(oldval_num, newval_num) as numval,
-    processed_dttm
+    processed_dttm as &processed
   from &ds1
   where is_pk=1
-  order by key_hash, processed_dttm;
+  order by &keyhash, &processed;
 
 /* grab pk values */
 %local pk;
-data _null_;
-  set &ds2;
-  by key_hash;
-  call symputx('pk',catx(' ',symget('pk'),tgtvar_nm),'l');
-  if last.key_hash then stop;
-run;
+select distinct upcase(tgtvar_nm) into: pk separated by ' ' from &ds2;
 
 %let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
 proc transpose data=&ds2(where=(tgtvar_type='C'))
     out=&ds3(drop=_name_);
-  by KEY_HASH PROCESSED_DTTM;
+  by &keyhash &processed;
   id TGTVAR_NM;
   var charval;
 run;
@@ -128,7 +131,7 @@ run;
 %let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
 proc transpose data=&ds2(where=(tgtvar_type='N'))
     out=&ds4(drop=_name_);
-  by KEY_HASH PROCESSED_DTTM;
+  by &keyhash &processed;
   id TGTVAR_NM;
   var numval;
 run;
@@ -136,20 +139,32 @@ run;
 %mp_ds2squeeze(&ds3,outds=&ds3)
 %mp_ds2squeeze(&ds4,outds=&ds4)
 
+/* now merge to get all key values and de-dup */
 %let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
 data &ds5;
-  length key_hash $32 processed_dttm 8;
+  length &keyhash $32 &processed 8;
   merge &ds3 &ds4;
-  by key_hash;
-  if not missing(key_hash);
+  by &keyhash &processed;
+  if not missing(&keyhash);
+run;
+proc sort data=&ds5 nodupkey;
+  by &pk;
 run;
 
 /* join to base table for preliminary stage DS */
 proc sql;
-create table &outds as select "No " as _____DELETE__THIS__RECORD_____,
-    b.*
+create table &outds as select "No " as _____DELETE__THIS__RECORD_____
+  %do x=1 %to %sysfunc(countw(&pk,%str( )));
+    ,a.%scan(&pk,&x,%str( ))
+  %end;
+  %local notpkcols;
+  %let notpkcols=%upcase(%mf_getvarlist(&libds));
+  %let notpkcols=%mf_wordsinstr1butnotstr2(str1=&notpkcols,str2=&pk);
+  %do x=1 %to %sysfunc(countw(&notpkcols,%str( )));
+    ,b.%scan(&notpkcols,&x,%str( ))
+  %end;
   from &ds5 a
-  inner join &libds b
+  left join &libds (where=(&filterstr)) b
   on 1=1
 %do x=1 %to %sysfunc(countw(&pk,%str( )));
   and a.%scan(&pk,&x,%str( ))=b.%scan(&pk,&x,%str( ))
@@ -198,13 +213,23 @@ data _null_;
   end;
   else if move_type='D' then do;
     if first.key_hash then do;
-      put "insert into &outds set _____DELETE__THIS__RECORD_____='No' " @@;
+      put "update &outds set _____DELETE__THIS__RECORD_____='No' " @@;
+    end;
+    if IS_PK=0 then do;
+      put "  ," tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
     end;
-    put "  ," tgtvar_nm '=' @@;
-    cnt=count(oldval_char,'"');
-    charval=quote(trim(substr(oldval_char,1,32765-cnt)));
-    if tgtvar_type='C' then put charval @@;
-    else put oldval_num @@;
   end;
   if last.key_hash then put ';';
 run;

fcmp/mcf_getfmttype.sas

@@ -80,8 +80,8 @@ function mcf_getfmttype(fmtnm $) $8;
     /* apply lookups */
     if cats(fmt) in ('DATETIME','B8601DN','B8601DN','B8601DT','B8601DT'
       ,'B8601DZ','B8601DZ','DATEAMPM','DTDATE','DTMONYY','DTWKDATX','DTYEAR'
-      ,'DTYYQC','E8601DN','E8601DN','E8601DT','E8601DT','E8601DZ','E8601DZ')
-      then return('DATETIME');
+      ,'DTYYQC','E8601DN','E8601DN','E8601DT','E8601DT','E8601DZ','E8601DZ'
+      ,'NLDATM') then return('DATETIME');
     else if fmt in ('DATE','YYMMDD','B8601DA','B8601DA','DAY','DDMMYY'
       ,'DDMMYYB','DDMMYYC','DDMMYYD','DDMMYYN','DDMMYYP','DDMMYYS','DDMMYYx'
       ,'DOWNAME','E8601DA','E8601DA','JULDAY','JULIAN','MMDDYY','MMDDYYB'
@@ -92,7 +92,7 @@ function mcf_getfmttype(fmtnm $) $8;
       ,'YYMMD','YYMMDDB','YYMMDDC','YYMMDDD','YYMMDDN','YYMMDDP','YYMMDDS'
       ,'YYMMDDx','YYMMN','YYMMP','YYMMS','YYMMx','YYMON','YYQ','YYQC','YYQD'
       ,'YYQN','YYQP','YYQR','YYQRC','YYQRD','YYQRN','YYQRP','YYQRS','YYQRx'
-      ,'YYQS','YYQx','YYQZ') then return('DATE');
+      ,'YYQS','YYQx','YYQZ','NLDATE') then return('DATE');
     else if fmt in ('TIME','B8601LZ','B8601LZ','B8601TM','B8601TM','B8601TZ'
       ,'B8601TZ','E8601LZ','E8601LZ','E8601TM','E8601TM','E8601TZ','E8601TZ'
       ,'HHMM','HOUR','MMSS','TIMEAMPM','TOD') then return('TIME');

meta/mm_assigndirectlib.sas

@@ -54,7 +54,8 @@
 
 %local cur_engine;
 %let cur_engine=%mf_getengine(&libref);
-%if &cur_engine ne META and &cur_engine ne %then %do;
+%if &cur_engine ne META and &cur_engine ne and %length(&open_passthrough)=0
+%then %do;
   %put NOTE:  &libref already has a direct (&cur_engine) libname connection;
   %return;
 %end;

meta/mm_getdetails.sas

@@ -1,22 +1,25 @@
 /**
   @file mm_getdetails.sas
   @brief extracts metadata attributes and associations for a particular uri
-
   @param [in] uri the metadata object for which to return
     attributes / associations
+  @param [in] sortoptions= Enables sorting of the output datasets, for example,
+    `SORTSEQ=LINGUISTIC`
   @param [out] outattrs= (work.attributes)
     The dataset to create that contains the list of attributes
   @param [out] outassocs= (work.associations)
     The dataset to contain the list of associations
 
-  @version 9.2
-  @author Allan Bowe
+  <h4> Related Files </h4>
+  @li mm_getobjects.sas
+  @li mm_gettypes.sas
 
 **/
 
 %macro mm_getdetails(uri
   ,outattrs=work.attributes
   ,outassocs=work.associations
+  ,sortoptions=
 )/*/STORE SOURCE*/;
 
 data &outassocs;
@@ -41,7 +44,7 @@ data &outassocs;
     n1+1;
   end;
 run;
-proc sort;
+proc sort &sortoptions;
   by assoc name;
 run;
 
@@ -61,7 +64,7 @@ data &outattrs;
     n1+1;
   end;
 run;
-proc sort;
+proc sort &sortoptions;
   by type name;
 run;
 

sasjs/utils/build.sh

@@ -1,22 +1,27 @@
 #!/bin/bash
 ####################################################################
-# PROJECT: Macro Core Docs Build                                   #
-# To execute, use the npm command (npm run docs)                   #
+# PROJECT: SASjs Core Docs Build
+# To execute, use the npm command (npm run docs)
+# Target repo will have github action to create sitemap
+# https://github.com/marketplace/actions/generate-sitemap
 ####################################################################
 
 # refresh github pages site
 rm -rf sasjsbuild/docsite
 git clone git@github.com:sasjs/core.github.io.git sasjsbuild/docsite
-rm -rf sasjsbuild/docsite/*
-mv sasjsbuild/docs/* sasjsbuild/docsite/
+rm -rf sasjsbuild/docsite/*.html
+rm -rf sasjsbuild/docsite/*.js
+rm -rf sasjsbuild/docsite/*.png
+rm -rf sasjsbuild/docsite/*.dot
+rm -rf sasjsbuild/docsite/*.css
+rm -rf sasjsbuild/docsite/*.svg
+rm -rf search
+cp -R sasjsbuild/docs/* sasjsbuild/docsite/
 cd sasjsbuild/docsite/
+git config user.name sasjs
 echo 'core.sasjs.io' > CNAME
 git add .
 git commit -m "build.sh build on $(date +%F:%H:%M:%S)"
 git push
-npx sitemap-generator-cli https://core.sasjs.io
-git add .
-git commit -m "adding sitemap"
-git push
 
 echo "check it out:  https://sasjs.github.io/core.github.io/files.html"

sasjs/utils/create_sas_package.sas

@@ -8,6 +8,12 @@
 
     Requires the server to have SSH keys.
 
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mp_gitadd.sas
+  @li mp_gitreleaseinfo.sas
+  @li mp_gitstatus.sas
+
   <h4> SAS Macros </h4>
   @li mp_gitadd.sas
   @li mp_gitreleaseinfo.sas
@@ -124,11 +130,15 @@ data members(compress=char);
   keep name name2 path;
 run;
 
+proc sort data=members;
+  by name name2;
+run;
+
 %let temp_options = %sysfunc(getoption(source)) %sysfunc(getoption(notes));
 options nosource nonotes;
 data _null_;
   set members;
-  by name notsorted;
+  by name;
 
   ord + first.name;
 

server/ms_triggerstp.sas

@@ -0,0 +1,231 @@
+/**
+  @file
+  @brief Triggers a SASjs Server STP using the /SASjsApi/stp/trigger endpoint
+  @details Triggers the STP and returns the sessionId
+
+  Example:
+
+      %ms_triggerstp(/some/stored/program
+        ,debug=131
+        ,outds=work.myresults
+      )
+
+  @param [in] pgm The full path to the Stored Program in SASjs Drive (_program
+    parameter)
+  @param [in] debug= (131) The value to supply to the _debug URL parameter
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] inputparams=(_null_) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$10000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+  @param [in] inputfiles= (_null_) A dataset containing fileref/name/filename in
+    the following format:
+    |fileref:$8|name:$32|filename:$256|
+    |---|---|--|
+    |someref|some_name|some_filename.xls|
+    |fref2|another_file|zyx_v2.csv|
+  @param [in] expiresaftermins= (15) The number of minutes to retain the session
+    folder after the session ends.
+
+  @param [out] outds= (work.ms_triggerstp) Set to the name of a dataset to
+  contain the sessionId. If this dataset already exists, and contains the
+  sessionId, it will be appended to.
+  Format:
+    |sessionId:$36|
+    |---|
+    |20241028074744-54132-1730101664824|
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+  @li mp_dropmembers.sas
+  @li mf_nobs.sas
+
+**/
+
+%macro ms_triggerstp(pgm
+    ,debug=131
+    ,inputparams=_null_
+    ,inputfiles=_null_
+    ,expiresAfterMins=15
+    ,outds=work.ms_triggerstp
+    ,mdebug=0
+  );
+  %local dbg mainref authref boundary libref triggered_sid;
+  %let mainref=%mf_getuniquefileref();
+  %let authref=%mf_getuniquefileref();
+  %let boundary=%mf_getuniquename();
+  %if &inputparams=0 %then %let inputparams=_null_;
+
+  %if &mdebug=1 %then %do;
+    %put &sysmacroname entry vars:;
+    %put _local_;
+  %end;
+  %else %let dbg=*;
+
+
+  %mp_abort(iftrue=("&pgm"="")
+    ,mac=&sysmacroname
+    ,msg=%str(Program not provided)
+  )
+  %mp_abort(iftrue=("&outds"="")
+    ,mac=&sysmacroname
+    ,msg=%str(Output dataset not provided)
+  )
+
+  /* avoid sending bom marker to API */
+  %local optval;
+  %let optval=%sysfunc(getoption(bomfile));
+  options nobomfile;
+
+  /* Add params to the content */
+  data _null_;
+    file &mainref termstr=crlf lrecl=32767 mod;
+    length line $1000 name $32 value $32767;
+    if _n_=1 then call missing(of _all_);
+    set &inputparams;
+    put "--&boundary";
+    line=cats('Content-Disposition: form-data; name="',name,'"');
+    put line;
+    put ;
+    put value;
+  run;
+
+  /* parse input file list */
+  %local webcount;
+  %let webcount=0;
+  data _null_;
+    set &inputfiles end=last;
+    length fileref $8 name $32 filename $256;
+    call symputx(cats('webref',_n_),fileref,'l');
+    call symputx(cats('webname',_n_),name,'l');
+    call symputx(cats('webfilename',_n_),filename,'l');
+    if last then do;
+      call symputx('webcount',_n_);
+      call missing(of _all_);
+    end;
+  run;
+
+  /* write out the input files to the content */
+  %local i;
+  %do i=1 %to &webcount;
+    data _null_;
+      file &mainref termstr=crlf lrecl=32767 mod;
+      infile &&webref&i lrecl=32767;
+      if _n_ = 1 then do;
+        length line $32767;
+        line=cats(
+          'Content-Disposition: form-data; name="'
+          ,"&&webname&i"
+          ,'"; filename="'
+          ,"&&webfilename&i"
+          ,'"'
+        );
+        put "--&boundary";
+        put line;
+        put "Content-Type: text/plain";
+        put ;
+      end;
+      input;
+      put _infile_; /* add the actual file to be sent */
+    run;
+  %end;
+
+  /* Add footer to the content */
+  data _null_;
+    file &mainref termstr=crlf mod;
+    put / "--&boundary--";
+  run;
+
+  data _null_;
+    file &authref lrecl=1000;
+    infile "&_sasjs_tokenfile" lrecl=1000;
+    input;
+    if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary";
+    put _infile_;
+  run;
+
+  %if &mdebug=1 %then %do;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** authref=&authref content *****";
+      infile &authref;
+      input;
+      put _infile_;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** mainref=&mainref content *****";
+      infile &mainref;
+      input;
+      put _infile_;
+    run;
+  %end;
+
+  %local resp_path outref;
+  %let resp_path=%sysfunc(pathname(work))/%mf_getuniquename();
+  %let outref=%mf_getuniquefileref();
+  filename &outref "&resp_path" lrecl=32767;
+
+  /* prepare request*/
+  proc http method='POST' headerin=&authref in=&mainref out=&outref
+    url="&_sasjs_apiserverurl/SASjsApi/stp/trigger?%trim(
+      )_program=&pgm%str(&)_debug=131%str(&)expiresAfterMins=&expiresaftermins";
+  %if &mdebug=1 %then %do;
+    debug level=2;
+  %end;
+  run;
+
+  %if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  or &mdebug=1
+  %then %do;
+    data _null_;
+      if _n_ eq 1 then putlog "NOTE: ***** outref=&outref content *****";
+      infile &outref;
+      input;
+      putlog _infile_;
+    run;
+  %end;
+  %mp_abort(
+    iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200
+        and &SYS_PROCHTTP_STATUS_CODE ne 201)
+    ,mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+
+  /* reset options */
+  options &optval;
+
+  %let libref=%mf_getuniquelibref();
+  libname &libref JSON fileref=&outref;
+  %let triggered_sid=%mf_getuniquename(prefix=triggered_sid_);
+
+  data work.&triggered_sid (keep=sessionid);
+    set &libref..root;
+
+    %if &mdebug=1 %then %do;
+      putlog (_all_)(=);
+    %end;
+  run;
+
+  %if %mf_nobs(work.&triggered_sid)>0 %then %do;
+    proc append base=&outds data=work.&triggered_sid;
+    run;
+  %end;
+
+  %if &mdebug=1 %then %do;
+    %put &sysmacroname exit vars:;
+    %put _local_;
+  %end;
+  %else %do;
+    /* clear refs */
+    filename &authref;
+    filename &mainref;
+    filename &outref;
+    libname &libref clear;
+    /* and remove temp dataset */
+    %mp_dropmembers(&triggered_sid,libref=work);
+  %end;
+
+%mend ms_triggerstp;

tests/serveronly/ms_triggerstp.test.sas

@@ -0,0 +1,90 @@
+/**
+  @file
+  @brief Testing ms_triggerstp.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquename.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li ms_createfile.sas
+  @li ms_triggerstp.sas
+  @li mf_existds.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertcols.sas
+  @li mf_getvartype.sas
+  @li ms_deletefile.sas
+
+**/
+
+/* first, create multiple STPs to run */
+filename stpcode1 temp;
+data _null_;
+  file stpcode1;
+  put '%put hello world;';
+  put '%put _all_;';
+  put 'data _null_; file _webout1; put "triggerstp test 1";run;';
+run;
+filename stpcode2 temp;
+data _null_;
+  file stpcode2;
+  put '%put Lorem Ipsum;';
+  put '%put _all_;';
+  put 'data _null_; file _webout2; put "triggerstp test 2";run;';
+run;
+options mprint;
+%let fname1=%mf_getuniquename();
+%let fname2=%mf_getuniquename();
+
+%ms_createfile(/sasjs/tests/&fname1..sas
+  ,inref=stpcode1
+  ,mdebug=1
+)
+%ms_createfile(/sasjs/tests/&fname2..sas
+  ,inref=stpcode2
+)
+
+%mp_assertscope(SNAPSHOT)
+  %ms_triggerstp(/sasjs/tests/&fname1
+    ,debug=131
+    ,outds=work.mySessions
+  )
+  %ms_triggerstp(/sasjs/tests/&fname2
+    ,outds=work.mySessions
+  )
+%mp_assertscope(COMPARE
+                ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN)
+
+%mp_assert(iftrue=%str(%mf_existds(work.mySessions)=1)
+          ,desc=Testing output exists
+          ,outds=work.test_results
+)
+
+%mp_assertdsobs(work.mySessions,
+  test=EQUALS 2,
+  desc=Testing observations,
+  outds=work.test_results
+)
+%mp_assertcols(work.mySessions,
+  cols=sessionid,
+  test=ALL,
+  desc=Testing column exists,
+  outds=work.test_results
+)
+
+data _null_;
+  retain contentCheck 1;
+  set work.mySessions end=last;
+  if missing(sessionID) then contentCheck = 0;
+  if last then do;
+    call symputx("contentCheck",contentCheck,"l");
+  end;
+run;
+%let typeCheck = %mf_getvartype(work.mySessions,sessionid);
+
+%mp_assert(iftrue=%str(&typeCheck = C and &contentCheck = 1)
+          ,desc=Testing type and content of output
+          ,outds=work.test_results
+)
+
+%ms_deletefile(/sasjs/tests/&fname1..sas)
+%ms_deletefile(/sasjs/tests/&fname2..sas)