chore: updating all.sas

fix: logic in mm_assigdirectlib to close #375

all.sas

@@ -4861,7 +4861,7 @@ data;run;
 data &out_ds(compress=no
     keep=file_or_folder filepath filename ext msg directory level
   );
-  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
+  length directory filepath $2000 fref fref2 $8 file_or_folder $6 filename $255
     ext $20 msg $200 foption $16;
   if _n_=1 then call missing(of _all_);
   retain level &level;
@@ -13393,7 +13393,7 @@ run;
 
     @li deleted rows - these are re-inserted
     @li changed rows - differences are reverted
-    @li added rows - these are marked with `_____DELETE_THIS_RECORD_____="YES"`
+    @li added rows - marked with `_____DELETE__THIS__RECORD_____="YES"`
 
     These changes are NOT applied to the base table - a staging dataset is
     simply prepared for an ETL process to action.  In Data Controller, this
@@ -13407,15 +13407,19 @@ run;
     change, plus ALL SUBSEQUENT CHANGES, will be reverted in the output table.
   @param [in] difftable The dataset containing the diffs.  Definition available
     in mddl_dc_difftable.sas
+  @param [in] filtervar= (0) If provided, the contents of this macro variable
+    will be applied as an additional filter against &libds
   @param [out] outds= (work.mp_stripdiffs) Output table containing the diffs.
     Has the same format as the base datset, plus a
-    `_____DELETE_THIS_RECORD_____` variable.
+    `_____DELETE__THIS__RECORD_____` variable.
   @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
   @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
   @li mf_islibds.sas
+  @li mf_wordsinstr1butnotstr2.sas
   @li mp_abort.sas
 
   <h4> Related Macros </h4>
@@ -13432,6 +13436,7 @@ run;
 %macro mp_stripdiffs(libds
   ,loadref
   ,difftable
+  ,filtervar=0
   ,outds=work.mp_stripdiffs
   ,mdebug=0
 )/*/STORE SOURCE*/;
@@ -13455,11 +13460,11 @@ run;
   ,msg=%str(Invalid library.dataset reference - %superq(libds))
 )
 
-
-
 /* set up unique and temporary vars */
-%local ds1 ds2 ds3 ds4 ds5 fref1;
+%local ds1 ds2 ds3 ds4 ds5 fref1 filterstr;
 %let fref1=%mf_getuniquefileref();
+%if &filtervar ne 0 %then %let filterstr=%superq(&filtervar);
+%else %let filterstr=%str(1=1);
 
 /* get timestamp of the diff to be reverted */
 %local ts;
@@ -13482,30 +13487,28 @@ create table &ds1 (drop=libref dsn) as
 
 /* extract key values only */
 %let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
+%local keyhash processed;
+%let keyhash=%upcase(%mf_getuniquename(prefix=mpsdvar_keyhash));
+%let processed=%upcase(%mf_getuniquename(prefix=mpsdvar_processed));
 create table &ds2 as
-  select key_hash,
+  select key_hash as &keyhash,
     tgtvar_nm,
     tgtvar_type,
     coalescec(oldval_char,newval_char) as charval,
     coalesce(oldval_num, newval_num) as numval,
-    processed_dttm
+    processed_dttm as &processed
   from &ds1
   where is_pk=1
-  order by key_hash, processed_dttm;
+  order by &keyhash, &processed;
 
 /* grab pk values */
 %local pk;
-data _null_;
-  set &ds2;
-  by key_hash;
-  call symputx('pk',catx(' ',symget('pk'),tgtvar_nm),'l');
-  if last.key_hash then stop;
-run;
+select distinct upcase(tgtvar_nm) into: pk separated by ' ' from &ds2;
 
 %let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
 proc transpose data=&ds2(where=(tgtvar_type='C'))
     out=&ds3(drop=_name_);
-  by KEY_HASH;
+  by &keyhash &processed;
   id TGTVAR_NM;
   var charval;
 run;
@@ -13513,7 +13516,7 @@ run;
 %let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
 proc transpose data=&ds2(where=(tgtvar_type='N'))
     out=&ds4(drop=_name_);
-  by KEY_HASH;
+  by &keyhash &processed;
   id TGTVAR_NM;
   var numval;
 run;
@@ -13521,20 +13524,32 @@ run;
 %mp_ds2squeeze(&ds3,outds=&ds3)
 %mp_ds2squeeze(&ds4,outds=&ds4)
 
+/* now merge to get all key values and de-dup */
 %let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
 data &ds5;
-  length key_hash $32;
+  length &keyhash $32 &processed 8;
   merge &ds3 &ds4;
-  by key_hash;
-  if not missing(key_hash);
+  by &keyhash &processed;
+  if not missing(&keyhash);
+run;
+proc sort data=&ds5 nodupkey;
+  by &pk;
 run;
 
 /* join to base table for preliminary stage DS */
 proc sql;
-create table &outds as select "No " as _____DELETE_THIS_RECORD_____,
-    b.*
+create table &outds as select "No " as _____DELETE__THIS__RECORD_____
+  %do x=1 %to %sysfunc(countw(&pk,%str( )));
+    ,a.%scan(&pk,&x,%str( ))
+  %end;
+  %local notpkcols;
+  %let notpkcols=%upcase(%mf_getvarlist(&libds));
+  %let notpkcols=%mf_wordsinstr1butnotstr2(str1=&notpkcols,str2=&pk);
+  %do x=1 %to %sysfunc(countw(&notpkcols,%str( )));
+    ,b.%scan(&notpkcols,&x,%str( ))
+  %end;
   from &ds5 a
-  inner join &libds b
+  left join &libds (where=(&filterstr)) b
   on 1=1
 %do x=1 %to %sysfunc(countw(&pk,%str( )));
   and a.%scan(&pk,&x,%str( ))=b.%scan(&pk,&x,%str( ))
@@ -13544,7 +13559,8 @@ create table &outds as select "No " as _____DELETE_THIS_RECORD_____,
 /* create SAS code to apply to stage_ds */
 data _null_;
   set &ds1;
-  file &fref1;
+  file &fref1 lrecl=33000;
+  length charval $32767;
   if _n_=1 then put 'proc sql noprint;';
   by descending processed_dttm key_hash is_pk;
   if move_type='M' then do;
@@ -13553,7 +13569,8 @@ data _null_;
     end;
     if IS_PK=0 then do;
       put "  " tgtvar_nm '=' @@;
-      charval=quote(cats(oldval_char));
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
       if tgtvar_type='C' then put charval @@;
       else put oldval_num @@;
       if not last.is_pk then put ',';
@@ -13561,36 +13578,49 @@ data _null_;
     else do;
       if first.is_pk then put "  where 1=1 " @@;
       put "  and " tgtvar_nm '=' @@;
-      charval=quote(cats(oldval_char));
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
       if tgtvar_type='C' then put charval @@;
       else put oldval_num @@;
     end;
   end;
   else if move_type='A' then do;
     if first.key_hash then do;
-      put "update &outds set _____DELETE_THIS_RECORD_____='Yes'  where 1=1 " @@;
+      put "update &outds set _____DELETE__THIS__RECORD_____='Yes' where 1=1 "@@;
     end;
     /* gating if - as only need PK now */
     if is_pk=1;
     put '  AND ' tgtvar_nm '=' @@;
-    charval=quote(cats(newval_char));
+    cnt=count(newval_char,'"');
+    charval=quote(trim(substr(newval_char,1,32765-cnt)));
     if tgtvar_type='C' then put charval @@;
     else put newval_num @@;
   end;
   else if move_type='D' then do;
     if first.key_hash then do;
-      put "insert into &outds set _____DELETE_THIS_RECORD_____='No' " @@;
+      put "update &outds set _____DELETE__THIS__RECORD_____='No' " @@;
+    end;
+    if IS_PK=0 then do;
+      put "  ," tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
     end;
-    put "  ," tgtvar_nm '=' @@;
-    charval=quote(cats(oldval_char));
-    if tgtvar_type='C' then put charval @@;
-    else put oldval_num @@;
   end;
   if last.key_hash then put ';';
 run;
 
 /* apply the modification statements */
-%inc &fref1/source2;
+%inc &fref1/source2 lrecl=33000;
 
 %if &mdebug=0 %then %do;
   proc sql;
@@ -14744,7 +14774,8 @@ filename __us2grp clear;
 
 %local cur_engine;
 %let cur_engine=%mf_getengine(&libref);
-%if &cur_engine ne META and &cur_engine ne %then %do;
+%if &cur_engine ne META and &cur_engine ne and %length(&open_passthrough)=0
+%then %do;
   %put NOTE:  &libref already has a direct (&cur_engine) libname connection;
   %return;
 %end;
@@ -23082,6 +23113,221 @@ run;
 %end;
 
 %mend ms_testservice;
+/**
+  @file
+  @brief Triggers a SASjs Server STP using the /SASjsApi/code/trigger endpoint
+  @details Triggers the STP and returns the sessionId
+
+  Example:
+
+      %ms_triggerstp(/some/stored/program
+        ,debug=131
+        ,outds=work.myresults
+      )
+
+  @param [in] pgm The full path to the Stored Program in SASjs Drive (_program
+    parameter)
+  @param [in] debug= (131) The value to supply to the _debug URL parameter
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] inputparams=(_null_) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$10000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+  @param [in] inputfiles= (_null_) A dataset containing fileref/name/filename in
+    the following format:
+    |fileref:$8|name:$32|filename:$256|
+    |---|---|--|
+    |someref|some_name|some_filename.xls|
+    |fref2|another_file|zyx_v2.csv|
+
+  @param [out] outds= (work.ms_triggerstp) Set to the name of a dataset to
+  contain the sessionId. If this dataset already exists, and contains the
+  sessionId, it will be appended to.
+  Format:
+    |sessionId:$36|
+    |---|
+    |20241028074744-54132-1730101664824|
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+
+**/
+
+%macro ms_triggerstp(pgm
+    ,debug=131
+    ,inputparams=_null_
+    ,inputfiles=_null_
+    ,outds=work.ms_triggerstp
+    ,mdebug=0
+  );
+%local dbg mainref authref;
+%let mainref=%mf_getuniquefileref();
+%let authref=%mf_getuniquefileref();
+%if &inputparams=0 %then %let inputparams=_null_;
+
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+
+%mp_abort(iftrue=("&pgm"="")
+  ,mac=&sysmacroname
+  ,msg=%str(Program not provided)
+)
+
+/* avoid sending bom marker to API */
+%local optval;
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
+/* add params */
+data _null_;
+  file &mainref termstr=crlf lrecl=32767 mod;
+  length line $1000 name $32 value $32767;
+  if _n_=1 then call missing(of _all_);
+  set &inputparams;
+  put "--&boundary";
+  line=cats('Content-Disposition: form-data; name="',name,'"');
+  put line;
+  put ;
+  put value;
+run;
+
+/* parse input file list */
+%local webcount;
+%let webcount=0;
+data _null_;
+  set &inputfiles end=last;
+  length fileref $8 name $32 filename $256;
+  call symputx(cats('webref',_n_),fileref,'l');
+  call symputx(cats('webname',_n_),name,'l');
+  call symputx(cats('webfilename',_n_),filename,'l');
+  if last then do;
+    call symputx('webcount',_n_);
+    call missing(of _all_);
+  end;
+run;
+
+/* write out the input files */
+%local i;
+%do i=1 %to &webcount;
+  data _null_;
+    file &mainref termstr=crlf lrecl=32767 mod;
+    infile &&webref&i lrecl=32767;
+    if _n_ = 1 then do;
+      length line $32767;
+      line=cats(
+        'Content-Disposition: form-data; name="'
+        ,"&&webname&i"
+        ,'"; filename="'
+        ,"&&webfilename&i"
+        ,'"'
+      );
+      put "--&boundary";
+      put line;
+      put "Content-Type: text/plain";
+      put ;
+    end;
+    input;
+    put _infile_; /* add the actual file to be sent */
+  run;
+%end;
+
+data _null_;
+  file &mainref termstr=crlf mod;
+  put "--&boundary--";
+run;
+
+data _null_;
+  file &authref lrecl=1000;
+  infile "&_sasjs_tokenfile" lrecl=1000;
+  input;
+  if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary";
+  put _infile_;
+run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &authref;
+    input;
+    put _infile_;
+  data _null_;
+    infile &mainref;
+    input;
+    put _infile_;
+  run;
+%end;
+
+%local resp_path;
+%let resp_path=%sysfunc(pathname(work))/%mf_getuniquename();
+filename &outref "&resp_path" lrecl=32767;
+
+/* prepare request*/
+proc http method='POST' headerin=&authref in=&mainref out=&outref
+  url="&_sasjs_apiserverurl/SASjsApi/stp/trigger?%trim(
+    )_program=&pgm%str(&)_debug=131";
+%if &mdebug=1 %then %do;
+  debug level=2;
+%end;
+run;
+
+%if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+or &mdebug=1
+%then %do;
+  data _null_;infile &outref;input;putlog _infile_;run;
+%end;
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+/* reset options */
+options &optval;
+
+data work.%mf_getuniquename();
+  infile "&resp_path";
+  input
+
+%if &outlogds ne _null_ or &mdebug=1 %then %do;
+  %local matchstr chopout;
+  %let matchstr=SASJS_LOGS_SEPARATOR_163ee17b6ff24f028928972d80a26784;
+  %let chopout=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop);
+
+  %mp_chop("&resp_path"
+    ,matchvar=matchstr
+    ,keep=LAST
+    ,matchpoint=END
+    ,outfile="&chopout"
+    ,mdebug=&mdebug
+  )
+
+  data &outlogds;
+    infile "&chopout" lrecl=2000;
+    length line $2000;
+    line=_infile_;
+  %if &mdebug=1 %then %do;
+    putlog line=;
+  %end;
+  run;
+%end;
+
+%if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%else %do;
+  /* clear refs */
+  filename &authref;
+  filename &mainref;
+%end;
+%mend ms_triggerstp;
 /**
   @file
   @brief Send data to/from sasjs/server

base/mp_dirlist.sas

@@ -85,7 +85,7 @@ data;run;
 data &out_ds(compress=no
     keep=file_or_folder filepath filename ext msg directory level
   );
-  length directory filepath $500 fref fref2 $8 file_or_folder $6 filename $80
+  length directory filepath $2000 fref fref2 $8 file_or_folder $6 filename $255
     ext $20 msg $200 foption $16;
   if _n_=1 then call missing(of _all_);
   retain level &level;

base/mp_stripdiffs.sas

@@ -8,7 +8,7 @@
 
     @li deleted rows - these are re-inserted
     @li changed rows - differences are reverted
-    @li added rows - these are marked with `_____DELETE_THIS_RECORD_____="YES"`
+    @li added rows - marked with `_____DELETE__THIS__RECORD_____="YES"`
 
     These changes are NOT applied to the base table - a staging dataset is
     simply prepared for an ETL process to action.  In Data Controller, this
@@ -22,15 +22,19 @@
     change, plus ALL SUBSEQUENT CHANGES, will be reverted in the output table.
   @param [in] difftable The dataset containing the diffs.  Definition available
     in mddl_dc_difftable.sas
+  @param [in] filtervar= (0) If provided, the contents of this macro variable
+    will be applied as an additional filter against &libds
   @param [out] outds= (work.mp_stripdiffs) Output table containing the diffs.
     Has the same format as the base datset, plus a
-    `_____DELETE_THIS_RECORD_____` variable.
+    `_____DELETE__THIS__RECORD_____` variable.
   @param [in] mdebug= set to 1 to enable DEBUG messages and preserve outputs
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
   @li mf_getuniquename.sas
+  @li mf_getvarlist.sas
   @li mf_islibds.sas
+  @li mf_wordsinstr1butnotstr2.sas
   @li mp_abort.sas
 
   <h4> Related Macros </h4>
@@ -47,6 +51,7 @@
 %macro mp_stripdiffs(libds
   ,loadref
   ,difftable
+  ,filtervar=0
   ,outds=work.mp_stripdiffs
   ,mdebug=0
 )/*/STORE SOURCE*/;
@@ -70,11 +75,11 @@
   ,msg=%str(Invalid library.dataset reference - %superq(libds))
 )
 
-
-
 /* set up unique and temporary vars */
-%local ds1 ds2 ds3 ds4 ds5 fref1;
+%local ds1 ds2 ds3 ds4 ds5 fref1 filterstr;
 %let fref1=%mf_getuniquefileref();
+%if &filtervar ne 0 %then %let filterstr=%superq(&filtervar);
+%else %let filterstr=%str(1=1);
 
 /* get timestamp of the diff to be reverted */
 %local ts;
@@ -97,30 +102,28 @@ create table &ds1 (drop=libref dsn) as
 
 /* extract key values only */
 %let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_pks));
+%local keyhash processed;
+%let keyhash=%upcase(%mf_getuniquename(prefix=mpsdvar_keyhash));
+%let processed=%upcase(%mf_getuniquename(prefix=mpsdvar_processed));
 create table &ds2 as
-  select key_hash,
+  select key_hash as &keyhash,
     tgtvar_nm,
     tgtvar_type,
     coalescec(oldval_char,newval_char) as charval,
     coalesce(oldval_num, newval_num) as numval,
-    processed_dttm
+    processed_dttm as &processed
   from &ds1
   where is_pk=1
-  order by key_hash, processed_dttm;
+  order by &keyhash, &processed;
 
 /* grab pk values */
 %local pk;
-data _null_;
-  set &ds2;
-  by key_hash;
-  call symputx('pk',catx(' ',symget('pk'),tgtvar_nm),'l');
-  if last.key_hash then stop;
-run;
+select distinct upcase(tgtvar_nm) into: pk separated by ' ' from &ds2;
 
 %let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_keychar));
 proc transpose data=&ds2(where=(tgtvar_type='C'))
     out=&ds3(drop=_name_);
-  by KEY_HASH;
+  by &keyhash &processed;
   id TGTVAR_NM;
   var charval;
 run;
@@ -128,7 +131,7 @@ run;
 %let ds4=%upcase(work.%mf_getuniquename(prefix=mpsd_keynum));
 proc transpose data=&ds2(where=(tgtvar_type='N'))
     out=&ds4(drop=_name_);
-  by KEY_HASH;
+  by &keyhash &processed;
   id TGTVAR_NM;
   var numval;
 run;
@@ -136,20 +139,32 @@ run;
 %mp_ds2squeeze(&ds3,outds=&ds3)
 %mp_ds2squeeze(&ds4,outds=&ds4)
 
+/* now merge to get all key values and de-dup */
 %let ds5=%upcase(work.%mf_getuniquename(prefix=mpsd_merged));
 data &ds5;
-  length key_hash $32;
+  length &keyhash $32 &processed 8;
   merge &ds3 &ds4;
-  by key_hash;
-  if not missing(key_hash);
+  by &keyhash &processed;
+  if not missing(&keyhash);
+run;
+proc sort data=&ds5 nodupkey;
+  by &pk;
 run;
 
 /* join to base table for preliminary stage DS */
 proc sql;
-create table &outds as select "No " as _____DELETE_THIS_RECORD_____,
-    b.*
+create table &outds as select "No " as _____DELETE__THIS__RECORD_____
+  %do x=1 %to %sysfunc(countw(&pk,%str( )));
+    ,a.%scan(&pk,&x,%str( ))
+  %end;
+  %local notpkcols;
+  %let notpkcols=%upcase(%mf_getvarlist(&libds));
+  %let notpkcols=%mf_wordsinstr1butnotstr2(str1=&notpkcols,str2=&pk);
+  %do x=1 %to %sysfunc(countw(&notpkcols,%str( )));
+    ,b.%scan(&notpkcols,&x,%str( ))
+  %end;
   from &ds5 a
-  inner join &libds b
+  left join &libds (where=(&filterstr)) b
   on 1=1
 %do x=1 %to %sysfunc(countw(&pk,%str( )));
   and a.%scan(&pk,&x,%str( ))=b.%scan(&pk,&x,%str( ))
@@ -159,7 +174,8 @@ create table &outds as select "No " as _____DELETE_THIS_RECORD_____,
 /* create SAS code to apply to stage_ds */
 data _null_;
   set &ds1;
-  file &fref1;
+  file &fref1 lrecl=33000;
+  length charval $32767;
   if _n_=1 then put 'proc sql noprint;';
   by descending processed_dttm key_hash is_pk;
   if move_type='M' then do;
@@ -168,7 +184,8 @@ data _null_;
     end;
     if IS_PK=0 then do;
       put "  " tgtvar_nm '=' @@;
-      charval=quote(cats(oldval_char));
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
       if tgtvar_type='C' then put charval @@;
       else put oldval_num @@;
       if not last.is_pk then put ',';
@@ -176,36 +193,49 @@ data _null_;
     else do;
       if first.is_pk then put "  where 1=1 " @@;
       put "  and " tgtvar_nm '=' @@;
-      charval=quote(cats(oldval_char));
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
       if tgtvar_type='C' then put charval @@;
       else put oldval_num @@;
     end;
   end;
   else if move_type='A' then do;
     if first.key_hash then do;
-      put "update &outds set _____DELETE_THIS_RECORD_____='Yes'  where 1=1 " @@;
+      put "update &outds set _____DELETE__THIS__RECORD_____='Yes' where 1=1 "@@;
     end;
     /* gating if - as only need PK now */
     if is_pk=1;
     put '  AND ' tgtvar_nm '=' @@;
-    charval=quote(cats(newval_char));
+    cnt=count(newval_char,'"');
+    charval=quote(trim(substr(newval_char,1,32765-cnt)));
     if tgtvar_type='C' then put charval @@;
     else put newval_num @@;
   end;
   else if move_type='D' then do;
     if first.key_hash then do;
-      put "insert into &outds set _____DELETE_THIS_RECORD_____='No' " @@;
+      put "update &outds set _____DELETE__THIS__RECORD_____='No' " @@;
+    end;
+    if IS_PK=0 then do;
+      put "  ," tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
+    end;
+    else do;
+      if first.is_pk then put "  where 1=1 " @@;
+      put "  and " tgtvar_nm '=' @@;
+      cnt=count(oldval_char,'"');
+      charval=quote(trim(substr(oldval_char,1,32765-cnt)));
+      if tgtvar_type='C' then put charval @@;
+      else put oldval_num @@;
     end;
-    put "  ," tgtvar_nm '=' @@;
-    charval=quote(cats(oldval_char));
-    if tgtvar_type='C' then put charval @@;
-    else put oldval_num @@;
   end;
   if last.key_hash then put ';';
 run;
 
 /* apply the modification statements */
-%inc &fref1/source2;
+%inc &fref1/source2 lrecl=33000;
 
 %if &mdebug=0 %then %do;
   proc sql;

meta/mm_assigndirectlib.sas

@@ -54,7 +54,8 @@
 
 %local cur_engine;
 %let cur_engine=%mf_getengine(&libref);
-%if &cur_engine ne META and &cur_engine ne %then %do;
+%if &cur_engine ne META and &cur_engine ne and %length(&open_passthrough)=0
+%then %do;
   %put NOTE:  &libref already has a direct (&cur_engine) libname connection;
   %return;
 %end;

sasjs/utils/create_sas_package.sas

@@ -8,6 +8,12 @@
 
     Requires the server to have SSH keys.
 
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mp_gitadd.sas
+  @li mp_gitreleaseinfo.sas
+  @li mp_gitstatus.sas
+
   <h4> SAS Macros </h4>
   @li mp_gitadd.sas
   @li mp_gitreleaseinfo.sas
@@ -124,11 +130,15 @@ data members(compress=char);
   keep name name2 path;
 run;
 
+proc sort data=members;
+  by name name2;
+run;
+
 %let temp_options = %sysfunc(getoption(source)) %sysfunc(getoption(notes));
 options nosource nonotes;
 data _null_;
   set members;
-  by name notsorted;
+  by name;
 
   ord + first.name;
 

server/ms_triggerstp.sas

@@ -0,0 +1,215 @@
+/**
+  @file
+  @brief Triggers a SASjs Server STP using the /SASjsApi/code/trigger endpoint
+  @details Triggers the STP and returns the sessionId
+
+  Example:
+
+      %ms_triggerstp(/some/stored/program
+        ,debug=131
+        ,outds=work.myresults
+      )
+
+  @param [in] pgm The full path to the Stored Program in SASjs Drive (_program
+    parameter)
+  @param [in] debug= (131) The value to supply to the _debug URL parameter
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [in] inputparams=(_null_) A dataset containing name/value pairs in the
+    following format:
+    |name:$32|value:$10000|
+    |---|---|
+    |stpmacname|some value|
+    |mustbevalidname|can be anything, oops, %abort!!|
+  @param [in] inputfiles= (_null_) A dataset containing fileref/name/filename in
+    the following format:
+    |fileref:$8|name:$32|filename:$256|
+    |---|---|--|
+    |someref|some_name|some_filename.xls|
+    |fref2|another_file|zyx_v2.csv|
+
+  @param [out] outds= (work.ms_triggerstp) Set to the name of a dataset to
+  contain the sessionId. If this dataset already exists, and contains the
+  sessionId, it will be appended to.
+  Format:
+    |sessionId:$36|
+    |---|
+    |20241028074744-54132-1730101664824|
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+
+**/
+
+%macro ms_triggerstp(pgm
+    ,debug=131
+    ,inputparams=_null_
+    ,inputfiles=_null_
+    ,outds=work.ms_triggerstp
+    ,mdebug=0
+  );
+%local dbg mainref authref;
+%let mainref=%mf_getuniquefileref();
+%let authref=%mf_getuniquefileref();
+%if &inputparams=0 %then %let inputparams=_null_;
+
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+
+
+%mp_abort(iftrue=("&pgm"="")
+  ,mac=&sysmacroname
+  ,msg=%str(Program not provided)
+)
+
+/* avoid sending bom marker to API */
+%local optval;
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
+/* add params */
+data _null_;
+  file &mainref termstr=crlf lrecl=32767 mod;
+  length line $1000 name $32 value $32767;
+  if _n_=1 then call missing(of _all_);
+  set &inputparams;
+  put "--&boundary";
+  line=cats('Content-Disposition: form-data; name="',name,'"');
+  put line;
+  put ;
+  put value;
+run;
+
+/* parse input file list */
+%local webcount;
+%let webcount=0;
+data _null_;
+  set &inputfiles end=last;
+  length fileref $8 name $32 filename $256;
+  call symputx(cats('webref',_n_),fileref,'l');
+  call symputx(cats('webname',_n_),name,'l');
+  call symputx(cats('webfilename',_n_),filename,'l');
+  if last then do;
+    call symputx('webcount',_n_);
+    call missing(of _all_);
+  end;
+run;
+
+/* write out the input files */
+%local i;
+%do i=1 %to &webcount;
+  data _null_;
+    file &mainref termstr=crlf lrecl=32767 mod;
+    infile &&webref&i lrecl=32767;
+    if _n_ = 1 then do;
+      length line $32767;
+      line=cats(
+        'Content-Disposition: form-data; name="'
+        ,"&&webname&i"
+        ,'"; filename="'
+        ,"&&webfilename&i"
+        ,'"'
+      );
+      put "--&boundary";
+      put line;
+      put "Content-Type: text/plain";
+      put ;
+    end;
+    input;
+    put _infile_; /* add the actual file to be sent */
+  run;
+%end;
+
+data _null_;
+  file &mainref termstr=crlf mod;
+  put "--&boundary--";
+run;
+
+data _null_;
+  file &authref lrecl=1000;
+  infile "&_sasjs_tokenfile" lrecl=1000;
+  input;
+  if _n_=1 then put "Content-Type: multipart/form-data; boundary=&boundary";
+  put _infile_;
+run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &authref;
+    input;
+    put _infile_;
+  data _null_;
+    infile &mainref;
+    input;
+    put _infile_;
+  run;
+%end;
+
+%local resp_path;
+%let resp_path=%sysfunc(pathname(work))/%mf_getuniquename();
+filename &outref "&resp_path" lrecl=32767;
+
+/* prepare request*/
+proc http method='POST' headerin=&authref in=&mainref out=&outref
+  url="&_sasjs_apiserverurl/SASjsApi/stp/trigger?%trim(
+    )_program=&pgm%str(&)_debug=131";
+%if &mdebug=1 %then %do;
+  debug level=2;
+%end;
+run;
+
+%if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+or &mdebug=1
+%then %do;
+  data _null_;infile &outref;input;putlog _infile_;run;
+%end;
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+/* reset options */
+options &optval;
+
+data work.%mf_getuniquename();
+  infile "&resp_path";
+  input
+
+%if &outlogds ne _null_ or &mdebug=1 %then %do;
+  %local matchstr chopout;
+  %let matchstr=SASJS_LOGS_SEPARATOR_163ee17b6ff24f028928972d80a26784;
+  %let chopout=%sysfunc(pathname(work))/%mf_getuniquename(prefix=chop);
+
+  %mp_chop("&resp_path"
+    ,matchvar=matchstr
+    ,keep=LAST
+    ,matchpoint=END
+    ,outfile="&chopout"
+    ,mdebug=&mdebug
+  )
+
+  data &outlogds;
+    infile "&chopout" lrecl=2000;
+    length line $2000;
+    line=_infile_;
+  %if &mdebug=1 %then %do;
+    putlog line=;
+  %end;
+  run;
+%end;
+
+%if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%else %do;
+  /* clear refs */
+  filename &authref;
+  filename &mainref;
+%end;
+%mend ms_triggerstp;

tests/base/mp_stripdiffs.test.sas

@@ -82,10 +82,10 @@ run;
 %let addpass=0;
 data _null_;
   set work.mp_stripdiffs;
-  if upcase(_____DELETE_THIS_RECORD_____)='NO' and name="&delname"
+  if upcase(_____DELETE__THIS__RECORD_____)='NO' and name="&delname"
   then call symputx('delpass',1);
   if name="&modname" and age=&modval then call symputx('modpass',1);
-  if upcase(_____DELETE_THIS_RECORD_____)='YES' and name="Newbie"
+  if upcase(_____DELETE__THIS__RECORD_____)='YES' and name="Newbie"
   then call symputx('addpass',1);
 run;