Merge pull request #199 from sasjs/server

fix: missing dependency in mv_deleteviyafolder

.git-hooks/pre-commit

@@ -1,5 +1,12 @@
 #!/bin/bash
-sasjs lint
+
+# Ensure lint is passing
+LINT=`sasjs lint`
+if [[ "$LINT" != "✔ All matched files use @sasjs/lint code style!" ]]; then
+  echo "$LINT"
+  echo "To commit in spite of these warnings, use the -n parameter."
+  exit 1
+fi
 
 # Avoid commits to the master branch
 BRANCH=`git rev-parse --abbrev-ref HEAD`

.gitpod.yml

@@ -1,7 +1,7 @@
 tasks:
-  - init: |
-      nvm install --lts
-      npm i -g @sasjs/cli
+  - init: npm install -g npm
+  - command: npm i
+  - command: npm i -g @sasjs/cli
 
 image:
   file: .gitpod.dockerfile
@@ -24,4 +24,4 @@ github:
     # add a "Review in Gitpod" button to pull requests (defaults to false)
     addBadge: false
     # add a label once the prebuild is ready to pull requests (defaults to false)
-    addLabel: prebuilt-in-gitpod
+    addLabel: prebuilt-in-gitpod

README.md

@@ -217,6 +217,15 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 ![](https://starchart.cc/sasjs/core.svg)
 
+## Other SAS Repositories
+
+The following repositories are also worth checking out:
+
+* [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
+* [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
+* [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)
+* [scottbass/sas](https://github.com/scottbass/SAS)
+* [yabwon/sas_packages](https://github.com/yabwon/SAS_PACKAGES)
 
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->

all.sas

@@ -95,6 +95,37 @@ options noquotelenmax;
 %mend mf_dedup;
 
 
+/**
+  @file
+  @brief Deletes a physical file, if it exists
+  @details Usage:
+
+      %mf_writefile(&sasjswork/myfile.txt,l1=some content)
+
+      %mf_deletefile(&sasjswork/myfile.txt)
+
+      %mf_deletefile(&sasjswork/myfile.txt)
+
+
+  @param filepath Full path to the target file
+
+  @returns The return code from the fdelete() invocation
+
+  <h4> Related Macros </h4>
+  @li mf_deletefile.test.sas
+  @li mf_writefile.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_deletefile(file
+)/*/STORE SOURCE*/;
+  %local rc fref;
+  %let rc= %sysfunc(filename(fref,&file));
+  %if %sysfunc(fdelete(&fref)) ne 0 %then %put %sysfunc(sysmsg());
+  %let rc= %sysfunc(filename(fref));
+%mend mf_deletefile;
 /**
   @file mf_existds.sas
   @brief Checks whether a dataset OR a view exists.
@@ -136,19 +167,17 @@ options noquotelenmax;
 
         %put %mf_existfeature(PROCLUA);
 
-  @param feature the feature to detect.  Leave blank to list all in log.
+  @param [in] feature The feature to detect.
 
   @return output returns 1 or 0 (or -1 if not found)
 
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
 
-
   @version 8
   @author Allan Bowe
 **/
 /** @cond */
-
 %macro mf_existfeature(feature
 )/*/STORE SOURCE*/;
   %let feature=%upcase(&feature);
@@ -156,7 +185,11 @@ options noquotelenmax;
   %let platform=%mf_getplatform();
 
   %if &feature= %then %do;
-    %put Supported features:  PROCLUA;
+    %put No feature was requested for detection;
+  %end;
+  %else %if &feature=COLCONSTRAINTS %then %do;
+    %if %substr(&sysver,1,1)=4 %then 0;
+    %else 1;
   %end;
   %else %if &feature=PROCLUA %then %do;
     /* https://blogs.sas.com/content/sasdummy/2015/08/03/using-lua-within-your-sas-programs */
@@ -170,8 +203,8 @@ options noquotelenmax;
     %put &sysmacroname: &feature not found;
   %end;
 %mend mf_existfeature;
-
-/** @endcond *//**
+/** @endcond */
+/**
   @file
   @brief Checks whether a fileref exists
   @details You can probably do without this macro as it is just a one liner.
@@ -1066,7 +1099,8 @@ or %index(&pgm,/tests/testteardown)
   %else %if %symexist(&metavar) %then %do;
     %if %length(&&&metavar)=0 %then %let user=&sysuserid;
     /* sometimes SAS will add @domain extension - remove for consistency */
-    %else %let user=%scan(&&&metavar,1,@);
+    /* but be sure to quote in case of usernames with commas */
+    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@));
   %end;
   %else %let user=&sysuserid;
 
@@ -1889,7 +1923,7 @@ Usage:
 
   %goto exit_success;
   %exit_err:
-    %put %str(ERR)OR: &abortmsg;
+    %put &abortmsg;
     %mf_abort(iftrue=(&mabort ne SOFT),
       mac=mf_verifymacvars,
       msg=%str(&abortmsg)
@@ -2160,7 +2194,7 @@ Usage:
   %end;
 
   /* Stored Process Server web app context */
-  %if %symexist(_metaperson)
+  %if %symexist(_metaport)
     or "&SYSPROCESSNAME "="Compute Server "
     or &mode=INCLUDE
   %then %do;
@@ -2335,7 +2369,8 @@ Usage:
   %end;
 %mend mp_abort;
 
-/** @endcond *//**
+/** @endcond */
+/**
   @file
   @brief Append (concatenate) two or more files.
   @details Will append one more more `appendrefs` (filerefs) to a `baseref`.
@@ -4213,6 +4248,13 @@ data &out_ds(compress=no
   %end;
   if rc = 0 then do;
     did = dopen(fref);
+    if did=0 then do;
+      putlog "NOTE: This directory is empty, or does not exist - &path";
+      msg=sysmsg();
+      put msg;
+      put _all_;
+      stop;
+    end;
     /* attribute is OS-dependent - could be "Directory" or "Directory Name" */
     numopts=doptnum(did);
     do i=1 to numopts;
@@ -4220,12 +4262,6 @@ data &out_ds(compress=no
       if foption=:'Directory' then i=numopts;
     end;
     directory=dinfo(did,foption);
-    if did=0 then do;
-      putlog "NOTE: This directory is empty - " directory;
-      msg=sysmsg();
-      put _all_;
-      stop;
-    end;
     rc = filename(fref);
   end;
   else do;
@@ -4361,7 +4397,8 @@ run;
 proc sql;
 drop table &out_ds;
 
-%mend mp_dirlist;/**
+%mend mp_dirlist;
+/**
   @file
   @brief Creates a dataset containing distinct _formatted_ values
   @details If no format is supplied, then the original value is used instead.
@@ -7937,7 +7974,7 @@ run;
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
   @param [in] iftrue= A condition under which the macro should be executed.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
-  will contain one column (hashkey) with one observation (a hex32.
+  will contain one column (hashkey) with one observation (a $hex32.
   representation of the input hash)
   |hashkey:$32.|
   |---|
@@ -8640,13 +8677,13 @@ select distinct lowcase(memname)
   @li mp_abort.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
+  @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas
   @li mddl_dc_locktable.sas
   @li mp_loadformat.test.sas
   @li mp_lockanytable.sas
-  @li mp_storediffs.sas
   @li mp_stackdiffs.sas
 
 
@@ -8734,7 +8771,7 @@ run;
   * First, extract only relevant formats from the catalog
   */
 proc sql noprint;
-select distinct fmtname into: fmtlist separated by ' ' from &libds;
+select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds;
 
 %mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
 
@@ -8744,8 +8781,11 @@ select distinct fmtname into: fmtlist separated by ' ' from &libds;
   */
 %mddl_sas_cntlout(libds=&template)
 data &inlibds;
+  length &delete_col $3;
   if 0 then set &template;
   set &libds;
+  if &delete_col='' then &delete_col='No';
+  fmtname=upcase(fmtname);
   if missing(type) then do;
     if substr(fmtname,1,1)='$' then type='C';
     else type='N';
@@ -8756,7 +8796,6 @@ data &inlibds;
   end;
 run;
 
-
 /**
   * Identify new records
   */
@@ -8863,7 +8902,7 @@ options ibufsize=&ibufsize;
     %end;
 
     %mp_storediffs(&libcat-FC
-      ,&inlibds
+      ,&base_fmts
       ,FMTNAME START
       ,delds=&outds_del
       ,modds=&outds_mod
@@ -8872,6 +8911,9 @@ options ibufsize=&ibufsize;
       ,mdebug=&mdebug
     )
 
+    proc append base=&auditlibds data=&storediffs;
+    run;
+
     %if &locklibds ne 0 %then %do;
       %mp_lockanytable(UNLOCK
         ,lib=%scan(&auditlibds,1,.)
@@ -8895,7 +8937,8 @@ options ibufsize=&ibufsize;
   %put &sysmacroname exit vars:;
   %put _local_;
 %end;
-%mend mp_loadformat;/**
+%mend mp_loadformat;
+/**
   @file
   @brief Mechanism for locking tables to prevent parallel modifications
   @details Uses a control table to enable ANY table to be locked for updates
@@ -9401,7 +9444,7 @@ put(md5(
     &sep put(md5(trim(put(ifn(missing(&var),&var,&var*1),binary64.))),$hex32.)
     %let sep=!!;
   %end;
-),hex32.)
+),$hex32.)
 %mend mp_md5;
 /**
   @file
@@ -9693,7 +9736,7 @@ filename &inref &infile lrecl=1 recfm=n;
 
 data &ds1;
   infile &inref;
-  input sourcechar $ 1. @@;
+  input sourcechar $char1. @@;
   format sourcechar hex2.;
 run;
 
@@ -15086,7 +15129,8 @@ data _null_;
   put '  %else %if %symexist(&metavar) %then %do; ';
   put '    %if %length(&&&metavar)=0 %then %let user=&sysuserid; ';
   put '    /* sometimes SAS will add @domain extension - remove for consistency */ ';
-  put '    %else %let user=%scan(&&&metavar,1,@); ';
+  put '    /* but be sure to quote in case of usernames with commas */ ';
+  put '    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@)); ';
   put '  %end; ';
   put '  %else %let user=&sysuserid; ';
   put ' ';
@@ -15307,6 +15351,7 @@ filename &fname2 clear;
 %local isgone;
 data _null_;
   length type uri $256;
+  call missing (of _all_);
   rc=metadata_resolve("omsobj:SASLibrary?@Id='&liburi'",type,uri);
   call symputx('isgone',type,'l');
 run;
@@ -17719,20 +17764,25 @@ filename __shake clear;
 
 
   <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_isblank.sas
   @li mf_loc.sas
   @li mm_tree.sas
-  @li mf_getuniquefileref.sas
-  @li mf_isblank.sas
   @li mp_abort.sas
 
-  @param metaloc= the metadata folder to export
-  @param secureref= fileref containing the username / password (should point to
-    a file in a secure location).  Leave blank to substitute $bash type vars.
-  @param outref= fileref to which to write the command
-  @param cmdoutloc= the directory to which the command will write the SPK
-    (default=WORK)
-  @param cmdoutname= the name of the spk / log files to create (will be
-    identical just with .spk or .log extension)
+
+  @param [in] metaloc= the metadata folder to export
+  @param [in] secureref= fileref containing the username / password (should
+    point to a file in a secure location). Leave blank to substitute $bash vars.
+  @param [in] excludevars= (0) A space seperated list of macro variable names,
+    each of which contains a value that should be used to filter the output
+    objects.
+  @param [out] outref= fileref to which to write the command
+  @param [out] cmdoutloc= (%sysfunc(pathname(work))) The directory to which the
+    command will write the SPK
+  @param [out] cmdoutname= (mmxport) The name of the spk / log files to create
+    (will be identical just with .spk or .log extension)
 
   @version 9.4
   @author Allan Bowe
@@ -17741,6 +17791,7 @@ filename __shake clear;
 
 %macro mm_spkexport(metaloc=
   ,secureref=
+  ,excludevars=0
   ,outref=
   ,cmdoutloc=%sysfunc(pathname(work))
   ,cmdoutname=mmxport
@@ -17752,7 +17803,7 @@ filename __shake clear;
 %end;
 
 /* set creds */
-%local mmxuser mmxpath;
+%local mmxuser mmxpath i var;
 %let mmxuser=$1;
 %let mmxpass=$2;
 %if %mf_isblank(&secureref)=0 %then %do;
@@ -17760,38 +17811,55 @@ filename __shake clear;
 %end;
 
 /* setup metadata connection options */
-%local host port platform_object_path connx_string;
+%local host port platform_object_path ds;
 %let host=%sysfunc(getoption(metaserver));
 %let port=%sysfunc(getoption(metaport));
 %let platform_object_path=%mf_loc(POF);
+%let ds=%mf_getuniquename(prefix=spkexportable);
 
-%let connx_string=%str(-host &host -port &port -user &mmxuser %trim(
-  )-password &mmxpass);
-
-%mm_tree(root=%str(&metaloc) ,types=EXPORTABLE ,outds=exportable)
+%mm_tree(root=%str(&metaloc),types=EXPORTABLE ,outds=&ds)
 
 %if %mf_isblank(&outref)=1 %then %let outref=%mf_getuniquefileref();
 
 data _null_;
-  set exportable end=last;
+  set &ds end=last;
   file &outref lrecl=32767;
   length str $32767;
   if _n_=1 then do;
+    put "# Script generated by &sysuserid on %sysfunc(datetime(),datetime19.)";
     put "cd ""&platform_object_path"" \";
-    put "; ./ExportPackage &connx_string -disableX11 \";
-    put " -package ""&cmdoutloc/&cmdoutname..spk"" \";
+    put "; ./ExportPackage -host &host -port &port -user &mmxuser \";
+    put "  -disableX11 -password &mmxpass \";
+    put "  -package ""&cmdoutloc/&cmdoutname..spk"" \";
   end;
+/* exclude particular patterns from the exported SPK */
+%if "&excludevars" ne "0" %then %do;
+  %do i=1 %to %sysfunc(countw(&excludevars));
+    %let var=%scan(&excludevars,&i);
+    if _n_=1 then do;
+      length excludestr&i $1000;
+      retain excludestr&i;
+      excludestr&i=symget("&var");
+      putlog excludestr&i=;
+      putlog path=;
+    end;
+    if index(path,cats(excludestr&i))=0 and index(name,cats(excludestr&i))=0;
+  %end;
+  /* ignore top level folder else all subcontent will be exported regardless */
+  if _n_>1;
+%end;
   str=' -objects '!!cats('"',path,'/',name,"(",publictype,')" \');
   put str;
   if last then put " -log ""&cmdoutloc/&cmdoutname..log"" 2>&1 ";
 run;
 
 %mp_abort(iftrue= (&syscc ne 0)
-  ,mac=&sysmacroname
+  ,mac=mm_spkexport
   ,msg=%str(syscc=&syscc)
 )
 
-%mend mm_spkexport;/**
+%mend mm_spkexport;
+/**
   @file mm_tree.sas
   @brief Returns all folders / subfolder content for a particular root
   @details Shows all members and SubTrees for a particular root.
@@ -18859,6 +18927,41 @@ run;
 )
 
 %mend ms_createfile;
+/**
+  @file
+  @brief Deletes a file from SASjs Drive
+  @details Deletes a file from SASjs Drive, if it exists.
+
+  Example:
+
+      filename stpcode temp;
+      data _null_;
+        file stpcode;
+        put '%put hello world;';
+      run;
+      %ms_createfile(/some/stored/program.sas, inref=stpcode)
+
+      %ms_deletefile(/some/stored/program.sas)
+
+  @param [in] driveloc The full path to the file in SASjs Drive
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+
+**/
+
+%macro ms_deletefile(driveloc
+    ,mdebug=0
+  );
+
+proc http method='DELETE'
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file?_filePath=&driveloc";
+%if &mdebug=1 %then %do;
+  debug level=2;
+%end;
+run;
+
+
+%mend ms_deletefile;
 /**
   @file
   @brief Gets a file from SASjs Drive
@@ -18883,7 +18986,7 @@ run;
 filename &outref temp;
 
 proc http method='GET' out=&outref
-  url="&_sasjs_apiserverurl/SASjsApi/drive/file?filePath=&driveloc";
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file?_filePath=&driveloc";
 %if &mdebug=1 %then %do;
   debug level=2;
 %end;
@@ -20536,7 +20639,8 @@ data _null_;
   put '  %else %if %symexist(&metavar) %then %do; ';
   put '    %if %length(&&&metavar)=0 %then %let user=&sysuserid; ';
   put '    /* sometimes SAS will add @domain extension - remove for consistency */ ';
-  put '    %else %let user=%scan(&&&metavar,1,@); ';
+  put '    /* but be sure to quote in case of usernames with commas */ ';
+  put '    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@)); ';
   put '  %end; ';
   put '  %else %let user=&sysuserid; ';
   put ' ';
@@ -21023,6 +21127,7 @@ libname &libref1a clear;
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
+  @li mf_existds.sas
   @li mf_getplatform.sas
   @li mf_getuniquefileref.sas
   @li mf_getuniquelibref.sas
@@ -21527,7 +21632,7 @@ filename &fname1 clear;
     ,grant_type=sas_services
     ,outds=work.viyagroups
   );
-%local oauth_bearer;
+%local oauth_bearer base_uri fname1 libref1;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
   %else %let grant_type=sas_services;
@@ -21545,11 +21650,10 @@ filename &fname1 clear;
 )
 
 options noquotelenmax;
-%local base_uri; /* location of rest apis */
+/* location of rest apis */
 %let base_uri=%mf_getplatform(VIYARESTAPI);
 
 /* fetching folder details for provided path */
-%local fname1;
 %let fname1=%mf_getuniquefileref();
 %let libref1=%mf_getuniquelibref();
 
@@ -21573,12 +21677,12 @@ data &outds;
 run;
 
 
-
 /* clear refs */
 filename &fname1 clear;
 libname &libref1 clear;
 
-%mend mv_getgroups;/**
+%mend mv_getgroups;
+/**
   @file
   @brief Extract the source code from a SAS Viya Job
   @details Extracts the SAS code from a Job into a fileref or physical file.
@@ -21625,7 +21729,7 @@ libname &libref1 clear;
   );
 %local dbg bufsize varcnt fname1 fname2 errmsg;
 %if &mdebug=1 %then %do;
-  %put &sysmacroname entry vars:;
+  %put &sysmacroname local entry vars:;
   %put _local_;
 %end;
 %else %let dbg=*;
@@ -21692,14 +21796,21 @@ proc http method='GET' out=&fname1 &oauth_bearer
   %end;
   ;
 run;
-%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
-%do;
-  data _null_;infile &fname1;input;putlog _infile_;run;
-  %mp_abort(mac=&sysmacroname
-    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
-  )
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fname1;
+    input;
+    putlog _infile_;
+  run;
 %end;
 
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
 %let fname2=%mf_getuniquefileref();
 filename &fname2 temp ;
 
@@ -21708,7 +21819,7 @@ filename &fname2 temp ;
 data _null_;
   file &fname2 recfm=n;
   infile &fname1 lrecl=1 recfm=n;
-  input sourcechar $ 1. @@;
+  input sourcechar $char1. @@;
   format sourcechar hex2.;
   retain startwrite 0;
   if startwrite=0 and sourcechar='"' then do;

base/mf_deletefile.sas

@@ -0,0 +1,31 @@
+/**
+  @file
+  @brief Deletes a physical file, if it exists
+  @details Usage:
+
+      %mf_writefile(&sasjswork/myfile.txt,l1=some content)
+
+      %mf_deletefile(&sasjswork/myfile.txt)
+
+      %mf_deletefile(&sasjswork/myfile.txt)
+
+
+  @param filepath Full path to the target file
+
+  @returns The return code from the fdelete() invocation
+
+  <h4> Related Macros </h4>
+  @li mf_deletefile.test.sas
+  @li mf_writefile.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mf_deletefile(file
+)/*/STORE SOURCE*/;
+  %local rc fref;
+  %let rc= %sysfunc(filename(fref,&file));
+  %if %sysfunc(fdelete(&fref)) ne 0 %then %put %sysfunc(sysmsg());
+  %let rc= %sysfunc(filename(fref));
+%mend mf_deletefile;

base/mf_existfeature.sas

@@ -9,19 +9,17 @@
 
         %put %mf_existfeature(PROCLUA);
 
-  @param feature the feature to detect.  Leave blank to list all in log.
+  @param [in] feature The feature to detect.
 
   @return output returns 1 or 0 (or -1 if not found)
 
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
 
-
   @version 8
   @author Allan Bowe
 **/
 /** @cond */
-
 %macro mf_existfeature(feature
 )/*/STORE SOURCE*/;
   %let feature=%upcase(&feature);
@@ -29,7 +27,11 @@
   %let platform=%mf_getplatform();
 
   %if &feature= %then %do;
-    %put Supported features:  PROCLUA;
+    %put No feature was requested for detection;
+  %end;
+  %else %if &feature=COLCONSTRAINTS %then %do;
+    %if %substr(&sysver,1,1)=4 %then 0;
+    %else 1;
   %end;
   %else %if &feature=PROCLUA %then %do;
     /* https://blogs.sas.com/content/sasdummy/2015/08/03/using-lua-within-your-sas-programs */
@@ -43,5 +45,4 @@
     %put &sysmacroname: &feature not found;
   %end;
 %mend mf_existfeature;
-
-/** @endcond */
+/** @endcond */

base/mf_getuser.sas

@@ -33,7 +33,8 @@
   %else %if %symexist(&metavar) %then %do;
     %if %length(&&&metavar)=0 %then %let user=&sysuserid;
     /* sometimes SAS will add @domain extension - remove for consistency */
-    %else %let user=%scan(&&&metavar,1,@);
+    /* but be sure to quote in case of usernames with commas */
+    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@));
   %end;
   %else %let user=&sysuserid;
 

base/mf_verifymacvars.sas

@@ -59,7 +59,7 @@
 
   %goto exit_success;
   %exit_err:
-    %put %str(ERR)OR: &abortmsg;
+    %put &abortmsg;
     %mf_abort(iftrue=(&mabort ne SOFT),
       mac=mf_verifymacvars,
       msg=%str(&abortmsg)

base/mp_abort.sas

@@ -85,7 +85,7 @@
   %end;
 
   /* Stored Process Server web app context */
-  %if %symexist(_metaperson)
+  %if %symexist(_metaport)
     or "&SYSPROCESSNAME "="Compute Server "
     or &mode=INCLUDE
   %then %do;
@@ -260,4 +260,4 @@
   %end;
 %mend mp_abort;
 
-/** @endcond */
+/** @endcond */

base/mp_dirlist.sas

@@ -94,6 +94,13 @@ data &out_ds(compress=no
   %end;
   if rc = 0 then do;
     did = dopen(fref);
+    if did=0 then do;
+      putlog "NOTE: This directory is empty, or does not exist - &path";
+      msg=sysmsg();
+      put msg;
+      put _all_;
+      stop;
+    end;
     /* attribute is OS-dependent - could be "Directory" or "Directory Name" */
     numopts=doptnum(did);
     do i=1 to numopts;
@@ -101,12 +108,6 @@ data &out_ds(compress=no
       if foption=:'Directory' then i=numopts;
     end;
     directory=dinfo(did,foption);
-    if did=0 then do;
-      putlog "NOTE: This directory is empty - " directory;
-      msg=sysmsg();
-      put _all_;
-      stop;
-    end;
     rc = filename(fref);
   end;
   else do;
@@ -242,4 +243,4 @@ run;
 proc sql;
 drop table &out_ds;
 
-%mend mp_dirlist;
+%mend mp_dirlist;

base/mp_hashdataset.sas

@@ -23,7 +23,7 @@
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
   @param [in] iftrue= A condition under which the macro should be executed.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
-  will contain one column (hashkey) with one observation (a hex32.
+  will contain one column (hashkey) with one observation (a $hex32.
   representation of the input hash)
   |hashkey:$32.|
   |---|

base/mp_loadformat.sas

@@ -40,13 +40,13 @@
   @li mp_abort.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
+  @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas
   @li mddl_dc_locktable.sas
   @li mp_loadformat.test.sas
   @li mp_lockanytable.sas
-  @li mp_storediffs.sas
   @li mp_stackdiffs.sas
 
 
@@ -134,7 +134,7 @@ run;
   * First, extract only relevant formats from the catalog
   */
 proc sql noprint;
-select distinct fmtname into: fmtlist separated by ' ' from &libds;
+select distinct upcase(fmtname) into: fmtlist separated by ' ' from &libds;
 
 %mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
 
@@ -144,8 +144,11 @@ select distinct fmtname into: fmtlist separated by ' ' from &libds;
   */
 %mddl_sas_cntlout(libds=&template)
 data &inlibds;
+  length &delete_col $3;
   if 0 then set &template;
   set &libds;
+  if &delete_col='' then &delete_col='No';
+  fmtname=upcase(fmtname);
   if missing(type) then do;
     if substr(fmtname,1,1)='$' then type='C';
     else type='N';
@@ -156,7 +159,6 @@ data &inlibds;
   end;
 run;
 
-
 /**
   * Identify new records
   */
@@ -263,7 +265,7 @@ options ibufsize=&ibufsize;
     %end;
 
     %mp_storediffs(&libcat-FC
-      ,&inlibds
+      ,&base_fmts
       ,FMTNAME START
       ,delds=&outds_del
       ,modds=&outds_mod
@@ -272,6 +274,9 @@ options ibufsize=&ibufsize;
       ,mdebug=&mdebug
     )
 
+    proc append base=&auditlibds data=&storediffs;
+    run;
+
     %if &locklibds ne 0 %then %do;
       %mp_lockanytable(UNLOCK
         ,lib=%scan(&auditlibds,1,.)
@@ -295,4 +300,4 @@ options ibufsize=&ibufsize;
   %put &sysmacroname exit vars:;
   %put _local_;
 %end;
-%mend mp_loadformat;
+%mend mp_loadformat;

base/mp_md5.sas

@@ -54,5 +54,5 @@ put(md5(
     &sep put(md5(trim(put(ifn(missing(&var),&var,&var*1),binary64.))),$hex32.)
     %let sep=!!;
   %end;
-),hex32.)
+),$hex32.)
 %mend mp_md5;

base/mp_replace.sas

@@ -70,7 +70,7 @@ filename &inref &infile lrecl=1 recfm=n;
 
 data &ds1;
   infile &inref;
-  input sourcechar $ 1. @@;
+  input sourcechar $char1. @@;
   format sourcechar hex2.;
 run;
 

meta/mm_createwebservice.sas

@@ -453,7 +453,8 @@ data _null_;
   put '  %else %if %symexist(&metavar) %then %do; ';
   put '    %if %length(&&&metavar)=0 %then %let user=&sysuserid; ';
   put '    /* sometimes SAS will add @domain extension - remove for consistency */ ';
-  put '    %else %let user=%scan(&&&metavar,1,@); ';
+  put '    /* but be sure to quote in case of usernames with commas */ ';
+  put '    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@)); ';
   put '  %end; ';
   put '  %else %let user=&sysuserid; ';
   put ' ';

meta/mm_deletelibrary.sas

@@ -78,6 +78,7 @@ filename &fname2 clear;
 %local isgone;
 data _null_;
   length type uri $256;
+  call missing (of _all_);
   rc=metadata_resolve("omsobj:SASLibrary?@Id='&liburi'",type,uri);
   call symputx('isgone',type,'l');
 run;

meta/mm_spkexport.sas

@@ -49,20 +49,25 @@
 
 
   <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_isblank.sas
   @li mf_loc.sas
   @li mm_tree.sas
-  @li mf_getuniquefileref.sas
-  @li mf_isblank.sas
   @li mp_abort.sas
 
-  @param metaloc= the metadata folder to export
-  @param secureref= fileref containing the username / password (should point to
-    a file in a secure location).  Leave blank to substitute $bash type vars.
-  @param outref= fileref to which to write the command
-  @param cmdoutloc= the directory to which the command will write the SPK
-    (default=WORK)
-  @param cmdoutname= the name of the spk / log files to create (will be
-    identical just with .spk or .log extension)
+
+  @param [in] metaloc= the metadata folder to export
+  @param [in] secureref= fileref containing the username / password (should
+    point to a file in a secure location). Leave blank to substitute $bash vars.
+  @param [in] excludevars= (0) A space seperated list of macro variable names,
+    each of which contains a value that should be used to filter the output
+    objects.
+  @param [out] outref= fileref to which to write the command
+  @param [out] cmdoutloc= (%sysfunc(pathname(work))) The directory to which the
+    command will write the SPK
+  @param [out] cmdoutname= (mmxport) The name of the spk / log files to create
+    (will be identical just with .spk or .log extension)
 
   @version 9.4
   @author Allan Bowe
@@ -71,6 +76,7 @@
 
 %macro mm_spkexport(metaloc=
   ,secureref=
+  ,excludevars=0
   ,outref=
   ,cmdoutloc=%sysfunc(pathname(work))
   ,cmdoutname=mmxport
@@ -82,7 +88,7 @@
 %end;
 
 /* set creds */
-%local mmxuser mmxpath;
+%local mmxuser mmxpath i var;
 %let mmxuser=$1;
 %let mmxpass=$2;
 %if %mf_isblank(&secureref)=0 %then %do;
@@ -90,35 +96,51 @@
 %end;
 
 /* setup metadata connection options */
-%local host port platform_object_path connx_string;
+%local host port platform_object_path ds;
 %let host=%sysfunc(getoption(metaserver));
 %let port=%sysfunc(getoption(metaport));
 %let platform_object_path=%mf_loc(POF);
+%let ds=%mf_getuniquename(prefix=spkexportable);
 
-%let connx_string=%str(-host &host -port &port -user &mmxuser %trim(
-  )-password &mmxpass);
-
-%mm_tree(root=%str(&metaloc) ,types=EXPORTABLE ,outds=exportable)
+%mm_tree(root=%str(&metaloc),types=EXPORTABLE ,outds=&ds)
 
 %if %mf_isblank(&outref)=1 %then %let outref=%mf_getuniquefileref();
 
 data _null_;
-  set exportable end=last;
+  set &ds end=last;
   file &outref lrecl=32767;
   length str $32767;
   if _n_=1 then do;
+    put "# Script generated by &sysuserid on %sysfunc(datetime(),datetime19.)";
     put "cd ""&platform_object_path"" \";
-    put "; ./ExportPackage &connx_string -disableX11 \";
-    put " -package ""&cmdoutloc/&cmdoutname..spk"" \";
+    put "; ./ExportPackage -host &host -port &port -user &mmxuser \";
+    put "  -disableX11 -password &mmxpass \";
+    put "  -package ""&cmdoutloc/&cmdoutname..spk"" \";
   end;
+/* exclude particular patterns from the exported SPK */
+%if "&excludevars" ne "0" %then %do;
+  %do i=1 %to %sysfunc(countw(&excludevars));
+    %let var=%scan(&excludevars,&i);
+    if _n_=1 then do;
+      length excludestr&i $1000;
+      retain excludestr&i;
+      excludestr&i=symget("&var");
+      putlog excludestr&i=;
+      putlog path=;
+    end;
+    if index(path,cats(excludestr&i))=0 and index(name,cats(excludestr&i))=0;
+  %end;
+  /* ignore top level folder else all subcontent will be exported regardless */
+  if _n_>1;
+%end;
   str=' -objects '!!cats('"',path,'/',name,"(",publictype,')" \');
   put str;
   if last then put " -log ""&cmdoutloc/&cmdoutname..log"" 2>&1 ";
 run;
 
 %mp_abort(iftrue= (&syscc ne 0)
-  ,mac=&sysmacroname
+  ,mac=mm_spkexport
   ,msg=%str(syscc=&syscc)
 )
 
-%mend mm_spkexport;
+%mend mm_spkexport;

server/ms_deletefile.sas

@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Deletes a file from SASjs Drive
+  @details Deletes a file from SASjs Drive, if it exists.
+
+  Example:
+
+      filename stpcode temp;
+      data _null_;
+        file stpcode;
+        put '%put hello world;';
+      run;
+      %ms_createfile(/some/stored/program.sas, inref=stpcode)
+
+      %ms_deletefile(/some/stored/program.sas)
+
+  @param [in] driveloc The full path to the file in SASjs Drive
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+
+**/
+
+%macro ms_deletefile(driveloc
+    ,mdebug=0
+  );
+
+proc http method='DELETE'
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file?_filePath=&driveloc";
+%if &mdebug=1 %then %do;
+  debug level=2;
+%end;
+run;
+
+
+%mend ms_deletefile;

server/ms_getfile.sas

@@ -22,7 +22,7 @@
 filename &outref temp;
 
 proc http method='GET' out=&outref
-  url="&_sasjs_apiserverurl/SASjsApi/drive/file?filePath=&driveloc";
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file?_filePath=&driveloc";
 %if &mdebug=1 %then %do;
   debug level=2;
 %end;

tests/crossplatform/mf_deletefile.test.sas

@@ -0,0 +1,29 @@
+/**
+  @file
+  @brief Testing mf_deletefile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_deletefile.sas
+  @li mf_writefile.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%let test1file=&sasjswork/myfile1.txt;
+
+%mf_writefile(&test1file,l1=some content)
+
+%mp_assert(
+  iftrue=(%sysfunc(fileexist(&test1file))=1),
+  desc=Check &test1file exists
+)
+
+%mp_assertscope(SNAPSHOT)
+%mf_deletefile(&test1file)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(%sysfunc(fileexist(&test1file))=0),
+  desc=Check &test1file no longer exists
+)

tests/crossplatform/mp_dirlist.test.sas

@@ -47,4 +47,18 @@ run;
   iftrue=(%mf_nobs(work.mytable3)=2),
   desc=Top level returned,
   outds=work.test_results
-)
+)
+
+%mp_dirlist(path=&root/b, outds=work.myTable4)
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable4)=0),
+  desc=Empty table for empty directory,
+  outds=work.test_results
+)
+
+%mp_dirlist(path=&root/notexisting, outds=work.myTable5)
+%mp_assert(
+  iftrue=(%mf_nobs(work.mytable5)=0),
+  desc=Empty table for non-existing directory,
+  outds=work.test_results
+)

tests/crossplatform/mp_loadformat.test.sas

@@ -3,6 +3,7 @@
   @brief Testing mp_loadformat.sas macro
 
   <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
   @li mp_loadformat.sas
   @li mp_assert.sas
   @li mp_assertscope.sas
@@ -12,6 +13,8 @@
 /* prep format catalog */
 libname perm (work);
 
+%mddl_dc_difftable(libds=perm.audit)
+
 data work.loadfmts;
   length fmtname $32;
   eexcl='Y';
@@ -49,7 +52,7 @@ run;
 %mp_loadformat(perm.testcat
   ,work.stagedata
   ,loadtarget=YES
-  ,auditlibds=0
+  ,auditlibds=perm.audit
   ,locklibds=0
   ,delete_col=deleteme
   ,outds_add=add_test1
@@ -73,4 +76,18 @@ run;
   iftrue=(%mf_nobs(mod_test1)=100),
   desc=Test 1 - mod obs,
   outds=work.test_results
-)
+)
+%mp_assert(
+  iftrue=(%mf_nobs(perm.audit)=7329),
+  desc=Test 1 - audit table updated,
+  outds=work.test_results
+)
+data work.difftest;
+  set perm.audit;
+  where is_diff=1;
+run;
+%mp_assert(
+  iftrue=(%mf_nobs(work.difftest)>0),
+  desc=Test 1 - diffs were found,
+  outds=work.test_results
+)

tests/crossplatform/mp_replace.test.sas

@@ -63,3 +63,33 @@ run;
   desc=Checking second replace 3rd row,
   outds=work.test_results
 )
+
+
+%let test3="&sasjswork/file3.txt";
+%let str=%str(replace.string.with.dots   );
+%let rep=%str( more.dots);
+data _null_;
+  file &test3;
+  put 'blahblah';
+  put "blahblah&str.blah&str. replace &str.X";
+  put "blahbreplacewith&str.spacesahblah";
+run;
+%mp_replace(&test3, findvar=str, replacevar=rep)
+
+data _null_;
+  infile &test3;
+  input;
+  if _n_=2 then call symputx('test3resulta',_infile_);
+  if _n_=3 then call symputx('test3resultb',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test3resulta" = "blahblah&rep.blah&rep. replace &rep.X"),
+  desc=Checking third replace 2nd row (dots),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test3resultb" = "blahbreplacewith&rep.spacesahblah"),
+  desc=Checking third replace 3rd row (dots),
+  outds=work.test_results
+)

tests/sas9only/mm_spkexport.test.sas

@@ -0,0 +1,41 @@
+/**
+  @file
+  @brief Testing mm_webout macro
+
+  <h4> SAS Macros </h4>
+  @li mm_spkexport.sas
+  @li mp_assert.sas
+  @li mp_as
+
+**/
+
+
+%* create sample text file as input to the macro;
+filename tmp temp;
+data _null_;
+  file tmp;
+  put '%let mmxuser="sasdemo";';
+  put '%let mmxpass="Mars321";';
+run;
+
+filename myref "%sysfunc(pathname(work))/mmxexport.sh"
+  permission='A::u::rwx,A::g::r-x,A::o::---';
+%mp_assertscope(SNAPSHOT)
+%mm_spkexport(metaloc=%str(/Shared Data)
+    ,outref=myref
+    ,secureref=tmp
+    ,cmdoutloc=%str(/tmp)
+)
+%mp_assertscope(COMPARE)
+
+data _null_;
+  infile tmp;
+  input;
+  putlog _infile_;
+  call symputx('nobs',_n_);
+run;
+
+%mp_assert(
+  iftrue=(&nobs>2),
+  desc=Check if content was created
+)

tests/serveronly/ms_deletefile.test.sas

@@ -0,0 +1,61 @@
+/**
+  @file
+  @brief Testing ms_deletefile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li ms_createfile.sas
+  @li ms_deletefile.sas
+  @li ms_getfile.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+
+/* first make a remote file */
+filename stpcode temp;
+%let fname=%mf_getuniquename();
+data _null_;
+  file stpcode;
+  put "data &fname;run;";
+run;
+%ms_createfile(/sasjs/tests/&fname..sas
+  ,inref=stpcode
+  ,mdebug=1
+)
+
+%ms_getfile(/sasjs/tests/&fname..sas,outref=testref)
+
+%let test1=0;
+data _null_;
+  infile testref;
+  input;
+  call symputx('test1',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test1"="data &fname;run;"),
+  desc=Make sure the file was created,
+  outds=work.test_results
+)
+
+%mp_assertscope(SNAPSHOT)
+%ms_deletefile(/sasjs/tests/&fname..sas,mdebug=1)
+%mp_assertscope(COMPARE)
+
+%ms_getfile(/sasjs/tests/&fname..sas,outref=testref2)
+
+%let test2=0;
+data _null_;
+  infile testref2;
+  input;
+  call symputx('test2',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test2"="%str(Err)or: File does not exist."),
+  desc=Make sure the file was deleted,
+  outds=work.test_results
+)
+
+

viya/mv_createwebservice.sas

@@ -655,7 +655,8 @@ data _null_;
   put '  %else %if %symexist(&metavar) %then %do; ';
   put '    %if %length(&&&metavar)=0 %then %let user=&sysuserid; ';
   put '    /* sometimes SAS will add @domain extension - remove for consistency */ ';
-  put '    %else %let user=%scan(&&&metavar,1,@); ';
+  put '    /* but be sure to quote in case of usernames with commas */ ';
+  put '    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@)); ';
   put '  %end; ';
   put '  %else %let user=&sysuserid; ';
   put ' ';

viya/mv_deleteviyafolder.sas

@@ -25,6 +25,7 @@
 
   <h4> SAS Macros </h4>
   @li mp_abort.sas
+  @li mf_existds.sas
   @li mf_getplatform.sas
   @li mf_getuniquefileref.sas
   @li mf_getuniquelibref.sas

viya/mv_getgroups.sas

@@ -32,7 +32,7 @@
     ,grant_type=sas_services
     ,outds=work.viyagroups
   );
-%local oauth_bearer;
+%local oauth_bearer base_uri fname1 libref1;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
   %else %let grant_type=sas_services;
@@ -50,11 +50,10 @@
 )
 
 options noquotelenmax;
-%local base_uri; /* location of rest apis */
+/* location of rest apis */
 %let base_uri=%mf_getplatform(VIYARESTAPI);
 
 /* fetching folder details for provided path */
-%local fname1;
 %let fname1=%mf_getuniquefileref();
 %let libref1=%mf_getuniquelibref();
 
@@ -78,9 +77,8 @@ data &outds;
 run;
 
 
-
 /* clear refs */
 filename &fname1 clear;
 libname &libref1 clear;
 
-%mend mv_getgroups;
+%mend mv_getgroups;

viya/mv_getjobcode.sas

@@ -45,7 +45,7 @@
   );
 %local dbg bufsize varcnt fname1 fname2 errmsg;
 %if &mdebug=1 %then %do;
-  %put &sysmacroname entry vars:;
+  %put &sysmacroname local entry vars:;
   %put _local_;
 %end;
 %else %let dbg=*;
@@ -112,14 +112,21 @@ proc http method='GET' out=&fname1 &oauth_bearer
   %end;
   ;
 run;
-%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
-%do;
-  data _null_;infile &fname1;input;putlog _infile_;run;
-  %mp_abort(mac=&sysmacroname
-    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
-  )
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fname1;
+    input;
+    putlog _infile_;
+  run;
 %end;
 
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
 %let fname2=%mf_getuniquefileref();
 filename &fname2 temp ;
 
@@ -128,7 +135,7 @@ filename &fname2 temp ;
 data _null_;
   file &fname2 recfm=n;
   infile &fname1 lrecl=1 recfm=n;
-  input sourcechar $ 1. @@;
+  input sourcechar $char1. @@;
   format sourcechar hex2.;
   retain startwrite 0;
   if startwrite=0 and sourcechar='"' then do;