feat: adding mv_jobexecute macro (and a fix for mv_getfoldermembers where there are no members)

all.sas

@@ -399,7 +399,7 @@ options noquotelenmax;
   @version 9.2
   @author Allan Bowe
 
-**/ /** \cond */
+**/
 
 %macro mf_getengine(libref
 )/*/STORE SOURCE*/;
@@ -419,9 +419,7 @@ options noquotelenmax;
 
  &engine
 
-%mend;
-
-/** \endcond *//**
+%mend;/**
   @file
   @brief Returns the size of a file in bytes.
   @details Provide full path/filename.extension to the file, eg:
@@ -1154,6 +1152,38 @@ Usage:
 
   %sysevalf(%superq(param)=,boolean)
 
+%mend;/**
+  @file
+  @brief Checks whether a path is a valid directory
+  @details
+  Usage:
+
+      %let isdir=%mf_isdir(/tmp);
+
+  With thanks and full credit to Andrea Defronzo - https://www.linkedin.com/in/andrea-defronzo-b1a47460/
+
+  @param path full path of the file/directory to be checked
+
+  @return output returns 1 if path is a directory, 0 if it is not
+
+  @version 9.2
+**/
+
+%macro mf_isdir(path
+)/*/STORE SOURCE*/;
+	%local rc did is_directory fref_t;
+
+	%let is_directory = 0;
+	%let rc = %sysfunc(filename(fref_t, %superq(path)));
+	%let did = %sysfunc(dopen(&fref_t.));
+	%if &did. ^= 0 %then %do;
+	   %let is_directory = 1;
+	   %let rc = %sysfunc(dclose(&did.));
+	%end;
+	%let rc = %sysfunc(filename(fref_t));
+
+	&is_directory
+
 %mend;/**
   @file
   @brief Returns physical location of various SAS items
@@ -1933,29 +1963,50 @@ Usage:
 
   @version 9.2
   @author Allan Bowe
+
+  <h4> Dependencies </h4>
+  @li mp_abort.sas
+  @li mf_existds.sas
+
 **/
 
-%macro mp_csv2ds(inref=0,outds=0,baseds=0);
-%if &inref=0 %then %do;
-  %put %str(ERR)OR: the INREF variable must be provided;
-  %let syscc=4;
-  %abort;
-%end;
-%if %superq(outds)=0 %then %do;
-  %put %str(ERR)OR: the OUTDS variable must be provided;
-  %let syscc=4;
-  %return;
-%end;
-%if &baseds=0 %then %do;
-  %put %str(ERR)OR: the BASEDS variable must be provided;
-  %let syscc=4;
-  %return;
-%end;
-%if %sysfunc(exist(&BASEDS)) ne 1 & %sysfunc(exist(&BASEDS,VIEW)) ne 1 %then %do;
-  %put %str(ERR)OR: the BASEDS dataset needs to be assigned, and to exist;
-  %let syscc=4;
-  %return;
-%end;
+%macro mp_csv2ds(inref=0,outds=0,baseds=0,view=NO);
+
+%mp_abort(iftrue=( &inref=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the INREF variable must be provided)
+)
+%mp_abort(iftrue=( %superq(outds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the OUTDS variable must be provided)
+)
+%mp_abort(iftrue=( &baseds=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS variable must be provided)
+)
+%mp_abort(iftrue=( &baseds=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS variable must be provided)
+)
+%mp_abort(iftrue=( %mf_existds(&baseds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS dataset (&baseds) needs to be assigned, and to exist)
+)
+
+/* count rows */
+%local hasheader; %let hasheader=0;
+data _null_;
+  if _N_ > 1 then do;
+     call symputx('hasheader',1,'l');
+     stop;
+  end;
+  infile &inref;
+  input;
+run;
+%mp_abort(iftrue=( &hasheader=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(No header row in &inref)
+)
 
 /* get the variables in the CSV */
 data _data_;
@@ -2004,6 +2055,7 @@ data _null_;
     if type=1 then informat='best.';
     else informat=cats('$',length,'.');
   end;
+  else informat=cats(informat,'.');
   in=catx(' ',in,name,':',informat);
   if last then do;
     call symputx('instat',in,'l');
@@ -2019,7 +2071,9 @@ data &outds
   ;
   infile &inref dsd firstobs=2;
   input &instat;
-  drop &dropvars;
+  %if %length(&dropvars)>0 %then %do;
+    drop &dropvars;
+  %end;
 run;
 
 %mend;/**
@@ -2628,6 +2682,329 @@ create table &outds as
   %end;
   ;
 
+%mend;/**
+  @file
+  @brief Extract DBML from SAS Libraries
+  @details DBML is an open source markup format to represent databases.
+  More details: https://www.dbml.org/home/
+
+  Usage:
+
+
+      %mp_getdbml(liblist=SASHELP WORK,outref=mydbml,showlog=YES)
+
+  Take the log output and paste it into the renderer at https://dbdiagram.io
+  to view your data model diagram.  The code takes a "best guess" at
+  the one to one and one to many relationships (based on constraints
+  and indexes, and assuming that the column names would match).
+
+  You may need to adjust the rendered DBML to suit your needs.
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getquotedstr.sas
+  @li mp_getconstraints.sas
+
+  @param liblist= Space seperated list of librefs to take as
+    input (Default=SASHELP)
+  @param outref= Fileref to contain the DBML (Default=getdbml)
+  @param showlog= set to YES to show the DBML in the log (Default is NO)
+
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_getdbml(liblist=SASHELP,outref=getdbml,showlog=NO
+)/*/STORE SOURCE*/;
+
+/* check fileref is assigned */
+%if %sysfunc(fileref(&outref)) > 0 %then %do;
+  filename &outref temp;
+%end;
+
+%let liblist=%upcase(&liblist);
+
+proc sql noprint;
+create table _data_ as
+  select * from dictionary.tables
+  where upcase(libname) in (%mf_getquotedstr(&liblist))
+  order by libname,memname;
+%local tabinfo; %let tabinfo=&syslast;
+
+create table _data_ as
+  select * from dictionary.columns
+  where upcase(libname) in (%mf_getquotedstr(&liblist))
+  order by libname,memname,varnum;
+%local colinfo; %let colinfo=&syslast;
+
+%local dsnlist;
+  select distinct upcase(cats(libname,'.',memname)) into: dsnlist
+  separated by ' '
+  from &syslast
+;
+
+create table _data_ as
+  select * from dictionary.indexes
+  where upcase(libname) in (%mf_getquotedstr(&liblist))
+  order by idxusage, indxname, indxpos;
+%local idxinfo; %let idxinfo=&syslast;
+
+/* Extract all Primary Key and Unique data constraints */
+%mp_getconstraints(lib=%scan(&liblist,1),outds=_data_)
+%local colconst; %let colconst=&syslast;
+
+%do x=2 %to %sysfunc(countw(&liblist));
+  %mp_getconstraints(lib=%scan(&liblist,&x),outds=_data_)
+  proc append base=&colconst data=&syslast;
+  run;
+%end;
+
+
+
+
+/* header info */
+data _null_;
+  file &outref;
+  put "// DBML generated by &sysuserid on %sysfunc(datetime(),datetime19.) ";
+  put "Project sasdbml {";
+  put "  database_type: 'SAS'";
+  put "  Note: 'Generated by the mp_getdbml() macro'";
+  put "}";
+run;
+
+/* create table groups */
+data _null_;
+  file &outref mod;
+  set &tabinfo;
+  by libname;
+  if first.libname then put "TableGroup " libname "{";
+  ds=quote(cats(libname,'.',memname));
+  put '   ' ds;
+  if last.libname then put "}";
+run;
+
+/* table for pks */
+data _data_;
+  length curds const col $39;
+  call missing (of _all_);
+  stop;
+run;
+%let pkds=&syslast;
+
+%local x curds constraints_used constcheck;
+%do x=1 %to %sysfunc(countw(&dsnlist,%str( )));
+  %let curds=%scan(&dsnlist,&x,%str( ));
+  %let constraints_used=;
+  %let constcheck=0;
+  data _null_;
+    file &outref mod;
+    length lab $1024 typ $20;
+    set &colinfo (where=(
+        libname="%scan(&curds,1,.)" and upcase(memname)="%scan(&curds,2,.)"
+    )) end=last;
+
+    if _n_=1 then do;
+      table='Table "'!!"&curds"!!'"{';
+      put table;
+    end;
+    name=upcase(name);
+    lab=" note:"!!quote(trim(tranwrd(label,'"',"'")));
+    if upcase(format)=:'DATETIME' then typ='datetime';
+    else if type='char' then typ=cats('char(',length,')');
+    else typ='num';
+
+    if notnull='yes' then notnul=' not null';
+    if notnull='no' and missing(label) then put '  ' name typ;
+    else if notnull='yes' and missing(label) then put '  ' name typ '[' notnul ']';
+    else if notnull='no' then put '  ' name typ '[' lab ']';
+    else put '  ' name typ '[' notnul ',' lab ']';
+
+  run;
+
+  data _data_(keep=curds const col);
+    length ctype $11 cols constraints_used $5000;
+    set &colconst (where=(
+      upcase(libref)="%scan(&curds,1,.)"
+      and upcase(table_name)="%scan(&curds,2,.)"
+      and constraint_type in ('PRIMARY','UNIQUE')
+    )) end=last;
+    file &outref mod;
+    by constraint_type constraint_name;
+    retain cols;
+    column_name=upcase(column_name);
+
+    if _n_=1 then put / '  indexes {';
+
+    if upcase(strip(constraint_type)) = 'PRIMARY' then ctype='[pk]';
+    else ctype='[unique]';
+
+    if first.constraint_name then cols = cats('(',column_name);
+    else cols=cats(cols,',',column_name);
+
+    if last.constraint_name then do;
+      cols=cats(cols,')',ctype)!!' //'!!constraint_name;
+      put '    ' cols;
+      constraints_used=catx(' ',constraints_used, constraint_name);
+      call symputx('constcheck',1);
+    end;
+
+    if last then call symputx('constraints_used',cats(upcase(constraints_used)));
+
+    length curds const col $39;
+    curds="&curds";
+    const=constraint_name;
+    col=column_name;
+  run;
+
+  proc append base=&pkds data=&syslast;run;
+
+  /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+  data _data_(keep=curds const col);
+    set &idxinfo (where=(
+      libname="%scan(&curds,1,.)"
+      and upcase(memname)="%scan(&curds,2,.)"
+      and unique='yes'
+      and upcase(indxname) not in (%mf_getquotedstr(&constraints_used))
+    ));
+    file &outref mod;
+    by idxusage indxname;
+    name=upcase(name);
+    if &constcheck=1 then stop; /* in fact we only care about PKs so stop if we have */
+    if _n_=1 and &constcheck=0 then put / '  indexes {';
+
+    length cols $5000;
+    retain cols;
+    if first.indxname then cols = cats('(',name);
+    else cols=cats(cols,',',name);
+
+    if last.indxname then do;
+      cols=cats(cols,')[unique]')!!' //'!!indxname;
+      put '    ' cols;
+      call symputx('constcheck',1);
+    end;
+
+    length curds const col $39;
+    curds="&curds";
+    const=indxname;
+    col=name;
+  run;
+  proc append base=&pkds data=&syslast;run;
+
+  data _null_;
+    file &outref mod;
+    if &constcheck =1 then put '  }';
+    put '}';
+  run;
+
+%end;
+
+/**
+ * now we need to figure out the relationships
+ */
+
+/* sort alphabetically so we can have one set of unique cols per table */
+proc sort data=&pkds nodupkey;
+  by curds const col;
+run;
+
+data &pkds.1 (keep=curds col)
+     &pkds.2 (keep=curds cols);
+  set &pkds;
+  by curds const;
+  length retconst $39 cols $5000;
+  retain retconst cols;
+  if first.curds then do;
+    retconst=const;
+    cols=upcase(col);
+  end;
+  else cols=catx(' ',cols,upcase(col));
+  if retconst=const then do;
+    output &pkds.1;
+    if last.const then output &pkds.2;
+  end;
+run;
+
+%let curdslist="0";
+%do x=1 %to %sysfunc(countw(&dsnlist,%str( )));
+  %let curds=%scan(&dsnlist,&x,%str( ));
+
+  %let pkcols=0;
+  data _null_;
+    set &pkds.2(where=(curds="&curds"));
+    call symputx('pkcols',cols);
+  run;
+  %if &pkcols ne 0 %then %do;
+    %let curdslist=&curdslist,"&curds";
+
+    /* start with one2one */
+    data &pkds.4;
+      file &outref mod;
+      set &pkds.2(where=(cols="&pkcols" and curds not in (&curdslist)));
+      line='Ref: "'!!"&curds"
+        !!cats('".(',"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))",')')
+        !!' - '
+        !!cats(quote(trim(curds)),'.(',"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))",')');
+      put line;
+    run;
+
+    /* now many2one */
+    /* get table with one row per col */
+    data &pkds.5;
+      set &pkds.1(where=(curds="&curds"));
+    run;
+    /* get tables which contain the PK columns */
+    proc sql;
+    create table &pkds.5a as
+      select upcase(cats(b.libname,'.',b.memname)) as curds
+        ,b.name
+      from &pkds.5 a
+      inner join &colinfo b
+      on a.col=upcase(b.name);
+    /* count to make sure those tables contain ALL the columns */
+    create table &pkds.5b as
+      select curds,count(*) as cnt
+      from &pkds.5a
+      where curds not in (select curds from &pkds.2 where cols="&pkcols") /* not a one to one match */
+        and curds ne "&curds" /* exclude self */
+      group by 1;
+    create table &pkds.6 as
+      select a.*
+        ,b.cols
+      from &pkds.5b a
+      left join &pkds.4 b
+      on a.curds=b.curds;
+
+    data _null_;
+      set &pkds.6;
+      file &outref mod;
+      colcnt=%sysfunc(countw(&pkcols));
+      if cnt=colcnt then do;
+        /* table contains all the PK cols, and was not a direct / 121 match */
+        line='Ref: "'!!"&curds"
+          !!'".('
+          !!"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))"
+          !!') > '
+          !!cats(quote(trim(curds))
+              ,'.('
+              ,"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))"
+              ,')'
+          );
+        put line;
+      end;
+    run;
+  %end;
+%end;
+
+
+%if %upcase(&showlog)=YES %then %do;
+  options ps=max;
+  data _null_;
+    infile &outref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
 %mend;/**
   @file mp_getddl.sas
   @brief Extract DDL in various formats, by table or library
@@ -4089,7 +4466,7 @@ proc sql
 
   options &etls_syntaxcheck;
 %mend;/**
-  @file mp_streamfile.sas
+  @file
   @brief Streams a file to _webout according to content type
   @details Will set headers using appropriate functions (SAS 9 vs Viya) and send
   content as a binary stream.
@@ -4107,6 +4484,7 @@ proc sql
 
   @param contenttype= Either TEXT, ZIP, CSV, EXCEL (default TEXT)
   @param inloc= /path/to/file.ext to be sent
+  @param inref= fileref of file to be sent (if provided, overrides `inloc`)
   @param outname= the name of the file, as downloaded by the browser
 
   @author Allan Bowe
@@ -4117,6 +4495,7 @@ proc sql
 %macro mp_streamfile(
   contenttype=TEXT
   ,inloc=
+  ,inref=0
   ,outname=
 )/*/STORE SOURCE*/;
 
@@ -4132,7 +4511,7 @@ proc sql
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.zip'
-      contenttype='application/zip' 
+      contenttype='application/zip'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
@@ -4146,7 +4525,7 @@ proc sql
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
-      contenttype='application/vnd.ms-excel' 
+      contenttype='application/vnd.ms-excel'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
@@ -4159,7 +4538,7 @@ proc sql
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
-      contenttype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' 
+      contenttype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
@@ -4192,7 +4571,7 @@ proc sql
 %else %if &contentype=HTML %then %do;
   %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
-      contenttype="text/html"; 
+      contenttype="text/html";
   %end;
 %end;
 %else %do;
@@ -4200,7 +4579,79 @@ proc sql
   %return;
 %end;
 
-%mp_binarycopy(inloc="&inloc",outref=_webout)
+%if &inref ne 0 %then %do;
+ %mp_binarycopy(inref=&inref,outref=_webout)
+%end;
+%else %do;
+ %mp_binarycopy(inloc="&inloc",outref=_webout)
+%end;
+
+%mend;/**
+  @file
+  @brief Recursively scans a directory tree to get all subfolders and content
+  @details
+  Usage:
+
+      %mp_tree(dir=/tmp, outds=work.tree)
+
+  Credits:
+
+  * Roger Deangelis, https://communities.sas.com/t5/SAS-Programming/listing-all-files-within-a-directory-and-subdirectories/m-p/332616/highlight/true#M74887
+  * Tom, https://communities.sas.com/t5/SAS-Programming/listing-all-files-of-all-types-from-all-subdirectories/m-p/334113/highlight/true#M75419
+
+
+  @param dir= Directory to be scanned (default=/tmp)
+  @param outds= Dataset to create (default=work.mp_tree)
+
+  @returns outds contains the following variables:
+
+   - `dir`: a flag (1/0) to say whether it is a directory or not.  This is not
+     reliable - folders that you do not have permission to open will be flagged
+     as directories.
+   - `ext`: file extension
+   - `filename`: file name
+   - `dirname`: directory name
+   - `fullpath`: directory + file name
+
+  @version 9.2
+**/
+
+%macro mp_tree(dir=/tmp
+  ,outds=work.mp_tree
+)/*/STORE SOURCE*/;
+
+data &outds ;
+  length dir 8 ext filename dirname $256 fullpath $512 ;
+  call missing(of _all_);
+  fullpath = "&dir";
+run;
+
+%local sep;
+%if &sysscp=WIN or &SYSSCP eq DNTHOST %then %let sep=\;
+%else %let sep=/;
+
+data &outds ;
+  modify &outds ;
+  retain sep "&sep";
+  rc=filename('tmp',fullpath);
+  dir_id=dopen('tmp');
+  dir = (dir_id ne 0) ;
+  if dir then dirname=fullpath;
+  else do;
+    filename=scan(fullpath,-1,sep) ;
+    dirname =substrn(fullpath,1,length(fullpath)-length(filename));
+    if index(filename,'.')>1 then ext=scan(filename,-1,'.');
+  end;
+  replace;
+  if dir then do;
+    do i=1 to dnum(dir_id);
+      fullpath=cats(dirname,sep,dread(dir_id,i));
+      output;
+    end;
+    rc=dclose(dir_id);
+  end;
+  rc=filename('tmp');
+run;
 
 %mend;/**
   @file mp_unzip.sas

base/mf_getengine.sas

@@ -22,7 +22,7 @@
   @version 9.2
   @author Allan Bowe
 
-**/ /** \cond */
+**/
 
 %macro mf_getengine(libref
 )/*/STORE SOURCE*/;
@@ -42,6 +42,4 @@
 
  &engine
 
-%mend;
-
-/** \endcond */
+%mend;

base/mf_isdir.sas

@@ -0,0 +1,33 @@
+/**
+  @file
+  @brief Checks whether a path is a valid directory
+  @details
+  Usage:
+
+      %let isdir=%mf_isdir(/tmp);
+
+  With thanks and full credit to Andrea Defronzo - https://www.linkedin.com/in/andrea-defronzo-b1a47460/
+
+  @param path full path of the file/directory to be checked
+
+  @return output returns 1 if path is a directory, 0 if it is not
+
+  @version 9.2
+**/
+
+%macro mf_isdir(path
+)/*/STORE SOURCE*/;
+	%local rc did is_directory fref_t;
+
+	%let is_directory = 0;
+	%let rc = %sysfunc(filename(fref_t, %superq(path)));
+	%let did = %sysfunc(dopen(&fref_t.));
+	%if &did. ^= 0 %then %do;
+	   %let is_directory = 1;
+	   %let rc = %sysfunc(dclose(&did.));
+	%end;
+	%let rc = %sysfunc(filename(fref_t));
+
+	&is_directory
+
+%mend;

base/mp_csv2ds.sas

@@ -28,29 +28,50 @@
 
   @version 9.2
   @author Allan Bowe
+
+  <h4> Dependencies </h4>
+  @li mp_abort.sas
+  @li mf_existds.sas
+
 **/
 
-%macro mp_csv2ds(inref=0,outds=0,baseds=0);
-%if &inref=0 %then %do;
-  %put %str(ERR)OR: the INREF variable must be provided;
-  %let syscc=4;
-  %abort;
-%end;
-%if %superq(outds)=0 %then %do;
-  %put %str(ERR)OR: the OUTDS variable must be provided;
-  %let syscc=4;
-  %return;
-%end;
-%if &baseds=0 %then %do;
-  %put %str(ERR)OR: the BASEDS variable must be provided;
-  %let syscc=4;
-  %return;
-%end;
-%if %sysfunc(exist(&BASEDS)) ne 1 & %sysfunc(exist(&BASEDS,VIEW)) ne 1 %then %do;
-  %put %str(ERR)OR: the BASEDS dataset needs to be assigned, and to exist;
-  %let syscc=4;
-  %return;
-%end;
+%macro mp_csv2ds(inref=0,outds=0,baseds=0,view=NO);
+
+%mp_abort(iftrue=( &inref=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the INREF variable must be provided)
+)
+%mp_abort(iftrue=( %superq(outds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the OUTDS variable must be provided)
+)
+%mp_abort(iftrue=( &baseds=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS variable must be provided)
+)
+%mp_abort(iftrue=( &baseds=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS variable must be provided)
+)
+%mp_abort(iftrue=( %mf_existds(&baseds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS dataset (&baseds) needs to be assigned, and to exist)
+)
+
+/* count rows */
+%local hasheader; %let hasheader=0;
+data _null_;
+  if _N_ > 1 then do;
+     call symputx('hasheader',1,'l');
+     stop;
+  end;
+  infile &inref;
+  input;
+run;
+%mp_abort(iftrue=( &hasheader=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(No header row in &inref)
+)
 
 /* get the variables in the CSV */
 data _data_;
@@ -99,6 +120,7 @@ data _null_;
     if type=1 then informat='best.';
     else informat=cats('$',length,'.');
   end;
+  else informat=cats(informat,'.');
   in=catx(' ',in,name,':',informat);
   if last then do;
     call symputx('instat',in,'l');
@@ -114,7 +136,9 @@ data &outds
   ;
   infile &inref dsd firstobs=2;
   input &instat;
-  drop &dropvars;
+  %if %length(&dropvars)>0 %then %do;
+    drop &dropvars;
+  %end;
 run;
 
 %mend;

base/mp_getdbml.sas

@@ -0,0 +1,324 @@
+/**
+  @file
+  @brief Extract DBML from SAS Libraries
+  @details DBML is an open source markup format to represent databases.
+  More details: https://www.dbml.org/home/
+
+  Usage:
+
+
+      %mp_getdbml(liblist=SASHELP WORK,outref=mydbml,showlog=YES)
+
+  Take the log output and paste it into the renderer at https://dbdiagram.io
+  to view your data model diagram.  The code takes a "best guess" at
+  the one to one and one to many relationships (based on constraints
+  and indexes, and assuming that the column names would match).
+
+  You may need to adjust the rendered DBML to suit your needs.
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getquotedstr.sas
+  @li mp_getconstraints.sas
+
+  @param liblist= Space seperated list of librefs to take as
+    input (Default=SASHELP)
+  @param outref= Fileref to contain the DBML (Default=getdbml)
+  @param showlog= set to YES to show the DBML in the log (Default is NO)
+
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_getdbml(liblist=SASHELP,outref=getdbml,showlog=NO
+)/*/STORE SOURCE*/;
+
+/* check fileref is assigned */
+%if %sysfunc(fileref(&outref)) > 0 %then %do;
+  filename &outref temp;
+%end;
+
+%let liblist=%upcase(&liblist);
+
+proc sql noprint;
+create table _data_ as
+  select * from dictionary.tables
+  where upcase(libname) in (%mf_getquotedstr(&liblist))
+  order by libname,memname;
+%local tabinfo; %let tabinfo=&syslast;
+
+create table _data_ as
+  select * from dictionary.columns
+  where upcase(libname) in (%mf_getquotedstr(&liblist))
+  order by libname,memname,varnum;
+%local colinfo; %let colinfo=&syslast;
+
+%local dsnlist;
+  select distinct upcase(cats(libname,'.',memname)) into: dsnlist
+  separated by ' '
+  from &syslast
+;
+
+create table _data_ as
+  select * from dictionary.indexes
+  where upcase(libname) in (%mf_getquotedstr(&liblist))
+  order by idxusage, indxname, indxpos;
+%local idxinfo; %let idxinfo=&syslast;
+
+/* Extract all Primary Key and Unique data constraints */
+%mp_getconstraints(lib=%scan(&liblist,1),outds=_data_)
+%local colconst; %let colconst=&syslast;
+
+%do x=2 %to %sysfunc(countw(&liblist));
+  %mp_getconstraints(lib=%scan(&liblist,&x),outds=_data_)
+  proc append base=&colconst data=&syslast;
+  run;
+%end;
+
+
+
+
+/* header info */
+data _null_;
+  file &outref;
+  put "// DBML generated by &sysuserid on %sysfunc(datetime(),datetime19.) ";
+  put "Project sasdbml {";
+  put "  database_type: 'SAS'";
+  put "  Note: 'Generated by the mp_getdbml() macro'";
+  put "}";
+run;
+
+/* create table groups */
+data _null_;
+  file &outref mod;
+  set &tabinfo;
+  by libname;
+  if first.libname then put "TableGroup " libname "{";
+  ds=quote(cats(libname,'.',memname));
+  put '   ' ds;
+  if last.libname then put "}";
+run;
+
+/* table for pks */
+data _data_;
+  length curds const col $39;
+  call missing (of _all_);
+  stop;
+run;
+%let pkds=&syslast;
+
+%local x curds constraints_used constcheck;
+%do x=1 %to %sysfunc(countw(&dsnlist,%str( )));
+  %let curds=%scan(&dsnlist,&x,%str( ));
+  %let constraints_used=;
+  %let constcheck=0;
+  data _null_;
+    file &outref mod;
+    length lab $1024 typ $20;
+    set &colinfo (where=(
+        libname="%scan(&curds,1,.)" and upcase(memname)="%scan(&curds,2,.)"
+    )) end=last;
+
+    if _n_=1 then do;
+      table='Table "'!!"&curds"!!'"{';
+      put table;
+    end;
+    name=upcase(name);
+    lab=" note:"!!quote(trim(tranwrd(label,'"',"'")));
+    if upcase(format)=:'DATETIME' then typ='datetime';
+    else if type='char' then typ=cats('char(',length,')');
+    else typ='num';
+
+    if notnull='yes' then notnul=' not null';
+    if notnull='no' and missing(label) then put '  ' name typ;
+    else if notnull='yes' and missing(label) then put '  ' name typ '[' notnul ']';
+    else if notnull='no' then put '  ' name typ '[' lab ']';
+    else put '  ' name typ '[' notnul ',' lab ']';
+
+  run;
+
+  data _data_(keep=curds const col);
+    length ctype $11 cols constraints_used $5000;
+    set &colconst (where=(
+      upcase(libref)="%scan(&curds,1,.)"
+      and upcase(table_name)="%scan(&curds,2,.)"
+      and constraint_type in ('PRIMARY','UNIQUE')
+    )) end=last;
+    file &outref mod;
+    by constraint_type constraint_name;
+    retain cols;
+    column_name=upcase(column_name);
+
+    if _n_=1 then put / '  indexes {';
+
+    if upcase(strip(constraint_type)) = 'PRIMARY' then ctype='[pk]';
+    else ctype='[unique]';
+
+    if first.constraint_name then cols = cats('(',column_name);
+    else cols=cats(cols,',',column_name);
+
+    if last.constraint_name then do;
+      cols=cats(cols,')',ctype)!!' //'!!constraint_name;
+      put '    ' cols;
+      constraints_used=catx(' ',constraints_used, constraint_name);
+      call symputx('constcheck',1);
+    end;
+
+    if last then call symputx('constraints_used',cats(upcase(constraints_used)));
+
+    length curds const col $39;
+    curds="&curds";
+    const=constraint_name;
+    col=column_name;
+  run;
+
+  proc append base=&pkds data=&syslast;run;
+
+  /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+  data _data_(keep=curds const col);
+    set &idxinfo (where=(
+      libname="%scan(&curds,1,.)"
+      and upcase(memname)="%scan(&curds,2,.)"
+      and unique='yes'
+      and upcase(indxname) not in (%mf_getquotedstr(&constraints_used))
+    ));
+    file &outref mod;
+    by idxusage indxname;
+    name=upcase(name);
+    if &constcheck=1 then stop; /* in fact we only care about PKs so stop if we have */
+    if _n_=1 and &constcheck=0 then put / '  indexes {';
+
+    length cols $5000;
+    retain cols;
+    if first.indxname then cols = cats('(',name);
+    else cols=cats(cols,',',name);
+
+    if last.indxname then do;
+      cols=cats(cols,')[unique]')!!' //'!!indxname;
+      put '    ' cols;
+      call symputx('constcheck',1);
+    end;
+
+    length curds const col $39;
+    curds="&curds";
+    const=indxname;
+    col=name;
+  run;
+  proc append base=&pkds data=&syslast;run;
+
+  data _null_;
+    file &outref mod;
+    if &constcheck =1 then put '  }';
+    put '}';
+  run;
+
+%end;
+
+/**
+ * now we need to figure out the relationships
+ */
+
+/* sort alphabetically so we can have one set of unique cols per table */
+proc sort data=&pkds nodupkey;
+  by curds const col;
+run;
+
+data &pkds.1 (keep=curds col)
+     &pkds.2 (keep=curds cols);
+  set &pkds;
+  by curds const;
+  length retconst $39 cols $5000;
+  retain retconst cols;
+  if first.curds then do;
+    retconst=const;
+    cols=upcase(col);
+  end;
+  else cols=catx(' ',cols,upcase(col));
+  if retconst=const then do;
+    output &pkds.1;
+    if last.const then output &pkds.2;
+  end;
+run;
+
+%let curdslist="0";
+%do x=1 %to %sysfunc(countw(&dsnlist,%str( )));
+  %let curds=%scan(&dsnlist,&x,%str( ));
+
+  %let pkcols=0;
+  data _null_;
+    set &pkds.2(where=(curds="&curds"));
+    call symputx('pkcols',cols);
+  run;
+  %if &pkcols ne 0 %then %do;
+    %let curdslist=&curdslist,"&curds";
+
+    /* start with one2one */
+    data &pkds.4;
+      file &outref mod;
+      set &pkds.2(where=(cols="&pkcols" and curds not in (&curdslist)));
+      line='Ref: "'!!"&curds"
+        !!cats('".(',"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))",')')
+        !!' - '
+        !!cats(quote(trim(curds)),'.(',"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))",')');
+      put line;
+    run;
+
+    /* now many2one */
+    /* get table with one row per col */
+    data &pkds.5;
+      set &pkds.1(where=(curds="&curds"));
+    run;
+    /* get tables which contain the PK columns */
+    proc sql;
+    create table &pkds.5a as
+      select upcase(cats(b.libname,'.',b.memname)) as curds
+        ,b.name
+      from &pkds.5 a
+      inner join &colinfo b
+      on a.col=upcase(b.name);
+    /* count to make sure those tables contain ALL the columns */
+    create table &pkds.5b as
+      select curds,count(*) as cnt
+      from &pkds.5a
+      where curds not in (select curds from &pkds.2 where cols="&pkcols") /* not a one to one match */
+        and curds ne "&curds" /* exclude self */
+      group by 1;
+    create table &pkds.6 as
+      select a.*
+        ,b.cols
+      from &pkds.5b a
+      left join &pkds.4 b
+      on a.curds=b.curds;
+
+    data _null_;
+      set &pkds.6;
+      file &outref mod;
+      colcnt=%sysfunc(countw(&pkcols));
+      if cnt=colcnt then do;
+        /* table contains all the PK cols, and was not a direct / 121 match */
+        line='Ref: "'!!"&curds"
+          !!'".('
+          !!"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))"
+          !!') > '
+          !!cats(quote(trim(curds))
+              ,'.('
+              ,"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))"
+              ,')'
+          );
+        put line;
+      end;
+    run;
+  %end;
+%end;
+
+
+%if %upcase(&showlog)=YES %then %do;
+  options ps=max;
+  data _null_;
+    infile &outref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend;

base/mp_streamfile.sas

@@ -1,5 +1,5 @@
 /**
-  @file mp_streamfile.sas
+  @file
   @brief Streams a file to _webout according to content type
   @details Will set headers using appropriate functions (SAS 9 vs Viya) and send
   content as a binary stream.
@@ -17,6 +17,7 @@
 
   @param contenttype= Either TEXT, ZIP, CSV, EXCEL (default TEXT)
   @param inloc= /path/to/file.ext to be sent
+  @param inref= fileref of file to be sent (if provided, overrides `inloc`)
   @param outname= the name of the file, as downloaded by the browser
 
   @author Allan Bowe
@@ -27,6 +28,7 @@
 %macro mp_streamfile(
   contenttype=TEXT
   ,inloc=
+  ,inref=0
   ,outname=
 )/*/STORE SOURCE*/;
 
@@ -42,7 +44,7 @@
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.zip'
-      contenttype='application/zip' 
+      contenttype='application/zip'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
@@ -56,7 +58,7 @@
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
-      contenttype='application/vnd.ms-excel' 
+      contenttype='application/vnd.ms-excel'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
@@ -69,7 +71,7 @@
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
-      contenttype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' 
+      contenttype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
@@ -102,7 +104,7 @@
 %else %if &contentype=HTML %then %do;
   %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
-      contenttype="text/html"; 
+      contenttype="text/html";
   %end;
 %end;
 %else %do;
@@ -110,6 +112,11 @@
   %return;
 %end;
 
-%mp_binarycopy(inloc="&inloc",outref=_webout)
+%if &inref ne 0 %then %do;
+ %mp_binarycopy(inref=&inref,outref=_webout)
+%end;
+%else %do;
+ %mp_binarycopy(inloc="&inloc",outref=_webout)
+%end;
 
 %mend;

base/mp_tree.sas

@@ -0,0 +1,68 @@
+/**
+  @file
+  @brief Recursively scans a directory tree to get all subfolders and content
+  @details
+  Usage:
+
+      %mp_tree(dir=/tmp, outds=work.tree)
+
+  Credits:
+
+  * Roger Deangelis, https://communities.sas.com/t5/SAS-Programming/listing-all-files-within-a-directory-and-subdirectories/m-p/332616/highlight/true#M74887
+  * Tom, https://communities.sas.com/t5/SAS-Programming/listing-all-files-of-all-types-from-all-subdirectories/m-p/334113/highlight/true#M75419
+
+
+  @param dir= Directory to be scanned (default=/tmp)
+  @param outds= Dataset to create (default=work.mp_tree)
+
+  @returns outds contains the following variables:
+
+   - `dir`: a flag (1/0) to say whether it is a directory or not.  This is not
+     reliable - folders that you do not have permission to open will be flagged
+     as directories.
+   - `ext`: file extension
+   - `filename`: file name
+   - `dirname`: directory name
+   - `fullpath`: directory + file name
+
+  @version 9.2
+**/
+
+%macro mp_tree(dir=/tmp
+  ,outds=work.mp_tree
+)/*/STORE SOURCE*/;
+
+data &outds ;
+  length dir 8 ext filename dirname $256 fullpath $512 ;
+  call missing(of _all_);
+  fullpath = "&dir";
+run;
+
+%local sep;
+%if &sysscp=WIN or &SYSSCP eq DNTHOST %then %let sep=\;
+%else %let sep=/;
+
+data &outds ;
+  modify &outds ;
+  retain sep "&sep";
+  rc=filename('tmp',fullpath);
+  dir_id=dopen('tmp');
+  dir = (dir_id ne 0) ;
+  if dir then dirname=fullpath;
+  else do;
+    filename=scan(fullpath,-1,sep) ;
+    dirname =substrn(fullpath,1,length(fullpath)-length(filename));
+    if index(filename,'.')>1 then ext=scan(filename,-1,'.');
+  end;
+  replace;
+  if dir then do;
+    do i=1 to dnum(dir_id);
+      fullpath=cats(dirname,sep,dread(dir_id,i));
+      output;
+    end;
+    rc=dclose(dir_id);
+  end;
+  rc=filename('tmp');
+run;
+
+%mend;

viya/mv_getfoldermembers.sas

@@ -42,7 +42,7 @@
     %let &access_token_var=;
 %end;
 %put &sysmacroname: grant_type=&grant_type;
-%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password 
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
   ,mac=&sysmacroname
@@ -85,10 +85,16 @@ options noquotelenmax;
   /*data _null_;infile &fname1;input;putlog _infile_;run;*/
   libname &libref1 JSON fileref=&fname1;
   /* now get the followon link to list members */
+  %local href;
+  %let href=0;
   data _null_;
     set &libref1..links;
     if rel='members' then call symputx('href',quote("&base_uri"!!trim(href)),'l');
   run;
+  %if &href=0 %then %do;
+    %put NOTE:;%put NOTE-  No members found in &root!!;%put NOTE-;
+    %return;
+  %end;
   %local fname2 libref2;
   %let fname2=%mf_getuniquefileref();
   %let libref2=%mf_getuniquelibref();

viya/mv_jobexecute.sas

@@ -0,0 +1,141 @@
+/**
+  @file
+  @brief Executes a SAS Viya Job
+  @details Triggers a SAS Viya Job, with optional URL parameters, using
+  the JES web app.
+
+  First, compile the macros:
+
+      filename mc url
+      "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Then, execute the job!
+
+      %mv_jobexecute(path=/Public/folder
+        ,name=somejob
+      )
+
+
+  @param access_token_var= The global macro variable to contain the access token
+  @param grant_type= valid values:
+   * password
+   * authorization_code
+   * detect - will check if access_token exists, if not will use sas_services if
+    a SASStudioV session else authorization_code.  Default option.
+   * sas_services - will use oauth_bearer=sas_services
+
+  @param path= The SAS Drive path to the job being executed
+  @param name= The name of the job to execute
+  @param params= A macro quoted string to append to the URL
+  @param contextName= Context name with which to run the job.
+    Default = `SAS Job Execution compute context`
+
+
+  @version VIYA V.03.04
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> Dependencies </h4>
+  @li mp_abort.sas
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mv_getfoldermembers.sas
+
+**/
+
+%macro mv_jobexecute(path=0
+    ,name=0
+    ,contextName=SAS Job Execution compute context
+    ,access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+  );
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+%put &sysmacroname: grant_type=&grant_type;
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+%mp_abort(iftrue=("&path"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(Path not provided)
+)
+%mp_abort(iftrue=("&name"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(Job Name not provided)
+)
+
+options noquotelenmax;
+
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+data;run;
+%local foldermembers;
+%let foldermembers=&syslast;
+%mv_getfoldermembers(root=&path
+    ,access_token_var=&access_token_var
+    ,grant_type=&grant_type
+    ,outds=&foldermembers
+)
+
+%local joburi;
+%let joburi=0;
+data _null_;
+  set &foldermembers;
+  if name="&name" and uri=:'/jobDefinitions/definitions'
+    then call symputx('joburi',uri);
+run;
+
+%mp_abort(iftrue=("&joburi"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(Job &path/&name not found)
+)
+
+/* prepare request*/
+%local fname0 fname1;
+%let fname0=%mf_getuniquefileref();
+%let fname1=%mf_getuniquefileref();
+
+data _null_;
+  file &fname0;
+  put '{"jobDefinitionUri": "'@@;
+  put "&joburi"@@;
+  put '","arguments":{"_contextName":"'@@;
+  put "&contextName"@@;
+  put '"}}';
+run;
+
+proc http method='POST' in=&fname0 out=&fname1 &oauth_bearer
+  url="&base_uri/jobExecution/jobs";
+  headers "Content-Type"="application/vnd.sas.job.execution.job.request+json"
+          "Accept"="application/vnd.sas.job.execution.job+json"
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;
+  ;
+run;
+/*data _null_;infile &fname1;input;putlog _infile_;run;*/
+
+%mp_abort(iftrue=(
+    &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+/* clear refs */
+filename &fname0 clear;
+filename &fname1 clear;
+
+%mend;