fix: adding outds and parameters to mv_jobexecute

LICENSE

@@ -1,4 +1,4 @@
-Copyright 2018 (Allan Bowe)
+Copyright 2020 (Allan Bowe)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

all.sas

@@ -39,7 +39,7 @@ options noquotelenmax;
 
   @version 9.2
   @author Allan Bowe
-**/
+**/  /** @cond */
 
 %macro mf_abort(mac=mf_abort.sas, type=, msg=, iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
@@ -157,7 +157,8 @@ options noquotelenmax;
     %abort cancel;
   %end;
 %mend;
-/**
+
+/** @endcond *//**
   @file mf_existds.sas
   @brief Checks whether a dataset OR a view exists.
   @details Can be used in open code, eg as follows:
@@ -183,7 +184,7 @@ options noquotelenmax;
   %else 1;
 
 %mend;/**
-  @file mf_existfeature.sas
+  @file
   @brief Checks whether a feature exists
   @details Check to see if a feature is supported in your environment.
     Run without arguments to see a list of detectable features.
@@ -191,10 +192,10 @@ options noquotelenmax;
     actual feature detection, as that is tricky / impossible to do
     without generating errors in most cases.
 
-      %put %mf_existfeature(PROCLUA);
+        %put %mf_existfeature(PROCLUA);
 
   @param feature the feature to detect.  Leave blank to list all in log.
-  
+
   @return output returns 1 or 0 (or -1 if not found)
 
   <h4> Dependencies </h4>
@@ -203,7 +204,7 @@ options noquotelenmax;
 
   @version 8
   @author Allan Bowe
-**/
+**/  /** @cond */
 
 %macro mf_existfeature(feature
 )/*/STORE SOURCE*/;
@@ -223,7 +224,9 @@ options noquotelenmax;
     -1
     %put &sysmacroname: &feature not found;
   %end;
-%mend;/**
+%mend;
+
+/** @endcond *//**
   @file
   @brief Checks if a variable exists in a data set.
   @details Returns 0 if the variable does NOT exist, and return the position of
@@ -236,7 +239,7 @@ options noquotelenmax;
   @param var (positional) - variable name
   @version 9.2
   @author Allan Bowe
-**/
+**/  /** @cond */
 
 %macro mf_existvar(libds /* 2 part dataset name */
       , var /* variable name */
@@ -254,7 +257,9 @@ options noquotelenmax;
       %let rc=%sysfunc(close(&dsid));
   %end;
 
-%mend;/**
+%mend;
+
+/** @endcond *//**
   @file
   @brief Checks if a set of variables ALL exist in a data set.
   @details Returns 0 if ANY of the variables do not exist, or 1 if they ALL do.
@@ -399,7 +404,7 @@ options noquotelenmax;
   @version 9.2
   @author Allan Bowe
 
-**/ /** \cond */
+**/  /** @cond */
 
 %macro mf_getengine(libref
 )/*/STORE SOURCE*/;
@@ -421,7 +426,7 @@ options noquotelenmax;
 
 %mend;
 
-/** \endcond *//**
+/** @endcond *//**
   @file
   @brief Returns the size of a file in bytes.
   @details Provide full path/filename.extension to the file, eg:
@@ -648,17 +653,18 @@ options noquotelenmax;
 /**
   @file
   @brief Assigns and returns an unused fileref
-  @details Use as follows:
+  @details
+  Use as follows:
 
-    %let fileref1=%mf_getuniquefileref();
-    %let fileref2=%mf_getuniquefileref();
-    %put &fileref1 &fileref2;
+      %let fileref1=%mf_getuniquefileref();
+      %let fileref2=%mf_getuniquefileref();
+      %put &fileref1 &fileref2;
 
   which returns:
 
 > mcref0 mcref1
 
-  @prefix= first part of fileref. Remember that filerefs can only be 8
+  @param prefix= first part of fileref. Remember that filerefs can only be 8
     characters, so a 7 letter prefix would mean that `maxtries` should be 10.
   @param maxtries= the last part of the libref.  Provide an integer value.
 
@@ -696,7 +702,7 @@ options noquotelenmax;
 
 > mclib3
 
-  @prefix= first part of libref.  Remember that librefs can only be 8 characters,
+  @param prefix= first part of libref.  Remember that librefs can only be 8 characters,
     so a 7 letter prefix would mean that maxtries should be 10.
   @param maxtries= the last part of the libref.  Provide an integer value.
 
@@ -1154,6 +1160,38 @@ Usage:
 
   %sysevalf(%superq(param)=,boolean)
 
+%mend;/**
+  @file
+  @brief Checks whether a path is a valid directory
+  @details
+  Usage:
+
+      %let isdir=%mf_isdir(/tmp);
+
+  With thanks and full credit to Andrea Defronzo - https://www.linkedin.com/in/andrea-defronzo-b1a47460/
+
+  @param path full path of the file/directory to be checked
+
+  @return output returns 1 if path is a directory, 0 if it is not
+
+  @version 9.2
+**/
+
+%macro mf_isdir(path
+)/*/STORE SOURCE*/;
+	%local rc did is_directory fref_t;
+
+	%let is_directory = 0;
+	%let rc = %sysfunc(filename(fref_t, %superq(path)));
+	%let did = %sysfunc(dopen(&fref_t.));
+	%if &did. ^= 0 %then %do;
+	   %let is_directory = 1;
+	   %let rc = %sysfunc(dclose(&did.));
+	%end;
+	%let rc = %sysfunc(filename(fref_t));
+
+	&is_directory
+
 %mend;/**
   @file
   @brief Returns physical location of various SAS items
@@ -1925,30 +1963,58 @@ Usage:
 
 
   @param inref= fileref to the CSV
-  @param outds= output ds.  Could also be a view (eg `outds=myds/view=myds`)
+  @param outds= output ds (lib.ds format)
+  @param view= Set to YES or NO to determine whether the output should be
+    a view or not.  Default is NO (not a view).
   @param baseds= Template dataset on which to create the input statement.
     Is used to determine types, lengths, and any informats.
 
   @version 9.2
   @author Allan Bowe
+
+  <h4> Dependencies </h4>
+  @li mp_abort.sas
+  @li mf_existds.sas
+
 **/
 
-%macro mp_csv2ds(inref=0,outds=0,baseds=0);
-%if &inref=0 %then %do;
-  %put %str(ERR)OR: the INREF variable must be provided;
-  %let syscc=4;
-  %abort;
-%end;
-%if &outds=0 %then %do;
-  %put %str(ERR)OR: the OUTDS variable must be provided;
-  %let syscc=4;
-  %return;
-%end;
-%if &baseds=0 %then %do;
-  %put %str(ERR)OR: the BASEDS variable must be provided;
-  %let syscc=4;
-  %return;
-%end;
+%macro mp_csv2ds(inref=0,outds=0,baseds=0,view=NO);
+
+%mp_abort(iftrue=( &inref=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the INREF variable must be provided)
+)
+%mp_abort(iftrue=( %superq(outds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the OUTDS variable must be provided)
+)
+%mp_abort(iftrue=( &baseds=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS variable must be provided)
+)
+%mp_abort(iftrue=( &baseds=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS variable must be provided)
+)
+%mp_abort(iftrue=( %mf_existds(&baseds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS dataset (&baseds) needs to be assigned, and to exist)
+)
+
+/* count rows */
+%local hasheader; %let hasheader=0;
+data _null_;
+  if _N_ > 1 then do;
+     call symputx('hasheader',1,'l');
+     stop;
+  end;
+  infile &inref;
+  input;
+run;
+%mp_abort(iftrue=( &hasheader=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(No header row in &inref)
+)
 
 /* get the variables in the CSV */
 data _data_;
@@ -1997,6 +2063,7 @@ data _null_;
     if type=1 then informat='best.';
     else informat=cats('$',length,'.');
   end;
+  else informat=cats(informat,'.');
   in=catx(' ',in,name,':',informat);
   if last then do;
     call symputx('instat',in,'l');
@@ -2004,10 +2071,17 @@ data _null_;
   end;
 run;
 
-data &outds;
+/* import the CSV */
+data &outds
+  %if %upcase(&view)=YES %then %do;
+   /view=&outds
+  %end;
+  ;
   infile &inref dsd firstobs=2;
   input &instat;
-  drop &dropvars;
+  %if %length(&dropvars)>0 %then %do;
+    drop &dropvars;
+  %end;
 run;
 
 %mend;/**
@@ -2616,6 +2690,329 @@ create table &outds as
   %end;
   ;
 
+%mend;/**
+  @file
+  @brief Extract DBML from SAS Libraries
+  @details DBML is an open source markup format to represent databases.
+  More details: https://www.dbml.org/home/
+
+  Usage:
+
+
+      %mp_getdbml(liblist=SASHELP WORK,outref=mydbml,showlog=YES)
+
+  Take the log output and paste it into the renderer at https://dbdiagram.io
+  to view your data model diagram.  The code takes a "best guess" at
+  the one to one and one to many relationships (based on constraints
+  and indexes, and assuming that the column names would match).
+
+  You may need to adjust the rendered DBML to suit your needs.
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getquotedstr.sas
+  @li mp_getconstraints.sas
+
+  @param liblist= Space seperated list of librefs to take as
+    input (Default=SASHELP)
+  @param outref= Fileref to contain the DBML (Default=getdbml)
+  @param showlog= set to YES to show the DBML in the log (Default is NO)
+
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_getdbml(liblist=SASHELP,outref=getdbml,showlog=NO
+)/*/STORE SOURCE*/;
+
+/* check fileref is assigned */
+%if %sysfunc(fileref(&outref)) > 0 %then %do;
+  filename &outref temp;
+%end;
+
+%let liblist=%upcase(&liblist);
+
+proc sql noprint;
+create table _data_ as
+  select * from dictionary.tables
+  where upcase(libname) in (%mf_getquotedstr(&liblist))
+  order by libname,memname;
+%local tabinfo; %let tabinfo=&syslast;
+
+create table _data_ as
+  select * from dictionary.columns
+  where upcase(libname) in (%mf_getquotedstr(&liblist))
+  order by libname,memname,varnum;
+%local colinfo; %let colinfo=&syslast;
+
+%local dsnlist;
+  select distinct upcase(cats(libname,'.',memname)) into: dsnlist
+  separated by ' '
+  from &syslast
+;
+
+create table _data_ as
+  select * from dictionary.indexes
+  where upcase(libname) in (%mf_getquotedstr(&liblist))
+  order by idxusage, indxname, indxpos;
+%local idxinfo; %let idxinfo=&syslast;
+
+/* Extract all Primary Key and Unique data constraints */
+%mp_getconstraints(lib=%scan(&liblist,1),outds=_data_)
+%local colconst; %let colconst=&syslast;
+
+%do x=2 %to %sysfunc(countw(&liblist));
+  %mp_getconstraints(lib=%scan(&liblist,&x),outds=_data_)
+  proc append base=&colconst data=&syslast;
+  run;
+%end;
+
+
+
+
+/* header info */
+data _null_;
+  file &outref;
+  put "// DBML generated by &sysuserid on %sysfunc(datetime(),datetime19.) ";
+  put "Project sasdbml {";
+  put "  database_type: 'SAS'";
+  put "  Note: 'Generated by the mp_getdbml() macro'";
+  put "}";
+run;
+
+/* create table groups */
+data _null_;
+  file &outref mod;
+  set &tabinfo;
+  by libname;
+  if first.libname then put "TableGroup " libname "{";
+  ds=quote(cats(libname,'.',memname));
+  put '   ' ds;
+  if last.libname then put "}";
+run;
+
+/* table for pks */
+data _data_;
+  length curds const col $39;
+  call missing (of _all_);
+  stop;
+run;
+%let pkds=&syslast;
+
+%local x curds constraints_used constcheck;
+%do x=1 %to %sysfunc(countw(&dsnlist,%str( )));
+  %let curds=%scan(&dsnlist,&x,%str( ));
+  %let constraints_used=;
+  %let constcheck=0;
+  data _null_;
+    file &outref mod;
+    length lab $1024 typ $20;
+    set &colinfo (where=(
+        libname="%scan(&curds,1,.)" and upcase(memname)="%scan(&curds,2,.)"
+    )) end=last;
+
+    if _n_=1 then do;
+      table='Table "'!!"&curds"!!'"{';
+      put table;
+    end;
+    name=upcase(name);
+    lab=" note:"!!quote(trim(tranwrd(label,'"',"'")));
+    if upcase(format)=:'DATETIME' then typ='datetime';
+    else if type='char' then typ=cats('char(',length,')');
+    else typ='num';
+
+    if notnull='yes' then notnul=' not null';
+    if notnull='no' and missing(label) then put '  ' name typ;
+    else if notnull='yes' and missing(label) then put '  ' name typ '[' notnul ']';
+    else if notnull='no' then put '  ' name typ '[' lab ']';
+    else put '  ' name typ '[' notnul ',' lab ']';
+
+  run;
+
+  data _data_(keep=curds const col);
+    length ctype $11 cols constraints_used $5000;
+    set &colconst (where=(
+      upcase(libref)="%scan(&curds,1,.)"
+      and upcase(table_name)="%scan(&curds,2,.)"
+      and constraint_type in ('PRIMARY','UNIQUE')
+    )) end=last;
+    file &outref mod;
+    by constraint_type constraint_name;
+    retain cols;
+    column_name=upcase(column_name);
+
+    if _n_=1 then put / '  indexes {';
+
+    if upcase(strip(constraint_type)) = 'PRIMARY' then ctype='[pk]';
+    else ctype='[unique]';
+
+    if first.constraint_name then cols = cats('(',column_name);
+    else cols=cats(cols,',',column_name);
+
+    if last.constraint_name then do;
+      cols=cats(cols,')',ctype)!!' //'!!constraint_name;
+      put '    ' cols;
+      constraints_used=catx(' ',constraints_used, constraint_name);
+      call symputx('constcheck',1);
+    end;
+
+    if last then call symputx('constraints_used',cats(upcase(constraints_used)));
+
+    length curds const col $39;
+    curds="&curds";
+    const=constraint_name;
+    col=column_name;
+  run;
+
+  proc append base=&pkds data=&syslast;run;
+
+  /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+  data _data_(keep=curds const col);
+    set &idxinfo (where=(
+      libname="%scan(&curds,1,.)"
+      and upcase(memname)="%scan(&curds,2,.)"
+      and unique='yes'
+      and upcase(indxname) not in (%mf_getquotedstr(&constraints_used))
+    ));
+    file &outref mod;
+    by idxusage indxname;
+    name=upcase(name);
+    if &constcheck=1 then stop; /* in fact we only care about PKs so stop if we have */
+    if _n_=1 and &constcheck=0 then put / '  indexes {';
+
+    length cols $5000;
+    retain cols;
+    if first.indxname then cols = cats('(',name);
+    else cols=cats(cols,',',name);
+
+    if last.indxname then do;
+      cols=cats(cols,')[unique]')!!' //'!!indxname;
+      put '    ' cols;
+      call symputx('constcheck',1);
+    end;
+
+    length curds const col $39;
+    curds="&curds";
+    const=indxname;
+    col=name;
+  run;
+  proc append base=&pkds data=&syslast;run;
+
+  data _null_;
+    file &outref mod;
+    if &constcheck =1 then put '  }';
+    put '}';
+  run;
+
+%end;
+
+/**
+ * now we need to figure out the relationships
+ */
+
+/* sort alphabetically so we can have one set of unique cols per table */
+proc sort data=&pkds nodupkey;
+  by curds const col;
+run;
+
+data &pkds.1 (keep=curds col)
+     &pkds.2 (keep=curds cols);
+  set &pkds;
+  by curds const;
+  length retconst $39 cols $5000;
+  retain retconst cols;
+  if first.curds then do;
+    retconst=const;
+    cols=upcase(col);
+  end;
+  else cols=catx(' ',cols,upcase(col));
+  if retconst=const then do;
+    output &pkds.1;
+    if last.const then output &pkds.2;
+  end;
+run;
+
+%let curdslist="0";
+%do x=1 %to %sysfunc(countw(&dsnlist,%str( )));
+  %let curds=%scan(&dsnlist,&x,%str( ));
+
+  %let pkcols=0;
+  data _null_;
+    set &pkds.2(where=(curds="&curds"));
+    call symputx('pkcols',cols);
+  run;
+  %if &pkcols ne 0 %then %do;
+    %let curdslist=&curdslist,"&curds";
+
+    /* start with one2one */
+    data &pkds.4;
+      file &outref mod;
+      set &pkds.2(where=(cols="&pkcols" and curds not in (&curdslist)));
+      line='Ref: "'!!"&curds"
+        !!cats('".(',"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))",')')
+        !!' - '
+        !!cats(quote(trim(curds)),'.(',"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))",')');
+      put line;
+    run;
+
+    /* now many2one */
+    /* get table with one row per col */
+    data &pkds.5;
+      set &pkds.1(where=(curds="&curds"));
+    run;
+    /* get tables which contain the PK columns */
+    proc sql;
+    create table &pkds.5a as
+      select upcase(cats(b.libname,'.',b.memname)) as curds
+        ,b.name
+      from &pkds.5 a
+      inner join &colinfo b
+      on a.col=upcase(b.name);
+    /* count to make sure those tables contain ALL the columns */
+    create table &pkds.5b as
+      select curds,count(*) as cnt
+      from &pkds.5a
+      where curds not in (select curds from &pkds.2 where cols="&pkcols") /* not a one to one match */
+        and curds ne "&curds" /* exclude self */
+      group by 1;
+    create table &pkds.6 as
+      select a.*
+        ,b.cols
+      from &pkds.5b a
+      left join &pkds.4 b
+      on a.curds=b.curds;
+
+    data _null_;
+      set &pkds.6;
+      file &outref mod;
+      colcnt=%sysfunc(countw(&pkcols));
+      if cnt=colcnt then do;
+        /* table contains all the PK cols, and was not a direct / 121 match */
+        line='Ref: "'!!"&curds"
+          !!'".('
+          !!"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))"
+          !!') > '
+          !!cats(quote(trim(curds))
+              ,'.('
+              ,"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))"
+              ,')'
+          );
+        put line;
+      end;
+    run;
+  %end;
+%end;
+
+
+%if %upcase(&showlog)=YES %then %do;
+  options ps=max;
+  data _null_;
+    infile &outref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
 %mend;/**
   @file mp_getddl.sas
   @brief Extract DDL in various formats, by table or library
@@ -4077,7 +4474,7 @@ proc sql
 
   options &etls_syntaxcheck;
 %mend;/**
-  @file mp_streamfile.sas
+  @file
   @brief Streams a file to _webout according to content type
   @details Will set headers using appropriate functions (SAS 9 vs Viya) and send
   content as a binary stream.
@@ -4095,6 +4492,7 @@ proc sql
 
   @param contenttype= Either TEXT, ZIP, CSV, EXCEL (default TEXT)
   @param inloc= /path/to/file.ext to be sent
+  @param inref= fileref of file to be sent (if provided, overrides `inloc`)
   @param outname= the name of the file, as downloaded by the browser
 
   @author Allan Bowe
@@ -4105,6 +4503,7 @@ proc sql
 %macro mp_streamfile(
   contenttype=TEXT
   ,inloc=
+  ,inref=0
   ,outname=
 )/*/STORE SOURCE*/;
 
@@ -4120,7 +4519,7 @@ proc sql
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.zip'
-      contenttype='application/zip' 
+      contenttype='application/zip'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
@@ -4134,7 +4533,7 @@ proc sql
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
-      contenttype='application/vnd.ms-excel' 
+      contenttype='application/vnd.ms-excel'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
@@ -4147,7 +4546,7 @@ proc sql
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
-      contenttype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' 
+      contenttype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
@@ -4180,7 +4579,7 @@ proc sql
 %else %if &contentype=HTML %then %do;
   %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
-      contenttype="text/html"; 
+      contenttype="text/html";
   %end;
 %end;
 %else %do;
@@ -4188,7 +4587,79 @@ proc sql
   %return;
 %end;
 
-%mp_binarycopy(inloc="&inloc",outref=_webout)
+%if &inref ne 0 %then %do;
+ %mp_binarycopy(inref=&inref,outref=_webout)
+%end;
+%else %do;
+ %mp_binarycopy(inloc="&inloc",outref=_webout)
+%end;
+
+%mend;/**
+  @file
+  @brief Recursively scans a directory tree to get all subfolders and content
+  @details
+  Usage:
+
+      %mp_tree(dir=/tmp, outds=work.tree)
+
+  Credits:
+
+  * Roger Deangelis, https://communities.sas.com/t5/SAS-Programming/listing-all-files-within-a-directory-and-subdirectories/m-p/332616/highlight/true#M74887
+  * Tom, https://communities.sas.com/t5/SAS-Programming/listing-all-files-of-all-types-from-all-subdirectories/m-p/334113/highlight/true#M75419
+
+
+  @param dir= Directory to be scanned (default=/tmp)
+  @param outds= Dataset to create (default=work.mp_tree)
+
+  @returns outds contains the following variables:
+
+   - `dir`: a flag (1/0) to say whether it is a directory or not.  This is not
+     reliable - folders that you do not have permission to open will be flagged
+     as directories.
+   - `ext`: file extension
+   - `filename`: file name
+   - `dirname`: directory name
+   - `fullpath`: directory + file name
+
+  @version 9.2
+**/
+
+%macro mp_tree(dir=/tmp
+  ,outds=work.mp_tree
+)/*/STORE SOURCE*/;
+
+data &outds ;
+  length dir 8 ext filename dirname $256 fullpath $512 ;
+  call missing(of _all_);
+  fullpath = "&dir";
+run;
+
+%local sep;
+%if &sysscp=WIN or &SYSSCP eq DNTHOST %then %let sep=\;
+%else %let sep=/;
+
+data &outds ;
+  modify &outds ;
+  retain sep "&sep";
+  rc=filename('tmp',fullpath);
+  dir_id=dopen('tmp');
+  dir = (dir_id ne 0) ;
+  if dir then dirname=fullpath;
+  else do;
+    filename=scan(fullpath,-1,sep) ;
+    dirname =substrn(fullpath,1,length(fullpath)-length(filename));
+    if index(filename,'.')>1 then ext=scan(filename,-1,'.');
+  end;
+  replace;
+  if dir then do;
+    do i=1 to dnum(dir_id);
+      fullpath=cats(dirname,sep,dread(dir_id,i));
+      output;
+    end;
+    rc=dclose(dir_id);
+  end;
+  rc=filename('tmp');
+run;
 
 %mend;/**
   @file mp_unzip.sas
@@ -5578,15 +6049,15 @@ run;
 
   Usage:
 
-    %mm_createlibrary(
-       libname=My New Library
-      ,libref=mynewlib
-      ,libdesc=Super & <fine>
-      ,engine=BASE
-      ,tree=/User Folders/sasdemo
-      ,servercontext=SASApp
-      ,directory=/tmp/tests
-      ,mDebug=1)
+      %mm_createlibrary(
+        libname=My New Library
+        ,libref=mynewlib
+        ,libdesc=Super & <fine>
+        ,engine=BASE
+        ,tree=/User Folders/sasdemo
+        ,servercontext=SASApp
+        ,directory=/tmp/tests
+        ,mDebug=1)
 
   <h4> Dependencies </h4>
   @li mf_verifymacvars.sas
@@ -8207,8 +8678,8 @@ filename __outdoc clear;
     combine with the <code>tree=</code> parameter.
   @param outds= the dataset to create that contains the list of stps.
   @param mDebug= set to 1 to show debug messages in the log
-  @showDesc= provide a non blank value to return stored process descriptions
-  @showUsageVersion= provide a non blank value to return the UsageVersion.  This
+  @param showDesc= provide a non blank value to return stored process descriptions
+  @param showUsageVersion= provide a non blank value to return the UsageVersion.  This
     is either 1000000 (type 1, 9.2) or 2000000 (type2, 9.3 onwards).
 
   @returns outds  dataset containing the following columns
@@ -8688,7 +9159,7 @@ libname _XML_ clear;
 /**
   @file
   @brief Retrieves properties of the SAS web app server
-  @details 
+  @details
   Usage:
 
       %mm_getwebappsrvprops(outds= some_ds)
@@ -8708,8 +9179,7 @@ libname _XML_ clear;
       libname __shake clear;
 
   @version 9.4
-  @author Allan Bowe
-  @source https://github.com/sasjs/core
+  @author Allan Bowe https://github.com/sasjs/core
 
 **/
 
@@ -11292,7 +11762,7 @@ libname &libref1 clear;
     %let &access_token_var=;
 %end;
 %put &sysmacroname: grant_type=&grant_type;
-%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password 
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
   ,mac=&sysmacroname
@@ -11335,10 +11805,16 @@ options noquotelenmax;
   /*data _null_;infile &fname1;input;putlog _infile_;run;*/
   libname &libref1 JSON fileref=&fname1;
   /* now get the followon link to list members */
+  %local href;
+  %let href=0;
   data _null_;
     set &libref1..links;
     if rel='members' then call symputx('href',quote("&base_uri"!!trim(href)),'l');
   run;
+  %if &href=0 %then %do;
+    %put NOTE:;%put NOTE-  No members found in &root!!;%put NOTE-;
+    %return;
+  %end;
   %local fname2 libref2;
   %let fname2=%mf_getuniquefileref();
   %let libref2=%mf_getuniquelibref();
@@ -11805,6 +12281,173 @@ run;
 filename &fname1 clear;
 libname &libref1 clear;
 
+%mend;/**
+  @file
+  @brief Executes a SAS Viya Job
+  @details Triggers a SAS Viya Job, with optional URL parameters, using
+  the JES web app.
+
+  First, compile the macros:
+
+      filename mc url
+      "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Then, execute the job!
+
+      %mv_jobexecute(path=/Public/folder
+        ,name=somejob
+      )
+
+  Example with parameters:
+
+     %mv_jobexecute(path=/Public/folder
+        ,name=somejob
+        ,paramstring=%str("macvarname":"macvarvalue","answer":42)
+      )
+
+  @param access_token_var= The global macro variable to contain the access token
+  @param grant_type= valid values:
+   * password
+   * authorization_code
+   * detect - will check if access_token exists, if not will use sas_services if
+    a SASStudioV session else authorization_code.  Default option.
+   * sas_services - will use oauth_bearer=sas_services
+
+  @param path= The SAS Drive path to the job being executed
+  @param name= The name of the job to execute
+  @param paramstring= A JSON fragment with name:value pairs, eg: `"name":"value"`
+  or "name":"value","name2":42`.  This will need to be wrapped in `%str()`.
+
+  @param contextName= Context name with which to run the job.
+    Default = `SAS Job Execution compute context`
+
+  @param outds= The output dataset containing links (Default=work.mv_jobexecute)
+
+
+  @version VIYA V.03.04
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> Dependencies </h4>
+  @li mp_abort.sas
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mv_getfoldermembers.sas
+
+**/
+
+%macro mv_jobexecute(path=0
+    ,name=0
+    ,contextName=SAS Job Execution compute context
+    ,access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+    ,paramstring=0
+    ,outds=work.mv_jobexecute
+  );
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+%put &sysmacroname: grant_type=&grant_type;
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+%mp_abort(iftrue=("&path"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(Path not provided)
+)
+%mp_abort(iftrue=("&name"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(Job Name not provided)
+)
+
+options noquotelenmax;
+
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+data;run;
+%local foldermembers;
+%let foldermembers=&syslast;
+%mv_getfoldermembers(root=&path
+    ,access_token_var=&access_token_var
+    ,grant_type=&grant_type
+    ,outds=&foldermembers
+)
+
+%local joburi;
+%let joburi=0;
+data _null_;
+  set &foldermembers;
+  if name="&name" and uri=:'/jobDefinitions/definitions'
+    then call symputx('joburi',uri);
+run;
+
+%mp_abort(iftrue=("&joburi"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(Job &path/&name not found)
+)
+
+/* prepare request*/
+%local fname0 fname1;
+%let fname0=%mf_getuniquefileref();
+%let fname1=%mf_getuniquefileref();
+
+data _null_;
+  file &fname0;
+  length joburi contextname $128 paramstring $32765;
+  joburi=quote(trim(symget('joburi')));
+  contextname=quote(trim(symget('contextname')));
+  paramstring=symget('paramstring');
+  put '{"jobDefinitionUri":' joburi ;
+  put '  ,"arguments":{"_contextName":' contextname;
+  if paramstring ne "0" then do;
+    put '    ,' paramstring;
+  end;
+  put '}}';
+run;
+
+proc http method='POST' in=&fname0 out=&fname1 &oauth_bearer
+  url="&base_uri/jobExecution/jobs";
+  headers "Content-Type"="application/vnd.sas.job.execution.job.request+json"
+          "Accept"="application/vnd.sas.job.execution.job+json"
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;
+  ;
+run;
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+%do;
+  data _null_;infile &fname0;input;putlog _infile_;run;
+  data _null_;infile &fname1;input;putlog _infile_;run;
+  %mp_abort(mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+%end;
+
+%local libref;
+%let libref=%mf_getuniquelibref();
+libname &libref JSON fileref=&fname1;
+
+data &outds;
+  set &libref..links;
+run;
+
+/* clear refs */
+filename &fname0 clear;
+filename &fname1 clear;
+libname &libref;
+
 %mend;/**
   @file mv_registerclient.sas
   @brief Register Client and Secret (admin task)

base/mf_abort.sas

@@ -21,7 +21,7 @@
 
   @version 9.2
   @author Allan Bowe
-**/
+**/  /** @cond */
 
 %macro mf_abort(mac=mf_abort.sas, type=, msg=, iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
@@ -139,3 +139,5 @@
     %abort cancel;
   %end;
 %mend;
+
+/** @endcond */

base/mf_existfeature.sas

@@ -1,5 +1,5 @@
 /**
-  @file mf_existfeature.sas
+  @file
   @brief Checks whether a feature exists
   @details Check to see if a feature is supported in your environment.
     Run without arguments to see a list of detectable features.
@@ -7,10 +7,10 @@
     actual feature detection, as that is tricky / impossible to do
     without generating errors in most cases.
 
-      %put %mf_existfeature(PROCLUA);
+        %put %mf_existfeature(PROCLUA);
 
   @param feature the feature to detect.  Leave blank to list all in log.
-  
+
   @return output returns 1 or 0 (or -1 if not found)
 
   <h4> Dependencies </h4>
@@ -19,7 +19,7 @@
 
   @version 8
   @author Allan Bowe
-**/
+**/  /** @cond */
 
 %macro mf_existfeature(feature
 )/*/STORE SOURCE*/;
@@ -39,4 +39,6 @@
     -1
     %put &sysmacroname: &feature not found;
   %end;
-%mend;
+%mend;
+
+/** @endcond */

base/mf_existvar.sas

@@ -11,7 +11,7 @@
   @param var (positional) - variable name
   @version 9.2
   @author Allan Bowe
-**/
+**/  /** @cond */
 
 %macro mf_existvar(libds /* 2 part dataset name */
       , var /* variable name */
@@ -29,4 +29,6 @@
       %let rc=%sysfunc(close(&dsid));
   %end;
 
-%mend;
+%mend;
+
+/** @endcond */

base/mf_getengine.sas

@@ -22,7 +22,7 @@
   @version 9.2
   @author Allan Bowe
 
-**/ /** \cond */
+**/  /** @cond */
 
 %macro mf_getengine(libref
 )/*/STORE SOURCE*/;
@@ -44,4 +44,4 @@
 
 %mend;
 
-/** \endcond */
+/** @endcond */

base/mf_getuniquefileref.sas

@@ -1,17 +1,18 @@
 /**
   @file
   @brief Assigns and returns an unused fileref
-  @details Use as follows:
+  @details
+  Use as follows:
 
-    %let fileref1=%mf_getuniquefileref();
-    %let fileref2=%mf_getuniquefileref();
-    %put &fileref1 &fileref2;
+      %let fileref1=%mf_getuniquefileref();
+      %let fileref2=%mf_getuniquefileref();
+      %put &fileref1 &fileref2;
 
   which returns:
 
 > mcref0 mcref1
 
-  @prefix= first part of fileref. Remember that filerefs can only be 8
+  @param prefix= first part of fileref. Remember that filerefs can only be 8
     characters, so a 7 letter prefix would mean that `maxtries` should be 10.
   @param maxtries= the last part of the libref.  Provide an integer value.
 

base/mf_getuniquelibref.sas

@@ -14,7 +14,7 @@
 
 > mclib3
 
-  @prefix= first part of libref.  Remember that librefs can only be 8 characters,
+  @param prefix= first part of libref.  Remember that librefs can only be 8 characters,
     so a 7 letter prefix would mean that maxtries should be 10.
   @param maxtries= the last part of the libref.  Provide an integer value.
 

base/mf_isdir.sas

@@ -0,0 +1,33 @@
+/**
+  @file
+  @brief Checks whether a path is a valid directory
+  @details
+  Usage:
+
+      %let isdir=%mf_isdir(/tmp);
+
+  With thanks and full credit to Andrea Defronzo - https://www.linkedin.com/in/andrea-defronzo-b1a47460/
+
+  @param path full path of the file/directory to be checked
+
+  @return output returns 1 if path is a directory, 0 if it is not
+
+  @version 9.2
+**/
+
+%macro mf_isdir(path
+)/*/STORE SOURCE*/;
+	%local rc did is_directory fref_t;
+
+	%let is_directory = 0;
+	%let rc = %sysfunc(filename(fref_t, %superq(path)));
+	%let did = %sysfunc(dopen(&fref_t.));
+	%if &did. ^= 0 %then %do;
+	   %let is_directory = 1;
+	   %let rc = %sysfunc(dclose(&did.));
+	%end;
+	%let rc = %sysfunc(filename(fref_t));
+
+	&is_directory
+
+%mend;

base/mp_csv2ds.sas

@@ -20,30 +20,58 @@
 
 
   @param inref= fileref to the CSV
-  @param outds= output ds.  Could also be a view (eg `outds=myds/view=myds`)
+  @param outds= output ds (lib.ds format)
+  @param view= Set to YES or NO to determine whether the output should be
+    a view or not.  Default is NO (not a view).
   @param baseds= Template dataset on which to create the input statement.
     Is used to determine types, lengths, and any informats.
 
   @version 9.2
   @author Allan Bowe
+
+  <h4> Dependencies </h4>
+  @li mp_abort.sas
+  @li mf_existds.sas
+
 **/
 
-%macro mp_csv2ds(inref=0,outds=0,baseds=0);
-%if &inref=0 %then %do;
-  %put %str(ERR)OR: the INREF variable must be provided;
-  %let syscc=4;
-  %abort;
-%end;
-%if &outds=0 %then %do;
-  %put %str(ERR)OR: the OUTDS variable must be provided;
-  %let syscc=4;
-  %return;
-%end;
-%if &baseds=0 %then %do;
-  %put %str(ERR)OR: the BASEDS variable must be provided;
-  %let syscc=4;
-  %return;
-%end;
+%macro mp_csv2ds(inref=0,outds=0,baseds=0,view=NO);
+
+%mp_abort(iftrue=( &inref=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the INREF variable must be provided)
+)
+%mp_abort(iftrue=( %superq(outds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the OUTDS variable must be provided)
+)
+%mp_abort(iftrue=( &baseds=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS variable must be provided)
+)
+%mp_abort(iftrue=( &baseds=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS variable must be provided)
+)
+%mp_abort(iftrue=( %mf_existds(&baseds)=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(the BASEDS dataset (&baseds) needs to be assigned, and to exist)
+)
+
+/* count rows */
+%local hasheader; %let hasheader=0;
+data _null_;
+  if _N_ > 1 then do;
+     call symputx('hasheader',1,'l');
+     stop;
+  end;
+  infile &inref;
+  input;
+run;
+%mp_abort(iftrue=( &hasheader=0 )
+  ,mac=&sysmacroname
+  ,msg=%str(No header row in &inref)
+)
 
 /* get the variables in the CSV */
 data _data_;
@@ -92,6 +120,7 @@ data _null_;
     if type=1 then informat='best.';
     else informat=cats('$',length,'.');
   end;
+  else informat=cats(informat,'.');
   in=catx(' ',in,name,':',informat);
   if last then do;
     call symputx('instat',in,'l');
@@ -100,10 +129,16 @@ data _null_;
 run;
 
 /* import the CSV */
-data &outds;
+data &outds
+  %if %upcase(&view)=YES %then %do;
+   /view=&outds
+  %end;
+  ;
   infile &inref dsd firstobs=2;
   input &instat;
-  drop &dropvars;
+  %if %length(&dropvars)>0 %then %do;
+    drop &dropvars;
+  %end;
 run;
 
 %mend;

base/mp_getdbml.sas

@@ -0,0 +1,324 @@
+/**
+  @file
+  @brief Extract DBML from SAS Libraries
+  @details DBML is an open source markup format to represent databases.
+  More details: https://www.dbml.org/home/
+
+  Usage:
+
+
+      %mp_getdbml(liblist=SASHELP WORK,outref=mydbml,showlog=YES)
+
+  Take the log output and paste it into the renderer at https://dbdiagram.io
+  to view your data model diagram.  The code takes a "best guess" at
+  the one to one and one to many relationships (based on constraints
+  and indexes, and assuming that the column names would match).
+
+  You may need to adjust the rendered DBML to suit your needs.
+
+
+  <h4> SAS Macros </h4>
+  @li mf_getquotedstr.sas
+  @li mp_getconstraints.sas
+
+  @param liblist= Space seperated list of librefs to take as
+    input (Default=SASHELP)
+  @param outref= Fileref to contain the DBML (Default=getdbml)
+  @param showlog= set to YES to show the DBML in the log (Default is NO)
+
+  @version 9.3
+  @author Allan Bowe
+**/
+
+%macro mp_getdbml(liblist=SASHELP,outref=getdbml,showlog=NO
+)/*/STORE SOURCE*/;
+
+/* check fileref is assigned */
+%if %sysfunc(fileref(&outref)) > 0 %then %do;
+  filename &outref temp;
+%end;
+
+%let liblist=%upcase(&liblist);
+
+proc sql noprint;
+create table _data_ as
+  select * from dictionary.tables
+  where upcase(libname) in (%mf_getquotedstr(&liblist))
+  order by libname,memname;
+%local tabinfo; %let tabinfo=&syslast;
+
+create table _data_ as
+  select * from dictionary.columns
+  where upcase(libname) in (%mf_getquotedstr(&liblist))
+  order by libname,memname,varnum;
+%local colinfo; %let colinfo=&syslast;
+
+%local dsnlist;
+  select distinct upcase(cats(libname,'.',memname)) into: dsnlist
+  separated by ' '
+  from &syslast
+;
+
+create table _data_ as
+  select * from dictionary.indexes
+  where upcase(libname) in (%mf_getquotedstr(&liblist))
+  order by idxusage, indxname, indxpos;
+%local idxinfo; %let idxinfo=&syslast;
+
+/* Extract all Primary Key and Unique data constraints */
+%mp_getconstraints(lib=%scan(&liblist,1),outds=_data_)
+%local colconst; %let colconst=&syslast;
+
+%do x=2 %to %sysfunc(countw(&liblist));
+  %mp_getconstraints(lib=%scan(&liblist,&x),outds=_data_)
+  proc append base=&colconst data=&syslast;
+  run;
+%end;
+
+
+
+
+/* header info */
+data _null_;
+  file &outref;
+  put "// DBML generated by &sysuserid on %sysfunc(datetime(),datetime19.) ";
+  put "Project sasdbml {";
+  put "  database_type: 'SAS'";
+  put "  Note: 'Generated by the mp_getdbml() macro'";
+  put "}";
+run;
+
+/* create table groups */
+data _null_;
+  file &outref mod;
+  set &tabinfo;
+  by libname;
+  if first.libname then put "TableGroup " libname "{";
+  ds=quote(cats(libname,'.',memname));
+  put '   ' ds;
+  if last.libname then put "}";
+run;
+
+/* table for pks */
+data _data_;
+  length curds const col $39;
+  call missing (of _all_);
+  stop;
+run;
+%let pkds=&syslast;
+
+%local x curds constraints_used constcheck;
+%do x=1 %to %sysfunc(countw(&dsnlist,%str( )));
+  %let curds=%scan(&dsnlist,&x,%str( ));
+  %let constraints_used=;
+  %let constcheck=0;
+  data _null_;
+    file &outref mod;
+    length lab $1024 typ $20;
+    set &colinfo (where=(
+        libname="%scan(&curds,1,.)" and upcase(memname)="%scan(&curds,2,.)"
+    )) end=last;
+
+    if _n_=1 then do;
+      table='Table "'!!"&curds"!!'"{';
+      put table;
+    end;
+    name=upcase(name);
+    lab=" note:"!!quote(trim(tranwrd(label,'"',"'")));
+    if upcase(format)=:'DATETIME' then typ='datetime';
+    else if type='char' then typ=cats('char(',length,')');
+    else typ='num';
+
+    if notnull='yes' then notnul=' not null';
+    if notnull='no' and missing(label) then put '  ' name typ;
+    else if notnull='yes' and missing(label) then put '  ' name typ '[' notnul ']';
+    else if notnull='no' then put '  ' name typ '[' lab ']';
+    else put '  ' name typ '[' notnul ',' lab ']';
+
+  run;
+
+  data _data_(keep=curds const col);
+    length ctype $11 cols constraints_used $5000;
+    set &colconst (where=(
+      upcase(libref)="%scan(&curds,1,.)"
+      and upcase(table_name)="%scan(&curds,2,.)"
+      and constraint_type in ('PRIMARY','UNIQUE')
+    )) end=last;
+    file &outref mod;
+    by constraint_type constraint_name;
+    retain cols;
+    column_name=upcase(column_name);
+
+    if _n_=1 then put / '  indexes {';
+
+    if upcase(strip(constraint_type)) = 'PRIMARY' then ctype='[pk]';
+    else ctype='[unique]';
+
+    if first.constraint_name then cols = cats('(',column_name);
+    else cols=cats(cols,',',column_name);
+
+    if last.constraint_name then do;
+      cols=cats(cols,')',ctype)!!' //'!!constraint_name;
+      put '    ' cols;
+      constraints_used=catx(' ',constraints_used, constraint_name);
+      call symputx('constcheck',1);
+    end;
+
+    if last then call symputx('constraints_used',cats(upcase(constraints_used)));
+
+    length curds const col $39;
+    curds="&curds";
+    const=constraint_name;
+    col=column_name;
+  run;
+
+  proc append base=&pkds data=&syslast;run;
+
+  /* Create Unique Indexes, but only if they were not already defined within the Constraints section. */
+  data _data_(keep=curds const col);
+    set &idxinfo (where=(
+      libname="%scan(&curds,1,.)"
+      and upcase(memname)="%scan(&curds,2,.)"
+      and unique='yes'
+      and upcase(indxname) not in (%mf_getquotedstr(&constraints_used))
+    ));
+    file &outref mod;
+    by idxusage indxname;
+    name=upcase(name);
+    if &constcheck=1 then stop; /* in fact we only care about PKs so stop if we have */
+    if _n_=1 and &constcheck=0 then put / '  indexes {';
+
+    length cols $5000;
+    retain cols;
+    if first.indxname then cols = cats('(',name);
+    else cols=cats(cols,',',name);
+
+    if last.indxname then do;
+      cols=cats(cols,')[unique]')!!' //'!!indxname;
+      put '    ' cols;
+      call symputx('constcheck',1);
+    end;
+
+    length curds const col $39;
+    curds="&curds";
+    const=indxname;
+    col=name;
+  run;
+  proc append base=&pkds data=&syslast;run;
+
+  data _null_;
+    file &outref mod;
+    if &constcheck =1 then put '  }';
+    put '}';
+  run;
+
+%end;
+
+/**
+ * now we need to figure out the relationships
+ */
+
+/* sort alphabetically so we can have one set of unique cols per table */
+proc sort data=&pkds nodupkey;
+  by curds const col;
+run;
+
+data &pkds.1 (keep=curds col)
+     &pkds.2 (keep=curds cols);
+  set &pkds;
+  by curds const;
+  length retconst $39 cols $5000;
+  retain retconst cols;
+  if first.curds then do;
+    retconst=const;
+    cols=upcase(col);
+  end;
+  else cols=catx(' ',cols,upcase(col));
+  if retconst=const then do;
+    output &pkds.1;
+    if last.const then output &pkds.2;
+  end;
+run;
+
+%let curdslist="0";
+%do x=1 %to %sysfunc(countw(&dsnlist,%str( )));
+  %let curds=%scan(&dsnlist,&x,%str( ));
+
+  %let pkcols=0;
+  data _null_;
+    set &pkds.2(where=(curds="&curds"));
+    call symputx('pkcols',cols);
+  run;
+  %if &pkcols ne 0 %then %do;
+    %let curdslist=&curdslist,"&curds";
+
+    /* start with one2one */
+    data &pkds.4;
+      file &outref mod;
+      set &pkds.2(where=(cols="&pkcols" and curds not in (&curdslist)));
+      line='Ref: "'!!"&curds"
+        !!cats('".(',"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))",')')
+        !!' - '
+        !!cats(quote(trim(curds)),'.(',"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))",')');
+      put line;
+    run;
+
+    /* now many2one */
+    /* get table with one row per col */
+    data &pkds.5;
+      set &pkds.1(where=(curds="&curds"));
+    run;
+    /* get tables which contain the PK columns */
+    proc sql;
+    create table &pkds.5a as
+      select upcase(cats(b.libname,'.',b.memname)) as curds
+        ,b.name
+      from &pkds.5 a
+      inner join &colinfo b
+      on a.col=upcase(b.name);
+    /* count to make sure those tables contain ALL the columns */
+    create table &pkds.5b as
+      select curds,count(*) as cnt
+      from &pkds.5a
+      where curds not in (select curds from &pkds.2 where cols="&pkcols") /* not a one to one match */
+        and curds ne "&curds" /* exclude self */
+      group by 1;
+    create table &pkds.6 as
+      select a.*
+        ,b.cols
+      from &pkds.5b a
+      left join &pkds.4 b
+      on a.curds=b.curds;
+
+    data _null_;
+      set &pkds.6;
+      file &outref mod;
+      colcnt=%sysfunc(countw(&pkcols));
+      if cnt=colcnt then do;
+        /* table contains all the PK cols, and was not a direct / 121 match */
+        line='Ref: "'!!"&curds"
+          !!'".('
+          !!"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))"
+          !!') > '
+          !!cats(quote(trim(curds))
+              ,'.('
+              ,"%mf_getquotedstr(&pkcols,dlm=%str(,),quote=%str( ))"
+              ,')'
+          );
+        put line;
+      end;
+    run;
+  %end;
+%end;
+
+
+%if %upcase(&showlog)=YES %then %do;
+  options ps=max;
+  data _null_;
+    infile &outref;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mend;

base/mp_streamfile.sas

@@ -1,5 +1,5 @@
 /**
-  @file mp_streamfile.sas
+  @file
   @brief Streams a file to _webout according to content type
   @details Will set headers using appropriate functions (SAS 9 vs Viya) and send
   content as a binary stream.
@@ -17,6 +17,7 @@
 
   @param contenttype= Either TEXT, ZIP, CSV, EXCEL (default TEXT)
   @param inloc= /path/to/file.ext to be sent
+  @param inref= fileref of file to be sent (if provided, overrides `inloc`)
   @param outname= the name of the file, as downloaded by the browser
 
   @author Allan Bowe
@@ -27,6 +28,7 @@
 %macro mp_streamfile(
   contenttype=TEXT
   ,inloc=
+  ,inref=0
   ,outname=
 )/*/STORE SOURCE*/;
 
@@ -42,7 +44,7 @@
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.zip'
-      contenttype='application/zip' 
+      contenttype='application/zip'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
@@ -56,7 +58,7 @@
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
-      contenttype='application/vnd.ms-excel' 
+      contenttype='application/vnd.ms-excel'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
@@ -69,7 +71,7 @@
   %end;
   %else %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name='_webout.xls'
-      contenttype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' 
+      contenttype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
       contentdisp="attachment; filename=&outname";
   %end;
 %end;
@@ -102,7 +104,7 @@
 %else %if &contentype=HTML %then %do;
   %if &platform=SASVIYA %then %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
-      contenttype="text/html"; 
+      contenttype="text/html";
   %end;
 %end;
 %else %do;
@@ -110,6 +112,11 @@
   %return;
 %end;
 
-%mp_binarycopy(inloc="&inloc",outref=_webout)
+%if &inref ne 0 %then %do;
+ %mp_binarycopy(inref=&inref,outref=_webout)
+%end;
+%else %do;
+ %mp_binarycopy(inloc="&inloc",outref=_webout)
+%end;
 
 %mend;

base/mp_tree.sas

@@ -0,0 +1,68 @@
+/**
+  @file
+  @brief Recursively scans a directory tree to get all subfolders and content
+  @details
+  Usage:
+
+      %mp_tree(dir=/tmp, outds=work.tree)
+
+  Credits:
+
+  * Roger Deangelis, https://communities.sas.com/t5/SAS-Programming/listing-all-files-within-a-directory-and-subdirectories/m-p/332616/highlight/true#M74887
+  * Tom, https://communities.sas.com/t5/SAS-Programming/listing-all-files-of-all-types-from-all-subdirectories/m-p/334113/highlight/true#M75419
+
+
+  @param dir= Directory to be scanned (default=/tmp)
+  @param outds= Dataset to create (default=work.mp_tree)
+
+  @returns outds contains the following variables:
+
+   - `dir`: a flag (1/0) to say whether it is a directory or not.  This is not
+     reliable - folders that you do not have permission to open will be flagged
+     as directories.
+   - `ext`: file extension
+   - `filename`: file name
+   - `dirname`: directory name
+   - `fullpath`: directory + file name
+
+  @version 9.2
+**/
+
+%macro mp_tree(dir=/tmp
+  ,outds=work.mp_tree
+)/*/STORE SOURCE*/;
+
+data &outds ;
+  length dir 8 ext filename dirname $256 fullpath $512 ;
+  call missing(of _all_);
+  fullpath = "&dir";
+run;
+
+%local sep;
+%if &sysscp=WIN or &SYSSCP eq DNTHOST %then %let sep=\;
+%else %let sep=/;
+
+data &outds ;
+  modify &outds ;
+  retain sep "&sep";
+  rc=filename('tmp',fullpath);
+  dir_id=dopen('tmp');
+  dir = (dir_id ne 0) ;
+  if dir then dirname=fullpath;
+  else do;
+    filename=scan(fullpath,-1,sep) ;
+    dirname =substrn(fullpath,1,length(fullpath)-length(filename));
+    if index(filename,'.')>1 then ext=scan(filename,-1,'.');
+  end;
+  replace;
+  if dir then do;
+    do i=1 to dnum(dir_id);
+      fullpath=cats(dirname,sep,dread(dir_id,i));
+      output;
+    end;
+    rc=dclose(dir_id);
+  end;
+  rc=filename('tmp');
+run;
+
+%mend;

doxy/Doxyfile

@@ -24,6 +24,7 @@ LAYOUT_FILE            = ./doxy/DoxygenLayout.xml
 MAX_INITIALIZER_LINES  = 0
 PROJECT_NAME           = Macro Core
 PROJECT_LOGO           = doxy/Macro_core_website_1.png
+PROJECT_BRIEF          = "Production Ready Macros for SAS Application Developers"
 RECURSIVE              = YES
 REPEAT_BRIEF           = NO
 SHOW_NAMESPACES        = NO

doxy/DoxygenLayout.xml

@@ -101,11 +101,11 @@
   <!-- Layout definition for a directory page -->
   <directory>
     <briefdescription visible="yes"/>
+    <detaileddescription visible="yes" title=""/>
     <directorygraph visible="yes"/>
     <memberdecl>
       <dirs visible="yes"/>
       <files visible="yes"/>
     </memberdecl>
-    <detaileddescription title=""/>
   </directory>
 </doxygenlayout>

doxy/build.sh

@@ -33,11 +33,11 @@ cd core.github.io
 rm -r *
 mv $BUILD_FOLDER/out/doxy/* .
 echo 'core.sasjs.io' > CNAME
-git add *
+git add .
 git commit -m "build.sh build on $(date +%F:%H:%M:%S)"
 git push
 npx sitemap-generator-cli https://core.sasjs.io
-git add *
+git add .
 git commit -m "adding sitemap"
 git push
 

doxy/new_footer.html

@@ -1,22 +1,23 @@
+<!-- HTML footer for doxygen 1.8.17-->
+<!-- start footer part -->
 <!--BEGIN GENERATE_TREEVIEW-->
-    <li class="footer"><b>$generatedby</b>
-    <a href="http://www.doxygen.org/index.html">
-    <img class="footer" src="doxygen.png" alt="doxygen"/></a>
+<div id="nav-path" class="navpath"><!-- id is needed for treeview function! -->
+  <ul>
+    $navpath
+    <li class="footer">$generatedby
+    <a href="https://www.doxygen.org/index.html">
+      <img class="footer" src="$relpath^doxygen.png" alt="doxygen"/></a> $doxygenversion </li>
    <i> For more information visit the </i> <a href="https://github.com/sasjs/core">Macro Core library</a>.</li>
 
    </ul>
  </div>
 <!--END GENERATE_TREEVIEW-->
 <!--BEGIN !GENERATE_TREEVIEW-->
-<hr class="footer"/>
-<table width="100%"><tbody><tr><td>
-For more information visit the <a href="https://github.com/sasjs/core">Macro Core library</a>.
-</td><td><address class="footer"><small>
-&copy;$year<br/>
-$generatedby <a href="http://www.doxygen.org/index.html">
-<!--<img class="footer" src="$relpath$doxygen.png" alt="doxygen"/>-->doxygen
-</a> $doxygenversion
-</small></address></tr></tbody></table>
+<hr class="footer"/><address class="footer"><small>
+  $generatedby &#160;<a href="http://www.doxygen.org/index.html">
+  <img class="footer" src="$relpath^doxygen.png" alt="doxygen"/>
+  </a> $doxygenversion
+  </small></address>
 <!--END !GENERATE_TREEVIEW-->
 </body>
 </html>

doxy/new_header.html

@@ -1,6 +1,6 @@
-<!-- HTML header for doxygen 1.8.14-->
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml">
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "https://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<!-- HTML header for doxygen 1.8.17-->
+<html xmlns="https://www.w3.org/1999/xhtml" lang="en">
 <head>
 <meta http-equiv="Content-Type" content="text/xhtml;charset=UTF-8"/>
 <meta http-equiv="X-UA-Compatible" content="IE=9"/>
@@ -16,51 +16,51 @@ $search
 $mathjax
 <link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
 <link REL="icon" HREF="https://sasjs.io/img/runningman.jpg">
-
 $extrastylesheet
 </head>
 <body>
 <div id="top"><!-- do not remove this div, it is closed by doxygen! -->
 
 <!--BEGIN TITLEAREA-->
-<div id="titlearea" style='background-color:white' >
+<div id="titlearea">
 <table cellspacing="0" cellpadding="0">
  <tbody>
- <tr style="height: 26px;">
+  <tr style="height: 56px;">
   <!--BEGIN PROJECT_LOGO-->
-  <td id="projectlogo"></td>
-
+  <td id="projectlogo">
+    <img alt="Logo" src="$relpath^$projectlogo"/></td>
   <!--END PROJECT_LOGO-->
 <!--BEGIN PROJECT_NAME-->
-  <td>
+<td id="projectalign" style="padding-left: 0.5em;">
   <div id="projectname">
-    <!--a href=".">
-      <img alt="Macro Core" src="https://macropeople.com/wp-content/uploads/2018/05/macropeople2014retina_V2.png" height=60/>
-     </a-->
-     <a href=".">
-        <img alt="Macro Core" src="./Macro_core_website_1.png" height=60/>
-       </a>
+   <!--BEGIN PROJECT_NUMBER-->&#160;<span id="projectnumber">$projectnumber</span><!--END PROJECT_NUMBER-->
    </div>
+   <!--BEGIN PROJECT_BRIEF--><div id="projectbrief">
+    Production Ready Macros for SAS Application Developers</br>
+    <a href="https://github.com/sasjs/core">
+        https://github.com/sasjs/core
+    </a>
+
+   </div>
+   <meta name="Description" content="$projectbrief">
+   <!--END PROJECT_BRIEF-->
   </td>
-  <td>
+  <!--END PROJECT_NAME-->
+  <!--BEGIN !PROJECT_NAME-->
+   <!--BEGIN PROJECT_BRIEF-->
+   <td style="padding-left: 0.5em;">
+    <div id="projectbrief">$projectbrief</div>
     <table style="padding-left: 2em;" cellspacing="0" cellpadding="0">
       <tr><td> Production Ready Macros for SAS Application Developers</td></tr>
       <tr><td><a href="https://github.com/sasjs/core">
           https://github.com/sasjs/core
       </a></td></tr>
     </table>
-  </td>
-  <!--END PROJECT_NAME-->
-  <!--BEGIN !PROJECT_NAME-->
-   <!--BEGIN PROJECT_BRIEF-->
+    </td>
    <!--END PROJECT_BRIEF-->
   <!--END !PROJECT_NAME-->
-  <div class="header">
-  <div class="headertitle">
-
   <!--BEGIN DISABLE_INDEX-->
    <!--BEGIN SEARCHENGINE-->
-
    <td>$searchbox</td>
    <!--END SEARCHENGINE-->
   <!--END DISABLE_INDEX-->

meta/mm_createlibrary.sas

@@ -9,15 +9,15 @@
 
   Usage:
 
-    %mm_createlibrary(
-       libname=My New Library
-      ,libref=mynewlib
-      ,libdesc=Super & <fine>
-      ,engine=BASE
-      ,tree=/User Folders/sasdemo
-      ,servercontext=SASApp
-      ,directory=/tmp/tests
-      ,mDebug=1)
+      %mm_createlibrary(
+        libname=My New Library
+        ,libref=mynewlib
+        ,libdesc=Super & <fine>
+        ,engine=BASE
+        ,tree=/User Folders/sasdemo
+        ,servercontext=SASApp
+        ,directory=/tmp/tests
+        ,mDebug=1)
 
   <h4> Dependencies </h4>
   @li mf_verifymacvars.sas

meta/mm_getstps.sas

@@ -23,8 +23,8 @@
     combine with the <code>tree=</code> parameter.
   @param outds= the dataset to create that contains the list of stps.
   @param mDebug= set to 1 to show debug messages in the log
-  @showDesc= provide a non blank value to return stored process descriptions
-  @showUsageVersion= provide a non blank value to return the UsageVersion.  This
+  @param showDesc= provide a non blank value to return stored process descriptions
+  @param showUsageVersion= provide a non blank value to return the UsageVersion.  This
     is either 1000000 (type 1, 9.2) or 2000000 (type2, 9.3 onwards).
 
   @returns outds  dataset containing the following columns

meta/mm_getwebappsrvprops.sas

@@ -1,7 +1,7 @@
 /**
   @file
   @brief Retrieves properties of the SAS web app server
-  @details 
+  @details
   Usage:
 
       %mm_getwebappsrvprops(outds= some_ds)
@@ -21,8 +21,7 @@
       libname __shake clear;
 
   @version 9.4
-  @author Allan Bowe
-  @source https://github.com/sasjs/core
+  @author Allan Bowe https://github.com/sasjs/core
 
 **/
 

viya/mv_getfoldermembers.sas

@@ -42,7 +42,7 @@
     %let &access_token_var=;
 %end;
 %put &sysmacroname: grant_type=&grant_type;
-%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password 
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
     and &grant_type ne sas_services
   )
   ,mac=&sysmacroname
@@ -85,10 +85,16 @@ options noquotelenmax;
   /*data _null_;infile &fname1;input;putlog _infile_;run;*/
   libname &libref1 JSON fileref=&fname1;
   /* now get the followon link to list members */
+  %local href;
+  %let href=0;
   data _null_;
     set &libref1..links;
     if rel='members' then call symputx('href',quote("&base_uri"!!trim(href)),'l');
   run;
+  %if &href=0 %then %do;
+    %put NOTE:;%put NOTE-  No members found in &root!!;%put NOTE-;
+    %return;
+  %end;
   %local fname2 libref2;
   %let fname2=%mf_getuniquefileref();
   %let libref2=%mf_getuniquelibref();

viya/mv_jobexecute.sas

@@ -0,0 +1,168 @@
+/**
+  @file
+  @brief Executes a SAS Viya Job
+  @details Triggers a SAS Viya Job, with optional URL parameters, using
+  the JES web app.
+
+  First, compile the macros:
+
+      filename mc url
+      "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+  Then, execute the job!
+
+      %mv_jobexecute(path=/Public/folder
+        ,name=somejob
+      )
+
+  Example with parameters:
+
+     %mv_jobexecute(path=/Public/folder
+        ,name=somejob
+        ,paramstring=%str("macvarname":"macvarvalue","answer":42)
+      )
+
+  @param access_token_var= The global macro variable to contain the access token
+  @param grant_type= valid values:
+   * password
+   * authorization_code
+   * detect - will check if access_token exists, if not will use sas_services if
+    a SASStudioV session else authorization_code.  Default option.
+   * sas_services - will use oauth_bearer=sas_services
+
+  @param path= The SAS Drive path to the job being executed
+  @param name= The name of the job to execute
+  @param paramstring= A JSON fragment with name:value pairs, eg: `"name":"value"`
+  or "name":"value","name2":42`.  This will need to be wrapped in `%str()`.
+
+  @param contextName= Context name with which to run the job.
+    Default = `SAS Job Execution compute context`
+
+  @param outds= The output dataset containing links (Default=work.mv_jobexecute)
+
+
+  @version VIYA V.03.04
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> Dependencies </h4>
+  @li mp_abort.sas
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mv_getfoldermembers.sas
+
+**/
+
+%macro mv_jobexecute(path=0
+    ,name=0
+    ,contextName=SAS Job Execution compute context
+    ,access_token_var=ACCESS_TOKEN
+    ,grant_type=sas_services
+    ,paramstring=0
+    ,outds=work.mv_jobexecute
+  );
+%local oauth_bearer;
+%if &grant_type=detect %then %do;
+  %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
+  %else %let grant_type=sas_services;
+%end;
+%if &grant_type=sas_services %then %do;
+    %let oauth_bearer=oauth_bearer=sas_services;
+    %let &access_token_var=;
+%end;
+%put &sysmacroname: grant_type=&grant_type;
+%mp_abort(iftrue=(&grant_type ne authorization_code and &grant_type ne password
+    and &grant_type ne sas_services
+  )
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid value for grant_type: &grant_type)
+)
+
+%mp_abort(iftrue=("&path"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(Path not provided)
+)
+%mp_abort(iftrue=("&name"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(Job Name not provided)
+)
+
+options noquotelenmax;
+
+%local base_uri; /* location of rest apis */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+data;run;
+%local foldermembers;
+%let foldermembers=&syslast;
+%mv_getfoldermembers(root=&path
+    ,access_token_var=&access_token_var
+    ,grant_type=&grant_type
+    ,outds=&foldermembers
+)
+
+%local joburi;
+%let joburi=0;
+data _null_;
+  set &foldermembers;
+  if name="&name" and uri=:'/jobDefinitions/definitions'
+    then call symputx('joburi',uri);
+run;
+
+%mp_abort(iftrue=("&joburi"="0")
+  ,mac=&sysmacroname
+  ,msg=%str(Job &path/&name not found)
+)
+
+/* prepare request*/
+%local fname0 fname1;
+%let fname0=%mf_getuniquefileref();
+%let fname1=%mf_getuniquefileref();
+
+data _null_;
+  file &fname0;
+  length joburi contextname $128 paramstring $32765;
+  joburi=quote(trim(symget('joburi')));
+  contextname=quote(trim(symget('contextname')));
+  paramstring=symget('paramstring');
+  put '{"jobDefinitionUri":' joburi ;
+  put '  ,"arguments":{"_contextName":' contextname;
+  if paramstring ne "0" then do;
+    put '    ,' paramstring;
+  end;
+  put '}}';
+run;
+
+proc http method='POST' in=&fname0 out=&fname1 &oauth_bearer
+  url="&base_uri/jobExecution/jobs";
+  headers "Content-Type"="application/vnd.sas.job.execution.job.request+json"
+          "Accept"="application/vnd.sas.job.execution.job+json"
+  %if &grant_type=authorization_code %then %do;
+          "Authorization"="Bearer &&&access_token_var"
+  %end;
+  ;
+run;
+%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
+%do;
+  data _null_;infile &fname0;input;putlog _infile_;run;
+  data _null_;infile &fname1;input;putlog _infile_;run;
+  %mp_abort(mac=&sysmacroname
+    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+  )
+%end;
+
+%local libref;
+%let libref=%mf_getuniquelibref();
+libname &libref JSON fileref=&fname1;
+
+data &outds;
+  set &libref..links;
+run;
+
+/* clear refs */
+filename &fname0 clear;
+filename &fname1 clear;
+libname &libref;
+
+%mend;