Merge pull request #57 from sasjs/mf_existfref

fix: showing filerefs that exist (even when underlying does not) in m…

all.sas

@@ -133,7 +133,13 @@ options noquotelenmax;
 %macro mf_existfileref(fref
 )/*/STORE SOURCE*/;
 
-  %if %sysfunc(fileref(&fref))=0 %then %do;
+  %local rc;
+  %let rc=%sysfunc(fileref(&fref));
+  %if &rc=0 %then %do;
+    1
+  %end;
+  %else %if &rc<0 %then %do;
+    %put &sysmacroname: Fileref &fref exists but the underlying file does not;
     1
   %end;
   %else %do;
@@ -3438,15 +3444,16 @@ run;
       %inc myref;
 
   @param [in] ds The dataset to be exported
+  @param [in] maxobs= (max) The max number of inserts to create
   @param [out] outref= (0) The output fileref.  If it does not exist, it is
     created. If it does exist, new records are APPENDED.
-  @param [out] outlib= (0) The library (or schema) in which the target table is
+  @param [out] schema= (0) The library (or schema) in which the target table is
     located.  If not provided, is ignored.
   @param [out] outds= (0) The output table to load.  If not provided, will
     default to the table in the &ds parameter.
-  @param [in] flavour= (BASE) The SQL flavour to be applied to the output. Valid
+  @param [in] flavour= (SAS) The SQL flavour to be applied to the output. Valid
     options:
-    @li BASE (default) - suitable for regular proc sql
+    @li SAS (default) - suitable for regular proc sql
     @li PGSQL - Used for Postgres databases
 
   <h4> SAS Macros </h4>
@@ -3459,7 +3466,7 @@ run;
   @author Allan Bowe (credit mjsq)
 **/
 
-%macro mp_ds2inserts(ds, outref=0,outlib=0,outds=0,flavour=BASE
+%macro mp_ds2inserts(ds, outref=0,schema=0,outds=0,flavour=SAS,maxobs=max
 )/*/STORE SOURCE*/;
 
 %if not %sysfunc(exist(&ds)) %then %do;
@@ -3475,7 +3482,7 @@ run;
 %if %index(&ds,.)=0 %then %let ds=WORK.&ds;
 
 %let flavour=%upcase(&flavour);
-%if &flavour ne BASE and &flavour ne PGSQL %then %do;
+%if &flavour ne SAS and &flavour ne PGSQL %then %do;
   %put %str(WAR)NING:  &flavour is not supported;
   %return;
 %end;
@@ -3488,8 +3495,8 @@ run;
   filename &outref temp lrecl=66000;
 %end;
 
-%if &outlib=0 %then %let outlib=;
-%else %let outlib=&outlib..;
+%if &schema=0 %then %let schema=;
+%else %let schema=&schema..;
 
 %if &outds=0 %then %let outds=%scan(&ds,2,.);
 
@@ -3508,8 +3515,18 @@ select count(*) into: nobs TRIMMED from &ds;
 %if &vars=0 %then %do;
   data _null_;
     file &outref mod;
-    put "/* No columns found in &ds */";
+    put "/* No columns found in &schema.&ds */";
   run;
+  %return;
+%end;
+%else %if &vars>1600 and &flavour=PGSQL %then %do;
+  data _null_;
+    file &fref mod;
+    put "/* &schema.&ds contains &vars vars */";
+    put "/* Postgres cannot handle tables with over 1600 vars */";
+    put "/* No inserts will be generated for this table */";
+  run;
+  %return;
 %end;
 
 %local varlist varlistcomma;
@@ -3519,8 +3536,11 @@ select count(*) into: nobs TRIMMED from &ds;
 /* next, export data */
 data _null_;
   file &outref mod ;
-  if _n_=1 then put "/* &outlib.&outds (&nobs rows, &vars columns) */";
+  if _n_=1 then put "/* &schema.&outds (&nobs rows, &vars columns) */";
   set &ds;
+  %if &maxobs ne max %then %do;
+    if _n_>&maxobs then stop;
+  %end;
   length _____str $32767;
   format _numeric_ best.;
   format _character_ ;
@@ -3529,13 +3549,13 @@ data _null_;
     %let var=%scan(&varlist,&i);
     %let vtype=%mf_getvartype(&ds,&var);
     %if &i=1 %then %do;
-      %if &flavour=BASE %then %do;
-        put "insert into &outlib.&outds set ";
+      %if &flavour=SAS %then %do;
+        put "insert into &schema.&outds set ";
         put "  &var="@;
       %end;
       %else %if &flavour=PGSQL %then %do;
         _____str=cats(
-          "INSERT INTO &outlib.&outds ("
+          "INSERT INTO &schema.&outds ("
           ,symget('varlistcomma')
           ,") VALUES ("
         );
@@ -3544,7 +3564,7 @@ data _null_;
       %end;
     %end;
     %else %do;
-      %if &flavour=BASE %then %do;
+      %if &flavour=SAS %then %do;
         put "  ,&var="@;
       %end;
       %else %if &flavour=PGSQL %then %do;
@@ -3552,7 +3572,7 @@ data _null_;
       %end;
     %end;
     %if &vtype=N %then %do;
-      %if &flavour=BASE %then %do;
+      %if &flavour=SAS %then %do;
         put &var;
       %end;
       %else %if &flavour=PGSQL %then %do;
@@ -3565,7 +3585,7 @@ data _null_;
       put _____str;
     %end;
   %end;
-  %if &flavour=BASE %then %do;
+  %if &flavour=SAS %then %do;
     put ';';
   %end;
   %else %if &flavour=PGSQL %then %do;
@@ -4386,6 +4406,8 @@ run;
     to create tables in SAS or a database.  The macro can be used at table or
     library level.  The default behaviour is to create DDL in SAS format.
 
+    Note - views are not currently supported.
+
   Usage:
 
       data test(index=(pk=(x y)/unique /nomiss));
@@ -4398,6 +4420,7 @@ run;
 
   <h4> SAS Macros </h4>
   @li mf_existfileref.sas
+  @li mf_getvarcount.sas
   @li mp_getconstraints.sas
 
   @param lib libref of the library to create DDL for.  Should be assigned.
@@ -4419,8 +4442,8 @@ run;
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %mf_existfileref(&outref)=0 %then %do;
-  filename &outref temp ;
+%if %mf_existfileref(&fref)=0 %then %do;
+  filename &fref temp ;
 %end;
 
 %if %length(&libref)=0 %then %let libref=WORK;
@@ -4430,6 +4453,7 @@ proc sql noprint;
 create table _data_ as
   select * from dictionary.tables
   where upcase(libname)="%upcase(&libref)"
+    and memtype='DATA' /* views not currently supported */
   %if %length(&ds)>0 %then %do;
     and upcase(memname)="%upcase(&ds)"
   %end;
@@ -4524,13 +4548,15 @@ run;
           put "create table &libref..&curds(";
         end;
         else do;
+          /* just a placeholder - we filter out views at the top */
           put "create view &libref..&curds(";
         end;
         put "    "@@;
       end;
       else put "   ,"@@;
       if length(format)>1 then fmt=" format="!!cats(format);
-      if length(label)>1 then lab=" label="!!quote(trim(label));
+      if length(label)>1 then
+        lab=" label="!!cats("'",tranwrd(label,"'","''"),"'");
       if notnull='yes' then notnul=' not null';
       if type='char' then typ=cats('char(',length,')');
       else if length ne 8 then typ='num length='!!left(length);
@@ -4602,6 +4628,7 @@ run;
           put "create table [&schema].[&curds](";
         end;
         else do;
+          /* just a placeholder - we filter out views at the top */
           put "create view [&schema].[&curds](";
         end;
         put "    "@@;
@@ -4685,71 +4712,81 @@ run;
     put "CREATE SCHEMA &schema;";
   %do x=1 %to %sysfunc(countw(&dsnlist));
     %let curds=%scan(&dsnlist,&x);
-    data _null_;
-      file &fref mod;
-      put "/* Postgres Flavour DDL for &schema..&curds */";
-    data _null_;
-      file &fref mod;
-      set &colinfo (where=(upcase(memname)="&curds")) end=last;
-      length fmt $32;
-      if _n_=1 then do;
-        if memtype='DATA' then do;
-          put "CREATE TABLE &schema..&curds (";
+    %local curdsvarcount;
+    %let curdsvarcount=%mf_getvarcount(&libref..&curds);
+    %if &curdsvarcount>1600 %then %do;
+      data _null_;
+        file &fref mod;
+        put "/* &libref..&curds contains &curdsvarcount vars */";
+        put "/* Postgres cannot create tables with over 1600 vars */";
+        put "/* No DDL will be generated for this table";
+      run;
+    %end;
+    %else %do;
+      data _null_;
+        file &fref mod;
+        put "/* Postgres Flavour DDL for &schema..&curds */";
+      data _null_;
+        file &fref mod;
+        set &colinfo (where=(upcase(memname)="&curds")) end=last;
+        length fmt $32;
+        if _n_=1 then do;
+          if memtype='DATA' then do;
+            put "CREATE TABLE &schema..&curds (";
+          end;
+          else do;
+            /* just a placeholder - we filter out views at the top */
+            put "CREATE VIEW &schema..&curds (";
+          end;
+          put "    "@@;
         end;
-        else do;
-          put "CREATE VIEW &schema..&curds (";
-        end;
-        put "    "@@;
-      end;
-      else put "   ,"@@;
-      format=upcase(format);
-      if 1=0 then; /* dummy if */
-      %if &applydttm=YES %then %do;
-        else if format=:'DATETIME' then fmt=' TIMESTAMP ';
-      %end;
-      else if type='num' then fmt=' DOUBLE PRECISION';
-      else fmt='VARCHAR('!!cats(length)!!')';
-      if notnull='yes' then notnul=' NOT NULL';
-      /* quote column names in case they represent reserved words */
-      name2=quote(trim(name));
-      put name2 fmt notnul;
-    run;
+        else put "   ,"@@;
+        format=upcase(format);
+        if 1=0 then; /* dummy if */
+        %if &applydttm=YES %then %do;
+          else if format=:'DATETIME' then fmt=' TIMESTAMP ';
+        %end;
+        else if type='num' then fmt=' DOUBLE PRECISION';
+        else fmt='VARCHAR('!!cats(length)!!')';
+        if notnull='yes' then notnul=' NOT NULL';
+        /* quote column names in case they represent reserved words */
+        name2=quote(trim(name));
+        put name2 fmt notnul;
+      run;
 
-    /* Extra step for data constraints */
-    %addConst()
+      /* Extra step for data constraints */
+      %addConst()
 
-    data _null_;
-      file &fref mod;
-      put ');';
-    run;
+      data _null_;
+        file &fref mod;
+        put ');';
+      run;
 
-    /* Create Unique Indexes, but only if they were not already defined within
-      the Constraints section. */
-    data _null_;
-      *length ds $128;
-      set &idxinfo(
-        where=(
-          memname="&curds"
-          and unique='yes'
-          and indxname not in (
-            %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+      /* Create Unique Indexes, but only if they were not already defined within
+        the Constraints section. */
+      data _null_;
+        *length ds $128;
+        set &idxinfo(
+          where=(
+            memname="&curds"
+            and unique='yes'
+            and indxname not in (
+              %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+            )
           )
-        )
-      );
-      file &fref mod;
-      by idxusage indxname;
-/*       ds=cats(libname,'.',memname); */
-      if first.indxname then do;
+        );
+        file &fref mod;
+        by idxusage indxname;
+        if first.indxname then do;
           put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds(";
           put '  "' name +(-1) '"' ;
-      end;
-      else put '  ,"' name +(-1) '"';
-      *else put '    ,' name ;
-      if last.indxname then do;
-        put ');';
-      end;
-    run;
-
+        end;
+        else put '  ,"' name +(-1) '"';
+        if last.indxname then do;
+          put ');';
+        end;
+      run;
+    %end;
   %end;
 %end;
 %if %upcase(&showlog)=YES %then %do;
@@ -5523,6 +5560,78 @@ select distinct lowcase(memname)
 %end;
 
 %mend mp_lib2cards;/**
+  @file
+  @brief Convert all data in a library to SQL insert statements
+  @details Gets list of members then calls the <code>%mp_ds2inserts()</code>
+  macro.
+  Usage:
+
+      %mp_getddl(sashelp, schema=work, fref=tempref)
+
+      %mp_lib2inserts(sashelp, schema=work, outref=tempref)
+
+      %inc tempref;
+
+
+  The output will be one file in the outref fileref.
+
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2inserts.sas
+
+
+  @param [in] lib Library in which to convert all datasets to inserts
+  @param [in] flavour= (SAS) The SQL flavour to be applied to the output. Valid
+    options:
+    @li SAS (default) - suitable for regular proc sql
+    @li PGSQL - Used for Postgres databases
+  @param [in] maxobs= (max) The max number of observations (per table) to create
+  @param [out] outref= Output fileref in which to create the insert statements.
+    If it exists, it will be appended to, otherwise it will be created.
+  @param [out] schema= (0) The schema of the target database, or the libref.
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_lib2inserts(lib
+    ,flavour=SAS
+    ,outref=0
+    ,schema=0
+    ,maxobs=max
+)/*/STORE SOURCE*/;
+
+/* Find the tables */
+%local x ds memlist;
+proc sql noprint;
+select distinct lowcase(memname)
+  into: memlist
+  separated by ' '
+  from dictionary.tables
+  where upcase(libname)="%upcase(&lib)"
+    and memtype='DATA'; /* exclude views */
+
+
+%let flavour=%upcase(&flavour);
+%if &flavour ne SAS and &flavour ne PGSQL %then %do;
+  %put %str(WAR)NING:  &flavour is not supported;
+  %return;
+%end;
+
+
+/* create the inserts */
+%do x=1 %to %sysfunc(countw(&memlist));
+  %let ds=%scan(&memlist,&x);
+  %mp_ds2inserts(&lib..&ds
+    ,outref=&outref
+    ,schema=&schema
+    ,outds=&ds
+    ,flavour=&flavour
+    ,maxobs=&maxobs
+  )
+%end;
+
+%mend mp_lib2inserts;/**
   @file
   @brief Create a Markdown Table from a dataset
   @details A markdown table is a simple table representation for use in
@@ -11562,18 +11671,20 @@ run;
 %mend mm_gettableid;/**
   @file
   @brief Creates a dataset with all metadata tables for a particular library
-  @details Will only show the tables to which a user has the requisite
-    metadata access.
+  @details Will only show the tables for which the executing user has the
+    requisite metadata access.
 
   usage:
 
-    %mm_gettables(uri=A5X8AHW1.B40001S5)
+      %mm_gettables(uri=A5X8AHW1.B40001S5)
 
-  @param outds the dataset to create that contains the list of tables
-  @param uri the uri of the library for which to return tables
-  @param getauth= YES|NO - fetch the authdomain used in database connections.
-  Set to NO to improve runtimes in larger environments, as there can be a
-  performance hit on the `metadata_getattr(domainuri, "Name", AuthDomain)` call.
+  @param [in] uri= the uri of the library for which to return tables
+  @param [out] outds= (work.mm_gettables) the dataset to contain the list of
+    tables
+  @param [in] getauth= (YES) Fetch the authdomain used in database connections.
+    Set to NO to improve runtimes in larger environments, as there can be a
+    performance hit on the `metadata_getattr(domainuri, "Name", AuthDomain)`
+    call.
 
   @returns outds  dataset containing all groups in a column named "metagroup"
     (defaults to work.mm_getlibs). The following columns are provided:
@@ -11601,8 +11712,8 @@ data &outds;
     libdesc $200 libref engine $8 IsDBMSLibname $1
     tablename $50 /* metadata table names can be longer than $32 */
     ;
-  keep libname libdesc libref engine ServerContext path_schema AuthDomain tableuri
-    tablename IsPreassigned IsDBMSLibname id;
+  keep libname libdesc libref engine ServerContext path_schema AuthDomain
+    tableuri tablename IsPreassigned IsDBMSLibname id;
   call missing (of _all_);
 
   uri=symget('uri');

base/mf_existfileref.sas

@@ -17,7 +17,13 @@
 %macro mf_existfileref(fref
 )/*/STORE SOURCE*/;
 
-  %if %sysfunc(fileref(&fref))=0 %then %do;
+  %local rc;
+  %let rc=%sysfunc(fileref(&fref));
+  %if &rc=0 %then %do;
+    1
+  %end;
+  %else %if &rc<0 %then %do;
+    %put &sysmacroname: Fileref &fref exists but the underlying file does not;
     1
   %end;
   %else %do;

base/mp_ds2inserts.sas

@@ -14,15 +14,16 @@
       %inc myref;
 
   @param [in] ds The dataset to be exported
+  @param [in] maxobs= (max) The max number of inserts to create
   @param [out] outref= (0) The output fileref.  If it does not exist, it is
     created. If it does exist, new records are APPENDED.
-  @param [out] outlib= (0) The library (or schema) in which the target table is
+  @param [out] schema= (0) The library (or schema) in which the target table is
     located.  If not provided, is ignored.
   @param [out] outds= (0) The output table to load.  If not provided, will
     default to the table in the &ds parameter.
-  @param [in] flavour= (BASE) The SQL flavour to be applied to the output. Valid
+  @param [in] flavour= (SAS) The SQL flavour to be applied to the output. Valid
     options:
-    @li BASE (default) - suitable for regular proc sql
+    @li SAS (default) - suitable for regular proc sql
     @li PGSQL - Used for Postgres databases
 
   <h4> SAS Macros </h4>
@@ -35,7 +36,7 @@
   @author Allan Bowe (credit mjsq)
 **/
 
-%macro mp_ds2inserts(ds, outref=0,outlib=0,outds=0,flavour=BASE
+%macro mp_ds2inserts(ds, outref=0,schema=0,outds=0,flavour=SAS,maxobs=max
 )/*/STORE SOURCE*/;
 
 %if not %sysfunc(exist(&ds)) %then %do;
@@ -51,7 +52,7 @@
 %if %index(&ds,.)=0 %then %let ds=WORK.&ds;
 
 %let flavour=%upcase(&flavour);
-%if &flavour ne BASE and &flavour ne PGSQL %then %do;
+%if &flavour ne SAS and &flavour ne PGSQL %then %do;
   %put %str(WAR)NING:  &flavour is not supported;
   %return;
 %end;
@@ -64,8 +65,8 @@
   filename &outref temp lrecl=66000;
 %end;
 
-%if &outlib=0 %then %let outlib=;
-%else %let outlib=&outlib..;
+%if &schema=0 %then %let schema=;
+%else %let schema=&schema..;
 
 %if &outds=0 %then %let outds=%scan(&ds,2,.);
 
@@ -84,8 +85,18 @@ select count(*) into: nobs TRIMMED from &ds;
 %if &vars=0 %then %do;
   data _null_;
     file &outref mod;
-    put "/* No columns found in &ds */";
+    put "/* No columns found in &schema.&ds */";
   run;
+  %return;
+%end;
+%else %if &vars>1600 and &flavour=PGSQL %then %do;
+  data _null_;
+    file &fref mod;
+    put "/* &schema.&ds contains &vars vars */";
+    put "/* Postgres cannot handle tables with over 1600 vars */";
+    put "/* No inserts will be generated for this table */";
+  run;
+  %return;
 %end;
 
 %local varlist varlistcomma;
@@ -95,8 +106,11 @@ select count(*) into: nobs TRIMMED from &ds;
 /* next, export data */
 data _null_;
   file &outref mod ;
-  if _n_=1 then put "/* &outlib.&outds (&nobs rows, &vars columns) */";
+  if _n_=1 then put "/* &schema.&outds (&nobs rows, &vars columns) */";
   set &ds;
+  %if &maxobs ne max %then %do;
+    if _n_>&maxobs then stop;
+  %end;
   length _____str $32767;
   format _numeric_ best.;
   format _character_ ;
@@ -105,13 +119,13 @@ data _null_;
     %let var=%scan(&varlist,&i);
     %let vtype=%mf_getvartype(&ds,&var);
     %if &i=1 %then %do;
-      %if &flavour=BASE %then %do;
-        put "insert into &outlib.&outds set ";
+      %if &flavour=SAS %then %do;
+        put "insert into &schema.&outds set ";
         put "  &var="@;
       %end;
       %else %if &flavour=PGSQL %then %do;
         _____str=cats(
-          "INSERT INTO &outlib.&outds ("
+          "INSERT INTO &schema.&outds ("
           ,symget('varlistcomma')
           ,") VALUES ("
         );
@@ -120,7 +134,7 @@ data _null_;
       %end;
     %end;
     %else %do;
-      %if &flavour=BASE %then %do;
+      %if &flavour=SAS %then %do;
         put "  ,&var="@;
       %end;
       %else %if &flavour=PGSQL %then %do;
@@ -128,7 +142,7 @@ data _null_;
       %end;
     %end;
     %if &vtype=N %then %do;
-      %if &flavour=BASE %then %do;
+      %if &flavour=SAS %then %do;
         put &var;
       %end;
       %else %if &flavour=PGSQL %then %do;
@@ -141,7 +155,7 @@ data _null_;
       put _____str;
     %end;
   %end;
-  %if &flavour=BASE %then %do;
+  %if &flavour=SAS %then %do;
     put ';';
   %end;
   %else %if &flavour=PGSQL %then %do;

base/mp_getddl.sas

@@ -5,6 +5,8 @@
     to create tables in SAS or a database.  The macro can be used at table or
     library level.  The default behaviour is to create DDL in SAS format.
 
+    Note - views are not currently supported.
+
   Usage:
 
       data test(index=(pk=(x y)/unique /nomiss));
@@ -17,6 +19,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_existfileref.sas
+  @li mf_getvarcount.sas
   @li mp_getconstraints.sas
 
   @param lib libref of the library to create DDL for.  Should be assigned.
@@ -38,8 +41,8 @@
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %mf_existfileref(&outref)=0 %then %do;
-  filename &outref temp ;
+%if %mf_existfileref(&fref)=0 %then %do;
+  filename &fref temp ;
 %end;
 
 %if %length(&libref)=0 %then %let libref=WORK;
@@ -49,6 +52,7 @@ proc sql noprint;
 create table _data_ as
   select * from dictionary.tables
   where upcase(libname)="%upcase(&libref)"
+    and memtype='DATA' /* views not currently supported */
   %if %length(&ds)>0 %then %do;
     and upcase(memname)="%upcase(&ds)"
   %end;
@@ -143,13 +147,15 @@ run;
           put "create table &libref..&curds(";
         end;
         else do;
+          /* just a placeholder - we filter out views at the top */
           put "create view &libref..&curds(";
         end;
         put "    "@@;
       end;
       else put "   ,"@@;
       if length(format)>1 then fmt=" format="!!cats(format);
-      if length(label)>1 then lab=" label="!!quote(trim(label));
+      if length(label)>1 then
+        lab=" label="!!cats("'",tranwrd(label,"'","''"),"'");
       if notnull='yes' then notnul=' not null';
       if type='char' then typ=cats('char(',length,')');
       else if length ne 8 then typ='num length='!!left(length);
@@ -221,6 +227,7 @@ run;
           put "create table [&schema].[&curds](";
         end;
         else do;
+          /* just a placeholder - we filter out views at the top */
           put "create view [&schema].[&curds](";
         end;
         put "    "@@;
@@ -304,71 +311,81 @@ run;
     put "CREATE SCHEMA &schema;";
   %do x=1 %to %sysfunc(countw(&dsnlist));
     %let curds=%scan(&dsnlist,&x);
-    data _null_;
-      file &fref mod;
-      put "/* Postgres Flavour DDL for &schema..&curds */";
-    data _null_;
-      file &fref mod;
-      set &colinfo (where=(upcase(memname)="&curds")) end=last;
-      length fmt $32;
-      if _n_=1 then do;
-        if memtype='DATA' then do;
-          put "CREATE TABLE &schema..&curds (";
+    %local curdsvarcount;
+    %let curdsvarcount=%mf_getvarcount(&libref..&curds);
+    %if &curdsvarcount>1600 %then %do;
+      data _null_;
+        file &fref mod;
+        put "/* &libref..&curds contains &curdsvarcount vars */";
+        put "/* Postgres cannot create tables with over 1600 vars */";
+        put "/* No DDL will be generated for this table";
+      run;
+    %end;
+    %else %do;
+      data _null_;
+        file &fref mod;
+        put "/* Postgres Flavour DDL for &schema..&curds */";
+      data _null_;
+        file &fref mod;
+        set &colinfo (where=(upcase(memname)="&curds")) end=last;
+        length fmt $32;
+        if _n_=1 then do;
+          if memtype='DATA' then do;
+            put "CREATE TABLE &schema..&curds (";
+          end;
+          else do;
+            /* just a placeholder - we filter out views at the top */
+            put "CREATE VIEW &schema..&curds (";
+          end;
+          put "    "@@;
         end;
-        else do;
-          put "CREATE VIEW &schema..&curds (";
-        end;
-        put "    "@@;
-      end;
-      else put "   ,"@@;
-      format=upcase(format);
-      if 1=0 then; /* dummy if */
-      %if &applydttm=YES %then %do;
-        else if format=:'DATETIME' then fmt=' TIMESTAMP ';
-      %end;
-      else if type='num' then fmt=' DOUBLE PRECISION';
-      else fmt='VARCHAR('!!cats(length)!!')';
-      if notnull='yes' then notnul=' NOT NULL';
-      /* quote column names in case they represent reserved words */
-      name2=quote(trim(name));
-      put name2 fmt notnul;
-    run;
+        else put "   ,"@@;
+        format=upcase(format);
+        if 1=0 then; /* dummy if */
+        %if &applydttm=YES %then %do;
+          else if format=:'DATETIME' then fmt=' TIMESTAMP ';
+        %end;
+        else if type='num' then fmt=' DOUBLE PRECISION';
+        else fmt='VARCHAR('!!cats(length)!!')';
+        if notnull='yes' then notnul=' NOT NULL';
+        /* quote column names in case they represent reserved words */
+        name2=quote(trim(name));
+        put name2 fmt notnul;
+      run;
 
-    /* Extra step for data constraints */
-    %addConst()
+      /* Extra step for data constraints */
+      %addConst()
 
-    data _null_;
-      file &fref mod;
-      put ');';
-    run;
+      data _null_;
+        file &fref mod;
+        put ');';
+      run;
 
-    /* Create Unique Indexes, but only if they were not already defined within
-      the Constraints section. */
-    data _null_;
-      *length ds $128;
-      set &idxinfo(
-        where=(
-          memname="&curds"
-          and unique='yes'
-          and indxname not in (
-            %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+      /* Create Unique Indexes, but only if they were not already defined within
+        the Constraints section. */
+      data _null_;
+        *length ds $128;
+        set &idxinfo(
+          where=(
+            memname="&curds"
+            and unique='yes'
+            and indxname not in (
+              %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+            )
           )
-        )
-      );
-      file &fref mod;
-      by idxusage indxname;
-/*       ds=cats(libname,'.',memname); */
-      if first.indxname then do;
+        );
+        file &fref mod;
+        by idxusage indxname;
+        if first.indxname then do;
           put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds(";
           put '  "' name +(-1) '"' ;
-      end;
-      else put '  ,"' name +(-1) '"';
-      *else put '    ,' name ;
-      if last.indxname then do;
-        put ');';
-      end;
-    run;
-
+        end;
+        else put '  ,"' name +(-1) '"';
+        if last.indxname then do;
+          put ');';
+        end;
+      run;
+    %end;
   %end;
 %end;
 %if %upcase(&showlog)=YES %then %do;

base/mp_lib2inserts.sas

@@ -0,0 +1,73 @@
+/**
+  @file
+  @brief Convert all data in a library to SQL insert statements
+  @details Gets list of members then calls the <code>%mp_ds2inserts()</code>
+  macro.
+  Usage:
+
+      %mp_getddl(sashelp, schema=work, fref=tempref)
+
+      %mp_lib2inserts(sashelp, schema=work, outref=tempref)
+
+      %inc tempref;
+
+
+  The output will be one file in the outref fileref.
+
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2inserts.sas
+
+
+  @param [in] lib Library in which to convert all datasets to inserts
+  @param [in] flavour= (SAS) The SQL flavour to be applied to the output. Valid
+    options:
+    @li SAS (default) - suitable for regular proc sql
+    @li PGSQL - Used for Postgres databases
+  @param [in] maxobs= (max) The max number of observations (per table) to create
+  @param [out] outref= Output fileref in which to create the insert statements.
+    If it exists, it will be appended to, otherwise it will be created.
+  @param [out] schema= (0) The schema of the target database, or the libref.
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_lib2inserts(lib
+    ,flavour=SAS
+    ,outref=0
+    ,schema=0
+    ,maxobs=max
+)/*/STORE SOURCE*/;
+
+/* Find the tables */
+%local x ds memlist;
+proc sql noprint;
+select distinct lowcase(memname)
+  into: memlist
+  separated by ' '
+  from dictionary.tables
+  where upcase(libname)="%upcase(&lib)"
+    and memtype='DATA'; /* exclude views */
+
+
+%let flavour=%upcase(&flavour);
+%if &flavour ne SAS and &flavour ne PGSQL %then %do;
+  %put %str(WAR)NING:  &flavour is not supported;
+  %return;
+%end;
+
+
+/* create the inserts */
+%do x=1 %to %sysfunc(countw(&memlist));
+  %let ds=%scan(&memlist,&x);
+  %mp_ds2inserts(&lib..&ds
+    ,outref=&outref
+    ,schema=&schema
+    ,outds=&ds
+    ,flavour=&flavour
+    ,maxobs=&maxobs
+  )
+%end;
+
+%mend mp_lib2inserts;

meta/mm_gettables.sas

@@ -1,18 +1,20 @@
 /**
   @file
   @brief Creates a dataset with all metadata tables for a particular library
-  @details Will only show the tables to which a user has the requisite
-    metadata access.
+  @details Will only show the tables for which the executing user has the
+    requisite metadata access.
 
   usage:
 
-    %mm_gettables(uri=A5X8AHW1.B40001S5)
+      %mm_gettables(uri=A5X8AHW1.B40001S5)
 
-  @param outds the dataset to create that contains the list of tables
-  @param uri the uri of the library for which to return tables
-  @param getauth= YES|NO - fetch the authdomain used in database connections.
-  Set to NO to improve runtimes in larger environments, as there can be a
-  performance hit on the `metadata_getattr(domainuri, "Name", AuthDomain)` call.
+  @param [in] uri= the uri of the library for which to return tables
+  @param [out] outds= (work.mm_gettables) the dataset to contain the list of
+    tables
+  @param [in] getauth= (YES) Fetch the authdomain used in database connections.
+    Set to NO to improve runtimes in larger environments, as there can be a
+    performance hit on the `metadata_getattr(domainuri, "Name", AuthDomain)`
+    call.
 
   @returns outds  dataset containing all groups in a column named "metagroup"
     (defaults to work.mm_getlibs). The following columns are provided:
@@ -40,8 +42,8 @@ data &outds;
     libdesc $200 libref engine $8 IsDBMSLibname $1
     tablename $50 /* metadata table names can be longer than $32 */
     ;
-  keep libname libdesc libref engine ServerContext path_schema AuthDomain tableuri
-    tablename IsPreassigned IsDBMSLibname id;
+  keep libname libdesc libref engine ServerContext path_schema AuthDomain
+    tableuri tablename IsPreassigned IsDBMSLibname id;
   call missing (of _all_);
 
   uri=symget('uri');

tests/base/mf_existfileref.test.sas

@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Testing mf_existfileref macro
+
+  <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mp_assert.sas
+
+**/
+
+filename ref1 temp;
+filename ref2 temp;
+
+data _null_;
+  file ref1;
+  put 'exists';
+run;
+
+%mp_assert(
+  iftrue=(%mf_existfileref(ref1)=1),
+  desc=Checking fileref WITH target file exists,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_existfileref(ref2)=1),
+  desc=Checking fileref WITHOUT target file exists,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_existfileref(ref3)=0),
+  desc=Checking non-existant fref does not exist,
+  outds=work.test_results
+)

tests/base/mp_ds2inserts.test.sas

@@ -13,7 +13,7 @@
   * Cars is a great dataset - it contains leading spaces, and formatted numerics
   */
 
-%mp_ds2inserts(sashelp.cars,outref=testref,outlib=work,outds=test)
+%mp_ds2inserts(sashelp.cars,outref=testref,schema=work,outds=test)
 
 data work.test;
   set sashelp.cars;

tests/base/mp_getddl.test.sas

@@ -0,0 +1,23 @@
+/**
+  @file
+  @brief Testing mp_getddl.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_getddl.sas
+  @li mp_assert.sas
+
+**/
+
+data test(index=(pk=(x y)/unique /nomiss));
+  x=1;
+  y='blah';
+  label x='blah';
+run;
+proc sql; describe table &syslast;
+%mp_getddl(work,test,flavour=tsql,showlog=YES)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=mp_getddl runs without errors,
+  outds=work.test_results
+)

tests/base/mp_lib2inserts.test.sas

@@ -0,0 +1,42 @@
+/**
+  @file
+  @brief Testing mp_ds2inserts.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_mkdir.sas
+  @li mp_getddl.sas
+  @li mp_lib2inserts.sas
+  @li mp_assert.sas
+
+**/
+
+/* grab 20 datasets from SASHELP */
+%let path=%sysfunc(pathname(work));
+%mf_mkdir(&path)
+libname sashlp "&path";
+proc sql noprint;
+create table members as
+  select distinct lowcase(memname) as memname
+  from dictionary.tables
+  where upcase(libname)="SASHELP"
+    and memtype='DATA'; /* exclude views */
+data _null_;
+  set work.members;
+  call execute(cats('data sashlp.',memname,';set sashelp.',memname,';run;'));
+  if _n_>20 then stop;
+run;
+
+/* export DDL and inserts */
+%mp_getddl(sashlp, schema=work, fref=tempref)
+%mp_lib2inserts(sashlp, schema=work, outref=tempref,maxobs=50)
+
+/* check if it actually runs */
+options source2;
+%inc tempref;
+
+/* without errors.. */
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Able to export 20 tables from sashelp using mp_lib2inserts,
+  outds=work.test_results
+)