fix: for PGSQL DDL generation, ignore tables with over 1600 columns (as they are not supported in Postgres)

all.sas

@@ -4398,6 +4398,7 @@ run;
 
   <h4> SAS Macros </h4>
   @li mf_existfileref.sas
+  @li mf_getvarcount.sas
   @li mp_getconstraints.sas
 
   @param lib libref of the library to create DDL for.  Should be assigned.
@@ -4419,8 +4420,8 @@ run;
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %mf_existfileref(&outref)=0 %then %do;
+%if %mf_existfileref(&fref)=0 %then %do;
-  filename &outref temp ;
+  filename &fref temp ;
 %end;
 
 %if %length(&libref)=0 %then %let libref=WORK;
@@ -4685,71 +4686,80 @@ run;
     put "CREATE SCHEMA &schema;";
   %do x=1 %to %sysfunc(countw(&dsnlist));
     %let curds=%scan(&dsnlist,&x);
-    data _null_;
+    %local curdsvarcount;
-      file &fref mod;
+    %let curdsvarcount=%mf_getvarcount(&libref..&curds);
-      put "/* Postgres Flavour DDL for &schema..&curds */";
+    %if &curdsvarcount>1600 %then %do;
-    data _null_;
+      data _null_;
-      file &fref mod;
+        file &fref mod;
-      set &colinfo (where=(upcase(memname)="&curds")) end=last;
+        put "/* &libref..&curds contains &curdsvarcount vars */";
-      length fmt $32;
+        put "/* Postgres cannot create tables with over 1600 vars */";
-      if _n_=1 then do;
+        put "/* No DDL will be generated for this table";
-        if memtype='DATA' then do;
+      run;
-          put "CREATE TABLE &schema..&curds (";
+    %end;
+    %else %do;
+      data _null_;
+        file &fref mod;
+        put "/* Postgres Flavour DDL for &schema..&curds */";
+      data _null_;
+        file &fref mod;
+        set &colinfo (where=(upcase(memname)="&curds")) end=last;
+        length fmt $32;
+        if _n_=1 then do;
+          if memtype='DATA' then do;
+            put "CREATE TABLE &schema..&curds (";
+          end;
+          else do;
+            put "CREATE VIEW &schema..&curds (";
+          end;
+          put "    "@@;
         end;
-        else do;
+        else put "   ,"@@;
-          put "CREATE VIEW &schema..&curds (";
+        format=upcase(format);
-        end;
+        if 1=0 then; /* dummy if */
-        put "    "@@;
+        %if &applydttm=YES %then %do;
-      end;
+          else if format=:'DATETIME' then fmt=' TIMESTAMP ';
-      else put "   ,"@@;
+        %end;
-      format=upcase(format);
+        else if type='num' then fmt=' DOUBLE PRECISION';
-      if 1=0 then; /* dummy if */
+        else fmt='VARCHAR('!!cats(length)!!')';
-      %if &applydttm=YES %then %do;
+        if notnull='yes' then notnul=' NOT NULL';
-        else if format=:'DATETIME' then fmt=' TIMESTAMP ';
+        /* quote column names in case they represent reserved words */
-      %end;
+        name2=quote(trim(name));
-      else if type='num' then fmt=' DOUBLE PRECISION';
+        put name2 fmt notnul;
-      else fmt='VARCHAR('!!cats(length)!!')';
+      run;
-      if notnull='yes' then notnul=' NOT NULL';
-      /* quote column names in case they represent reserved words */
-      name2=quote(trim(name));
-      put name2 fmt notnul;
-    run;
 
-    /* Extra step for data constraints */
+      /* Extra step for data constraints */
-    %addConst()
+      %addConst()
 
-    data _null_;
+      data _null_;
-      file &fref mod;
+        file &fref mod;
-      put ');';
+        put ');';
-    run;
+      run;
 
-    /* Create Unique Indexes, but only if they were not already defined within
+      /* Create Unique Indexes, but only if they were not already defined within
-      the Constraints section. */
+        the Constraints section. */
-    data _null_;
+      data _null_;
-      *length ds $128;
+        *length ds $128;
-      set &idxinfo(
+        set &idxinfo(
-        where=(
+          where=(
-          memname="&curds"
+            memname="&curds"
-          and unique='yes'
+            and unique='yes'
-          and indxname not in (
+            and indxname not in (
-            %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+              %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+            )
           )
-        )
+        );
-      );
+        file &fref mod;
-      file &fref mod;
+        by idxusage indxname;
-      by idxusage indxname;
+        if first.indxname then do;
-/*       ds=cats(libname,'.',memname); */
-      if first.indxname then do;
           put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds(";
           put '  "' name +(-1) '"' ;
-      end;
+        end;
-      else put '  ,"' name +(-1) '"';
+        else put '  ,"' name +(-1) '"';
-      *else put '    ,' name ;
+        if last.indxname then do;
-      if last.indxname then do;
+          put ');';
-        put ');';
+        end;
-      end;
+      run;
-    run;
+    %end;
-
   %end;
 %end;
 %if %upcase(&showlog)=YES %then %do;
@@ -5158,8 +5168,7 @@ create table &outds (rename=(
   @li mf_getvartype.sas
 
   @param [in] libds dataset to hash
-  @param [in] salt= Provide a salt (could be, for instance, the name of the
+  @param [in] salt= Provide a salt (could be, for instance, the dataset name)
-    dataset). Max 32 chars.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
   will contain one column (hashkey) with one observation (a hex32.
   representation of the input hash)
@@ -5194,7 +5203,7 @@ create table &outds (rename=(
     %let varlist=%mf_getvarlist(&libds);
     data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
       length &prevkeyvar &keyvar $32;
-      retain &prevkeyvar "&salt";
+      retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
       set &libds end=&lastvar;
       /* hash should include previous row */
       &keyvar=put(md5(&prevkeyvar

base/mp_getddl.sas

@@ -17,6 +17,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_existfileref.sas
+  @li mf_getvarcount.sas
   @li mp_getconstraints.sas
 
   @param lib libref of the library to create DDL for.  Should be assigned.
@@ -38,8 +39,8 @@
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %mf_existfileref(&outref)=0 %then %do;
+%if %mf_existfileref(&fref)=0 %then %do;
-  filename &outref temp ;
+  filename &fref temp ;
 %end;
 
 %if %length(&libref)=0 %then %let libref=WORK;
@@ -304,71 +305,80 @@ run;
     put "CREATE SCHEMA &schema;";
   %do x=1 %to %sysfunc(countw(&dsnlist));
     %let curds=%scan(&dsnlist,&x);
-    data _null_;
+    %local curdsvarcount;
-      file &fref mod;
+    %let curdsvarcount=%mf_getvarcount(&libref..&curds);
-      put "/* Postgres Flavour DDL for &schema..&curds */";
+    %if &curdsvarcount>1600 %then %do;
-    data _null_;
+      data _null_;
-      file &fref mod;
+        file &fref mod;
-      set &colinfo (where=(upcase(memname)="&curds")) end=last;
+        put "/* &libref..&curds contains &curdsvarcount vars */";
-      length fmt $32;
+        put "/* Postgres cannot create tables with over 1600 vars */";
-      if _n_=1 then do;
+        put "/* No DDL will be generated for this table";
-        if memtype='DATA' then do;
+      run;
-          put "CREATE TABLE &schema..&curds (";
+    %end;
+    %else %do;
+      data _null_;
+        file &fref mod;
+        put "/* Postgres Flavour DDL for &schema..&curds */";
+      data _null_;
+        file &fref mod;
+        set &colinfo (where=(upcase(memname)="&curds")) end=last;
+        length fmt $32;
+        if _n_=1 then do;
+          if memtype='DATA' then do;
+            put "CREATE TABLE &schema..&curds (";
+          end;
+          else do;
+            put "CREATE VIEW &schema..&curds (";
+          end;
+          put "    "@@;
         end;
-        else do;
+        else put "   ,"@@;
-          put "CREATE VIEW &schema..&curds (";
+        format=upcase(format);
-        end;
+        if 1=0 then; /* dummy if */
-        put "    "@@;
+        %if &applydttm=YES %then %do;
-      end;
+          else if format=:'DATETIME' then fmt=' TIMESTAMP ';
-      else put "   ,"@@;
+        %end;
-      format=upcase(format);
+        else if type='num' then fmt=' DOUBLE PRECISION';
-      if 1=0 then; /* dummy if */
+        else fmt='VARCHAR('!!cats(length)!!')';
-      %if &applydttm=YES %then %do;
+        if notnull='yes' then notnul=' NOT NULL';
-        else if format=:'DATETIME' then fmt=' TIMESTAMP ';
+        /* quote column names in case they represent reserved words */
-      %end;
+        name2=quote(trim(name));
-      else if type='num' then fmt=' DOUBLE PRECISION';
+        put name2 fmt notnul;
-      else fmt='VARCHAR('!!cats(length)!!')';
+      run;
-      if notnull='yes' then notnul=' NOT NULL';
-      /* quote column names in case they represent reserved words */
-      name2=quote(trim(name));
-      put name2 fmt notnul;
-    run;
 
-    /* Extra step for data constraints */
+      /* Extra step for data constraints */
-    %addConst()
+      %addConst()
 
-    data _null_;
+      data _null_;
-      file &fref mod;
+        file &fref mod;
-      put ');';
+        put ');';
-    run;
+      run;
 
-    /* Create Unique Indexes, but only if they were not already defined within
+      /* Create Unique Indexes, but only if they were not already defined within
-      the Constraints section. */
+        the Constraints section. */
-    data _null_;
+      data _null_;
-      *length ds $128;
+        *length ds $128;
-      set &idxinfo(
+        set &idxinfo(
-        where=(
+          where=(
-          memname="&curds"
+            memname="&curds"
-          and unique='yes'
+            and unique='yes'
-          and indxname not in (
+            and indxname not in (
-            %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+              %sysfunc(tranwrd("&constraints_used",%str( ),%str(",")))
+            )
           )
-        )
+        );
-      );
+        file &fref mod;
-      file &fref mod;
+        by idxusage indxname;
-      by idxusage indxname;
+        if first.indxname then do;
-/*       ds=cats(libname,'.',memname); */
-      if first.indxname then do;
           put 'CREATE UNIQUE INDEX "' indxname +(-1) '" ' "ON &schema..&curds(";
           put '  "' name +(-1) '"' ;
-      end;
+        end;
-      else put '  ,"' name +(-1) '"';
+        else put '  ,"' name +(-1) '"';
-      *else put '    ,' name ;
+        if last.indxname then do;
-      if last.indxname then do;
+          put ');';
-        put ');';
+        end;
-      end;
+      run;
-    run;
+    %end;
-
   %end;
 %end;
 %if %upcase(&showlog)=YES %then %do;

base/mp_hashdataset.sas

@@ -20,8 +20,7 @@
   @li mf_getvartype.sas
 
   @param [in] libds dataset to hash
-  @param [in] salt= Provide a salt (could be, for instance, the name of the
+  @param [in] salt= Provide a salt (could be, for instance, the dataset name)
-    dataset). Max 32 chars.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
   will contain one column (hashkey) with one observation (a hex32.
   representation of the input hash)
@@ -56,7 +55,7 @@
     %let varlist=%mf_getvarlist(&libds);
     data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
       length &prevkeyvar &keyvar $32;
-      retain &prevkeyvar "&salt";
+      retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
       set &libds end=&lastvar;
       /* hash should include previous row */
       &keyvar=put(md5(&prevkeyvar

tests/base/mp_getddl.test.sas

@@ -0,0 +1,23 @@
+/**
+  @file
+  @brief Testing mp_getddl.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_getddl.sas
+  @li mp_assert.sas
+
+**/
+
+data test(index=(pk=(x y)/unique /nomiss));
+  x=1;
+  y='blah';
+  label x='blah';
+run;
+proc sql; describe table &syslast;
+%mp_getddl(work,test,flavour=tsql,showlog=YES)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=mp_getddl runs without errors,
+  outds=work.test_results
+)