Merge pull request #69 from sasjs/issue68

feat: supporting postgres timestamps for mp_ds2inserts and mp_lib2inserts

all.sas

@@ -2413,10 +2413,29 @@ run;
 
         %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
 
-  @param inloc full, quoted "path/and/filename.ext" of the object to be copied
+  To append to a file, use the mode option, eg:
-  @param outloc full, quoted "path/and/filename.ext" of object to be created
+
-  @param inref can override default input fileref to avoid naming clash
+      filename tmp1 temp;
-  @param outref an override default output fileref to avoid naming clash
+      filename tmp2 temp;
+      data _null_;
+        file tmp1;
+        put 'stacking';
+      run;
+
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+
+
+  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
+  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
+  @param [in] inref (____in) If provided, this fileref will take precedence over
+    the `inloc` parameter
+  @param [out] outref (____in) If provided, this fileref will take precedence
+    over the `outloc` parameter.  It must already exist!
+  @param [in] mode (CREATE) Valid values:
+    @li CREATE - Create the file (even if it already exists)
+    @li APPEND - Append to the file (don't overwrite)
+
   @returns nothing
 
   @version 9.2
@@ -2428,20 +2447,29 @@ run;
     ,outloc=          /* full path and filename of object to be created */
     ,inref=____in   /* override default to use own filerefs */
     ,outref=____out /* override default to use own filerefs */
+    ,mode=CREATE
 )/*/STORE SOURCE*/;
+  %local mod outmode;
+  %if &mode=APPEND %then %do;
+    %let mod=mod;
+    %let outmode='a';
+  %end;
+  %else %do;
+    %let outmode='o';
+  %end;
   /* these IN and OUT filerefs can point to anything */
   %if &inref = ____in %then %do;
     filename &inref &inloc lrecl=1048576 ;
   %end;
   %if &outref=____out %then %do;
-    filename &outref &outloc lrecl=1048576 ;
+    filename &outref &outloc lrecl=1048576 &mod;
   %end;
 
   /* copy the file byte-for-byte  */
   data _null_;
     length filein 8 fileid 8;
     filein = fopen("&inref",'I',1,'B');
-    fileid = fopen("&outref",'O',1,'B');
+    fileid = fopen("&outref",&outmode,1,'B');
     rec = '20'x;
     do while(fread(filein)=0);
       rc = fget(filein,rec,1);
@@ -3563,10 +3591,13 @@ run;
     options:
     @li SAS (default) - suitable for regular proc sql
     @li PGSQL - Used for Postgres databases
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals
 
   <h4> SAS Macros </h4>
   @li mf_existfileref.sas
   @li mf_getvarcount.sas
+  @li mf_getvarformat.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
 
@@ -3575,6 +3606,7 @@ run;
 **/
 
 %macro mp_ds2inserts(ds, outref=0,schema=0,outds=0,flavour=SAS,maxobs=max
+  ,applydttm=YES
 )/*/STORE SOURCE*/;
 
 %if not %sysfunc(exist(&ds)) %then %do;
@@ -3652,10 +3684,11 @@ data _null_;
   length _____str $32767;
   format _numeric_ best.;
   format _character_ ;
-  %local i comma var vtype;
+  %local i comma var vtype vfmt;
   %do i=1 %to %sysfunc(countw(&varlist));
     %let var=%scan(&varlist,&i);
     %let vtype=%mf_getvartype(&ds,&var);
+    %let vfmt=%upcase(%mf_getvarformat(&ds,&var,force=1));
     %if &i=1 %then %do;
       %if &flavour=SAS %then %do;
         put "insert into &schema.&outds set ";
@@ -3685,7 +3718,13 @@ data _null_;
       %end;
       %else %if &flavour=PGSQL %then %do;
         if missing(&var) then put 'NULL';
-        else put &var;
+        %if &applydttm=YES and "%substr(&vfmt.xxxxxxxx,1,8)"="DATETIME"
+        %then %do;
+          else put "TIMESTAMP '" &var E8601DT25.6 "'";
+        %end;
+        %else %do;
+          else put &var;
+        %end;
       %end;
     %end;
     %else %do;
@@ -5697,6 +5736,8 @@ select distinct lowcase(memname)
   @param [out] outref= Output fileref in which to create the insert statements.
     If it exists, it will be appended to, otherwise it will be created.
   @param [out] schema= (0) The schema of the target database, or the libref.
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals
 
   @version 9.2
   @author Allan Bowe
@@ -5707,6 +5748,7 @@ select distinct lowcase(memname)
     ,outref=0
     ,schema=0
     ,maxobs=max
+    ,applydttm=YES
 )/*/STORE SOURCE*/;
 
 /* Find the tables */
@@ -5736,6 +5778,7 @@ select distinct lowcase(memname)
     ,outds=&ds
     ,flavour=&flavour
     ,maxobs=&maxobs
+    ,applydttm=&applydttm
   )
 %end;
 

base/mp_binarycopy.sas

@@ -9,10 +9,29 @@
 
         %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
 
-  @param inloc full, quoted "path/and/filename.ext" of the object to be copied
+  To append to a file, use the mode option, eg:
-  @param outloc full, quoted "path/and/filename.ext" of object to be created
+
-  @param inref can override default input fileref to avoid naming clash
+      filename tmp1 temp;
-  @param outref an override default output fileref to avoid naming clash
+      filename tmp2 temp;
+      data _null_;
+        file tmp1;
+        put 'stacking';
+      run;
+
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+      %mp_binarycopy(inref=tmp1, outref=tmp2, mode=APPEND)
+
+
+  @param [in] inloc quoted "path/and/filename.ext" of the file to be copied
+  @param [out] outloc quoted "path/and/filename.ext" of the file to be created
+  @param [in] inref (____in) If provided, this fileref will take precedence over
+    the `inloc` parameter
+  @param [out] outref (____in) If provided, this fileref will take precedence
+    over the `outloc` parameter.  It must already exist!
+  @param [in] mode (CREATE) Valid values:
+    @li CREATE - Create the file (even if it already exists)
+    @li APPEND - Append to the file (don't overwrite)
+
   @returns nothing
 
   @version 9.2
@@ -24,20 +43,29 @@
     ,outloc=          /* full path and filename of object to be created */
     ,inref=____in   /* override default to use own filerefs */
     ,outref=____out /* override default to use own filerefs */
+    ,mode=CREATE
 )/*/STORE SOURCE*/;
+  %local mod outmode;
+  %if &mode=APPEND %then %do;
+    %let mod=mod;
+    %let outmode='a';
+  %end;
+  %else %do;
+    %let outmode='o';
+  %end;
   /* these IN and OUT filerefs can point to anything */
   %if &inref = ____in %then %do;
     filename &inref &inloc lrecl=1048576 ;
   %end;
   %if &outref=____out %then %do;
-    filename &outref &outloc lrecl=1048576 ;
+    filename &outref &outloc lrecl=1048576 &mod;
   %end;
 
   /* copy the file byte-for-byte  */
   data _null_;
     length filein 8 fileid 8;
     filein = fopen("&inref",'I',1,'B');
-    fileid = fopen("&outref",'O',1,'B');
+    fileid = fopen("&outref",&outmode,1,'B');
     rec = '20'x;
     do while(fread(filein)=0);
       rc = fget(filein,rec,1);

base/mp_ds2inserts.sas

@@ -25,10 +25,13 @@
     options:
     @li SAS (default) - suitable for regular proc sql
     @li PGSQL - Used for Postgres databases
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals
 
   <h4> SAS Macros </h4>
   @li mf_existfileref.sas
   @li mf_getvarcount.sas
+  @li mf_getvarformat.sas
   @li mf_getvarlist.sas
   @li mf_getvartype.sas
 
@@ -37,6 +40,7 @@
 **/
 
 %macro mp_ds2inserts(ds, outref=0,schema=0,outds=0,flavour=SAS,maxobs=max
+  ,applydttm=YES
 )/*/STORE SOURCE*/;
 
 %if not %sysfunc(exist(&ds)) %then %do;
@@ -114,10 +118,11 @@ data _null_;
   length _____str $32767;
   format _numeric_ best.;
   format _character_ ;
-  %local i comma var vtype;
+  %local i comma var vtype vfmt;
   %do i=1 %to %sysfunc(countw(&varlist));
     %let var=%scan(&varlist,&i);
     %let vtype=%mf_getvartype(&ds,&var);
+    %let vfmt=%upcase(%mf_getvarformat(&ds,&var,force=1));
     %if &i=1 %then %do;
       %if &flavour=SAS %then %do;
         put "insert into &schema.&outds set ";
@@ -147,7 +152,13 @@ data _null_;
       %end;
       %else %if &flavour=PGSQL %then %do;
         if missing(&var) then put 'NULL';
-        else put &var;
+        %if &applydttm=YES and "%substr(&vfmt.xxxxxxxx,1,8)"="DATETIME"
+        %then %do;
+          else put "TIMESTAMP '" &var E8601DT25.6 "'";
+        %end;
+        %else %do;
+          else put &var;
+        %end;
       %end;
     %end;
     %else %do;

base/mp_lib2inserts.sas

@@ -28,6 +28,8 @@
   @param [out] outref= Output fileref in which to create the insert statements.
     If it exists, it will be appended to, otherwise it will be created.
   @param [out] schema= (0) The schema of the target database, or the libref.
+  @param [in] applydttm= (YES) If YES, any columns using datetime formats will
+    be converted to native DB datetime literals
 
   @version 9.2
   @author Allan Bowe
@@ -38,6 +40,7 @@
     ,outref=0
     ,schema=0
     ,maxobs=max
+    ,applydttm=YES
 )/*/STORE SOURCE*/;
 
 /* Find the tables */
@@ -67,6 +70,7 @@ select distinct lowcase(memname)
     ,outds=&ds
     ,flavour=&flavour
     ,maxobs=&maxobs
+    ,applydttm=&applydttm
   )
 %end;
 

tests/crossplatform/mp_binarycopy.test.sas

@@ -0,0 +1,99 @@
+/**
+  @file
+  @brief Testing mp_binarycopy.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_binarycopy.sas
+  @li mp_assert.sas
+
+**/
+
+
+/* TEST 1 - regular file copy */
+%let string1=test1;
+filename tmp temp;
+filename myref temp;
+data _null_;
+  file tmp;
+  put "&string1";
+run;
+%mp_binarycopy(inref=tmp, outref=myref)
+data _null_;
+  infile myref;
+  input;
+  put _infile_;
+  call symputx('string1_check',_infile_);
+  stop;
+run;
+%mp_assert(
+  iftrue=("&string1"="&string1_check"),
+  desc=Basic String Compare,
+  outds=work.test_results
+)
+
+
+/* TEST 2 - File append */
+%let string2=test2;
+%let path2=%sysfunc(pathname(work))/somefile.txt;
+data _null_;
+  file "&path2";
+  put "&string2";
+run;
+%mp_binarycopy(inloc="&path2", outref=myref, mode=APPEND)
+data _null_;
+  infile myref;
+  input;
+  put _infile_;
+  if _n_=2 then call symputx('string2_check',_infile_);
+run;
+%mp_assert(
+  iftrue=("&string2"="&string2_check"),
+  desc=Append Check (file to ref),
+  outds=work.test_results
+)
+
+/* TEST 3 - File create (ref to existing file) */
+%let string3=test3;
+%let path3=%sysfunc(pathname(work))/somefile3.txt;
+filename tmp3 temp;
+data _null_;
+  file tmp3;
+  put "&string3";
+run;
+data _null_;
+  file "&path3";
+  put "this should not be returned";
+run;
+%mp_binarycopy(inref=tmp3, outloc="&path3")
+data _null_;
+  infile "&path3";
+  input;
+  put _infile_;
+  if _n_=1 then call symputx('string3_check',_infile_);
+run;
+%mp_assert(
+  iftrue=("&string3"="&string3_check"),
+  desc=Append Check (ref to existing file),
+  outds=work.test_results
+)
+
+/* TEST 4 - File append (ref to file) */
+%let string4=test4;
+%let string4_check=;
+filename tmp4 temp;
+data _null_;
+  file tmp4;
+  put "&string4";
+run;
+%mp_binarycopy(inref=tmp4, outloc="&path3",mode=APPEND)
+data _null_;
+  infile "&path3";
+  input;
+  put _infile_;
+  if _n_=2 then call symputx('string4_check',_infile_);
+run;
+%mp_assert(
+  iftrue=("&string4"="&string4_check"),
+  desc=Append Check (ref to file),
+  outds=work.test_results
+)