fix: removing bug introduced to mp_getddl and adding a test to prevent regression

feat: updating mp_ds2inserts to support postgres database

.all-contributorsrc

@@ -0,0 +1,104 @@
+{
+  "projectName": "core",
+  "projectOwner": "sasjs",
+  "repoType": "github",
+  "repoHost": "https://github.com",
+  "files": [
+    "README.md"
+  ],
+  "imageSize": 100,
+  "commit": false,
+  "commitConvention": "angular",
+  "contributors": [
+    {
+      "login": "allanbowe",
+      "name": "Allan Bowe",
+      "avatar_url": "https://avatars.githubusercontent.com/u/4420615?v=4",
+      "profile": "https://github.com/allanbowe",
+      "contributions": [
+        "business",
+        "code",
+        "content",
+        "doc",
+        "infra",
+        "maintenance",
+        "mentoring",
+        "question",
+        "review",
+        "test"
+      ]
+    },
+    {
+      "login": "rafgag",
+      "name": "rafgag",
+      "avatar_url": "https://avatars.githubusercontent.com/u/69139928?v=4",
+      "profile": "https://github.com/rafgag",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "tmoody",
+      "name": "Trevor Moody",
+      "avatar_url": "https://avatars.githubusercontent.com/u/79837106?v=4",
+      "profile": "https://github.com/tmoody",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "krishna-acondy",
+      "name": "Krishna Acondy",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2980428?v=4",
+      "profile": "https://krishna-acondy.io/",
+      "contributions": [
+        "code",
+        "infra",
+        "blog",
+        "content",
+        "ideas",
+        "video"
+      ]
+    },
+    {
+      "login": "saadjutt01",
+      "name": "Muhammad Saad ",
+      "avatar_url": "https://avatars.githubusercontent.com/u/8914650?v=4",
+      "profile": "https://github.com/saadjutt01",
+      "contributions": [
+        "code",
+        "ideas"
+      ]
+    },
+    {
+      "login": "YuryShkoda",
+      "name": "Yury Shkoda",
+      "avatar_url": "https://avatars.githubusercontent.com/u/25773492?v=4",
+      "profile": "https://www.erudicat.com/",
+      "contributions": [
+        "code",
+        "infra",
+        "video"
+      ]
+    },
+    {
+      "login": "medjedovicm",
+      "name": "Mihajlo Medjedovic",
+      "avatar_url": "https://avatars.githubusercontent.com/u/18329105?v=4",
+      "profile": "https://github.com/medjedovicm",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "kkchandok",
+      "name": "kkchandok",
+      "avatar_url": "https://avatars.githubusercontent.com/u/46090627?v=4",
+      "profile": "https://github.com/kkchandok",
+      "contributions": [
+        "ideas"
+      ]
+    }
+  ],
+  "contributorsPerLine": 7
+}

README.md

@@ -179,3 +179,34 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 
 
+
+## Contributors ✨
+<!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
+[![All Contributors](https://img.shields.io/badge/all_contributors-8-orange.svg?style=flat-square)](#contributors-)
+<!-- ALL-CONTRIBUTORS-BADGE:END -->
+Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
+
+<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-disable -->
+<table>
+  <tr>
+    <td align="center"><a href="https://github.com/allanbowe"><img src="https://avatars.githubusercontent.com/u/4420615?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Allan Bowe</b></sub></a><br /><a href="#business-allanbowe" title="Business development">💼</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Code">💻</a> <a href="#content-allanbowe" title="Content">🖋</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Documentation">📖</a> <a href="#infra-allanbowe" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#maintenance-allanbowe" title="Maintenance">🚧</a> <a href="#mentoring-allanbowe" title="Mentoring">🧑‍🏫</a> <a href="#question-allanbowe" title="Answering Questions">💬</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3Aallanbowe" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Tests">⚠️</a></td>
+    <td align="center"><a href="https://github.com/rafgag"><img src="https://avatars.githubusercontent.com/u/69139928?v=4?s=100" width="100px;" alt=""/><br /><sub><b>rafgag</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rafgag" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/tmoody"><img src="https://avatars.githubusercontent.com/u/79837106?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Trevor Moody</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=tmoody" title="Code">💻</a></td>
+    <td align="center"><a href="https://krishna-acondy.io/"><img src="https://avatars.githubusercontent.com/u/2980428?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Krishna Acondy</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=krishna-acondy" title="Code">💻</a> <a href="#infra-krishna-acondy" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#blog-krishna-acondy" title="Blogposts">📝</a> <a href="#content-krishna-acondy" title="Content">🖋</a> <a href="#ideas-krishna-acondy" title="Ideas, Planning, & Feedback">🤔</a> <a href="#video-krishna-acondy" title="Videos">📹</a></td>
+    <td align="center"><a href="https://github.com/saadjutt01"><img src="https://avatars.githubusercontent.com/u/8914650?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Muhammad Saad </b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=saadjutt01" title="Code">💻</a> <a href="#ideas-saadjutt01" title="Ideas, Planning, & Feedback">🤔</a></td>
+    <td align="center"><a href="https://www.erudicat.com/"><img src="https://avatars.githubusercontent.com/u/25773492?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Yury Shkoda</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=YuryShkoda" title="Code">💻</a> <a href="#infra-YuryShkoda" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#video-YuryShkoda" title="Videos">📹</a></td>
+    <td align="center"><a href="https://github.com/medjedovicm"><img src="https://avatars.githubusercontent.com/u/18329105?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Mihajlo Medjedovic</b></sub></a><br /><a href="#infra-medjedovicm" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+  </tr>
+  <tr>
+    <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
+  </tr>
+</table>
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+
+<!-- ALL-CONTRIBUTORS-LIST:END -->
+
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!

all.sas

@@ -2250,7 +2250,7 @@ run;
 
 %if &outfound=0 %then %do;
   filename &outref temp lrecl=2097088;
-%%end;
+%end;
 
 %if &action=ENCODE %then %do;
   data _null_;
@@ -2258,12 +2258,12 @@ run;
     retain line "";
     infile &inref recfm=F lrecl= 1 end=eof;
     input @1 stream $char1.;
-    file &outref lrecl=76;
+    file &outref recfm=N;
     substr(line,(_N_-(CEIL(_N_/57)-1)*57),1) = byte(rank(stream));
     if mod(_N_,57)=0 or EOF then do;
       if eof then b64=put(trim(line),$base64X76.);
       else b64=put(line, $base64X76.);
-      put b64;
+      put b64 + (-1) @;
       line="";
     end;
   run;
@@ -2275,19 +2275,13 @@ run;
     fileout = fopen("&outref",'O',3,'B');
     char= '20'x;
     do while(fread(filein)=0);
-      raw="1234";
+      length raw $4;
       do i=1 to 4;
         rc=fget(filein,char,1);
         substr(raw,i,1)=char;
       end;
-      val="123";
+      rc = fput(fileout,input(raw,$base64X4.));
-      val=input(raw,$base64X4.);
+      rc = fwrite(fileout);
-      do i=1 to 3;
-        length byte $1;
-        byte=byte(rank(substr(val,i,1)));
-        rc = fput(fileout, byte);
-      end;
-      rc =fwrite(fileout);
     end;
     rc = fclose(filein);
     rc = fclose(fileout);
@@ -3429,6 +3423,159 @@ data &outds;
 run;
 
 %mend mp_ds2fmtds;/**
+  @file
+  @brief Export a dataset to SQL insert statements
+  @details Converts dataset values to SQL insert statements for use across
+  multiple database types.
+
+  Usage:
+
+      %mp_ds2inserts(sashelp.class,outref=myref,outds=class)
+      data class;
+        set sashelp.class;
+        stop;
+      proc sql;
+      %inc myref;
+
+  @param [in] ds The dataset to be exported
+  @param [out] outref= (0) The output fileref.  If it does not exist, it is
+    created. If it does exist, new records are APPENDED.
+  @param [out] outlib= (0) The library (or schema) in which the target table is
+    located.  If not provided, is ignored.
+  @param [out] outds= (0) The output table to load.  If not provided, will
+    default to the table in the &ds parameter.
+  @param [in] flavour= (BASE) The SQL flavour to be applied to the output. Valid
+    options:
+    @li BASE (default) - suitable for regular proc sql
+    @li PGSQL - Used for Postgres databases
+
+  <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mf_getvarcount.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+
+  @version 9.2
+  @author Allan Bowe (credit mjsq)
+**/
+
+%macro mp_ds2inserts(ds, outref=0,outlib=0,outds=0,flavour=BASE
+)/*/STORE SOURCE*/;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if %index(&ds,.)=0 %then %let ds=WORK.&ds;
+
+%let flavour=%upcase(&flavour);
+%if &flavour ne BASE and &flavour ne PGSQL %then %do;
+  %put %str(WAR)NING:  &flavour is not supported;
+  %return;
+%end;
+
+%if &outref=0 %then %do;
+  %put %str(WAR)NING:  Please provide a fileref;
+  %return;
+%end;
+%if %mf_existfileref(&outref)=0 %then %do;
+  filename &outref temp lrecl=66000;
+%end;
+
+%if &outlib=0 %then %let outlib=;
+%else %let outlib=&outlib..;
+
+%if &outds=0 %then %let outds=%scan(&ds,2,.);
+
+%local nobs;
+proc sql noprint;
+select count(*) into: nobs TRIMMED from &ds;
+%if &nobs=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No rows found in &ds */";
+  run;
+%end;
+
+%local vars;
+%let vars=%mf_getvarcount(&ds);
+%if &vars=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No columns found in &ds */";
+  run;
+%end;
+
+%local varlist varlistcomma;
+%let varlist=%mf_getvarlist(&ds);
+%let varlistcomma=%mf_getvarlist(&ds,dlm=%str(,),quote=double);
+
+/* next, export data */
+data _null_;
+  file &outref mod ;
+  if _n_=1 then put "/* &outlib.&outds (&nobs rows, &vars columns) */";
+  set &ds;
+  length _____str $32767;
+  format _numeric_ best.;
+  format _character_ ;
+  %local i comma var vtype;
+  %do i=1 %to %sysfunc(countw(&varlist));
+    %let var=%scan(&varlist,&i);
+    %let vtype=%mf_getvartype(&ds,&var);
+    %if &i=1 %then %do;
+      %if &flavour=BASE %then %do;
+        put "insert into &outlib.&outds set ";
+        put "  &var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        _____str=cats(
+          "INSERT INTO &outlib.&outds ("
+          ,symget('varlistcomma')
+          ,") VALUES ("
+        );
+        put _____str;
+        put "  "@;
+      %end;
+    %end;
+    %else %do;
+      %if &flavour=BASE %then %do;
+        put "  ,&var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        put "  ,"@;
+      %end;
+    %end;
+    %if &vtype=N %then %do;
+      %if &flavour=BASE %then %do;
+        put &var;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        if missing(&var) then put 'NULL';
+        else put &var;
+      %end;
+    %end;
+    %else %do;
+      _____str="'"!!trim(tranwrd(&var,"'","''"))!!"'";
+      put _____str;
+    %end;
+  %end;
+  %if &flavour=BASE %then %do;
+    put ';';
+  %end;
+  %else %if &flavour=PGSQL %then %do;
+    put ');';
+  %end;
+
+  if _n_=&nobs then put /;
+run;
+
+%mend mp_ds2inserts;/**
   @file
   @brief Checks an input filter table for validity
   @details Performs checks on the input table to ensure it arrives in the
@@ -4250,12 +4397,13 @@ run;
       %mp_getddl(work,test,flavour=tsql,showlog=YES)
 
   <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
   @li mp_getconstraints.sas
 
   @param lib libref of the library to create DDL for.  Should be assigned.
   @param ds dataset to create ddl for (optional)
-  @param fref= the fileref to which to write the DDL.  If not preassigned, will
+  @param fref= the fileref to which to _append_ the DDL.  If it does not exist,
-    be assigned to TEMP.
+    it will be created.
   @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
   @param showlog= Set to YES to show the DDL in the log
   @param schema= Choose a preferred schema name (default is to use actual schema
@@ -4271,9 +4419,10 @@ run;
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %sysfunc(fileref(&fref)) > 0 %then %do;
+%if %mf_existfileref(&fref)=0 %then %do;
-  filename &fref temp;
+  filename &fref temp ;
 %end;
+
 %if %length(&libref)=0 %then %let libref=WORK;
 %let flavour=%upcase(&flavour);
 
@@ -4352,7 +4501,7 @@ create table _data_ as
 %mend addConst;
 
 data _null_;
-  file &fref;
+  file &fref mod;
   put "/* DDL generated by &sysuserid on %sysfunc(datetime(),datetime19.) */";
 run;
 
@@ -5009,6 +5158,7 @@ create table &outds (rename=(
   @li mf_getvartype.sas
 
   @param [in] libds dataset to hash
+  @param [in] salt= Provide a salt (could be, for instance, the dataset name)
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
   will contain one column (hashkey) with one observation (a hex32.
   representation of the input hash)
@@ -5022,7 +5172,8 @@ create table &outds (rename=(
 
 %macro mp_hashdataset(
   libds,
-  outds=
+  outds=,
+  salt=
 )/*/STORE SOURCE*/;
   %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
     %put %str(WARN)ING: Dataset &libds is empty;, or is not a dataset;
@@ -5042,7 +5193,7 @@ create table &outds (rename=(
     %let varlist=%mf_getvarlist(&libds);
     data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
       length &prevkeyvar &keyvar $32;
-      retain &prevkeyvar;
+      retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
       set &libds end=&lastvar;
       /* hash should include previous row */
       &keyvar=put(md5(&prevkeyvar

base/mp_base64copy.sas

@@ -76,7 +76,7 @@ run;
 
 %if &outfound=0 %then %do;
   filename &outref temp lrecl=2097088;
-%%end;
+%end;
 
 %if &action=ENCODE %then %do;
   data _null_;
@@ -84,12 +84,12 @@ run;
     retain line "";
     infile &inref recfm=F lrecl= 1 end=eof;
     input @1 stream $char1.;
-    file &outref lrecl=76;
+    file &outref recfm=N;
     substr(line,(_N_-(CEIL(_N_/57)-1)*57),1) = byte(rank(stream));
     if mod(_N_,57)=0 or EOF then do;
       if eof then b64=put(trim(line),$base64X76.);
       else b64=put(line, $base64X76.);
-      put b64;
+      put b64 + (-1) @;
       line="";
     end;
   run;
@@ -101,19 +101,13 @@ run;
     fileout = fopen("&outref",'O',3,'B');
     char= '20'x;
     do while(fread(filein)=0);
-      raw="1234";
+      length raw $4;
       do i=1 to 4;
         rc=fget(filein,char,1);
         substr(raw,i,1)=char;
       end;
-      val="123";
+      rc = fput(fileout,input(raw,$base64X4.));
-      val=input(raw,$base64X4.);
+      rc = fwrite(fileout);
-      do i=1 to 3;
-        length byte $1;
-        byte=byte(rank(substr(val,i,1)));
-        rc = fput(fileout, byte);
-      end;
-      rc =fwrite(fileout);
     end;
     rc = fclose(filein);
     rc = fclose(fileout);

base/mp_ds2inserts.sas

@@ -0,0 +1,154 @@
+/**
+  @file
+  @brief Export a dataset to SQL insert statements
+  @details Converts dataset values to SQL insert statements for use across
+  multiple database types.
+
+  Usage:
+
+      %mp_ds2inserts(sashelp.class,outref=myref,outds=class)
+      data class;
+        set sashelp.class;
+        stop;
+      proc sql;
+      %inc myref;
+
+  @param [in] ds The dataset to be exported
+  @param [out] outref= (0) The output fileref.  If it does not exist, it is
+    created. If it does exist, new records are APPENDED.
+  @param [out] outlib= (0) The library (or schema) in which the target table is
+    located.  If not provided, is ignored.
+  @param [out] outds= (0) The output table to load.  If not provided, will
+    default to the table in the &ds parameter.
+  @param [in] flavour= (BASE) The SQL flavour to be applied to the output. Valid
+    options:
+    @li BASE (default) - suitable for regular proc sql
+    @li PGSQL - Used for Postgres databases
+
+  <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mf_getvarcount.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+
+  @version 9.2
+  @author Allan Bowe (credit mjsq)
+**/
+
+%macro mp_ds2inserts(ds, outref=0,outlib=0,outds=0,flavour=BASE
+)/*/STORE SOURCE*/;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if %index(&ds,.)=0 %then %let ds=WORK.&ds;
+
+%let flavour=%upcase(&flavour);
+%if &flavour ne BASE and &flavour ne PGSQL %then %do;
+  %put %str(WAR)NING:  &flavour is not supported;
+  %return;
+%end;
+
+%if &outref=0 %then %do;
+  %put %str(WAR)NING:  Please provide a fileref;
+  %return;
+%end;
+%if %mf_existfileref(&outref)=0 %then %do;
+  filename &outref temp lrecl=66000;
+%end;
+
+%if &outlib=0 %then %let outlib=;
+%else %let outlib=&outlib..;
+
+%if &outds=0 %then %let outds=%scan(&ds,2,.);
+
+%local nobs;
+proc sql noprint;
+select count(*) into: nobs TRIMMED from &ds;
+%if &nobs=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No rows found in &ds */";
+  run;
+%end;
+
+%local vars;
+%let vars=%mf_getvarcount(&ds);
+%if &vars=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No columns found in &ds */";
+  run;
+%end;
+
+%local varlist varlistcomma;
+%let varlist=%mf_getvarlist(&ds);
+%let varlistcomma=%mf_getvarlist(&ds,dlm=%str(,),quote=double);
+
+/* next, export data */
+data _null_;
+  file &outref mod ;
+  if _n_=1 then put "/* &outlib.&outds (&nobs rows, &vars columns) */";
+  set &ds;
+  length _____str $32767;
+  format _numeric_ best.;
+  format _character_ ;
+  %local i comma var vtype;
+  %do i=1 %to %sysfunc(countw(&varlist));
+    %let var=%scan(&varlist,&i);
+    %let vtype=%mf_getvartype(&ds,&var);
+    %if &i=1 %then %do;
+      %if &flavour=BASE %then %do;
+        put "insert into &outlib.&outds set ";
+        put "  &var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        _____str=cats(
+          "INSERT INTO &outlib.&outds ("
+          ,symget('varlistcomma')
+          ,") VALUES ("
+        );
+        put _____str;
+        put "  "@;
+      %end;
+    %end;
+    %else %do;
+      %if &flavour=BASE %then %do;
+        put "  ,&var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        put "  ,"@;
+      %end;
+    %end;
+    %if &vtype=N %then %do;
+      %if &flavour=BASE %then %do;
+        put &var;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        if missing(&var) then put 'NULL';
+        else put &var;
+      %end;
+    %end;
+    %else %do;
+      _____str="'"!!trim(tranwrd(&var,"'","''"))!!"'";
+      put _____str;
+    %end;
+  %end;
+  %if &flavour=BASE %then %do;
+    put ';';
+  %end;
+  %else %if &flavour=PGSQL %then %do;
+    put ');';
+  %end;
+
+  if _n_=&nobs then put /;
+run;
+
+%mend mp_ds2inserts;

base/mp_getddl.sas

@@ -16,12 +16,13 @@
       %mp_getddl(work,test,flavour=tsql,showlog=YES)
 
   <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
   @li mp_getconstraints.sas
 
   @param lib libref of the library to create DDL for.  Should be assigned.
   @param ds dataset to create ddl for (optional)
-  @param fref= the fileref to which to write the DDL.  If not preassigned, will
+  @param fref= the fileref to which to _append_ the DDL.  If it does not exist,
-    be assigned to TEMP.
+    it will be created.
   @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
   @param showlog= Set to YES to show the DDL in the log
   @param schema= Choose a preferred schema name (default is to use actual schema
@@ -37,9 +38,10 @@
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %sysfunc(fileref(&fref)) > 0 %then %do;
+%if %mf_existfileref(&fref)=0 %then %do;
-  filename &fref temp;
+  filename &fref temp ;
 %end;
+
 %if %length(&libref)=0 %then %let libref=WORK;
 %let flavour=%upcase(&flavour);
 
@@ -118,7 +120,7 @@ create table _data_ as
 %mend addConst;
 
 data _null_;
-  file &fref;
+  file &fref mod;
   put "/* DDL generated by &sysuserid on %sysfunc(datetime(),datetime19.) */";
 run;
 

base/mp_hashdataset.sas

@@ -20,6 +20,7 @@
   @li mf_getvartype.sas
 
   @param [in] libds dataset to hash
+  @param [in] salt= Provide a salt (could be, for instance, the dataset name)
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
   will contain one column (hashkey) with one observation (a hex32.
   representation of the input hash)
@@ -33,7 +34,8 @@
 
 %macro mp_hashdataset(
   libds,
-  outds=
+  outds=,
+  salt=
 )/*/STORE SOURCE*/;
   %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
     %put %str(WARN)ING: Dataset &libds is empty;, or is not a dataset;
@@ -53,7 +55,7 @@
     %let varlist=%mf_getvarlist(&libds);
     data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
       length &prevkeyvar &keyvar $32;
-      retain &prevkeyvar;
+      retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
       set &libds end=&lastvar;
       /* hash should include previous row */
       &keyvar=put(md5(&prevkeyvar

package-lock.json

@@ -5,38 +5,37 @@
   "packages": {
     "": {
       "name": "@sasjs/core",
-      "hasInstallScript": true,
       "license": "MIT",
       "devDependencies": {
-        "@sasjs/cli": "^2.27.0"
+        "@sasjs/cli": "2.33.3"
       }
     },
     "node_modules/@sasjs/adapter": {
-      "version": "2.6.3",
+      "version": "2.8.9",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.6.3.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.8.9.tgz",
-      "integrity": "sha512-Mg1AIDd0JDa0/UjvvmrGxjkpYmGEpqmD1dOYmS55EJG5loAENaVKJg9Ht8bU2XFyVaMbFpgB2yqDd0VWPP4eBA==",
+      "integrity": "sha512-8UChcZlqqlmaMMaKOCr2Bc1h+i2KVDY0FINlPXQN5PdAgEMd8dbxI2p9bsiI1yjYjOBO9LuMl7B79/mwYCtyEw==",
       "dev": true,
-      "hasInstallScript": true,
       "dependencies": {
-        "@sasjs/utils": "^2.18.0",
+        "@sasjs/utils": "^2.21.0",
         "axios": "^0.21.1",
         "axios-cookiejar-support": "^1.0.1",
         "form-data": "^4.0.0",
         "https": "^1.0.0",
+        "jwt-decode": "^3.1.2",
         "tough-cookie": "^4.0.0",
         "url": "^0.11.0"
       }
     },
     "node_modules/@sasjs/cli": {
-      "version": "2.27.0",
+      "version": "2.33.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.27.0.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.33.3.tgz",
-      "integrity": "sha512-jdQBNFbqvkidCL2+V6kuGk66YGH9BshHetPIzYSUCJPYrsKdWYRk0+OP+dZZ4qcNk8yLbcwZH+Z6Y1xh9yroFg==",
+      "integrity": "sha512-y1uFM5MEE6eoKLrPUJbweDt4njSy9Y3CS5w5/U2xwNbiAyhiPXgkwCHjCQ8Qg+0rQnMvyNEn6qJuJZ3udc6T7w==",
       "dev": true,
       "dependencies": {
-        "@sasjs/adapter": "2.6.3",
+        "@sasjs/adapter": "2.8.9",
-        "@sasjs/core": "2.34.5",
+        "@sasjs/core": "2.35.3",
-        "@sasjs/lint": "1.11.1",
+        "@sasjs/lint": "1.11.2",
-        "@sasjs/utils": "2.21.0",
+        "@sasjs/utils": "2.23.3",
         "@types/url-parse": "1.4.3",
         "btoa": "1.2.1",
         "chalk": "4.1.1",
@@ -61,34 +60,32 @@
       }
     },
     "node_modules/@sasjs/core": {
-      "version": "2.34.5",
+      "version": "2.35.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.34.5.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.35.3.tgz",
-      "integrity": "sha512-aq84ce9zmyAnkhKlSQ2SGZSQyJ5+EQAaUwktWfPhVdV9B6FJJSrQL3s1K/Dw9IuMczJcZcBlS4HO/w+goQBgyw==",
+      "integrity": "sha512-3o5PU6DkihpA+Aibt1lRy4USqJI0VFa+wNsKCD+bUD2DLZICU3JablZQxwAPH70VWJGXAUJtDFj0T/iRo5Devg==",
-      "dev": true,
+      "dev": true
-      "hasInstallScript": true
     },
     "node_modules/@sasjs/lint": {
-      "version": "1.11.1",
+      "version": "1.11.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.1.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.2.tgz",
-      "integrity": "sha512-nb0xFwXSDrE1qQLYV8whUoaXX2lh4nN3jbA/dOlLpjmF+8uvpgZo9GHy5vz5cpIhtguxiFd7lOGN/zyrzKO8yg==",
+      "integrity": "sha512-zEonhvha9kwrD+hxhG0hEhtfqpXwffH4vRDIr6eDiXkC7S8M3yImpjyFBvX/THJO5+8iuY8TYkOXKl7+nK/wAg==",
       "dev": true,
-      "hasInstallScript": true,
       "dependencies": {
         "@sasjs/utils": "^2.19.0"
       }
     },
     "node_modules/@sasjs/utils": {
-      "version": "2.21.0",
+      "version": "2.23.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.21.0.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.23.3.tgz",
-      "integrity": "sha512-ehRF1VHBiF8FNg2w4VocjBlb+ZkyMasUZOvUn19bsdqpF0otJVYu66gFOJF0rIpykh/vbpLi6dhqYOxgCRAYIw==",
+      "integrity": "sha512-tEh4mGG80eUxSLpbPivA0vl4akMdauL+yZrLn1uUM8EyiXPvlcWPkQTeN6oHbyyAH108D9cfEBidTePZh1p5VQ==",
       "dev": true,
-      "hasInstallScript": true,
       "dependencies": {
         "@types/prompts": "^2.0.13",
         "chalk": "^4.1.1",
         "cli-table": "^0.3.6",
         "consola": "^2.15.0",
         "fs-extra": "^10.0.0",
+        "jwt-decode": "^3.1.2",
         "prompts": "^2.4.1",
         "valid-url": "^1.0.9"
       }
@@ -103,24 +100,24 @@
       }
     },
     "node_modules/@types/node": {
-      "version": "15.12.4",
+      "version": "16.3.1",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.4.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.3.1.tgz",
-      "integrity": "sha512-zrNj1+yqYF4WskCMOHwN+w9iuD12+dGm0rQ35HLl9/Ouuq52cEtd0CH9qMgrdNmi5ejC1/V7vKEXYubB+65DkA==",
+      "integrity": "sha512-N87VuQi7HEeRJkhzovao/JviiqKjDKMVKxKMfUvSKw+MbkbW8R0nA3fi/MQhhlxV2fQ+2ReM+/Nt4efdrJx3zA==",
       "dev": true
     },
     "node_modules/@types/prompts": {
-      "version": "2.0.13",
+      "version": "2.0.14",
-      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.13.tgz",
+      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.14.tgz",
-      "integrity": "sha512-jwMOIGy49VruR/gYehhJYgpVzB+EVpEE7t7j9m1oTo4HMpOe7KmsyqdBuoxAzA5B4caUgx0cKrWr7wUEqMXJ7Q==",
+      "integrity": "sha512-HZBd99fKxRWpYCErtm2/yxUZv6/PBI9J7N4TNFffl5JbrYMHBwF25DjQGTW3b3jmXq+9P6/8fCIb2ee57BFfYA==",
       "dev": true,
       "dependencies": {
         "@types/node": "*"
       }
     },
     "node_modules/@types/tough-cookie": {
-      "version": "4.0.0",
+      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.1.tgz",
-      "integrity": "sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==",
+      "integrity": "sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==",
       "dev": true,
       "peer": true
     },
@@ -1334,6 +1331,7 @@
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
       "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=",
+      "deprecated": "The",
       "dev": true,
       "engines": {
         "node": ">=0.4.x"
@@ -1762,30 +1760,31 @@
   },
   "dependencies": {
     "@sasjs/adapter": {
-      "version": "2.6.3",
+      "version": "2.8.9",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.6.3.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.8.9.tgz",
-      "integrity": "sha512-Mg1AIDd0JDa0/UjvvmrGxjkpYmGEpqmD1dOYmS55EJG5loAENaVKJg9Ht8bU2XFyVaMbFpgB2yqDd0VWPP4eBA==",
+      "integrity": "sha512-8UChcZlqqlmaMMaKOCr2Bc1h+i2KVDY0FINlPXQN5PdAgEMd8dbxI2p9bsiI1yjYjOBO9LuMl7B79/mwYCtyEw==",
       "dev": true,
       "requires": {
-        "@sasjs/utils": "^2.18.0",
+        "@sasjs/utils": "^2.21.0",
         "axios": "^0.21.1",
         "axios-cookiejar-support": "^1.0.1",
         "form-data": "^4.0.0",
         "https": "^1.0.0",
+        "jwt-decode": "^3.1.2",
         "tough-cookie": "^4.0.0",
         "url": "^0.11.0"
       }
     },
     "@sasjs/cli": {
-      "version": "2.27.0",
+      "version": "2.33.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.27.0.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.33.3.tgz",
-      "integrity": "sha512-jdQBNFbqvkidCL2+V6kuGk66YGH9BshHetPIzYSUCJPYrsKdWYRk0+OP+dZZ4qcNk8yLbcwZH+Z6Y1xh9yroFg==",
+      "integrity": "sha512-y1uFM5MEE6eoKLrPUJbweDt4njSy9Y3CS5w5/U2xwNbiAyhiPXgkwCHjCQ8Qg+0rQnMvyNEn6qJuJZ3udc6T7w==",
       "dev": true,
       "requires": {
-        "@sasjs/adapter": "2.6.3",
+        "@sasjs/adapter": "2.8.9",
-        "@sasjs/core": "2.34.5",
+        "@sasjs/core": "2.35.3",
-        "@sasjs/lint": "1.11.1",
+        "@sasjs/lint": "1.11.2",
-        "@sasjs/utils": "2.21.0",
+        "@sasjs/utils": "2.23.3",
         "@types/url-parse": "1.4.3",
         "btoa": "1.2.1",
         "chalk": "4.1.1",
@@ -1807,24 +1806,24 @@
       }
     },
     "@sasjs/core": {
-      "version": "2.34.5",
+      "version": "2.35.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.34.5.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.35.3.tgz",
-      "integrity": "sha512-aq84ce9zmyAnkhKlSQ2SGZSQyJ5+EQAaUwktWfPhVdV9B6FJJSrQL3s1K/Dw9IuMczJcZcBlS4HO/w+goQBgyw==",
+      "integrity": "sha512-3o5PU6DkihpA+Aibt1lRy4USqJI0VFa+wNsKCD+bUD2DLZICU3JablZQxwAPH70VWJGXAUJtDFj0T/iRo5Devg==",
       "dev": true
     },
     "@sasjs/lint": {
-      "version": "1.11.1",
+      "version": "1.11.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.1.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.2.tgz",
-      "integrity": "sha512-nb0xFwXSDrE1qQLYV8whUoaXX2lh4nN3jbA/dOlLpjmF+8uvpgZo9GHy5vz5cpIhtguxiFd7lOGN/zyrzKO8yg==",
+      "integrity": "sha512-zEonhvha9kwrD+hxhG0hEhtfqpXwffH4vRDIr6eDiXkC7S8M3yImpjyFBvX/THJO5+8iuY8TYkOXKl7+nK/wAg==",
       "dev": true,
       "requires": {
         "@sasjs/utils": "^2.19.0"
       }
     },
     "@sasjs/utils": {
-      "version": "2.21.0",
+      "version": "2.23.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.21.0.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.23.3.tgz",
-      "integrity": "sha512-ehRF1VHBiF8FNg2w4VocjBlb+ZkyMasUZOvUn19bsdqpF0otJVYu66gFOJF0rIpykh/vbpLi6dhqYOxgCRAYIw==",
+      "integrity": "sha512-tEh4mGG80eUxSLpbPivA0vl4akMdauL+yZrLn1uUM8EyiXPvlcWPkQTeN6oHbyyAH108D9cfEBidTePZh1p5VQ==",
       "dev": true,
       "requires": {
         "@types/prompts": "^2.0.13",
@@ -1832,6 +1831,7 @@
         "cli-table": "^0.3.6",
         "consola": "^2.15.0",
         "fs-extra": "^10.0.0",
+        "jwt-decode": "^3.1.2",
         "prompts": "^2.4.1",
         "valid-url": "^1.0.9"
       }
@@ -1843,24 +1843,24 @@
       "dev": true
     },
     "@types/node": {
-      "version": "15.12.4",
+      "version": "16.3.1",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.4.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.3.1.tgz",
-      "integrity": "sha512-zrNj1+yqYF4WskCMOHwN+w9iuD12+dGm0rQ35HLl9/Ouuq52cEtd0CH9qMgrdNmi5ejC1/V7vKEXYubB+65DkA==",
+      "integrity": "sha512-N87VuQi7HEeRJkhzovao/JviiqKjDKMVKxKMfUvSKw+MbkbW8R0nA3fi/MQhhlxV2fQ+2ReM+/Nt4efdrJx3zA==",
       "dev": true
     },
     "@types/prompts": {
-      "version": "2.0.13",
+      "version": "2.0.14",
-      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.13.tgz",
+      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.14.tgz",
-      "integrity": "sha512-jwMOIGy49VruR/gYehhJYgpVzB+EVpEE7t7j9m1oTo4HMpOe7KmsyqdBuoxAzA5B4caUgx0cKrWr7wUEqMXJ7Q==",
+      "integrity": "sha512-HZBd99fKxRWpYCErtm2/yxUZv6/PBI9J7N4TNFffl5JbrYMHBwF25DjQGTW3b3jmXq+9P6/8fCIb2ee57BFfYA==",
       "dev": true,
       "requires": {
         "@types/node": "*"
       }
     },
     "@types/tough-cookie": {
-      "version": "4.0.0",
+      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.1.tgz",
-      "integrity": "sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==",
+      "integrity": "sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==",
       "dev": true,
       "peer": true
     },

package.json

@@ -30,9 +30,9 @@
     "docs": "sasjs doc && ./sasjs/utils/build.sh",
     "test": "sasjs test -t viya",
     "lint": "sasjs lint",
-    "prepare": "[ -d .git ] && git config core.hooksPath ./.git-hooks || true"
+    "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "2.27.0"
+    "@sasjs/cli": "2.33.3"
   }
 }

tests/base/mp_base64copy.test.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Testing mp_base64copy.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_base64copy.sas
+  @li mp_assert.sas
+
+**/
+
+
+/* TEST 1 - regular base64 decode */
+
+%let string1=base ik ally;
+filename tmp temp;
+data _null_;
+  file tmp;
+  put "&string1";
+run;
+%mp_base64copy(inref=tmp, outref=myref, action=ENCODE)
+
+data _null_;
+  infile myref;
+  input;
+  put _infile_;
+run;
+%mp_base64copy(inref=myref, outref=mynewref, action=DECODE)
+data _null_;
+  infile mynewref lrecl=5000;
+  input;
+  put _infile_;
+  call symputx('string1_check',_infile_);
+  stop;
+run;
+%mp_assert(
+  iftrue=("&string1"="&string1_check"),
+  desc=Basic String Compare,
+  outds=work.test_results
+)
+
+
+/* multibyte string check */
+
+filename tmp2 temp;
+data _null_;
+  file tmp2;
+  put "'╤', '╔', '╗', '═', '╧', '╚', '╝', '║', '╟', '─', '┼', '║', '╢', '│'";
+run;
+%mp_base64copy(inref=tmp2, outref=myref2, action=ENCODE)
+
+%mp_base64copy(inref=myref2, outref=newref2, action=DECODE)
+data _null_;
+  infile newref2 lrecl=5000;
+  input;
+  list;
+  /* do not print the string to the log else viya 3.5 throws exception */
+  if trim(_infile_)=
+    "'╤', '╔', '╗', '═', '╧', '╚', '╝', '║', '╟', '─', '┼', '║', '╢', '│'"
+  then call symputx('check2',1);
+  else call symputx('check2',0);
+  stop;
+run;
+%mp_assert(
+  iftrue=("&check2"="1"),
+  desc=Double Byte String Compare,
+  outds=work.test_results
+)

tests/base/mp_ds2inserts.test.sas

@@ -0,0 +1,31 @@
+/**
+  @file
+  @brief Testing mp_ds2inserts.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2inserts.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * test 1 - rebuild an existing dataset
+  * Cars is a great dataset - it contains leading spaces, and formatted numerics
+  */
+
+%mp_ds2inserts(sashelp.cars,outref=testref,outlib=work,outds=test)
+
+data work.test;
+  set sashelp.cars;
+  stop;
+proc sql;
+%inc testref;
+
+proc compare base=sashelp.cars compare=work.test;
+quit;
+
+%mp_assert(
+  iftrue=(&sysinfo=1),
+  desc=sashelp.cars is identical except for ds label,
+  outds=work.test_results
+)

tests/base/mp_getddl.test.sas

@@ -0,0 +1,23 @@
+/**
+  @file
+  @brief Testing mp_getddl.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_getddl.sas
+  @li mp_assert.sas
+
+**/
+
+data test(index=(pk=(x y)/unique /nomiss));
+  x=1;
+  y='blah';
+  label x='blah';
+run;
+proc sql; describe table &syslast;
+%mp_getddl(work,test,flavour=tsql,showlog=YES)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=mp_getddl runs without errors,
+  outds=work.test_results
+)