Merge pull request #55 from sasjs/insertforpg

feat: updating mp_ds2inserts to support postgres database

.all-contributorsrc

@@ -0,0 +1,104 @@
+{
+  "projectName": "core",
+  "projectOwner": "sasjs",
+  "repoType": "github",
+  "repoHost": "https://github.com",
+  "files": [
+    "README.md"
+  ],
+  "imageSize": 100,
+  "commit": false,
+  "commitConvention": "angular",
+  "contributors": [
+    {
+      "login": "allanbowe",
+      "name": "Allan Bowe",
+      "avatar_url": "https://avatars.githubusercontent.com/u/4420615?v=4",
+      "profile": "https://github.com/allanbowe",
+      "contributions": [
+        "business",
+        "code",
+        "content",
+        "doc",
+        "infra",
+        "maintenance",
+        "mentoring",
+        "question",
+        "review",
+        "test"
+      ]
+    },
+    {
+      "login": "rafgag",
+      "name": "rafgag",
+      "avatar_url": "https://avatars.githubusercontent.com/u/69139928?v=4",
+      "profile": "https://github.com/rafgag",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "tmoody",
+      "name": "Trevor Moody",
+      "avatar_url": "https://avatars.githubusercontent.com/u/79837106?v=4",
+      "profile": "https://github.com/tmoody",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "krishna-acondy",
+      "name": "Krishna Acondy",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2980428?v=4",
+      "profile": "https://krishna-acondy.io/",
+      "contributions": [
+        "code",
+        "infra",
+        "blog",
+        "content",
+        "ideas",
+        "video"
+      ]
+    },
+    {
+      "login": "saadjutt01",
+      "name": "Muhammad Saad ",
+      "avatar_url": "https://avatars.githubusercontent.com/u/8914650?v=4",
+      "profile": "https://github.com/saadjutt01",
+      "contributions": [
+        "code",
+        "ideas"
+      ]
+    },
+    {
+      "login": "YuryShkoda",
+      "name": "Yury Shkoda",
+      "avatar_url": "https://avatars.githubusercontent.com/u/25773492?v=4",
+      "profile": "https://www.erudicat.com/",
+      "contributions": [
+        "code",
+        "infra",
+        "video"
+      ]
+    },
+    {
+      "login": "medjedovicm",
+      "name": "Mihajlo Medjedovic",
+      "avatar_url": "https://avatars.githubusercontent.com/u/18329105?v=4",
+      "profile": "https://github.com/medjedovicm",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "kkchandok",
+      "name": "kkchandok",
+      "avatar_url": "https://avatars.githubusercontent.com/u/46090627?v=4",
+      "profile": "https://github.com/kkchandok",
+      "contributions": [
+        "ideas"
+      ]
+    }
+  ],
+  "contributorsPerLine": 7
+}

.github/vpn/config.ovpn

@@ -0,0 +1,30 @@
+cipher AES-256-CBC
+setenv FORWARD_COMPATIBLE 1
+client
+server-poll-timeout 4
+nobind
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 443 tcp
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 1194 udp
+remote vpn.analytium.co.uk 1194 udp
+dev tun
+dev-type tun
+ns-cert-type server
+setenv opt tls-version-min 1.0 or-highest
+reneg-sec 604800
+sndbuf 0
+rcvbuf 0
+# NOTE: LZO commands are pushed by the Access Server at connect time.
+# NOTE: The below line doesn't disable LZO.
+comp-lzo no
+verb 3
+setenv PUSH_PEER_INFO
+
+ca ca.crt
+cert user.crt
+key user.key
+tls-auth tls.key 1

.github/workflows/run-tests.yml

@@ -21,6 +21,31 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
 
+      - name: Write VPN Files
+        run: |
+          echo "$CA_CRT" > .github/vpn/ca.crt
+          echo "$USER_CRT" > .github/vpn/user.crt
+          echo "$USER_KEY" > .github/vpn/user.key
+          echo "$TLS_KEY" > .github/vpn/tls.key
+        shell: bash
+        env:
+          CA_CRT: ${{ secrets.CA_CRT}}
+          USER_CRT: ${{ secrets.USER_CRT }}
+          USER_KEY: ${{ secrets.USER_KEY }}
+          TLS_KEY: ${{ secrets.TLS_KEY }}
+
+      - name: Install Open VPN
+        run: |
+          sudo apt install apt-transport-https
+          sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
+          sudo apt-key add openvpn-repo-pkg-key.pub
+          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-bionic.list
+          sudo apt update
+          sudo apt install openvpn3
+
+      - name: Start Open VPN 3
+        run: openvpn3 session-start --config .github/vpn/config.ovpn
+
       - name: Install Doxygen
         run: sudo apt-get install doxygen
 
@@ -31,16 +56,16 @@ jobs:
         run: npm run lint
 
       - name: Add client
-        run:  echo "CLIENT=${{secrets.CLIENT}}"> .env.viya
+        run: echo "CLIENT=${{secrets.CLIENT}}"> .env.viya
 
       - name: Add secret
-        run:  echo "SECRET=${{secrets.SECRET}}" >> .env.viya
+        run: echo "SECRET=${{secrets.SECRET}}" >> .env.viya
 
       - name: Add access token
-        run:  echo "ACCESS_TOKEN=${{secrets.ACCESS_TOKEN}}" >> .env.viya
+        run: echo "ACCESS_TOKEN=${{secrets.ACCESS_TOKEN}}" >> .env.viya
 
       - name: Add refresh token
-        run:  echo "REFRESH_TOKEN=${{secrets.REFRESH_TOKEN}}" >> .env.viya
+        run: echo "REFRESH_TOKEN=${{secrets.REFRESH_TOKEN}}" >> .env.viya
 
       - name: Build Project
         run: npm run build

.sasjslint

@@ -2,7 +2,7 @@
     "noTrailingSpaces": true,
     "noEncodedPasswords": true,
     "hasDoxygenHeader": true,
-    "hasMacroNameInMend": false,
+    "hasMacroNameInMend": true,
     "hasMacroParentheses": true,
     "noNestedMacros": false,
     "noSpacesInFileNames": true,

README.md

@@ -179,3 +179,34 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 
 
+
+## Contributors ✨
+<!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
+[![All Contributors](https://img.shields.io/badge/all_contributors-8-orange.svg?style=flat-square)](#contributors-)
+<!-- ALL-CONTRIBUTORS-BADGE:END -->
+Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
+
+<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-disable -->
+<table>
+  <tr>
+    <td align="center"><a href="https://github.com/allanbowe"><img src="https://avatars.githubusercontent.com/u/4420615?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Allan Bowe</b></sub></a><br /><a href="#business-allanbowe" title="Business development">💼</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Code">💻</a> <a href="#content-allanbowe" title="Content">🖋</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Documentation">📖</a> <a href="#infra-allanbowe" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#maintenance-allanbowe" title="Maintenance">🚧</a> <a href="#mentoring-allanbowe" title="Mentoring">🧑‍🏫</a> <a href="#question-allanbowe" title="Answering Questions">💬</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3Aallanbowe" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Tests">⚠️</a></td>
+    <td align="center"><a href="https://github.com/rafgag"><img src="https://avatars.githubusercontent.com/u/69139928?v=4?s=100" width="100px;" alt=""/><br /><sub><b>rafgag</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rafgag" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/tmoody"><img src="https://avatars.githubusercontent.com/u/79837106?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Trevor Moody</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=tmoody" title="Code">💻</a></td>
+    <td align="center"><a href="https://krishna-acondy.io/"><img src="https://avatars.githubusercontent.com/u/2980428?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Krishna Acondy</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=krishna-acondy" title="Code">💻</a> <a href="#infra-krishna-acondy" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#blog-krishna-acondy" title="Blogposts">📝</a> <a href="#content-krishna-acondy" title="Content">🖋</a> <a href="#ideas-krishna-acondy" title="Ideas, Planning, & Feedback">🤔</a> <a href="#video-krishna-acondy" title="Videos">📹</a></td>
+    <td align="center"><a href="https://github.com/saadjutt01"><img src="https://avatars.githubusercontent.com/u/8914650?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Muhammad Saad </b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=saadjutt01" title="Code">💻</a> <a href="#ideas-saadjutt01" title="Ideas, Planning, & Feedback">🤔</a></td>
+    <td align="center"><a href="https://www.erudicat.com/"><img src="https://avatars.githubusercontent.com/u/25773492?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Yury Shkoda</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=YuryShkoda" title="Code">💻</a> <a href="#infra-YuryShkoda" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#video-YuryShkoda" title="Videos">📹</a></td>
+    <td align="center"><a href="https://github.com/medjedovicm"><img src="https://avatars.githubusercontent.com/u/18329105?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Mihajlo Medjedovic</b></sub></a><br /><a href="#infra-medjedovicm" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+  </tr>
+  <tr>
+    <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
+  </tr>
+</table>
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+
+<!-- ALL-CONTRIBUTORS-LIST:END -->
+
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!

base/mf_existfeature.sas

@@ -42,6 +42,6 @@
     -1
     %put &sysmacroname: &feature not found;
   %end;
-%mend;
+%mend mf_existfeature;
 
 /** @endcond */

base/mf_existfileref.sas

@@ -24,4 +24,4 @@
     0
   %end;
 
-%mend;
+%mend mf_existfileref;

base/mf_existvar.sas

@@ -30,6 +30,6 @@
       %let rc=%sysfunc(close(&dsid));
   %end;
 
-%mend;
+%mend mf_existvar;
 
 /** @endcond */

base/mf_existvarlist.sas

@@ -54,6 +54,6 @@
     0
     %put Vars not found: &found;
   %end;
-%mend;
+%mend mf_existvarlist;
 
 /** @endcond */

base/mf_getattrc.sas

@@ -31,4 +31,4 @@
     %sysfunc(attrc(&dsid,&attr))
     %let rc=%sysfunc(close(&dsid));
   %end;
-%mend;
+%mend mf_getattrc;

base/mf_getattrn.sas

@@ -31,4 +31,4 @@
     %sysfunc(attrn(&dsid,&attr))
     %let rc=%sysfunc(close(&dsid));
   %end;
-%mend;
+%mend mf_getattrn;

base/mf_getengine.sas

@@ -48,6 +48,6 @@
 
   &engine
 
-%mend;
+%mend mf_getengine;
 
 /** @endcond */

base/mf_getfilesize.sas

@@ -44,4 +44,4 @@
     %sysfunc(INPUTN(&bytes, best.),sizekmg.)
   %end;
 
-%mend ;
+%mend mf_getfilesize ;

base/mf_getkeyvalue.sas

@@ -29,4 +29,4 @@
   &valc
 %end;
 %else %put %str(ERR)OR: Unable to find key &key in ds &libds;
-%mend;
+%mend mf_getkeyvalue;

base/mf_getplatform.sas

@@ -62,4 +62,4 @@
 %else %if &switch=VIYARESTAPI %then %do;
   %mf_trimstr(%sysfunc(getoption(servicesbaseurl)),/)
 %end;
-%mend;
+%mend mf_getplatform;

base/mf_getquotedstr.sas

@@ -50,4 +50,4 @@
 
   &buffer
 
-%mend;
+%mend mf_getquotedstr;

base/mf_getschema.sas

@@ -38,6 +38,6 @@
 
   &schema
 
-%mend;
+%mend mf_getschema;
 
 /** @endcond */

base/mf_getuniquefileref.sas

@@ -34,4 +34,4 @@
   %end;
   %end;
   %put unable to find available fileref in range &prefix.0-&maxtries;
-%mend;
+%mend mf_getuniquefileref;

base/mf_getuniquelibref.sas

@@ -37,4 +37,4 @@
   %end;
   %end;
   %put unable to find available libref in range &prefix.0-&maxtries;
-%mend;
+%mend mf_getuniquelibref;

base/mf_getuser.sas

@@ -39,4 +39,4 @@
 
   %quote(&user)
 
-%mend;
+%mend mf_getuser;

base/mf_getvalue.sas

@@ -30,4 +30,4 @@
     %trim(&&&variable)
 
   %end;
-%mend;
+%mend mf_getvalue;

base/mf_getvarcount.sas

@@ -29,4 +29,4 @@
     %let rc=%sysfunc(close(&dsid));
   %end;
   &nvars
-%mend;
+%mend mf_getvarcount;

base/mf_getvarlen.sas

@@ -49,4 +49,4 @@
   %let rc = %sysfunc(close(&dsid));
   /* Return variable format */
   &vlen
-%mend;
+%mend mf_getVarLen;

base/mf_getvarnum.sas

@@ -51,4 +51,4 @@ returns:
   /* Return variable number */
     &vnum.
 
-%mend;
+%mend mf_getVarNum;

base/mf_getxengine.sas

@@ -40,4 +40,4 @@
 
   &engine
 
-%mend;
+%mend mf_getxengine;

base/mf_isblank.sas

@@ -24,4 +24,4 @@
 
   %sysevalf(%superq(param)=,boolean)
 
-%mend;
+%mend mf_isblank;

base/mf_isdir.sas

@@ -31,4 +31,4 @@
 
   &is_directory
 
-%mend;
+%mend mf_isdir;

base/mf_loc.sas

@@ -26,4 +26,4 @@
   &root
 %end;
 
-%mend;
+%mend mf_loc;

base/mf_mkdir.sas

@@ -64,4 +64,4 @@ Usage:
     %end;
   %end;
   /* exit quietly if directory did exist.*/
-%mend;
+%mend mf_mkdir;

base/mf_mval.sas

@@ -16,4 +16,4 @@
   %if %symexist(&var) %then %do;
     %superq(&var)
   %end;
-%mend;
+%mend mf_mval;

base/mf_nobs.sas

@@ -23,4 +23,4 @@
 %macro mf_nobs(libds
 )/*/STORE SOURCE*/;
   %mf_getattrn(&libds,NLOBS)
-%mend;
+%mend mf_nobs;

base/mf_trimstr.sas

@@ -47,4 +47,4 @@
   &basestr
 %end;
 
-%mend;
+%mend mf_trimstr;

base/mf_verifymacvars.sas

@@ -62,4 +62,4 @@
     %else %mf_abort(mac=mf_verifymacvars,type=&mabort,msg=&abortmsg);
   %exit_success:
 
-%mend;
+%mend mf_verifymacvars;

base/mf_wordsinstr1butnotstr2.sas

@@ -50,5 +50,5 @@
 
   &outvar
 
-%mend;
+%mend mf_wordsInStr1ButNotStr2;
 

base/mp_abort.sas

@@ -165,6 +165,7 @@
         sysuserid=symget('sysuserid');
         iftrue=symget('iftrue');
         put (_all_)(/=);
+        call symputx('syscc',0);
         abort cancel nolist;
       run;
     %end;

base/mp_assertcols.sas

@@ -142,4 +142,4 @@
   proc sql;
   drop table &ds;
 
-%mend;
+%mend mp_assertcols;

base/mp_assertcolvals.sas

@@ -144,4 +144,4 @@
   proc sql;
   drop table &ds;
 
-%mend;
+%mend mp_assertcolvals;

base/mp_base64copy.sas

@@ -0,0 +1,117 @@
+/**
+  @file
+  @brief Convert a file to/from base64 format
+  @details Creates a new version of a file either encoded or decoded using
+  Base64.  Inspired by this post by Michael Dixon:
+  https://support.selerity.com.au/hc/en-us/articles/223345708-Tip-SAS-and-Base64
+
+  Usage:
+
+        filename tmp temp;
+        data _null_;
+          file tmp;
+          put 'base ik ally';
+        run;
+        %mp_base64copy(inref=tmp, outref=myref, action=ENCODE)
+
+        data _null_;
+          infile myref;
+          input;
+          put _infile_;
+        run;
+
+        %mp_base64copy(inref=myref, outref=mynewref, action=DECODE)
+
+        data _null_;
+          infile mynewref;
+          input;
+          put _infile_;
+        run;
+
+  @param [in] inref= Fileref of the input file (should exist)
+  @param [out] outref= Output fileref. If it does not exist, it is created.
+  @param [in] action= (ENCODE) The action to take. Valid values:
+    @li ENCODE - Convert the file to base64 format
+    @li DECODE - Decode the file from base64 format
+
+  @version 9.2
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+
+
+**/
+
+%macro mp_base64copy(
+  inref=0,
+  outref=0,
+  action=ENCODE
+)/*/STORE SOURCE*/;
+
+%let inref=%upcase(&inref);
+%let outref=%upcase(&outref);
+%let action=%upcase(&action);
+%local infound outfound;
+%let infound=0;
+%let outfound=0;
+data _null_;
+  set sashelp.vextfl(where=(fileref="&inref" or fileref="&outref"));
+  if fileref="&inref" then call symputx('infound',1,'l');
+  if fileref="&outref" then call symputx('outfound',1,'l');
+run;
+
+%mp_abort(iftrue= (&infound=0)
+  ,mac=&sysmacroname
+  ,msg=%str(INREF &inref NOT FOUND!)
+)
+%mp_abort(iftrue= (&outref=0)
+  ,mac=&sysmacroname
+  ,msg=%str(OUTREF NOT PROVIDED!)
+)
+%mp_abort(iftrue= (&action ne ENCODE and &action ne DECODE)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid action! Should be ENCODE OR DECODE)
+)
+
+%if &outfound=0 %then %do;
+  filename &outref temp lrecl=2097088;
+%end;
+
+%if &action=ENCODE %then %do;
+  data _null_;
+    length b64 $ 76 line $ 57;
+    retain line "";
+    infile &inref recfm=F lrecl= 1 end=eof;
+    input @1 stream $char1.;
+    file &outref recfm=N;
+    substr(line,(_N_-(CEIL(_N_/57)-1)*57),1) = byte(rank(stream));
+    if mod(_N_,57)=0 or EOF then do;
+      if eof then b64=put(trim(line),$base64X76.);
+      else b64=put(line, $base64X76.);
+      put b64 + (-1) @;
+      line="";
+    end;
+  run;
+%end;
+%else %if &action=DECODE %then %do;
+  data _null_;
+    length filein 8 fileout 8;
+    filein = fopen("&inref",'I',4,'B');
+    fileout = fopen("&outref",'O',3,'B');
+    char= '20'x;
+    do while(fread(filein)=0);
+      length raw $4;
+      do i=1 to 4;
+        rc=fget(filein,char,1);
+        substr(raw,i,1)=char;
+      end;
+      rc = fput(fileout,input(raw,$base64X4.));
+      rc = fwrite(fileout);
+    end;
+    rc = fclose(filein);
+    rc = fclose(fileout);
+  run;
+%end;
+
+%mend mp_base64copy;

base/mp_binarycopy.sas

@@ -53,4 +53,4 @@
   %if &outref=____out %then %do;
     filename &outref clear;
   %end;
-%mend;
+%mend mp_binarycopy;

base/mp_cleancsv.sas

@@ -67,5 +67,5 @@
       else put inchar $char1.;
     end;
   run;
-%mend;
+%mend mp_cleancsv;
 /** @endcond */

base/mp_createconstraints.sas

@@ -64,4 +64,4 @@ data &outds;
   output;
 run;
 
-%mend;
+%mend mp_createconstraints;

base/mp_createwebservice.sas

@@ -80,4 +80,4 @@ Usage:
   )
 %end;
 
-%mend;
+%mend mp_createwebservice;

base/mp_csv2ds.sas

@@ -141,4 +141,4 @@ data &outds
   %end;
 run;
 
-%mend;
+%mend mp_csv2ds;

base/mp_deleteconstraints.sas

@@ -49,4 +49,4 @@ data &outds;
   end;
 run;
 
-%mend;
+%mend mp_deleteconstraints;

base/mp_dirlist.sas

@@ -167,4 +167,4 @@ run;
     by filepath file_or_folder filename ext ;
   run;
 %end;
-%mend;
+%mend mp_dirlist;

base/mp_distinctfmtvalues.sas

@@ -47,4 +47,4 @@
     %end;
         as &outvar length=&varlen
     from &libds;
-%mend;
+%mend mp_distinctfmtvalues;

base/mp_ds2cards.sas

@@ -251,4 +251,4 @@ quit;
 %put NOTE-;%put NOTE-;
 %put NOTE- %sysfunc(dequote(&cards_file.));
 %put NOTE-;%put NOTE-;
-%mend;
+%mend mp_ds2cards;

base/mp_ds2csv.sas

@@ -55,4 +55,4 @@ data _null_;
 run;
 
 
-%mend;
+%mend mp_ds2csv;

base/mp_ds2inserts.sas

@@ -0,0 +1,154 @@
+/**
+  @file
+  @brief Export a dataset to SQL insert statements
+  @details Converts dataset values to SQL insert statements for use across
+  multiple database types.
+
+  Usage:
+
+      %mp_ds2inserts(sashelp.class,outref=myref,outds=class)
+      data class;
+        set sashelp.class;
+        stop;
+      proc sql;
+      %inc myref;
+
+  @param [in] ds The dataset to be exported
+  @param [out] outref= (0) The output fileref.  If it does not exist, it is
+    created. If it does exist, new records are APPENDED.
+  @param [out] outlib= (0) The library (or schema) in which the target table is
+    located.  If not provided, is ignored.
+  @param [out] outds= (0) The output table to load.  If not provided, will
+    default to the table in the &ds parameter.
+  @param [in] flavour= (BASE) The SQL flavour to be applied to the output. Valid
+    options:
+    @li BASE (default) - suitable for regular proc sql
+    @li PGSQL - Used for Postgres databases
+
+  <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mf_getvarcount.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+
+  @version 9.2
+  @author Allan Bowe (credit mjsq)
+**/
+
+%macro mp_ds2inserts(ds, outref=0,outlib=0,outds=0,flavour=BASE
+)/*/STORE SOURCE*/;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if %index(&ds,.)=0 %then %let ds=WORK.&ds;
+
+%let flavour=%upcase(&flavour);
+%if &flavour ne BASE and &flavour ne PGSQL %then %do;
+  %put %str(WAR)NING:  &flavour is not supported;
+  %return;
+%end;
+
+%if &outref=0 %then %do;
+  %put %str(WAR)NING:  Please provide a fileref;
+  %return;
+%end;
+%if %mf_existfileref(&outref)=0 %then %do;
+  filename &outref temp lrecl=66000;
+%end;
+
+%if &outlib=0 %then %let outlib=;
+%else %let outlib=&outlib..;
+
+%if &outds=0 %then %let outds=%scan(&ds,2,.);
+
+%local nobs;
+proc sql noprint;
+select count(*) into: nobs TRIMMED from &ds;
+%if &nobs=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No rows found in &ds */";
+  run;
+%end;
+
+%local vars;
+%let vars=%mf_getvarcount(&ds);
+%if &vars=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No columns found in &ds */";
+  run;
+%end;
+
+%local varlist varlistcomma;
+%let varlist=%mf_getvarlist(&ds);
+%let varlistcomma=%mf_getvarlist(&ds,dlm=%str(,),quote=double);
+
+/* next, export data */
+data _null_;
+  file &outref mod ;
+  if _n_=1 then put "/* &outlib.&outds (&nobs rows, &vars columns) */";
+  set &ds;
+  length _____str $32767;
+  format _numeric_ best.;
+  format _character_ ;
+  %local i comma var vtype;
+  %do i=1 %to %sysfunc(countw(&varlist));
+    %let var=%scan(&varlist,&i);
+    %let vtype=%mf_getvartype(&ds,&var);
+    %if &i=1 %then %do;
+      %if &flavour=BASE %then %do;
+        put "insert into &outlib.&outds set ";
+        put "  &var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        _____str=cats(
+          "INSERT INTO &outlib.&outds ("
+          ,symget('varlistcomma')
+          ,") VALUES ("
+        );
+        put _____str;
+        put "  "@;
+      %end;
+    %end;
+    %else %do;
+      %if &flavour=BASE %then %do;
+        put "  ,&var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        put "  ,"@;
+      %end;
+    %end;
+    %if &vtype=N %then %do;
+      %if &flavour=BASE %then %do;
+        put &var;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        if missing(&var) then put 'NULL';
+        else put &var;
+      %end;
+    %end;
+    %else %do;
+      _____str="'"!!trim(tranwrd(&var,"'","''"))!!"'";
+      put _____str;
+    %end;
+  %end;
+  %if &flavour=BASE %then %do;
+    put ';';
+  %end;
+  %else %if &flavour=PGSQL %then %do;
+    put ');';
+  %end;
+
+  if _n_=&nobs then put /;
+run;
+
+%mend mp_ds2inserts;

base/mp_filtergenerate.sas

@@ -99,4 +99,4 @@ filename &outref temp;
   run;
 %end;
 
-%mend;
+%mend mp_filtergenerate;

base/mp_getconstraints.sas

@@ -57,4 +57,4 @@ create table &outds as
   %end;
   ;
 
-%mend;
+%mend mp_getconstraints;

base/mp_getdbml.sas

@@ -332,4 +332,4 @@ run;
   run;
 %end;
 
-%mend;
+%mend mp_getdbml;

base/mp_getddl.sas

@@ -16,12 +16,13 @@
       %mp_getddl(work,test,flavour=tsql,showlog=YES)
 
   <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
   @li mp_getconstraints.sas
 
   @param lib libref of the library to create DDL for.  Should be assigned.
   @param ds dataset to create ddl for (optional)
-  @param fref= the fileref to which to write the DDL.  If not preassigned, will
-    be assigned to TEMP.
+  @param fref= the fileref to which to _append_ the DDL.  If it does not exist,
+    it will be created.
   @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
   @param showlog= Set to YES to show the DDL in the log
   @param schema= Choose a preferred schema name (default is to use actual schema
@@ -37,9 +38,10 @@
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %sysfunc(fileref(&fref)) > 0 %then %do;
-  filename &fref temp;
+%if %mf_existfileref(&outref)=0 %then %do;
+  filename &outref temp ;
 %end;
+
 %if %length(&libref)=0 %then %let libref=WORK;
 %let flavour=%upcase(&flavour);
 
@@ -115,10 +117,10 @@ create table _data_ as
   end;
   run;
   %put &=constraints_used;
-%mend;
+%mend addConst;
 
 data _null_;
-  file &fref;
+  file &fref mod;
   put "/* DDL generated by &sysuserid on %sysfunc(datetime(),datetime19.) */";
 run;
 
@@ -378,4 +380,4 @@ run;
   run;
 %end;
 
-%mend;
+%mend mp_getddl;

base/mp_getmaxvarlengths.sas

@@ -70,4 +70,4 @@ create table &outds (rename=(
     out=&outds(rename=(_name_=NAME COL1=MAXLEN));
   run;
 
-%mend;
+%mend mp_getmaxvarlengths;

base/mp_guesspk.sas

@@ -301,4 +301,4 @@
     %return;
   %end;
 
-%mend;
+%mend mp_guesspk;

base/mp_hashdataset.sas

@@ -20,6 +20,8 @@
   @li mf_getvartype.sas
 
   @param [in] libds dataset to hash
+  @param [in] salt= Provide a salt (could be, for instance, the name of the
+    dataset). Max 32 chars.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
   will contain one column (hashkey) with one observation (a hex32.
   representation of the input hash)
@@ -33,7 +35,8 @@
 
 %macro mp_hashdataset(
   libds,
-  outds=
+  outds=,
+  salt=
 )/*/STORE SOURCE*/;
   %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
     %put %str(WARN)ING: Dataset &libds is empty;, or is not a dataset;
@@ -53,7 +56,7 @@
     %let varlist=%mf_getvarlist(&libds);
     data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
       length &prevkeyvar &keyvar $32;
-      retain &prevkeyvar;
+      retain &prevkeyvar "&salt";
       set &libds end=&lastvar;
       /* hash should include previous row */
       &keyvar=put(md5(&prevkeyvar
@@ -72,4 +75,4 @@
       if &lastvar then output;
     run;
   %end;
-%mend;
+%mend mp_hashdataset;

base/mp_lib2cards.sas

@@ -75,4 +75,4 @@ select distinct lowcase(memname)
   )
 %end;
 
-%mend;
+%mend mp_lib2cards;

base/mp_perflog.sas

@@ -39,4 +39,4 @@
         ,dttm=%sysfunc(datetime());
   quit;
 
-%mend;
+%mend mp_perflog;

base/mp_prevobs.sas

@@ -85,4 +85,4 @@
     "with record &record and " _n_=;
 %end;
 
-%mend;
+%mend mp_prevobs;

base/mp_recursivejoin.sas

@@ -87,4 +87,4 @@ insert into &outds select distinct * from &append_ds;
     )
 %end;
 
-%mend;
+%mend mp_recursivejoin;

base/mp_resetoption.sas

@@ -30,4 +30,4 @@ data _null_;
   end;
 run;
 
-%mend;
+%mend mp_resetoption;

base/mp_runddl.sas

@@ -46,4 +46,4 @@
 
 
 
-%mend;
+%mend mp_runddl;

base/mp_searchdata.sas

@@ -117,4 +117,4 @@ proc sql
 
 %put process finished at %sysfunc(datetime(),datetime19.);
 
-%mend;
+%mend mp_searchdata;

base/mp_setkeyvalue.sas

@@ -49,4 +49,4 @@
 
   quit;
 
-%mend;
+%mend mp_setkeyvalue;

base/mp_stprequests.sas

@@ -71,4 +71,4 @@
   proc append base=&libds data=&syslast nowarn;run;
 
   options &etls_syntaxcheck;
-%mend;
+%mend mp_stprequests;

base/mp_streamfile.sas

@@ -134,4 +134,4 @@ run;
   %mp_binarycopy(inloc="&inloc",outref=_webout)
 %end;
 
-%mend;
+%mend mp_streamfile;

base/mp_testjob.sas

@@ -89,4 +89,4 @@ quit;
 libname &lib clear;
 
 
-%mend;
+%mend mp_testjob;

base/mp_testservice.sas

@@ -22,8 +22,11 @@
     |mustbevalidname|can be anything, oops, %abort!!|
 
   @param [in] debug= (log) Provide the _debug value
-  @param [in] viyaresult=(WEBOUT_JSON) The Viya result type to return.  For
+  @param [in] mdebug= (0) Set to 1 to provide macro debugging
+  @param [in] viyaresult= (WEBOUT_JSON) The Viya result type to return.  For
     more info, see mv_getjobresult.sas
+  @param [in] viyacontext= (SAS Job Execution compute context) The Viya compute
+    context on which to run the service
   @param [out] outlib= (0) Output libref to contain the final tables.  Set to
     0 if the service output is not in JSON format.
   @param [out] outref= (0) Output fileref to create, to contain the full _webout
@@ -47,17 +50,18 @@
   inputfiles=0,
   inputparams=0,
   debug=log,
+  mdebug=0,
   outlib=0,
   outref=0,
-  viyaresult=WEBOUT_JSON
+  viyaresult=WEBOUT_JSON,
+  viyacontext=SAS Job Execution compute context
 )/*/STORE SOURCE*/;
-%local mdebug;
-%if &debug ne 0 %then %do;
-  %let mdebug=1;
+%local dbg;
+%if &mdebug=1 %then %do;
   %put &sysmacroname entry vars:;
   %put _local_;
 %end;
-%else %let mdebug=0;
+%else %let dbg=*;
 
 /* sanitise inputparams */
 %local pcnt;
@@ -212,6 +216,7 @@
 
   data &ds1;
     retain _program "&program";
+    retain _contextname "&viyacontext";
     set &ds1;
     putlog "&sysmacroname inputparams:";
     putlog (_all_)(=);

base/mp_testwritespeedlibrary.sas

@@ -56,4 +56,4 @@ data &outds;
   duration_seconds=end_dttm-start_dttm;
 run;
 
-%mend;
+%mend mp_testwritespeedlibrary;

base/mp_tree.sas

@@ -68,4 +68,4 @@ data &outds ;
   rc=filename('tmp');
 run;
 
-%mend;
+%mend mp_tree;

base/mp_unzip.sas

@@ -63,4 +63,4 @@ data _null_;
     !!'filename &fname2 clear; filename &fname3 clear;');
 run;
 
-%mend;
+%mend mp_unzip;

base/mp_updatevarlength.sas

@@ -90,4 +90,4 @@ alter table &libds modify &var char(&len);
 
 %mp_createconstraints(inds=&dsconst,outds=&dsconst._addd,execute=YES)
 
-%mend;
+%mend mp_updatevarlength;

base/mp_zip.sas

@@ -76,4 +76,4 @@ ods package publish archive properties
   (archive_name="&outname..zip" archive_path="&outpath");
 ods package close;
 
-%mend;
+%mend mp_zip;

build.py

@@ -23,7 +23,7 @@ for file in files:
           ml.write("  put '" + line.rstrip().replace("'","''") + " ';\n")
     ml.write("run;\n\n")
     ml.write("%inc \"%sysfunc(pathname(work))/" + name + ".lua\";\n\n")
-    ml.write("%mend;\n")
+    ml.write("%mend " + name + ";\n")
 
 ml.close()
 

lua/ml_json.sas

@@ -391,4 +391,4 @@ run;
 
 %inc "%sysfunc(pathname(work))/ml_json.lua";
 
-%mend;
+%mend ml_json;

meta/mm_adduser2group.sas

@@ -95,4 +95,4 @@ run;
 
 filename __us2grp clear;
 
-%mend;
+%mend mm_adduser2group;

meta/mm_assigndirectlib.sas

@@ -460,4 +460,4 @@ run;
   %return;
 %end;
 
-%mend;
+%mend mm_assigndirectlib;

meta/mm_assignlib.sas

@@ -75,4 +75,4 @@
 %else %do;
   %put NOTE: Library &libref is already assigned;
 %end;
-%mend;
+%mend mm_assignlib;

meta/mm_createapplication.sas

@@ -152,4 +152,4 @@ run;
 %else %put NOTE: Application (&name) successfully created in (&tree)!;
 
 
-%mend;
+%mend mm_createapplication;

meta/mm_createdataset.sas

@@ -80,4 +80,4 @@ data _null_;
   if last then call execute('call missing(of _all_);stop;run;');
 run;
 
-%mend;
+%mend mm_createdataset;

meta/mm_createdocument.sas

@@ -122,4 +122,4 @@ run;
   run;
 %end;
 
-%mend;
+%mend mm_createdocument;

meta/mm_createfolder.sas

@@ -157,4 +157,4 @@ run;
 %end;
 
 %put &sysmacroname: execution finished for &path;
-%mend;
+%mend mm_createfolder;

meta/mm_createlibrary.sas

@@ -318,4 +318,4 @@ filename &frefout temp;
   filename &frefout clear;
 %end;
 
-%mend;
+%mend mm_createlibrary;

meta/mm_createstp.sas

@@ -385,4 +385,4 @@ run;
   %put %str(WARN)ING:  STPTYPE=*&stptype* not recognised!;
 %end;
 
-%mend;
+%mend mm_createstp;

meta/mm_createwebservice.sas

@@ -415,7 +415,7 @@ data _null_;
   put ' ';
   put '  %quote(&user) ';
   put ' ';
-  put '%mend; ';
+  put '%mend mf_getuser; ';
 /* WEBOUT END */
   put '%macro webout(action,ds,dslabel=,fmt=);';
   put '  %mm_webout(&action,ds=&ds,dslabel=&dslabel,fmt=&fmt)';

meta/mm_deletedocument.sas

@@ -68,4 +68,4 @@ run;
   %return;
 %end;
 
-%mend;
+%mend mm_deletedocument;

meta/mm_deletelibrary.sas

@@ -89,4 +89,4 @@ run;
 
 %put &sysmacroname: Library &name (&liburi) was successfully deleted;
 
-%mend;
+%mend mm_deletelibrary;

meta/mm_deletestp.sas

@@ -67,4 +67,4 @@ run;
   %return;
 %end;
 
-%mend;
+%mend mm_deletestp;

meta/mm_getauthinfo.sas

@@ -113,4 +113,4 @@ run;
 
 filename &fileref clear;
 
-%mend;
+%mend mm_getauthinfo;

meta/mm_getcols.sas

@@ -50,4 +50,4 @@ proc sort;
   by colname;
 run;
 
-%mend;
+%mend mm_getcols;

meta/mm_getdetails.sas

@@ -62,4 +62,4 @@ proc sort;
   by type name;
 run;
 
-%mend;
+%mend mm_getdetails;

meta/mm_getdirectories.sas

@@ -54,4 +54,4 @@ data &outds (keep=directoryuri name directoryname directorydesc );
   end;
 run;
 
-%mend;
+%mend mm_getDirectories;

meta/mm_getdocument.sas

@@ -143,4 +143,4 @@ run;
 filename __getdoc clear;
 filename __outdoc clear;
 
-%mend;
+%mend mm_getdocument;

meta/mm_getfoldermembers.sas

@@ -93,4 +93,4 @@
   run;
 %end;
 
-%mend;
+%mend mm_getfoldermembers;

meta/mm_getfoldertree.sas

@@ -88,4 +88,4 @@ data _null_;
     !!",level=%eval(&level+1),append=YES)");
 run;
 
-%mend;
+%mend mm_getfoldertree;

meta/mm_getgroupmembers.sas

@@ -69,4 +69,4 @@
     end;
   run;
 
-%mend;
+%mend mm_getgroupmembers;

meta/mm_getgroups.sas

@@ -92,4 +92,4 @@
   options metarepository=&oldrepo;
 %end;
 
-%mend;
+%mend mm_getGroups;

meta/mm_getlibs.sas

@@ -94,4 +94,4 @@ filename sxlemap clear;
 filename response clear;
 libname _XML_ clear;
 
-%mend;
+%mend mm_getlibs;

meta/mm_getobjects.sas

@@ -67,4 +67,4 @@ filename sxlemap clear;
 filename response clear;
 libname _XML_ clear;
 
-%mend;
+%mend mm_getobjects;