Merge pull request #55 from sasjs/insertforpg

feat: updating mp_ds2inserts to support postgres database

.all-contributorsrc

@@ -0,0 +1,104 @@
+{
+  "projectName": "core",
+  "projectOwner": "sasjs",
+  "repoType": "github",
+  "repoHost": "https://github.com",
+  "files": [
+    "README.md"
+  ],
+  "imageSize": 100,
+  "commit": false,
+  "commitConvention": "angular",
+  "contributors": [
+    {
+      "login": "allanbowe",
+      "name": "Allan Bowe",
+      "avatar_url": "https://avatars.githubusercontent.com/u/4420615?v=4",
+      "profile": "https://github.com/allanbowe",
+      "contributions": [
+        "business",
+        "code",
+        "content",
+        "doc",
+        "infra",
+        "maintenance",
+        "mentoring",
+        "question",
+        "review",
+        "test"
+      ]
+    },
+    {
+      "login": "rafgag",
+      "name": "rafgag",
+      "avatar_url": "https://avatars.githubusercontent.com/u/69139928?v=4",
+      "profile": "https://github.com/rafgag",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "tmoody",
+      "name": "Trevor Moody",
+      "avatar_url": "https://avatars.githubusercontent.com/u/79837106?v=4",
+      "profile": "https://github.com/tmoody",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "krishna-acondy",
+      "name": "Krishna Acondy",
+      "avatar_url": "https://avatars.githubusercontent.com/u/2980428?v=4",
+      "profile": "https://krishna-acondy.io/",
+      "contributions": [
+        "code",
+        "infra",
+        "blog",
+        "content",
+        "ideas",
+        "video"
+      ]
+    },
+    {
+      "login": "saadjutt01",
+      "name": "Muhammad Saad ",
+      "avatar_url": "https://avatars.githubusercontent.com/u/8914650?v=4",
+      "profile": "https://github.com/saadjutt01",
+      "contributions": [
+        "code",
+        "ideas"
+      ]
+    },
+    {
+      "login": "YuryShkoda",
+      "name": "Yury Shkoda",
+      "avatar_url": "https://avatars.githubusercontent.com/u/25773492?v=4",
+      "profile": "https://www.erudicat.com/",
+      "contributions": [
+        "code",
+        "infra",
+        "video"
+      ]
+    },
+    {
+      "login": "medjedovicm",
+      "name": "Mihajlo Medjedovic",
+      "avatar_url": "https://avatars.githubusercontent.com/u/18329105?v=4",
+      "profile": "https://github.com/medjedovicm",
+      "contributions": [
+        "infra"
+      ]
+    },
+    {
+      "login": "kkchandok",
+      "name": "kkchandok",
+      "avatar_url": "https://avatars.githubusercontent.com/u/46090627?v=4",
+      "profile": "https://github.com/kkchandok",
+      "contributions": [
+        "ideas"
+      ]
+    }
+  ],
+  "contributorsPerLine": 7
+}

README.md

@@ -179,3 +179,34 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 
 
+
+## Contributors ✨
+<!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
+[![All Contributors](https://img.shields.io/badge/all_contributors-8-orange.svg?style=flat-square)](#contributors-)
+<!-- ALL-CONTRIBUTORS-BADGE:END -->
+Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
+
+<!-- ALL-CONTRIBUTORS-LIST:START - Do not remove or modify this section -->
+<!-- prettier-ignore-start -->
+<!-- markdownlint-disable -->
+<table>
+  <tr>
+    <td align="center"><a href="https://github.com/allanbowe"><img src="https://avatars.githubusercontent.com/u/4420615?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Allan Bowe</b></sub></a><br /><a href="#business-allanbowe" title="Business development">💼</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Code">💻</a> <a href="#content-allanbowe" title="Content">🖋</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Documentation">📖</a> <a href="#infra-allanbowe" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#maintenance-allanbowe" title="Maintenance">🚧</a> <a href="#mentoring-allanbowe" title="Mentoring">🧑‍🏫</a> <a href="#question-allanbowe" title="Answering Questions">💬</a> <a href="https://github.com/sasjs/core/pulls?q=is%3Apr+reviewed-by%3Aallanbowe" title="Reviewed Pull Requests">👀</a> <a href="https://github.com/sasjs/core/commits?author=allanbowe" title="Tests">⚠️</a></td>
+    <td align="center"><a href="https://github.com/rafgag"><img src="https://avatars.githubusercontent.com/u/69139928?v=4?s=100" width="100px;" alt=""/><br /><sub><b>rafgag</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=rafgag" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/tmoody"><img src="https://avatars.githubusercontent.com/u/79837106?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Trevor Moody</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=tmoody" title="Code">💻</a></td>
+    <td align="center"><a href="https://krishna-acondy.io/"><img src="https://avatars.githubusercontent.com/u/2980428?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Krishna Acondy</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=krishna-acondy" title="Code">💻</a> <a href="#infra-krishna-acondy" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#blog-krishna-acondy" title="Blogposts">📝</a> <a href="#content-krishna-acondy" title="Content">🖋</a> <a href="#ideas-krishna-acondy" title="Ideas, Planning, & Feedback">🤔</a> <a href="#video-krishna-acondy" title="Videos">📹</a></td>
+    <td align="center"><a href="https://github.com/saadjutt01"><img src="https://avatars.githubusercontent.com/u/8914650?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Muhammad Saad </b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=saadjutt01" title="Code">💻</a> <a href="#ideas-saadjutt01" title="Ideas, Planning, & Feedback">🤔</a></td>
+    <td align="center"><a href="https://www.erudicat.com/"><img src="https://avatars.githubusercontent.com/u/25773492?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Yury Shkoda</b></sub></a><br /><a href="https://github.com/sasjs/core/commits?author=YuryShkoda" title="Code">💻</a> <a href="#infra-YuryShkoda" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a> <a href="#video-YuryShkoda" title="Videos">📹</a></td>
+    <td align="center"><a href="https://github.com/medjedovicm"><img src="https://avatars.githubusercontent.com/u/18329105?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Mihajlo Medjedovic</b></sub></a><br /><a href="#infra-medjedovicm" title="Infrastructure (Hosting, Build-Tools, etc)">🚇</a></td>
+  </tr>
+  <tr>
+    <td align="center"><a href="https://github.com/kkchandok"><img src="https://avatars.githubusercontent.com/u/46090627?v=4?s=100" width="100px;" alt=""/><br /><sub><b>kkchandok</b></sub></a><br /><a href="#ideas-kkchandok" title="Ideas, Planning, & Feedback">🤔</a></td>
+  </tr>
+</table>
+
+<!-- markdownlint-restore -->
+<!-- prettier-ignore-end -->
+
+<!-- ALL-CONTRIBUTORS-LIST:END -->
+
+This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!

all.sas

@@ -1678,6 +1678,7 @@ Usage:
         sysuserid=symget('sysuserid');
         iftrue=symget('iftrue');
         put (_all_)(/=);
+        call symputx('syscc',0);
         abort cancel nolist;
       run;
     %end;
@@ -2172,6 +2173,122 @@ Usage:
   drop table &ds;
 
 %mend mp_assertdsobs;/**
+  @file
+  @brief Convert a file to/from base64 format
+  @details Creates a new version of a file either encoded or decoded using
+  Base64.  Inspired by this post by Michael Dixon:
+  https://support.selerity.com.au/hc/en-us/articles/223345708-Tip-SAS-and-Base64
+
+  Usage:
+
+        filename tmp temp;
+        data _null_;
+          file tmp;
+          put 'base ik ally';
+        run;
+        %mp_base64copy(inref=tmp, outref=myref, action=ENCODE)
+
+        data _null_;
+          infile myref;
+          input;
+          put _infile_;
+        run;
+
+        %mp_base64copy(inref=myref, outref=mynewref, action=DECODE)
+
+        data _null_;
+          infile mynewref;
+          input;
+          put _infile_;
+        run;
+
+  @param [in] inref= Fileref of the input file (should exist)
+  @param [out] outref= Output fileref. If it does not exist, it is created.
+  @param [in] action= (ENCODE) The action to take. Valid values:
+    @li ENCODE - Convert the file to base64 format
+    @li DECODE - Decode the file from base64 format
+
+  @version 9.2
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+
+
+**/
+
+%macro mp_base64copy(
+  inref=0,
+  outref=0,
+  action=ENCODE
+)/*/STORE SOURCE*/;
+
+%let inref=%upcase(&inref);
+%let outref=%upcase(&outref);
+%let action=%upcase(&action);
+%local infound outfound;
+%let infound=0;
+%let outfound=0;
+data _null_;
+  set sashelp.vextfl(where=(fileref="&inref" or fileref="&outref"));
+  if fileref="&inref" then call symputx('infound',1,'l');
+  if fileref="&outref" then call symputx('outfound',1,'l');
+run;
+
+%mp_abort(iftrue= (&infound=0)
+  ,mac=&sysmacroname
+  ,msg=%str(INREF &inref NOT FOUND!)
+)
+%mp_abort(iftrue= (&outref=0)
+  ,mac=&sysmacroname
+  ,msg=%str(OUTREF NOT PROVIDED!)
+)
+%mp_abort(iftrue= (&action ne ENCODE and &action ne DECODE)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid action! Should be ENCODE OR DECODE)
+)
+
+%if &outfound=0 %then %do;
+  filename &outref temp lrecl=2097088;
+%end;
+
+%if &action=ENCODE %then %do;
+  data _null_;
+    length b64 $ 76 line $ 57;
+    retain line "";
+    infile &inref recfm=F lrecl= 1 end=eof;
+    input @1 stream $char1.;
+    file &outref recfm=N;
+    substr(line,(_N_-(CEIL(_N_/57)-1)*57),1) = byte(rank(stream));
+    if mod(_N_,57)=0 or EOF then do;
+      if eof then b64=put(trim(line),$base64X76.);
+      else b64=put(line, $base64X76.);
+      put b64 + (-1) @;
+      line="";
+    end;
+  run;
+%end;
+%else %if &action=DECODE %then %do;
+  data _null_;
+    length filein 8 fileout 8;
+    filein = fopen("&inref",'I',4,'B');
+    fileout = fopen("&outref",'O',3,'B');
+    char= '20'x;
+    do while(fread(filein)=0);
+      length raw $4;
+      do i=1 to 4;
+        rc=fget(filein,char,1);
+        substr(raw,i,1)=char;
+      end;
+      rc = fput(fileout,input(raw,$base64X4.));
+      rc = fwrite(fileout);
+    end;
+    rc = fclose(filein);
+    rc = fclose(fileout);
+  run;
+%end;
+
+%mend mp_base64copy;/**
   @file
   @brief Copy any file using binary input / output streams
   @details Reads in a file byte by byte and writes it back out.  Is an
@@ -3306,6 +3423,159 @@ data &outds;
 run;
 
 %mend mp_ds2fmtds;/**
+  @file
+  @brief Export a dataset to SQL insert statements
+  @details Converts dataset values to SQL insert statements for use across
+  multiple database types.
+
+  Usage:
+
+      %mp_ds2inserts(sashelp.class,outref=myref,outds=class)
+      data class;
+        set sashelp.class;
+        stop;
+      proc sql;
+      %inc myref;
+
+  @param [in] ds The dataset to be exported
+  @param [out] outref= (0) The output fileref.  If it does not exist, it is
+    created. If it does exist, new records are APPENDED.
+  @param [out] outlib= (0) The library (or schema) in which the target table is
+    located.  If not provided, is ignored.
+  @param [out] outds= (0) The output table to load.  If not provided, will
+    default to the table in the &ds parameter.
+  @param [in] flavour= (BASE) The SQL flavour to be applied to the output. Valid
+    options:
+    @li BASE (default) - suitable for regular proc sql
+    @li PGSQL - Used for Postgres databases
+
+  <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mf_getvarcount.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+
+  @version 9.2
+  @author Allan Bowe (credit mjsq)
+**/
+
+%macro mp_ds2inserts(ds, outref=0,outlib=0,outds=0,flavour=BASE
+)/*/STORE SOURCE*/;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if %index(&ds,.)=0 %then %let ds=WORK.&ds;
+
+%let flavour=%upcase(&flavour);
+%if &flavour ne BASE and &flavour ne PGSQL %then %do;
+  %put %str(WAR)NING:  &flavour is not supported;
+  %return;
+%end;
+
+%if &outref=0 %then %do;
+  %put %str(WAR)NING:  Please provide a fileref;
+  %return;
+%end;
+%if %mf_existfileref(&outref)=0 %then %do;
+  filename &outref temp lrecl=66000;
+%end;
+
+%if &outlib=0 %then %let outlib=;
+%else %let outlib=&outlib..;
+
+%if &outds=0 %then %let outds=%scan(&ds,2,.);
+
+%local nobs;
+proc sql noprint;
+select count(*) into: nobs TRIMMED from &ds;
+%if &nobs=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No rows found in &ds */";
+  run;
+%end;
+
+%local vars;
+%let vars=%mf_getvarcount(&ds);
+%if &vars=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No columns found in &ds */";
+  run;
+%end;
+
+%local varlist varlistcomma;
+%let varlist=%mf_getvarlist(&ds);
+%let varlistcomma=%mf_getvarlist(&ds,dlm=%str(,),quote=double);
+
+/* next, export data */
+data _null_;
+  file &outref mod ;
+  if _n_=1 then put "/* &outlib.&outds (&nobs rows, &vars columns) */";
+  set &ds;
+  length _____str $32767;
+  format _numeric_ best.;
+  format _character_ ;
+  %local i comma var vtype;
+  %do i=1 %to %sysfunc(countw(&varlist));
+    %let var=%scan(&varlist,&i);
+    %let vtype=%mf_getvartype(&ds,&var);
+    %if &i=1 %then %do;
+      %if &flavour=BASE %then %do;
+        put "insert into &outlib.&outds set ";
+        put "  &var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        _____str=cats(
+          "INSERT INTO &outlib.&outds ("
+          ,symget('varlistcomma')
+          ,") VALUES ("
+        );
+        put _____str;
+        put "  "@;
+      %end;
+    %end;
+    %else %do;
+      %if &flavour=BASE %then %do;
+        put "  ,&var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        put "  ,"@;
+      %end;
+    %end;
+    %if &vtype=N %then %do;
+      %if &flavour=BASE %then %do;
+        put &var;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        if missing(&var) then put 'NULL';
+        else put &var;
+      %end;
+    %end;
+    %else %do;
+      _____str="'"!!trim(tranwrd(&var,"'","''"))!!"'";
+      put _____str;
+    %end;
+  %end;
+  %if &flavour=BASE %then %do;
+    put ';';
+  %end;
+  %else %if &flavour=PGSQL %then %do;
+    put ');';
+  %end;
+
+  if _n_=&nobs then put /;
+run;
+
+%mend mp_ds2inserts;/**
   @file
   @brief Checks an input filter table for validity
   @details Performs checks on the input table to ensure it arrives in the
@@ -4127,12 +4397,13 @@ run;
       %mp_getddl(work,test,flavour=tsql,showlog=YES)
 
   <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
   @li mp_getconstraints.sas
 
   @param lib libref of the library to create DDL for.  Should be assigned.
   @param ds dataset to create ddl for (optional)
-  @param fref= the fileref to which to write the DDL.  If not preassigned, will
+  @param fref= the fileref to which to _append_ the DDL.  If it does not exist,
-    be assigned to TEMP.
+    it will be created.
   @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
   @param showlog= Set to YES to show the DDL in the log
   @param schema= Choose a preferred schema name (default is to use actual schema
@@ -4148,9 +4419,10 @@ run;
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %sysfunc(fileref(&fref)) > 0 %then %do;
+%if %mf_existfileref(&outref)=0 %then %do;
-  filename &fref temp;
+  filename &outref temp ;
 %end;
+
 %if %length(&libref)=0 %then %let libref=WORK;
 %let flavour=%upcase(&flavour);
 
@@ -4229,7 +4501,7 @@ create table _data_ as
 %mend addConst;
 
 data _null_;
-  file &fref;
+  file &fref mod;
   put "/* DDL generated by &sysuserid on %sysfunc(datetime(),datetime19.) */";
 run;
 
@@ -4886,6 +5158,8 @@ create table &outds (rename=(
   @li mf_getvartype.sas
 
   @param [in] libds dataset to hash
+  @param [in] salt= Provide a salt (could be, for instance, the name of the
+    dataset). Max 32 chars.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
   will contain one column (hashkey) with one observation (a hex32.
   representation of the input hash)
@@ -4899,7 +5173,8 @@ create table &outds (rename=(
 
 %macro mp_hashdataset(
   libds,
-  outds=
+  outds=,
+  salt=
 )/*/STORE SOURCE*/;
   %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
     %put %str(WARN)ING: Dataset &libds is empty;, or is not a dataset;
@@ -4919,7 +5194,7 @@ create table &outds (rename=(
     %let varlist=%mf_getvarlist(&libds);
     data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
       length &prevkeyvar &keyvar $32;
-      retain &prevkeyvar;
+      retain &prevkeyvar "&salt";
       set &libds end=&lastvar;
       /* hash should include previous row */
       &keyvar=put(md5(&prevkeyvar
@@ -6241,8 +6516,11 @@ libname &lib clear;
     |mustbevalidname|can be anything, oops, %abort!!|
 
   @param [in] debug= (log) Provide the _debug value
-  @param [in] viyaresult=(WEBOUT_JSON) The Viya result type to return.  For
+  @param [in] mdebug= (0) Set to 1 to provide macro debugging
+  @param [in] viyaresult= (WEBOUT_JSON) The Viya result type to return.  For
     more info, see mv_getjobresult.sas
+  @param [in] viyacontext= (SAS Job Execution compute context) The Viya compute
+    context on which to run the service
   @param [out] outlib= (0) Output libref to contain the final tables.  Set to
     0 if the service output is not in JSON format.
   @param [out] outref= (0) Output fileref to create, to contain the full _webout
@@ -6266,17 +6544,18 @@ libname &lib clear;
   inputfiles=0,
   inputparams=0,
   debug=log,
+  mdebug=0,
   outlib=0,
   outref=0,
-  viyaresult=WEBOUT_JSON
+  viyaresult=WEBOUT_JSON,
+  viyacontext=SAS Job Execution compute context
 )/*/STORE SOURCE*/;
-%local mdebug;
+%local dbg;
-%if &debug ne 0 %then %do;
+%if &mdebug=1 %then %do;
-  %let mdebug=1;
   %put &sysmacroname entry vars:;
   %put _local_;
 %end;
-%else %let mdebug=0;
+%else %let dbg=*;
 
 /* sanitise inputparams */
 %local pcnt;
@@ -6431,6 +6710,7 @@ libname &lib clear;
 
   data &ds1;
     retain _program "&program";
+    retain _contextname "&viyacontext";
     set &ds1;
     putlog "&sysmacroname inputparams:";
     putlog (_all_)(=);
@@ -12702,9 +12982,7 @@ data _null_;
   putlog _infile_;
 run;
 
-
+%mend mmx_deletemetafolder;/**
-%mend mmx_deletemetafolder;
-/**
   @file mmx_spkexport.sas
   @brief Exports everything in a particular metadata folder
   @details Will export everything in a metadata folder to a specified location.
@@ -12912,9 +13190,13 @@ run;
   @param [in] path= The parent folder in which to create the file
   @param [in] name= The name of the file to be created
   @param [in] inref= The fileref pointing to the file to be uploaded
+  @param [in] intype= (BINARY) The type of the input data.  Valid values:
+    @li BINARY File is copied byte for byte using the mp_binarycopy.sas macro.
+    @li BASE64 File will be first decoded using the mp_base64.sas macro, then
+      loaded byte by byte to SAS Drive.
   @param [in] contentdisp= (inline) Content Disposition. Example values:
-      @li inline
+    @li inline
-      @li attachment
+    @li attachment
 
   @param [in] access_token_var= The global macro variable to contain the access
     token, if using authorization_code grant type.
@@ -12932,6 +13214,7 @@ run;
   @li mf_getuniquefileref.sas
   @li mf_isblank.sas
   @li mp_abort.sas
+  @li mp_base64copy.sas
   @li mp_binarycopy.sas
   @li mv_createfolder.sas
 
@@ -12940,6 +13223,7 @@ run;
 %macro mv_createfile(path=
     ,name=
     ,inref=
+    ,intype=BINARY
     ,contentdisp=inline
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
@@ -12994,7 +13278,12 @@ filename &fref filesrvc
   cdisp="&contentdisp"
   lrecl=1048544;
 
-%mp_binarycopy(inref=&inref, outref=&fref)
+%if &intype=BINARY %then %do;
+  %mp_binarycopy(inref=&inref, outref=&fref)
+%end;
+%else %if &intype=BASE64 %then %do;
+  %mp_base64copy(inref=&inref, outref=&fref, action=DECODE)
+%end;
 
 filename &fref clear;
 
@@ -15439,7 +15728,7 @@ run;
 
   @param [in] access_token_var= The global macro variable to contain the access
     token
-  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
   @param [in] grant_type= valid values:
     @li password
     @li authorization_code
@@ -15464,7 +15753,6 @@ run;
 **/
 
 %macro mv_getjoblog(uri=0,outref=0
-    ,contextName=SAS Job Execution compute context
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
     ,mdebug=0
@@ -15785,7 +16073,6 @@ run;
 **/
 
 %macro mv_getjobresult(uri=0
-    ,contextName=SAS Job Execution compute context
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
     ,mdebug=0
@@ -15899,10 +16186,11 @@ proc http method='GET' out=&fname2 &oauth_bearer
   ;
 run;
 %if &mdebug=1 %then %do;
+  /* send one char at a time as the json can be very wide */
   data _null_;
-    infile &fname2 lrecl=32767;
+    infile &fname2 recfm=n;
-    input;
+    input char $char1. ;
-    putlog _infile_;
+    putlog char $char1. @;
   run;
 %end;
 
@@ -16659,6 +16947,11 @@ run;
 %if &mdebug=1 %then %do;
   %put &sysmacroname entry vars:;
   %put _local_;
+  %put inds vars:;
+  data _null_;
+    set &inds;
+    putlog (_all_)(=);
+  run;
 %end;
 %else %let dbg=*;
 
@@ -16703,6 +16996,7 @@ run;
       retain FLOW_ID 0;
     %end;
     set &inds;
+    &dbg. putlog (_all_)(=);
   run;
 %end;
 
@@ -16767,6 +17061,8 @@ data;run;%let jdswaitfor=&syslast;
       call symputx(cats('job',_n_),_program,'l');
       call symputx(cats('context',_n_),_contextName,'l');
       call symputx('jcnt',_n_,'l');
+      &dbg. if _n_= 1 then putlog "Loop &fid";
+      &dbg. putlog (_all_)(=);
     run;
     %put exporting job variables in json format;
     %do jid=1 %to &jcnt;
@@ -16828,6 +17124,7 @@ data;run;%let jdswaitfor=&syslast;
               ,name=&jobname
               ,paramstring=%superq(jparams&jid)
               ,outds=&jdsapp
+              ,contextname=&&context&jid
             )
             data &jdsapp;
               format jobparams $32767.;

base/mp_abort.sas

@@ -165,6 +165,7 @@
         sysuserid=symget('sysuserid');
         iftrue=symget('iftrue');
         put (_all_)(/=);
+        call symputx('syscc',0);
         abort cancel nolist;
       run;
     %end;

base/mp_base64copy.sas

@@ -0,0 +1,117 @@
+/**
+  @file
+  @brief Convert a file to/from base64 format
+  @details Creates a new version of a file either encoded or decoded using
+  Base64.  Inspired by this post by Michael Dixon:
+  https://support.selerity.com.au/hc/en-us/articles/223345708-Tip-SAS-and-Base64
+
+  Usage:
+
+        filename tmp temp;
+        data _null_;
+          file tmp;
+          put 'base ik ally';
+        run;
+        %mp_base64copy(inref=tmp, outref=myref, action=ENCODE)
+
+        data _null_;
+          infile myref;
+          input;
+          put _infile_;
+        run;
+
+        %mp_base64copy(inref=myref, outref=mynewref, action=DECODE)
+
+        data _null_;
+          infile mynewref;
+          input;
+          put _infile_;
+        run;
+
+  @param [in] inref= Fileref of the input file (should exist)
+  @param [out] outref= Output fileref. If it does not exist, it is created.
+  @param [in] action= (ENCODE) The action to take. Valid values:
+    @li ENCODE - Convert the file to base64 format
+    @li DECODE - Decode the file from base64 format
+
+  @version 9.2
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+
+
+**/
+
+%macro mp_base64copy(
+  inref=0,
+  outref=0,
+  action=ENCODE
+)/*/STORE SOURCE*/;
+
+%let inref=%upcase(&inref);
+%let outref=%upcase(&outref);
+%let action=%upcase(&action);
+%local infound outfound;
+%let infound=0;
+%let outfound=0;
+data _null_;
+  set sashelp.vextfl(where=(fileref="&inref" or fileref="&outref"));
+  if fileref="&inref" then call symputx('infound',1,'l');
+  if fileref="&outref" then call symputx('outfound',1,'l');
+run;
+
+%mp_abort(iftrue= (&infound=0)
+  ,mac=&sysmacroname
+  ,msg=%str(INREF &inref NOT FOUND!)
+)
+%mp_abort(iftrue= (&outref=0)
+  ,mac=&sysmacroname
+  ,msg=%str(OUTREF NOT PROVIDED!)
+)
+%mp_abort(iftrue= (&action ne ENCODE and &action ne DECODE)
+  ,mac=&sysmacroname
+  ,msg=%str(Invalid action! Should be ENCODE OR DECODE)
+)
+
+%if &outfound=0 %then %do;
+  filename &outref temp lrecl=2097088;
+%end;
+
+%if &action=ENCODE %then %do;
+  data _null_;
+    length b64 $ 76 line $ 57;
+    retain line "";
+    infile &inref recfm=F lrecl= 1 end=eof;
+    input @1 stream $char1.;
+    file &outref recfm=N;
+    substr(line,(_N_-(CEIL(_N_/57)-1)*57),1) = byte(rank(stream));
+    if mod(_N_,57)=0 or EOF then do;
+      if eof then b64=put(trim(line),$base64X76.);
+      else b64=put(line, $base64X76.);
+      put b64 + (-1) @;
+      line="";
+    end;
+  run;
+%end;
+%else %if &action=DECODE %then %do;
+  data _null_;
+    length filein 8 fileout 8;
+    filein = fopen("&inref",'I',4,'B');
+    fileout = fopen("&outref",'O',3,'B');
+    char= '20'x;
+    do while(fread(filein)=0);
+      length raw $4;
+      do i=1 to 4;
+        rc=fget(filein,char,1);
+        substr(raw,i,1)=char;
+      end;
+      rc = fput(fileout,input(raw,$base64X4.));
+      rc = fwrite(fileout);
+    end;
+    rc = fclose(filein);
+    rc = fclose(fileout);
+  run;
+%end;
+
+%mend mp_base64copy;

base/mp_ds2inserts.sas

@@ -0,0 +1,154 @@
+/**
+  @file
+  @brief Export a dataset to SQL insert statements
+  @details Converts dataset values to SQL insert statements for use across
+  multiple database types.
+
+  Usage:
+
+      %mp_ds2inserts(sashelp.class,outref=myref,outds=class)
+      data class;
+        set sashelp.class;
+        stop;
+      proc sql;
+      %inc myref;
+
+  @param [in] ds The dataset to be exported
+  @param [out] outref= (0) The output fileref.  If it does not exist, it is
+    created. If it does exist, new records are APPENDED.
+  @param [out] outlib= (0) The library (or schema) in which the target table is
+    located.  If not provided, is ignored.
+  @param [out] outds= (0) The output table to load.  If not provided, will
+    default to the table in the &ds parameter.
+  @param [in] flavour= (BASE) The SQL flavour to be applied to the output. Valid
+    options:
+    @li BASE (default) - suitable for regular proc sql
+    @li PGSQL - Used for Postgres databases
+
+  <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
+  @li mf_getvarcount.sas
+  @li mf_getvarlist.sas
+  @li mf_getvartype.sas
+
+  @version 9.2
+  @author Allan Bowe (credit mjsq)
+**/
+
+%macro mp_ds2inserts(ds, outref=0,outlib=0,outds=0,flavour=BASE
+)/*/STORE SOURCE*/;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+  %put %str(WAR)NING:  &ds does not exist;
+  %return;
+%end;
+
+%if %index(&ds,.)=0 %then %let ds=WORK.&ds;
+
+%let flavour=%upcase(&flavour);
+%if &flavour ne BASE and &flavour ne PGSQL %then %do;
+  %put %str(WAR)NING:  &flavour is not supported;
+  %return;
+%end;
+
+%if &outref=0 %then %do;
+  %put %str(WAR)NING:  Please provide a fileref;
+  %return;
+%end;
+%if %mf_existfileref(&outref)=0 %then %do;
+  filename &outref temp lrecl=66000;
+%end;
+
+%if &outlib=0 %then %let outlib=;
+%else %let outlib=&outlib..;
+
+%if &outds=0 %then %let outds=%scan(&ds,2,.);
+
+%local nobs;
+proc sql noprint;
+select count(*) into: nobs TRIMMED from &ds;
+%if &nobs=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No rows found in &ds */";
+  run;
+%end;
+
+%local vars;
+%let vars=%mf_getvarcount(&ds);
+%if &vars=0 %then %do;
+  data _null_;
+    file &outref mod;
+    put "/* No columns found in &ds */";
+  run;
+%end;
+
+%local varlist varlistcomma;
+%let varlist=%mf_getvarlist(&ds);
+%let varlistcomma=%mf_getvarlist(&ds,dlm=%str(,),quote=double);
+
+/* next, export data */
+data _null_;
+  file &outref mod ;
+  if _n_=1 then put "/* &outlib.&outds (&nobs rows, &vars columns) */";
+  set &ds;
+  length _____str $32767;
+  format _numeric_ best.;
+  format _character_ ;
+  %local i comma var vtype;
+  %do i=1 %to %sysfunc(countw(&varlist));
+    %let var=%scan(&varlist,&i);
+    %let vtype=%mf_getvartype(&ds,&var);
+    %if &i=1 %then %do;
+      %if &flavour=BASE %then %do;
+        put "insert into &outlib.&outds set ";
+        put "  &var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        _____str=cats(
+          "INSERT INTO &outlib.&outds ("
+          ,symget('varlistcomma')
+          ,") VALUES ("
+        );
+        put _____str;
+        put "  "@;
+      %end;
+    %end;
+    %else %do;
+      %if &flavour=BASE %then %do;
+        put "  ,&var="@;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        put "  ,"@;
+      %end;
+    %end;
+    %if &vtype=N %then %do;
+      %if &flavour=BASE %then %do;
+        put &var;
+      %end;
+      %else %if &flavour=PGSQL %then %do;
+        if missing(&var) then put 'NULL';
+        else put &var;
+      %end;
+    %end;
+    %else %do;
+      _____str="'"!!trim(tranwrd(&var,"'","''"))!!"'";
+      put _____str;
+    %end;
+  %end;
+  %if &flavour=BASE %then %do;
+    put ';';
+  %end;
+  %else %if &flavour=PGSQL %then %do;
+    put ');';
+  %end;
+
+  if _n_=&nobs then put /;
+run;
+
+%mend mp_ds2inserts;

base/mp_getddl.sas

@@ -16,12 +16,13 @@
       %mp_getddl(work,test,flavour=tsql,showlog=YES)
 
   <h4> SAS Macros </h4>
+  @li mf_existfileref.sas
   @li mp_getconstraints.sas
 
   @param lib libref of the library to create DDL for.  Should be assigned.
   @param ds dataset to create ddl for (optional)
-  @param fref= the fileref to which to write the DDL.  If not preassigned, will
+  @param fref= the fileref to which to _append_ the DDL.  If it does not exist,
-    be assigned to TEMP.
+    it will be created.
   @param flavour= The type of DDL to create (default=SAS). Supported=TSQL
   @param showlog= Set to YES to show the DDL in the log
   @param schema= Choose a preferred schema name (default is to use actual schema
@@ -37,9 +38,10 @@
 )/*/STORE SOURCE*/;
 
 /* check fileref is assigned */
-%if %sysfunc(fileref(&fref)) > 0 %then %do;
+%if %mf_existfileref(&outref)=0 %then %do;
-  filename &fref temp;
+  filename &outref temp ;
 %end;
+
 %if %length(&libref)=0 %then %let libref=WORK;
 %let flavour=%upcase(&flavour);
 
@@ -118,7 +120,7 @@ create table _data_ as
 %mend addConst;
 
 data _null_;
-  file &fref;
+  file &fref mod;
   put "/* DDL generated by &sysuserid on %sysfunc(datetime(),datetime19.) */";
 run;
 

base/mp_hashdataset.sas

@@ -20,6 +20,8 @@
   @li mf_getvartype.sas
 
   @param [in] libds dataset to hash
+  @param [in] salt= Provide a salt (could be, for instance, the name of the
+    dataset). Max 32 chars.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
   will contain one column (hashkey) with one observation (a hex32.
   representation of the input hash)
@@ -33,7 +35,8 @@
 
 %macro mp_hashdataset(
   libds,
-  outds=
+  outds=,
+  salt=
 )/*/STORE SOURCE*/;
   %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
     %put %str(WARN)ING: Dataset &libds is empty;, or is not a dataset;
@@ -53,7 +56,7 @@
     %let varlist=%mf_getvarlist(&libds);
     data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
       length &prevkeyvar &keyvar $32;
-      retain &prevkeyvar;
+      retain &prevkeyvar "&salt";
       set &libds end=&lastvar;
       /* hash should include previous row */
       &keyvar=put(md5(&prevkeyvar

base/mp_testservice.sas

@@ -22,8 +22,11 @@
     |mustbevalidname|can be anything, oops, %abort!!|
 
   @param [in] debug= (log) Provide the _debug value
-  @param [in] viyaresult=(WEBOUT_JSON) The Viya result type to return.  For
+  @param [in] mdebug= (0) Set to 1 to provide macro debugging
+  @param [in] viyaresult= (WEBOUT_JSON) The Viya result type to return.  For
     more info, see mv_getjobresult.sas
+  @param [in] viyacontext= (SAS Job Execution compute context) The Viya compute
+    context on which to run the service
   @param [out] outlib= (0) Output libref to contain the final tables.  Set to
     0 if the service output is not in JSON format.
   @param [out] outref= (0) Output fileref to create, to contain the full _webout
@@ -47,17 +50,18 @@
   inputfiles=0,
   inputparams=0,
   debug=log,
+  mdebug=0,
   outlib=0,
   outref=0,
-  viyaresult=WEBOUT_JSON
+  viyaresult=WEBOUT_JSON,
+  viyacontext=SAS Job Execution compute context
 )/*/STORE SOURCE*/;
-%local mdebug;
+%local dbg;
-%if &debug ne 0 %then %do;
+%if &mdebug=1 %then %do;
-  %let mdebug=1;
   %put &sysmacroname entry vars:;
   %put _local_;
 %end;
-%else %let mdebug=0;
+%else %let dbg=*;
 
 /* sanitise inputparams */
 %local pcnt;
@@ -212,6 +216,7 @@
 
   data &ds1;
     retain _program "&program";
+    retain _contextname "&viyacontext";
     set &ds1;
     putlog "&sysmacroname inputparams:";
     putlog (_all_)(=);

package-lock.json

@@ -5,38 +5,37 @@
   "packages": {
     "": {
       "name": "@sasjs/core",
-      "hasInstallScript": true,
       "license": "MIT",
       "devDependencies": {
-        "@sasjs/cli": "^2.27.0"
+        "@sasjs/cli": "2.33.3"
       }
     },
     "node_modules/@sasjs/adapter": {
-      "version": "2.6.3",
+      "version": "2.8.9",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.6.3.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.8.9.tgz",
-      "integrity": "sha512-Mg1AIDd0JDa0/UjvvmrGxjkpYmGEpqmD1dOYmS55EJG5loAENaVKJg9Ht8bU2XFyVaMbFpgB2yqDd0VWPP4eBA==",
+      "integrity": "sha512-8UChcZlqqlmaMMaKOCr2Bc1h+i2KVDY0FINlPXQN5PdAgEMd8dbxI2p9bsiI1yjYjOBO9LuMl7B79/mwYCtyEw==",
       "dev": true,
-      "hasInstallScript": true,
       "dependencies": {
-        "@sasjs/utils": "^2.18.0",
+        "@sasjs/utils": "^2.21.0",
         "axios": "^0.21.1",
         "axios-cookiejar-support": "^1.0.1",
         "form-data": "^4.0.0",
         "https": "^1.0.0",
+        "jwt-decode": "^3.1.2",
         "tough-cookie": "^4.0.0",
         "url": "^0.11.0"
       }
     },
     "node_modules/@sasjs/cli": {
-      "version": "2.27.0",
+      "version": "2.33.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.27.0.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.33.3.tgz",
-      "integrity": "sha512-jdQBNFbqvkidCL2+V6kuGk66YGH9BshHetPIzYSUCJPYrsKdWYRk0+OP+dZZ4qcNk8yLbcwZH+Z6Y1xh9yroFg==",
+      "integrity": "sha512-y1uFM5MEE6eoKLrPUJbweDt4njSy9Y3CS5w5/U2xwNbiAyhiPXgkwCHjCQ8Qg+0rQnMvyNEn6qJuJZ3udc6T7w==",
       "dev": true,
       "dependencies": {
-        "@sasjs/adapter": "2.6.3",
+        "@sasjs/adapter": "2.8.9",
-        "@sasjs/core": "2.34.5",
+        "@sasjs/core": "2.35.3",
-        "@sasjs/lint": "1.11.1",
+        "@sasjs/lint": "1.11.2",
-        "@sasjs/utils": "2.21.0",
+        "@sasjs/utils": "2.23.3",
         "@types/url-parse": "1.4.3",
         "btoa": "1.2.1",
         "chalk": "4.1.1",
@@ -61,34 +60,32 @@
       }
     },
     "node_modules/@sasjs/core": {
-      "version": "2.34.5",
+      "version": "2.35.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.34.5.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.35.3.tgz",
-      "integrity": "sha512-aq84ce9zmyAnkhKlSQ2SGZSQyJ5+EQAaUwktWfPhVdV9B6FJJSrQL3s1K/Dw9IuMczJcZcBlS4HO/w+goQBgyw==",
+      "integrity": "sha512-3o5PU6DkihpA+Aibt1lRy4USqJI0VFa+wNsKCD+bUD2DLZICU3JablZQxwAPH70VWJGXAUJtDFj0T/iRo5Devg==",
-      "dev": true,
+      "dev": true
-      "hasInstallScript": true
     },
     "node_modules/@sasjs/lint": {
-      "version": "1.11.1",
+      "version": "1.11.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.1.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.2.tgz",
-      "integrity": "sha512-nb0xFwXSDrE1qQLYV8whUoaXX2lh4nN3jbA/dOlLpjmF+8uvpgZo9GHy5vz5cpIhtguxiFd7lOGN/zyrzKO8yg==",
+      "integrity": "sha512-zEonhvha9kwrD+hxhG0hEhtfqpXwffH4vRDIr6eDiXkC7S8M3yImpjyFBvX/THJO5+8iuY8TYkOXKl7+nK/wAg==",
       "dev": true,
-      "hasInstallScript": true,
       "dependencies": {
         "@sasjs/utils": "^2.19.0"
       }
     },
     "node_modules/@sasjs/utils": {
-      "version": "2.21.0",
+      "version": "2.23.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.21.0.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.23.3.tgz",
-      "integrity": "sha512-ehRF1VHBiF8FNg2w4VocjBlb+ZkyMasUZOvUn19bsdqpF0otJVYu66gFOJF0rIpykh/vbpLi6dhqYOxgCRAYIw==",
+      "integrity": "sha512-tEh4mGG80eUxSLpbPivA0vl4akMdauL+yZrLn1uUM8EyiXPvlcWPkQTeN6oHbyyAH108D9cfEBidTePZh1p5VQ==",
       "dev": true,
-      "hasInstallScript": true,
       "dependencies": {
         "@types/prompts": "^2.0.13",
         "chalk": "^4.1.1",
         "cli-table": "^0.3.6",
         "consola": "^2.15.0",
         "fs-extra": "^10.0.0",
+        "jwt-decode": "^3.1.2",
         "prompts": "^2.4.1",
         "valid-url": "^1.0.9"
       }
@@ -103,24 +100,24 @@
       }
     },
     "node_modules/@types/node": {
-      "version": "15.12.4",
+      "version": "16.3.1",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.4.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.3.1.tgz",
-      "integrity": "sha512-zrNj1+yqYF4WskCMOHwN+w9iuD12+dGm0rQ35HLl9/Ouuq52cEtd0CH9qMgrdNmi5ejC1/V7vKEXYubB+65DkA==",
+      "integrity": "sha512-N87VuQi7HEeRJkhzovao/JviiqKjDKMVKxKMfUvSKw+MbkbW8R0nA3fi/MQhhlxV2fQ+2ReM+/Nt4efdrJx3zA==",
       "dev": true
     },
     "node_modules/@types/prompts": {
-      "version": "2.0.13",
+      "version": "2.0.14",
-      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.13.tgz",
+      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.14.tgz",
-      "integrity": "sha512-jwMOIGy49VruR/gYehhJYgpVzB+EVpEE7t7j9m1oTo4HMpOe7KmsyqdBuoxAzA5B4caUgx0cKrWr7wUEqMXJ7Q==",
+      "integrity": "sha512-HZBd99fKxRWpYCErtm2/yxUZv6/PBI9J7N4TNFffl5JbrYMHBwF25DjQGTW3b3jmXq+9P6/8fCIb2ee57BFfYA==",
       "dev": true,
       "dependencies": {
         "@types/node": "*"
       }
     },
     "node_modules/@types/tough-cookie": {
-      "version": "4.0.0",
+      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.1.tgz",
-      "integrity": "sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==",
+      "integrity": "sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==",
       "dev": true,
       "peer": true
     },
@@ -1334,6 +1331,7 @@
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz",
       "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=",
+      "deprecated": "The",
       "dev": true,
       "engines": {
         "node": ">=0.4.x"
@@ -1762,30 +1760,31 @@
   },
   "dependencies": {
     "@sasjs/adapter": {
-      "version": "2.6.3",
+      "version": "2.8.9",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.6.3.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.8.9.tgz",
-      "integrity": "sha512-Mg1AIDd0JDa0/UjvvmrGxjkpYmGEpqmD1dOYmS55EJG5loAENaVKJg9Ht8bU2XFyVaMbFpgB2yqDd0VWPP4eBA==",
+      "integrity": "sha512-8UChcZlqqlmaMMaKOCr2Bc1h+i2KVDY0FINlPXQN5PdAgEMd8dbxI2p9bsiI1yjYjOBO9LuMl7B79/mwYCtyEw==",
       "dev": true,
       "requires": {
-        "@sasjs/utils": "^2.18.0",
+        "@sasjs/utils": "^2.21.0",
         "axios": "^0.21.1",
         "axios-cookiejar-support": "^1.0.1",
         "form-data": "^4.0.0",
         "https": "^1.0.0",
+        "jwt-decode": "^3.1.2",
         "tough-cookie": "^4.0.0",
         "url": "^0.11.0"
       }
     },
     "@sasjs/cli": {
-      "version": "2.27.0",
+      "version": "2.33.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.27.0.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/cli/-/cli-2.33.3.tgz",
-      "integrity": "sha512-jdQBNFbqvkidCL2+V6kuGk66YGH9BshHetPIzYSUCJPYrsKdWYRk0+OP+dZZ4qcNk8yLbcwZH+Z6Y1xh9yroFg==",
+      "integrity": "sha512-y1uFM5MEE6eoKLrPUJbweDt4njSy9Y3CS5w5/U2xwNbiAyhiPXgkwCHjCQ8Qg+0rQnMvyNEn6qJuJZ3udc6T7w==",
       "dev": true,
       "requires": {
-        "@sasjs/adapter": "2.6.3",
+        "@sasjs/adapter": "2.8.9",
-        "@sasjs/core": "2.34.5",
+        "@sasjs/core": "2.35.3",
-        "@sasjs/lint": "1.11.1",
+        "@sasjs/lint": "1.11.2",
-        "@sasjs/utils": "2.21.0",
+        "@sasjs/utils": "2.23.3",
         "@types/url-parse": "1.4.3",
         "btoa": "1.2.1",
         "chalk": "4.1.1",
@@ -1807,24 +1806,24 @@
       }
     },
     "@sasjs/core": {
-      "version": "2.34.5",
+      "version": "2.35.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.34.5.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.35.3.tgz",
-      "integrity": "sha512-aq84ce9zmyAnkhKlSQ2SGZSQyJ5+EQAaUwktWfPhVdV9B6FJJSrQL3s1K/Dw9IuMczJcZcBlS4HO/w+goQBgyw==",
+      "integrity": "sha512-3o5PU6DkihpA+Aibt1lRy4USqJI0VFa+wNsKCD+bUD2DLZICU3JablZQxwAPH70VWJGXAUJtDFj0T/iRo5Devg==",
       "dev": true
     },
     "@sasjs/lint": {
-      "version": "1.11.1",
+      "version": "1.11.2",
-      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.1.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/lint/-/lint-1.11.2.tgz",
-      "integrity": "sha512-nb0xFwXSDrE1qQLYV8whUoaXX2lh4nN3jbA/dOlLpjmF+8uvpgZo9GHy5vz5cpIhtguxiFd7lOGN/zyrzKO8yg==",
+      "integrity": "sha512-zEonhvha9kwrD+hxhG0hEhtfqpXwffH4vRDIr6eDiXkC7S8M3yImpjyFBvX/THJO5+8iuY8TYkOXKl7+nK/wAg==",
       "dev": true,
       "requires": {
         "@sasjs/utils": "^2.19.0"
       }
     },
     "@sasjs/utils": {
-      "version": "2.21.0",
+      "version": "2.23.3",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.21.0.tgz",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.23.3.tgz",
-      "integrity": "sha512-ehRF1VHBiF8FNg2w4VocjBlb+ZkyMasUZOvUn19bsdqpF0otJVYu66gFOJF0rIpykh/vbpLi6dhqYOxgCRAYIw==",
+      "integrity": "sha512-tEh4mGG80eUxSLpbPivA0vl4akMdauL+yZrLn1uUM8EyiXPvlcWPkQTeN6oHbyyAH108D9cfEBidTePZh1p5VQ==",
       "dev": true,
       "requires": {
         "@types/prompts": "^2.0.13",
@@ -1832,6 +1831,7 @@
         "cli-table": "^0.3.6",
         "consola": "^2.15.0",
         "fs-extra": "^10.0.0",
+        "jwt-decode": "^3.1.2",
         "prompts": "^2.4.1",
         "valid-url": "^1.0.9"
       }
@@ -1843,24 +1843,24 @@
       "dev": true
     },
     "@types/node": {
-      "version": "15.12.4",
+      "version": "16.3.1",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.4.tgz",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-16.3.1.tgz",
-      "integrity": "sha512-zrNj1+yqYF4WskCMOHwN+w9iuD12+dGm0rQ35HLl9/Ouuq52cEtd0CH9qMgrdNmi5ejC1/V7vKEXYubB+65DkA==",
+      "integrity": "sha512-N87VuQi7HEeRJkhzovao/JviiqKjDKMVKxKMfUvSKw+MbkbW8R0nA3fi/MQhhlxV2fQ+2ReM+/Nt4efdrJx3zA==",
       "dev": true
     },
     "@types/prompts": {
-      "version": "2.0.13",
+      "version": "2.0.14",
-      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.13.tgz",
+      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.14.tgz",
-      "integrity": "sha512-jwMOIGy49VruR/gYehhJYgpVzB+EVpEE7t7j9m1oTo4HMpOe7KmsyqdBuoxAzA5B4caUgx0cKrWr7wUEqMXJ7Q==",
+      "integrity": "sha512-HZBd99fKxRWpYCErtm2/yxUZv6/PBI9J7N4TNFffl5JbrYMHBwF25DjQGTW3b3jmXq+9P6/8fCIb2ee57BFfYA==",
       "dev": true,
       "requires": {
         "@types/node": "*"
       }
     },
     "@types/tough-cookie": {
-      "version": "4.0.0",
+      "version": "4.0.1",
-      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.1.tgz",
-      "integrity": "sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==",
+      "integrity": "sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==",
       "dev": true,
       "peer": true
     },

package.json

@@ -30,9 +30,9 @@
     "docs": "sasjs doc && ./sasjs/utils/build.sh",
     "test": "sasjs test -t viya",
     "lint": "sasjs lint",
-    "postinstall": "[ -d .git ] && git config core.hooksPath ./.git-hooks || true"
+    "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "^2.27.0"
+    "@sasjs/cli": "2.33.3"
   }
 }

tests/base/mp_base64copy.test.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Testing mp_base64copy.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_base64copy.sas
+  @li mp_assert.sas
+
+**/
+
+
+/* TEST 1 - regular base64 decode */
+
+%let string1=base ik ally;
+filename tmp temp;
+data _null_;
+  file tmp;
+  put "&string1";
+run;
+%mp_base64copy(inref=tmp, outref=myref, action=ENCODE)
+
+data _null_;
+  infile myref;
+  input;
+  put _infile_;
+run;
+%mp_base64copy(inref=myref, outref=mynewref, action=DECODE)
+data _null_;
+  infile mynewref lrecl=5000;
+  input;
+  put _infile_;
+  call symputx('string1_check',_infile_);
+  stop;
+run;
+%mp_assert(
+  iftrue=("&string1"="&string1_check"),
+  desc=Basic String Compare,
+  outds=work.test_results
+)
+
+
+/* multibyte string check */
+
+filename tmp2 temp;
+data _null_;
+  file tmp2;
+  put "'╤', '╔', '╗', '═', '╧', '╚', '╝', '║', '╟', '─', '┼', '║', '╢', '│'";
+run;
+%mp_base64copy(inref=tmp2, outref=myref2, action=ENCODE)
+
+%mp_base64copy(inref=myref2, outref=newref2, action=DECODE)
+data _null_;
+  infile newref2 lrecl=5000;
+  input;
+  list;
+  /* do not print the string to the log else viya 3.5 throws exception */
+  if trim(_infile_)=
+    "'╤', '╔', '╗', '═', '╧', '╚', '╝', '║', '╟', '─', '┼', '║', '╢', '│'"
+  then call symputx('check2',1);
+  else call symputx('check2',0);
+  stop;
+run;
+%mp_assert(
+  iftrue=("&check2"="1"),
+  desc=Double Byte String Compare,
+  outds=work.test_results
+)

tests/base/mp_ds2inserts.test.sas

@@ -0,0 +1,31 @@
+/**
+  @file
+  @brief Testing mp_ds2inserts.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_ds2inserts.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * test 1 - rebuild an existing dataset
+  * Cars is a great dataset - it contains leading spaces, and formatted numerics
+  */
+
+%mp_ds2inserts(sashelp.cars,outref=testref,outlib=work,outds=test)
+
+data work.test;
+  set sashelp.cars;
+  stop;
+proc sql;
+%inc testref;
+
+proc compare base=sashelp.cars compare=work.test;
+quit;
+
+%mp_assert(
+  iftrue=(&sysinfo=1),
+  desc=sashelp.cars is identical except for ds label,
+  outds=work.test_results
+)

viya/mv_createfile.sas

@@ -17,9 +17,13 @@
   @param [in] path= The parent folder in which to create the file
   @param [in] name= The name of the file to be created
   @param [in] inref= The fileref pointing to the file to be uploaded
+  @param [in] intype= (BINARY) The type of the input data.  Valid values:
+    @li BINARY File is copied byte for byte using the mp_binarycopy.sas macro.
+    @li BASE64 File will be first decoded using the mp_base64.sas macro, then
+      loaded byte by byte to SAS Drive.
   @param [in] contentdisp= (inline) Content Disposition. Example values:
-      @li inline
+    @li inline
-      @li attachment
+    @li attachment
 
   @param [in] access_token_var= The global macro variable to contain the access
     token, if using authorization_code grant type.
@@ -37,6 +41,7 @@
   @li mf_getuniquefileref.sas
   @li mf_isblank.sas
   @li mp_abort.sas
+  @li mp_base64copy.sas
   @li mp_binarycopy.sas
   @li mv_createfolder.sas
 
@@ -45,6 +50,7 @@
 %macro mv_createfile(path=
     ,name=
     ,inref=
+    ,intype=BINARY
     ,contentdisp=inline
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
@@ -99,7 +105,12 @@ filename &fref filesrvc
   cdisp="&contentdisp"
   lrecl=1048544;
 
-%mp_binarycopy(inref=&inref, outref=&fref)
+%if &intype=BINARY %then %do;
+  %mp_binarycopy(inref=&inref, outref=&fref)
+%end;
+%else %if &intype=BASE64 %then %do;
+  %mp_base64copy(inref=&inref, outref=&fref, action=DECODE)
+%end;
 
 filename &fref clear;
 

viya/mv_getjoblog.sas

@@ -66,7 +66,7 @@
 
   @param [in] access_token_var= The global macro variable to contain the access
     token
-  @param [in] mdebug= set to 1 to enable DEBUG messages
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
   @param [in] grant_type= valid values:
     @li password
     @li authorization_code
@@ -91,7 +91,6 @@
 **/
 
 %macro mv_getjoblog(uri=0,outref=0
-    ,contextName=SAS Job Execution compute context
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
     ,mdebug=0

viya/mv_getjobresult.sas

@@ -90,7 +90,6 @@
 **/
 
 %macro mv_getjobresult(uri=0
-    ,contextName=SAS Job Execution compute context
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
     ,mdebug=0
@@ -204,10 +203,11 @@ proc http method='GET' out=&fname2 &oauth_bearer
   ;
 run;
 %if &mdebug=1 %then %do;
+  /* send one char at a time as the json can be very wide */
   data _null_;
-    infile &fname2 lrecl=32767;
+    infile &fname2 recfm=n;
-    input;
+    input char $char1. ;
-    putlog _infile_;
+    putlog char $char1. @;
   run;
 %end;
 

viya/mv_jobflow.sas

@@ -140,6 +140,11 @@
 %if &mdebug=1 %then %do;
   %put &sysmacroname entry vars:;
   %put _local_;
+  %put inds vars:;
+  data _null_;
+    set &inds;
+    putlog (_all_)(=);
+  run;
 %end;
 %else %let dbg=*;
 
@@ -184,6 +189,7 @@
       retain FLOW_ID 0;
     %end;
     set &inds;
+    &dbg. putlog (_all_)(=);
   run;
 %end;
 
@@ -248,6 +254,8 @@ data;run;%let jdswaitfor=&syslast;
       call symputx(cats('job',_n_),_program,'l');
       call symputx(cats('context',_n_),_contextName,'l');
       call symputx('jcnt',_n_,'l');
+      &dbg. if _n_= 1 then putlog "Loop &fid";
+      &dbg. putlog (_all_)(=);
     run;
     %put exporting job variables in json format;
     %do jid=1 %to &jcnt;
@@ -309,6 +317,7 @@ data;run;%let jdswaitfor=&syslast;
               ,name=&jobname
               ,paramstring=%superq(jparams&jid)
               ,outds=&jdsapp
+              ,contextname=&&context&jid
             )
             data &jdsapp;
               format jobparams $32767.;