Merge pull request #219 from sasjs/issue218

fix: check 4 special chars b4 replacing. Closes #218
chore: alignment
2025-12-11 06:24:35 +00:00 · 2022-04-16 22:44:35 +03:00 · 2022-04-16 20:37:47 +01:00 · 2022-04-16 20:33:21 +01:00 · 2022-04-14 21:32:25 +03:00 · 2022-04-14 19:31:36 +01:00
20 changed files with 816 additions and 2436 deletions
--- a/all.sas
+++ b/all.sas
@@ -4308,6 +4308,7 @@ run;
  data &out_ds;
    set &out_ds;
    length infoname infoval $60 fref $8;
+    if _n_=1 then call missing(fref);
    rc=filename(fref,filepath);
    drop rc infoname fid i close fref;
    if file_or_folder='file' then do;
@@ -4544,6 +4545,7 @@ drop table &out_ds;

  @version 9.2
  @author Allan Bowe
+  @cond
 **/

 %macro mp_ds2cards(base_ds=, tgt_ds=
@@ -4715,7 +4717,8 @@ data _null_;
    put '  @file';
    put "  @brief Datalines for %upcase(%scan(&base_ds,2)) dataset";
    put "  @details Generated by %nrstr(%%)mp_ds2cards()";
-    put "  Available on github.com/sasjs/core";
+    put "    Source: https://github.com/sasjs/core";
+    put '  @cond ';
    put '**/';
    put "data &tgt_ds &indexes;";
    put "attrib ";
@@ -4760,6 +4763,7 @@ data _null_;
  if __lastobs then do;
    put ';;;;';
    put 'run;';
+    put '/** @endcond **/';
    stop;
  end;
 run;
@@ -4779,7 +4783,9 @@ quit;
 %put NOTE-;%put NOTE-;
 %put NOTE- %sysfunc(dequote(&cards_file.));
 %put NOTE-;%put NOTE-;
-%mend mp_ds2cards;/**
+%mend mp_ds2cards;
+/** @endcond **/
+/**
  @file
  @brief Export a dataset to a CSV file WITH leading blanks
  @details Export a dataset to a file or fileref, retaining leading blanks.
@@ -7361,7 +7367,8 @@ create table &outds (rename=(
 )/*/STORE SOURCE*/;


-%local engine schema ds1 ds2 ds3 dsn tabs1 tabs2 sum pk4sure pkdefault finalpks;
+%local engine schema ds1 ds2 ds3 dsn tabs1 tabs2 sum pk4sure pkdefault finalpks
+  pkfromindex;

 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
@@ -7376,6 +7383,7 @@ create table &outds (rename=(
 %let sum=%mf_getuniquename(prefix=getpk_sum);
 %let pk4sure=%mf_getuniquename(prefix=getpk_pk4sure);
 %let pkdefault=%mf_getuniquename(prefix=getpk_pkdefault);
+%let pkfromindex=%mf_getuniquename(prefix=getpk_pkfromindex);
 %let finalpks=%mf_getuniquename(prefix=getpk_finalpks);

 %local dbg;
@@ -7486,9 +7494,23 @@ create table &ds1 as
      and a.constraint_name=b.constraint_name
    order by 1,2,3,4;

+  /* extract cols from the relevant unique INDEXES */
+  create table &pkfromindex as
+    select libname as libref
+      ,memname as table_name
+      ,indxname as constraint_name
+      ,indxpos as constraint_order
+      ,name
+    from dictionary.indexes
+    where nomiss='yes' and unique='yes' and upcase(libname)="&lib"
+  %if &ds ne 0 %then %do;
+      and upcase(memname)="&ds"
+  %end;
+    order by 1,2,3,4;
+
  /* create one table */
  data &finalpks;
-    set &pkdefault &pk4sure ;
+    set &pkdefault &pk4sure &pkfromindex;
    pk_ind=1;
    /* if there are multiple unique constraints, take the first */
    by libref table_name constraint_name;
@@ -7562,7 +7584,8 @@ create table &outds as
  iftrue=(&mdebug=0)
 )

-%mend mp_getpk;/**
+%mend mp_getpk;
+/**
  @file
  @brief Performs a text substitution on a file
  @details Makes use of the GSUB function in LUA to perform a text substitution
@@ -8216,7 +8239,8 @@ options
 %end;
 ;

-%mend mp_init;/**
+%mend mp_init;
+/**
  @file mp_jsonout.sas
  @brief Writes JSON in SASjs format to a fileref
  @details PROC JSON is faster but will produce errs like the ones below if
@@ -8417,19 +8441,27 @@ options
      format _numeric_ bart.;
    %do i=1 %to &numcols;
      %if &&typelong&i=char or &fmt=Y %then %do;
-        &&name&i='"'!!trim(prxchange('s/"/\"/',-1,
-                    prxchange('s/'!!'0A'x!!'/\n/',-1,
-                    prxchange('s/'!!'0D'x!!'/\r/',-1,
-                    prxchange('s/'!!'09'x!!'/\t/',-1,
-                    prxchange('s/'!!'00'x!!'/\\u0000/',-1, /* NUL */
-                    prxchange('s/'!!'0E'x!!'/\\u000E/',-1, /* SS  */
-                    prxchange('s/'!!'0F'x!!'/\\u000F/',-1, /* SF  */
-                    prxchange('s/'!!'01'x!!'/\\u0001/',-1, /* SOH */
-                    prxchange('s/'!!'02'x!!'/\\u0002/',-1, /* STX */
-                    prxchange('s/'!!'02'x!!'/\\u0010/',-1, /* DLE */
-                    prxchange('s/'!!'11'x!!'/\\u0011/',-1, /* DC1 */
-                    prxchange('s/\\/\\\\/',-1,&&name&i)
-        ))))))))))))!!'"';
+        if prxmatch(
+            'm/\"|\x0A|\x0D|\x09|\x00|\x0E|\x0F|\x01|\x02|\x10|\x11|\\/o',
+            &&name&i
+          )>0
+        then do;
+          &&name&i='"'!!trim(
+            prxchange('s/"/\\"/',-1,        /* double quote */
+            prxchange('s/\x0A/\n/',-1,      /* new line */
+            prxchange('s/\x0D/\r/',-1,      /* carriage return */
+            prxchange('s/\x09/\\t/',-1,     /* tab */
+            prxchange('s/\x00/\\u0000/',-1, /* NUL */
+            prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+            prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+            prxchange('s/\x01/\\u0001/',-1, /* SOH */
+            prxchange('s/\x02/\\u0002/',-1, /* STX */
+            prxchange('s/\x10/\\u0010/',-1, /* DLE */
+            prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+            prxchange('s/\\/\\\\/',-1,&&name&i)
+          ))))))))))))!!'"';
+        end;
+        else &&name&i=quote(cats(&&name&i));
      %end;
    %end;
    run;
@@ -11586,12 +11618,20 @@ run;
  %end;
 %end;
 %else %if &contentype=HTML %then %do;
-  %if &platform=SASVIYA %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type','text/html');
+      rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
-      contenttype="text/html";
+      contenttype="text/html"
+      contentdisp="attachment; filename=&outname";
  %end;
  %else %if &platform=SASJS %then %do;
    %mfs_httpheader(Content-type,text/html)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
  %end;
 %end;
 %else %if &contentype=TEXT %then %do;
@@ -14934,19 +14974,27 @@ data _null_;
  put '      format _numeric_ bart.; ';
  put '    %do i=1 %to &numcols; ';
  put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
-  put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
-  put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
-  put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
-  put '                    prxchange(''s/''!!''00''x!!''/\\u0000/'',-1, /* NUL */ ';
-  put '                    prxchange(''s/''!!''0E''x!!''/\\u000E/'',-1, /* SS  */ ';
-  put '                    prxchange(''s/''!!''0F''x!!''/\\u000F/'',-1, /* SF  */ ';
-  put '                    prxchange(''s/''!!''01''x!!''/\\u0001/'',-1, /* SOH */ ';
-  put '                    prxchange(''s/''!!''02''x!!''/\\u0002/'',-1, /* STX */ ';
-  put '                    prxchange(''s/''!!''02''x!!''/\\u0010/'',-1, /* DLE */ ';
-  put '                    prxchange(''s/''!!''11''x!!''/\\u0011/'',-1, /* DC1 */ ';
-  put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        ))))))))))))!!''"''; ';
+  put '        if prxmatch( ';
+  put '            ''m/\"|\x0A|\x0D|\x09|\x00|\x0E|\x0F|\x01|\x02|\x10|\x11|\\/o'', ';
+  put '            &&name&i ';
+  put '          )>0 ';
+  put '        then do; ';
+  put '          &&name&i=''"''!!trim( ';
+  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '          ))))))))))))!!''"''; ';
+  put '        end; ';
+  put '        else &&name&i=quote(cats(&&name&i)); ';
  put '      %end; ';
  put '    %end; ';
  put '    run; ';
@@ -15130,14 +15178,16 @@ data _null_;
  put '    put '',"_METAPERSON": '' _METAPERSON; ';
  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
  put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
+  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
  put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
  put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
  put '    put ",""SYSSITE"" : ""&syssite"" "; ';
  put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
  put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
+  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
  put '    length memsize $32; ';
  put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
@@ -18667,14 +18717,16 @@ run;
    put ',"_METAPERSON": ' _METAPERSON;
    put ',"_PROGRAM" : ' _PROGRAM ;
    put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
    put ",""SYSJOBID"" : ""&sysjobid"" ";
    put ",""SYSSCPL"" : ""&sysscpl"" ";
    put ",""SYSSITE"" : ""&syssite"" ";
    sysvlong=quote(trim(symget('sysvlong')));
    put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
    length memsize $32;
    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
@@ -18902,12 +18954,16 @@ run;
    ,mdebug=0
  );

-%local fname0 fname1 fname2 boundary fname statcd msg;
+%local fname0 fname1 fname2 boundary fname statcd msg optval;
 %let fname0=%mf_getuniquefileref();
 %let fname1=%mf_getuniquefileref();
 %let fname2=%mf_getuniquefileref();
 %let boundary=%mf_getuniquename();

+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
 data _null_;
  file &fname0 termstr=crlf;
  infile &inref end=eof;
@@ -18968,6 +19024,9 @@ run;
  ,msg=%superq(msg)
 )

+/* reset options */
+options &optval;
+
 %mend ms_createfile;
 /**
  @file
@@ -19123,7 +19182,7 @@ run;
 %end;
 %mend ms_runstp;/**
  @file
-  @brief Send data to/from @sasjs/server
+  @brief Send data to/from sasjs/server
  @details This macro should be added to the start of each web service,
  **immediately** followed by a call to:

@@ -19270,7 +19329,8 @@ run;
    put ",""_DEBUG"" : ""&_debug"" ";
    put ',"_PROGRAM" : ' _PROGRAM ;
    put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
    SYSHOSTINFOLONG=quote(trim(symget('SYSHOSTINFOLONG')));
    put ',"SYSHOSTINFOLONG" : ' SYSHOSTINFOLONG;
    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
@@ -19285,7 +19345,8 @@ run;
    put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" ";
    sysvlong=quote(trim(symget('sysvlong')));
    put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
    length autoexec $512;
    autoexec=quote(urlencode(trim(getoption('autoexec'))));
@@ -20412,19 +20473,27 @@ data _null_;
  put '      format _numeric_ bart.; ';
  put '    %do i=1 %to &numcols; ';
  put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
-  put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
-  put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
-  put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
-  put '                    prxchange(''s/''!!''00''x!!''/\\u0000/'',-1, /* NUL */ ';
-  put '                    prxchange(''s/''!!''0E''x!!''/\\u000E/'',-1, /* SS  */ ';
-  put '                    prxchange(''s/''!!''0F''x!!''/\\u000F/'',-1, /* SF  */ ';
-  put '                    prxchange(''s/''!!''01''x!!''/\\u0001/'',-1, /* SOH */ ';
-  put '                    prxchange(''s/''!!''02''x!!''/\\u0002/'',-1, /* STX */ ';
-  put '                    prxchange(''s/''!!''02''x!!''/\\u0010/'',-1, /* DLE */ ';
-  put '                    prxchange(''s/''!!''11''x!!''/\\u0011/'',-1, /* DC1 */ ';
-  put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        ))))))))))))!!''"''; ';
+  put '        if prxmatch( ';
+  put '            ''m/\"|\x0A|\x0D|\x09|\x00|\x0E|\x0F|\x01|\x02|\x10|\x11|\\/o'', ';
+  put '            &&name&i ';
+  put '          )>0 ';
+  put '        then do; ';
+  put '          &&name&i=''"''!!trim( ';
+  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '          ))))))))))))!!''"''; ';
+  put '        end; ';
+  put '        else &&name&i=quote(cats(&&name&i)); ';
  put '      %end; ';
  put '    %end; ';
  put '    run; ';
@@ -20497,7 +20566,7 @@ data _null_;
  put '  sasjs_tables SYS_JES_JOB_URI; ';
  put '%if %index("&_debug",log) %then %let _debug=131; ';
  put ' ';
-  put '%local i tempds; ';
+  put '%local i tempds table; ';
  put '%let action=%upcase(&action); ';
  put ' ';
  put '%if &action=FETCH %then %do; ';
@@ -20512,60 +20581,35 @@ data _null_;
  put '  %end; ';
  put ' ';
  put '  /* if the sasjs_tables param is passed, we expect param based upload */ ';
-  put '  %if %length(&sasjs_tables.XX)>2 %then %do; ';
-  put '    filename _sasjs "%sysfunc(pathname(work))/sasjs.lua"; ';
-  put '    data _null_; ';
-  put '      file _sasjs; ';
-  put '      put ''s=sas.symget("sasjs_tables")''; ';
-  put '      put ''if(s:sub(1,7) == "%nrstr(")''; ';
-  put '      put ''then''; ';
-  put '      put '' tablist=s:sub(8,s:len()-1)''; ';
-  put '      put ''else''; ';
-  put '      put '' tablist=s''; ';
-  put '      put ''end''; ';
-  put '      put ''for i = 1,sas.countw(tablist) ''; ';
-  put '      put ''do ''; ';
-  put '      put ''  tab=sas.scan(tablist,i)''; ';
-  put '      put ''  sasdata=""''; ';
-  put '      put ''  if (sas.symexist("sasjs"..i.."data0")==0)''; ';
-  put '      put ''  then''; ';
-  put '      /* TODO - condense this logic */ ';
-  put '      put ''    s=sas.symget("sasjs"..i.."data")''; ';
-  put '      put ''    if(s:sub(1,7) == "%nrstr(")''; ';
-  put '      put ''    then''; ';
-  put '      put ''      sasdata=s:sub(8,s:len()-1)''; ';
-  put '      put ''    else''; ';
-  put '      put ''      sasdata=s''; ';
-  put '      put ''    end''; ';
-  put '      put ''  else''; ';
-  put '      put ''    for d = 1, sas.symget("sasjs"..i.."data0")''; ';
-  put '      put ''    do''; ';
-  put '      put ''      s=sas.symget("sasjs"..i.."data"..d)''; ';
-  put '      put ''      if(s:sub(1,7) == "%nrstr(")''; ';
-  put '      put ''      then''; ';
-  put '      put ''        sasdata=sasdata..s:sub(8,s:len()-1)''; ';
-  put '      put ''      else''; ';
-  put '      put ''        sasdata=sasdata..s''; ';
-  put '      put ''      end''; ';
-  put '      put ''    end''; ';
-  put '      put ''  end''; ';
-  put '      put ''  file = io.open(sas.pathname("work").."/"..tab..".csv", "a")''; ';
-  put '      put ''  io.output(file)''; ';
-  put '      put ''  io.write(sasdata)''; ';
-  put '      put ''  io.close(file)''; ';
-  put '      put ''end''; ';
-  put '    run; ';
-  put '    %inc _sasjs; ';
+  put '  %if %length(&sasjs_tables.X)>1 %then %do; ';
  put ' ';
-  put '    /* now read in the data */ ';
+  put '    /* convert data from macro variables to datasets */ ';
  put '    %do i=1 %to %sysfunc(countw(&sasjs_tables)); ';
-  put '      %local table; %let table=%scan(&sasjs_tables,&i); ';
+  put '      %let table=%scan(&sasjs_tables,&i,%str( )); ';
+  put '      %if %symexist(sasjs&i.data0)=0 %then %let sasjs&i.data0=1; ';
+  put '      data _null_; ';
+  put '        file "%sysfunc(pathname(work))/&table..csv" recfm=n; ';
+  put '        retain nrflg 0; ';
+  put '        length line $32767; ';
+  put '        do i=1 to &&sasjs&i.data0; ';
+  put '          if &&sasjs&i.data0=1 then line=symget("sasjs&i.data"); ';
+  put '          else line=symget(cats("sasjs&i.data",i)); ';
+  put '          if i=1 and substr(line,1,7)=''%nrstr('' then do; ';
+  put '            nrflg=1; ';
+  put '            line=substr(line,8); ';
+  put '          end; ';
+  put '          if i=&&sasjs&i.data0 and nrflg=1 then do; ';
+  put '            line=substr(line,1,length(line)-1); ';
+  put '          end; ';
+  put '          put line +(-1) @; ';
+  put '        end; ';
+  put '      run; ';
  put '      data _null_; ';
  put '        infile "%sysfunc(pathname(work))/&table..csv" termstr=crlf ; ';
  put '        input; ';
  put '        if _n_=1 then call symputx(''input_statement'',_infile_); ';
  put '        list; ';
-  put '      data &table; ';
+  put '      data work.&table; ';
  put '        infile "%sysfunc(pathname(work))/&table..csv" firstobs=2 dsd ';
  put '          termstr=crlf; ';
  put '        input &input_statement; ';
@@ -20666,13 +20710,15 @@ data _null_;
  put '    put ",""_DEBUG"" : ""&_debug"" "; ';
  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
  put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
+  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
  put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
  put '    put ",""SYSSITE"" : ""&syssite"" "; ';
  put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
  put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
+  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
  put '    length memsize $32; ';
  put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
@@ -24329,7 +24375,7 @@ filename &fref1 clear;
  sasjs_tables SYS_JES_JOB_URI;
 %if %index("&_debug",log) %then %let _debug=131;

-%local i tempds;
+%local i tempds table;
 %let action=%upcase(&action);

 %if &action=FETCH %then %do;
@@ -24344,60 +24390,35 @@ filename &fref1 clear;
  %end;

  /* if the sasjs_tables param is passed, we expect param based upload */
-  %if %length(&sasjs_tables.XX)>2 %then %do;
-    filename _sasjs "%sysfunc(pathname(work))/sasjs.lua";
-    data _null_;
-      file _sasjs;
-      put 's=sas.symget("sasjs_tables")';
-      put 'if(s:sub(1,7) == "%nrstr(")';
-      put 'then';
-      put ' tablist=s:sub(8,s:len()-1)';
-      put 'else';
-      put ' tablist=s';
-      put 'end';
-      put 'for i = 1,sas.countw(tablist) ';
-      put 'do ';
-      put '  tab=sas.scan(tablist,i)';
-      put '  sasdata=""';
-      put '  if (sas.symexist("sasjs"..i.."data0")==0)';
-      put '  then';
-      /* TODO - condense this logic */
-      put '    s=sas.symget("sasjs"..i.."data")';
-      put '    if(s:sub(1,7) == "%nrstr(")';
-      put '    then';
-      put '      sasdata=s:sub(8,s:len()-1)';
-      put '    else';
-      put '      sasdata=s';
-      put '    end';
-      put '  else';
-      put '    for d = 1, sas.symget("sasjs"..i.."data0")';
-      put '    do';
-      put '      s=sas.symget("sasjs"..i.."data"..d)';
-      put '      if(s:sub(1,7) == "%nrstr(")';
-      put '      then';
-      put '        sasdata=sasdata..s:sub(8,s:len()-1)';
-      put '      else';
-      put '        sasdata=sasdata..s';
-      put '      end';
-      put '    end';
-      put '  end';
-      put '  file = io.open(sas.pathname("work").."/"..tab..".csv", "a")';
-      put '  io.output(file)';
-      put '  io.write(sasdata)';
-      put '  io.close(file)';
-      put 'end';
-    run;
-    %inc _sasjs;
+  %if %length(&sasjs_tables.X)>1 %then %do;

-    /* now read in the data */
+    /* convert data from macro variables to datasets */
    %do i=1 %to %sysfunc(countw(&sasjs_tables));
-      %local table; %let table=%scan(&sasjs_tables,&i);
+      %let table=%scan(&sasjs_tables,&i,%str( ));
+      %if %symexist(sasjs&i.data0)=0 %then %let sasjs&i.data0=1;
+      data _null_;
+        file "%sysfunc(pathname(work))/&table..csv" recfm=n;
+        retain nrflg 0;
+        length line $32767;
+        do i=1 to &&sasjs&i.data0;
+          if &&sasjs&i.data0=1 then line=symget("sasjs&i.data");
+          else line=symget(cats("sasjs&i.data",i));
+          if i=1 and substr(line,1,7)='%nrstr(' then do;
+            nrflg=1;
+            line=substr(line,8);
+          end;
+          if i=&&sasjs&i.data0 and nrflg=1 then do;
+            line=substr(line,1,length(line)-1);
+          end;
+          put line +(-1) @;
+        end;
+      run;
      data _null_;
        infile "%sysfunc(pathname(work))/&table..csv" termstr=crlf ;
        input;
        if _n_=1 then call symputx('input_statement',_infile_);
        list;
-      data &table;
+      data work.&table;
        infile "%sysfunc(pathname(work))/&table..csv" firstobs=2 dsd
          termstr=crlf;
        input &input_statement;
@@ -24498,13 +24519,15 @@ filename &fref1 clear;
    put ",""_DEBUG"" : ""&_debug"" ";
    put ',"_PROGRAM" : ' _PROGRAM ;
    put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
    put ",""SYSSCPL"" : ""&sysscpl"" ";
    put ",""SYSSITE"" : ""&syssite"" ";
    sysvlong=quote(trim(symget('sysvlong')));
    put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
    length memsize $32;
    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
--- a/base/mp_dirlist.sas
+++ b/base/mp_dirlist.sas
@@ -152,6 +152,7 @@ run;
  data &out_ds;
    set &out_ds;
    length infoname infoval $60 fref $8;
+    if _n_=1 then call missing(fref);
    rc=filename(fref,filepath);
    drop rc infoname fid i close fref;
    if file_or_folder='file' then do;
--- a/base/mp_ds2cards.sas
+++ b/base/mp_ds2cards.sas
@@ -48,6 +48,7 @@

  @version 9.2
  @author Allan Bowe
+  @cond
 **/

 %macro mp_ds2cards(base_ds=, tgt_ds=
@@ -219,7 +220,8 @@ data _null_;
    put '  @file';
    put "  @brief Datalines for %upcase(%scan(&base_ds,2)) dataset";
    put "  @details Generated by %nrstr(%%)mp_ds2cards()";
-    put "  Available on github.com/sasjs/core";
+    put "    Source: https://github.com/sasjs/core";
+    put '  @cond ';
    put '**/';
    put "data &tgt_ds &indexes;";
    put "attrib ";
@@ -264,6 +266,7 @@ data _null_;
  if __lastobs then do;
    put ';;;;';
    put 'run;';
+    put '/** @endcond **/';
    stop;
  end;
 run;
@@ -283,4 +286,5 @@ quit;
 %put NOTE-;%put NOTE-;
 %put NOTE- %sysfunc(dequote(&cards_file.));
 %put NOTE-;%put NOTE-;
-%mend mp_ds2cards;
+%mend mp_ds2cards;
+/** @endcond **/
--- a/base/mp_getpk.sas
+++ b/base/mp_getpk.sas
@@ -55,7 +55,8 @@
 )/*/STORE SOURCE*/;


-%local engine schema ds1 ds2 ds3 dsn tabs1 tabs2 sum pk4sure pkdefault finalpks;
+%local engine schema ds1 ds2 ds3 dsn tabs1 tabs2 sum pk4sure pkdefault finalpks
+  pkfromindex;

 %let lib=%upcase(&lib);
 %let ds=%upcase(&ds);
@@ -70,6 +71,7 @@
 %let sum=%mf_getuniquename(prefix=getpk_sum);
 %let pk4sure=%mf_getuniquename(prefix=getpk_pk4sure);
 %let pkdefault=%mf_getuniquename(prefix=getpk_pkdefault);
+%let pkfromindex=%mf_getuniquename(prefix=getpk_pkfromindex);
 %let finalpks=%mf_getuniquename(prefix=getpk_finalpks);

 %local dbg;
@@ -180,9 +182,23 @@ create table &ds1 as
      and a.constraint_name=b.constraint_name
    order by 1,2,3,4;

+  /* extract cols from the relevant unique INDEXES */
+  create table &pkfromindex as
+    select libname as libref
+      ,memname as table_name
+      ,indxname as constraint_name
+      ,indxpos as constraint_order
+      ,name
+    from dictionary.indexes
+    where nomiss='yes' and unique='yes' and upcase(libname)="&lib"
+  %if &ds ne 0 %then %do;
+      and upcase(memname)="&ds"
+  %end;
+    order by 1,2,3,4;
+
  /* create one table */
  data &finalpks;
-    set &pkdefault &pk4sure ;
+    set &pkdefault &pk4sure &pkfromindex;
    pk_ind=1;
    /* if there are multiple unique constraints, take the first */
    by libref table_name constraint_name;
@@ -256,4 +272,4 @@ create table &outds as
  iftrue=(&mdebug=0)
 )

-%mend mp_getpk;
+%mend mp_getpk;
--- a/base/mp_init.sas
+++ b/base/mp_init.sas
@@ -73,4 +73,4 @@ options
 %end;
 ;

-%mend mp_init;
+%mend mp_init;
--- a/base/mp_jsonout.sas
+++ b/base/mp_jsonout.sas
@@ -199,19 +199,27 @@
      format _numeric_ bart.;
    %do i=1 %to &numcols;
      %if &&typelong&i=char or &fmt=Y %then %do;
-        &&name&i='"'!!trim(prxchange('s/"/\"/',-1,
-                    prxchange('s/'!!'0A'x!!'/\n/',-1,
-                    prxchange('s/'!!'0D'x!!'/\r/',-1,
-                    prxchange('s/'!!'09'x!!'/\t/',-1,
-                    prxchange('s/'!!'00'x!!'/\\u0000/',-1, /* NUL */
-                    prxchange('s/'!!'0E'x!!'/\\u000E/',-1, /* SS  */
-                    prxchange('s/'!!'0F'x!!'/\\u000F/',-1, /* SF  */
-                    prxchange('s/'!!'01'x!!'/\\u0001/',-1, /* SOH */
-                    prxchange('s/'!!'02'x!!'/\\u0002/',-1, /* STX */
-                    prxchange('s/'!!'02'x!!'/\\u0010/',-1, /* DLE */
-                    prxchange('s/'!!'11'x!!'/\\u0011/',-1, /* DC1 */
-                    prxchange('s/\\/\\\\/',-1,&&name&i)
-        ))))))))))))!!'"';
+        if prxmatch(
+            'm/\"|\x0A|\x0D|\x09|\x00|\x0E|\x0F|\x01|\x02|\x10|\x11|\\/o',
+            &&name&i
+          )>0
+        then do;
+          &&name&i='"'!!trim(
+            prxchange('s/"/\\"/',-1,        /* double quote */
+            prxchange('s/\x0A/\n/',-1,      /* new line */
+            prxchange('s/\x0D/\r/',-1,      /* carriage return */
+            prxchange('s/\x09/\\t/',-1,     /* tab */
+            prxchange('s/\x00/\\u0000/',-1, /* NUL */
+            prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+            prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+            prxchange('s/\x01/\\u0001/',-1, /* SOH */
+            prxchange('s/\x02/\\u0002/',-1, /* STX */
+            prxchange('s/\x10/\\u0010/',-1, /* DLE */
+            prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+            prxchange('s/\\/\\\\/',-1,&&name&i)
+          ))))))))))))!!'"';
+        end;
+        else &&name&i=quote(cats(&&name&i));
      %end;
    %end;
    run;
--- a/base/mp_streamfile.sas
+++ b/base/mp_streamfile.sas
@@ -105,12 +105,20 @@ run;
  %end;
 %end;
 %else %if &contentype=HTML %then %do;
-  %if &platform=SASVIYA %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
+    data _null_;
+      rc=stpsrv_header('Content-type','text/html');
+      rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
+    run;
+  %end;
+  %else %if &platform=SASVIYA %then %do;
    filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
-      contenttype="text/html";
+      contenttype="text/html"
+      contentdisp="attachment; filename=&outname";
  %end;
  %else %if &platform=SASJS %then %do;
    %mfs_httpheader(Content-type,text/html)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
  %end;
 %end;
 %else %if &contentype=TEXT %then %do;
--- a/meta/mm_createwebservice.sas
+++ b/meta/mm_createwebservice.sas
@@ -232,19 +232,27 @@ data _null_;
  put '      format _numeric_ bart.; ';
  put '    %do i=1 %to &numcols; ';
  put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
-  put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
-  put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
-  put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
-  put '                    prxchange(''s/''!!''00''x!!''/\\u0000/'',-1, /* NUL */ ';
-  put '                    prxchange(''s/''!!''0E''x!!''/\\u000E/'',-1, /* SS  */ ';
-  put '                    prxchange(''s/''!!''0F''x!!''/\\u000F/'',-1, /* SF  */ ';
-  put '                    prxchange(''s/''!!''01''x!!''/\\u0001/'',-1, /* SOH */ ';
-  put '                    prxchange(''s/''!!''02''x!!''/\\u0002/'',-1, /* STX */ ';
-  put '                    prxchange(''s/''!!''02''x!!''/\\u0010/'',-1, /* DLE */ ';
-  put '                    prxchange(''s/''!!''11''x!!''/\\u0011/'',-1, /* DC1 */ ';
-  put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        ))))))))))))!!''"''; ';
+  put '        if prxmatch( ';
+  put '            ''m/\"|\x0A|\x0D|\x09|\x00|\x0E|\x0F|\x01|\x02|\x10|\x11|\\/o'', ';
+  put '            &&name&i ';
+  put '          )>0 ';
+  put '        then do; ';
+  put '          &&name&i=''"''!!trim( ';
+  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '          ))))))))))))!!''"''; ';
+  put '        end; ';
+  put '        else &&name&i=quote(cats(&&name&i)); ';
  put '      %end; ';
  put '    %end; ';
  put '    run; ';
@@ -428,14 +436,16 @@ data _null_;
  put '    put '',"_METAPERSON": '' _METAPERSON; ';
  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
  put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
+  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
  put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
  put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
  put '    put ",""SYSSITE"" : ""&syssite"" "; ';
  put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
  put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
+  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
  put '    length memsize $32; ';
  put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
--- a/meta/mm_webout.sas
+++ b/meta/mm_webout.sas
@@ -157,14 +157,16 @@
    put ',"_METAPERSON": ' _METAPERSON;
    put ',"_PROGRAM" : ' _PROGRAM ;
    put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
    put ",""SYSJOBID"" : ""&sysjobid"" ";
    put ",""SYSSCPL"" : ""&sysscpl"" ";
    put ",""SYSSITE"" : ""&syssite"" ";
    sysvlong=quote(trim(symget('sysvlong')));
    put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
    length memsize $32;
    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
--- a/package-lock.json
+++ b/package-lock.json
--- a/package.json
+++ b/package.json
@@ -26,14 +26,13 @@
  "homepage": "https://core.sasjs.io",
  "main": "index.js",
  "scripts": {
-    "build": "sasjs cbd -t viya",
+    "build": "sasjs cbd -t server",
    "docs": "sasjs doc -t docsonly && ./sasjs/utils/build.sh",
-    "test": "sasjs test -t viya",
+    "test": "sasjs test -t server",
    "lint": "sasjs lint",
    "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
  },
  "devDependencies": {
-    "@sasjs/cli": "3.6.0",
-    "@sasjs/core": "4.4.7"
+    "@sasjs/cli": "3.13.0"
  }
 }
--- a/sasjs/sasjsconfig.json
+++ b/sasjs/sasjsconfig.json
@@ -25,26 +25,20 @@
      "mcTestAppLoc": "/Public/temp/macrocore"
    }
  },
-  "defaultTarget": "viya",
+  "defaultTarget": "server",
  "targets": [
    {
      "name": "viya",
-      "serverUrl": "https://sas.analytium.co.uk",
+      "serverUrl": "",
      "serverType": "SASVIYA",
      "httpsAgentOptions": {
        "allowInsecureRequests": false
      },
-      "appLoc": "/Public/temp/macrocore",
+      "appLoc": "/Public/app/macrocore",
      "macroFolders": [
        "viya",
        "tests/viyaonly"
      ],
-      "programFolders": [],
-      "binaryFolders": [],
-      "deployConfig": {
-        "deployServicePack": true,
-        "deployScripts": []
-      },
      "contextName": "SAS Job Execution compute context"
    },
    {
--- a/server/ms_createfile.sas
+++ b/server/ms_createfile.sas
@@ -29,12 +29,16 @@
    ,mdebug=0
  );

-%local fname0 fname1 fname2 boundary fname statcd msg;
+%local fname0 fname1 fname2 boundary fname statcd msg optval;
 %let fname0=%mf_getuniquefileref();
 %let fname1=%mf_getuniquefileref();
 %let fname2=%mf_getuniquefileref();
 %let boundary=%mf_getuniquename();

+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
 data _null_;
  file &fname0 termstr=crlf;
  infile &inref end=eof;
@@ -95,4 +99,7 @@ run;
  ,msg=%superq(msg)
 )

+/* reset options */
+options &optval;
+
 %mend ms_createfile;
--- a/server/ms_webout.sas
+++ b/server/ms_webout.sas
@@ -1,6 +1,6 @@
 /**
  @file
-  @brief Send data to/from @sasjs/server
+  @brief Send data to/from sasjs/server
  @details This macro should be added to the start of each web service,
  **immediately** followed by a call to:

@@ -147,7 +147,8 @@
    put ",""_DEBUG"" : ""&_debug"" ";
    put ',"_PROGRAM" : ' _PROGRAM ;
    put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
    SYSHOSTINFOLONG=quote(trim(symget('SYSHOSTINFOLONG')));
    put ',"SYSHOSTINFOLONG" : ' SYSHOSTINFOLONG;
    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
@@ -162,7 +163,8 @@
    put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" ";
    sysvlong=quote(trim(symget('sysvlong')));
    put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
    length autoexec $512;
    autoexec=quote(urlencode(trim(getoption('autoexec'))));
--- a/tests/crossplatform/mp_getpk.test.sas
+++ b/tests/crossplatform/mp_getpk.test.sas
@@ -88,4 +88,50 @@ run;
 /* constraint capture at library level is functional - uses first 2 tests */
 %mp_getpk(work,outds=test4)

-%mp_assertdsobs(work.test4,test=ATLEAST 2)
+%mp_assertdsobs(work.test4,test=ATLEAST 2)
+
+/* unique & not null INDEX captured */
+proc sql;
+create table work.example5(
+  TX_FROM float format=datetime19.,
+  DD_TYPE char(16),
+  DD_SOURCE char(2048),
+  DD_SHORTDESC char(256)
+);
+proc datasets lib=work noprint;
+  modify example5;
+  index create tx_from /nomiss unique;
+quit;
+%mp_getpk(work,ds=example5,outds=test5)
+data _null_;
+  set work.test5;
+  call symputx('test5',pk_fields);
+run;
+%mp_assert(
+  iftrue=("&test5"="TX_FROM"),
+  desc=mp_getpk captures single column not null unique index,
+  outds=work.test_results
+)
+
+/* unique & not null COMPOSITE INDEX captured */
+proc sql;
+create table work.example6(
+  TX_FROM float format=datetime19.,
+  DD_TYPE char(16),
+  DD_SOURCE char(2048),
+  DD_SHORTDESC char(256)
+);
+proc datasets lib=work noprint;
+  modify example6;
+  index create pk_6=(tx_from dd_type) /nomiss unique;
+quit;
+%mp_getpk(work,ds=example6,outds=test6)
+data _null_;
+  set work.test6;
+  call symputx('test6',pk_fields);
+run;
+%mp_assert(
+  iftrue=("&test6"="TX_FROM DD_TYPE"),
+  desc=mp_getpk captures multiple column not null unique index,
+  outds=work.test_results
+)
--- a/tests/crossplatform/mp_hashdataset.test.sas
+++ b/tests/crossplatform/mp_hashdataset.test.sas
@@ -3,6 +3,7 @@
  @brief Testing mp_hashdataset.sas macro

  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
  @li mp_hashdataset.sas
  @li mp_assert.sas
  @li mp_assertscope.sas
--- a/tests/crossplatform/mp_jsonout.test.3.sas
+++ b/tests/crossplatform/mp_jsonout.test.3.sas
@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Testing mp_jsonout.sas macro with non-standard chars
+
+  <h4> SAS Macros </h4>
+  @li mp_jsonout.sas
+  @li mp_assert.sas
+
+**/
+
+filename webref temp;
+
+data demo;
+  do x='"','0A'x,'0D'x,'09'x,'00'x,'0E'x,'0F'x,'01'x,'02'x,'10'x,'11'x,'\';
+    output;
+  end;
+run;
+%mp_jsonout(OPEN,jref=webref)
+%mp_jsonout(OBJ,demo,jref=webref)
+%mp_jsonout(CLOSE,jref=webref)
+
+data _null_;
+  infile webref;
+  input;
+  putlog _infile_;
+run;
+
+libname web JSON fileref=webref;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking for error condition with special chars export,
+  outds=work.test_results
+)
+
+/*
+data _null_;
+  set work.demo (in=start) web.demo (in=end);
+  put (_all_)(=);
+run;
+proc sql;
+describe table work.demo;
+describe table web.demo;
+*/
+
+proc compare base=work.demo compare=web.demo(keep=x);
+quit;
+
+%mp_assert(
+  iftrue=(&sysinfo=0),
+  desc=Returned json is identical to input table for all special chars,
+  outds=work.test_results
+)
--- a/tests/viyaonly/mv_webout.test.sas
+++ b/tests/viyaonly/mv_webout.test.sas
@@ -1,15 +1,33 @@
 /**
  @file
-  @brief Testing mm_webout macro
+  @brief Testing mv_webout macro

  <h4> SAS Macros </h4>
  @li mf_getuniquefileref.sas
  @li mv_webout.sas
  @li mp_assert.sas
+  @li mp_assertdsobs.sas

 **/


+/* testing FETCHing (WEB approach) */
+
+data _null_;
+  call symputx('sasjs1data','area:$char4.'!!'0d0a'x!!'Adak');
+  call symputx('sasjs_tables','areas');
+run;
+%put &=sasjs1data;
+
+%mv_webout(FETCH)
+
+%mp_assertdsobs(work.areas,
+  desc=Test input table has 1 row,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+
 %let fref=%mf_getuniquefileref();
 %global _metaperson;
 data some datasets;
@@ -38,4 +56,4 @@ run;
 %mp_assert(
  iftrue=(%str(&checkval)=%str(&sysvlong)),
  desc=Check if the sysvlong value was created
-)
+)
--- a/viya/mv_createwebservice.sas
+++ b/viya/mv_createwebservice.sas
@@ -376,19 +376,27 @@ data _null_;
  put '      format _numeric_ bart.; ';
  put '    %do i=1 %to &numcols; ';
  put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
-  put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
-  put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
-  put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
-  put '                    prxchange(''s/''!!''00''x!!''/\\u0000/'',-1, /* NUL */ ';
-  put '                    prxchange(''s/''!!''0E''x!!''/\\u000E/'',-1, /* SS  */ ';
-  put '                    prxchange(''s/''!!''0F''x!!''/\\u000F/'',-1, /* SF  */ ';
-  put '                    prxchange(''s/''!!''01''x!!''/\\u0001/'',-1, /* SOH */ ';
-  put '                    prxchange(''s/''!!''02''x!!''/\\u0002/'',-1, /* STX */ ';
-  put '                    prxchange(''s/''!!''02''x!!''/\\u0010/'',-1, /* DLE */ ';
-  put '                    prxchange(''s/''!!''11''x!!''/\\u0011/'',-1, /* DC1 */ ';
-  put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        ))))))))))))!!''"''; ';
+  put '        if prxmatch( ';
+  put '            ''m/\"|\x0A|\x0D|\x09|\x00|\x0E|\x0F|\x01|\x02|\x10|\x11|\\/o'', ';
+  put '            &&name&i ';
+  put '          )>0 ';
+  put '        then do; ';
+  put '          &&name&i=''"''!!trim( ';
+  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '          ))))))))))))!!''"''; ';
+  put '        end; ';
+  put '        else &&name&i=quote(cats(&&name&i)); ';
  put '      %end; ';
  put '    %end; ';
  put '    run; ';
@@ -461,7 +469,7 @@ data _null_;
  put '  sasjs_tables SYS_JES_JOB_URI; ';
  put '%if %index("&_debug",log) %then %let _debug=131; ';
  put ' ';
-  put '%local i tempds; ';
+  put '%local i tempds table; ';
  put '%let action=%upcase(&action); ';
  put ' ';
  put '%if &action=FETCH %then %do; ';
@@ -476,60 +484,35 @@ data _null_;
  put '  %end; ';
  put ' ';
  put '  /* if the sasjs_tables param is passed, we expect param based upload */ ';
-  put '  %if %length(&sasjs_tables.XX)>2 %then %do; ';
-  put '    filename _sasjs "%sysfunc(pathname(work))/sasjs.lua"; ';
-  put '    data _null_; ';
-  put '      file _sasjs; ';
-  put '      put ''s=sas.symget("sasjs_tables")''; ';
-  put '      put ''if(s:sub(1,7) == "%nrstr(")''; ';
-  put '      put ''then''; ';
-  put '      put '' tablist=s:sub(8,s:len()-1)''; ';
-  put '      put ''else''; ';
-  put '      put '' tablist=s''; ';
-  put '      put ''end''; ';
-  put '      put ''for i = 1,sas.countw(tablist) ''; ';
-  put '      put ''do ''; ';
-  put '      put ''  tab=sas.scan(tablist,i)''; ';
-  put '      put ''  sasdata=""''; ';
-  put '      put ''  if (sas.symexist("sasjs"..i.."data0")==0)''; ';
-  put '      put ''  then''; ';
-  put '      /* TODO - condense this logic */ ';
-  put '      put ''    s=sas.symget("sasjs"..i.."data")''; ';
-  put '      put ''    if(s:sub(1,7) == "%nrstr(")''; ';
-  put '      put ''    then''; ';
-  put '      put ''      sasdata=s:sub(8,s:len()-1)''; ';
-  put '      put ''    else''; ';
-  put '      put ''      sasdata=s''; ';
-  put '      put ''    end''; ';
-  put '      put ''  else''; ';
-  put '      put ''    for d = 1, sas.symget("sasjs"..i.."data0")''; ';
-  put '      put ''    do''; ';
-  put '      put ''      s=sas.symget("sasjs"..i.."data"..d)''; ';
-  put '      put ''      if(s:sub(1,7) == "%nrstr(")''; ';
-  put '      put ''      then''; ';
-  put '      put ''        sasdata=sasdata..s:sub(8,s:len()-1)''; ';
-  put '      put ''      else''; ';
-  put '      put ''        sasdata=sasdata..s''; ';
-  put '      put ''      end''; ';
-  put '      put ''    end''; ';
-  put '      put ''  end''; ';
-  put '      put ''  file = io.open(sas.pathname("work").."/"..tab..".csv", "a")''; ';
-  put '      put ''  io.output(file)''; ';
-  put '      put ''  io.write(sasdata)''; ';
-  put '      put ''  io.close(file)''; ';
-  put '      put ''end''; ';
-  put '    run; ';
-  put '    %inc _sasjs; ';
+  put '  %if %length(&sasjs_tables.X)>1 %then %do; ';
  put ' ';
-  put '    /* now read in the data */ ';
+  put '    /* convert data from macro variables to datasets */ ';
  put '    %do i=1 %to %sysfunc(countw(&sasjs_tables)); ';
-  put '      %local table; %let table=%scan(&sasjs_tables,&i); ';
+  put '      %let table=%scan(&sasjs_tables,&i,%str( )); ';
+  put '      %if %symexist(sasjs&i.data0)=0 %then %let sasjs&i.data0=1; ';
+  put '      data _null_; ';
+  put '        file "%sysfunc(pathname(work))/&table..csv" recfm=n; ';
+  put '        retain nrflg 0; ';
+  put '        length line $32767; ';
+  put '        do i=1 to &&sasjs&i.data0; ';
+  put '          if &&sasjs&i.data0=1 then line=symget("sasjs&i.data"); ';
+  put '          else line=symget(cats("sasjs&i.data",i)); ';
+  put '          if i=1 and substr(line,1,7)=''%nrstr('' then do; ';
+  put '            nrflg=1; ';
+  put '            line=substr(line,8); ';
+  put '          end; ';
+  put '          if i=&&sasjs&i.data0 and nrflg=1 then do; ';
+  put '            line=substr(line,1,length(line)-1); ';
+  put '          end; ';
+  put '          put line +(-1) @; ';
+  put '        end; ';
+  put '      run; ';
  put '      data _null_; ';
  put '        infile "%sysfunc(pathname(work))/&table..csv" termstr=crlf ; ';
  put '        input; ';
  put '        if _n_=1 then call symputx(''input_statement'',_infile_); ';
  put '        list; ';
-  put '      data &table; ';
+  put '      data work.&table; ';
  put '        infile "%sysfunc(pathname(work))/&table..csv" firstobs=2 dsd ';
  put '          termstr=crlf; ';
  put '        input &input_statement; ';
@@ -630,13 +613,15 @@ data _null_;
  put '    put ",""_DEBUG"" : ""&_debug"" "; ';
  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
  put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
+  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
  put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
  put '    put ",""SYSSITE"" : ""&syssite"" "; ';
  put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
  put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
+  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
  put '    length memsize $32; ';
  put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
--- a/viya/mv_webout.sas
+++ b/viya/mv_webout.sas
@@ -49,7 +49,7 @@
  sasjs_tables SYS_JES_JOB_URI;
 %if %index("&_debug",log) %then %let _debug=131;

-%local i tempds;
+%local i tempds table;
 %let action=%upcase(&action);

 %if &action=FETCH %then %do;
@@ -64,60 +64,35 @@
  %end;

  /* if the sasjs_tables param is passed, we expect param based upload */
-  %if %length(&sasjs_tables.XX)>2 %then %do;
-    filename _sasjs "%sysfunc(pathname(work))/sasjs.lua";
-    data _null_;
-      file _sasjs;
-      put 's=sas.symget("sasjs_tables")';
-      put 'if(s:sub(1,7) == "%nrstr(")';
-      put 'then';
-      put ' tablist=s:sub(8,s:len()-1)';
-      put 'else';
-      put ' tablist=s';
-      put 'end';
-      put 'for i = 1,sas.countw(tablist) ';
-      put 'do ';
-      put '  tab=sas.scan(tablist,i)';
-      put '  sasdata=""';
-      put '  if (sas.symexist("sasjs"..i.."data0")==0)';
-      put '  then';
-      /* TODO - condense this logic */
-      put '    s=sas.symget("sasjs"..i.."data")';
-      put '    if(s:sub(1,7) == "%nrstr(")';
-      put '    then';
-      put '      sasdata=s:sub(8,s:len()-1)';
-      put '    else';
-      put '      sasdata=s';
-      put '    end';
-      put '  else';
-      put '    for d = 1, sas.symget("sasjs"..i.."data0")';
-      put '    do';
-      put '      s=sas.symget("sasjs"..i.."data"..d)';
-      put '      if(s:sub(1,7) == "%nrstr(")';
-      put '      then';
-      put '        sasdata=sasdata..s:sub(8,s:len()-1)';
-      put '      else';
-      put '        sasdata=sasdata..s';
-      put '      end';
-      put '    end';
-      put '  end';
-      put '  file = io.open(sas.pathname("work").."/"..tab..".csv", "a")';
-      put '  io.output(file)';
-      put '  io.write(sasdata)';
-      put '  io.close(file)';
-      put 'end';
-    run;
-    %inc _sasjs;
+  %if %length(&sasjs_tables.X)>1 %then %do;

-    /* now read in the data */
+    /* convert data from macro variables to datasets */
    %do i=1 %to %sysfunc(countw(&sasjs_tables));
-      %local table; %let table=%scan(&sasjs_tables,&i);
+      %let table=%scan(&sasjs_tables,&i,%str( ));
+      %if %symexist(sasjs&i.data0)=0 %then %let sasjs&i.data0=1;
+      data _null_;
+        file "%sysfunc(pathname(work))/&table..csv" recfm=n;
+        retain nrflg 0;
+        length line $32767;
+        do i=1 to &&sasjs&i.data0;
+          if &&sasjs&i.data0=1 then line=symget("sasjs&i.data");
+          else line=symget(cats("sasjs&i.data",i));
+          if i=1 and substr(line,1,7)='%nrstr(' then do;
+            nrflg=1;
+            line=substr(line,8);
+          end;
+          if i=&&sasjs&i.data0 and nrflg=1 then do;
+            line=substr(line,1,length(line)-1);
+          end;
+          put line +(-1) @;
+        end;
+      run;
      data _null_;
        infile "%sysfunc(pathname(work))/&table..csv" termstr=crlf ;
        input;
        if _n_=1 then call symputx('input_statement',_infile_);
        list;
-      data &table;
+      data work.&table;
        infile "%sysfunc(pathname(work))/&table..csv" firstobs=2 dsd
          termstr=crlf;
        input &input_statement;
@@ -218,13 +193,15 @@
    put ",""_DEBUG"" : ""&_debug"" ";
    put ',"_PROGRAM" : ' _PROGRAM ;
    put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
    put ",""SYSSCPL"" : ""&sysscpl"" ";
    put ",""SYSSITE"" : ""&syssite"" ";
    sysvlong=quote(trim(symget('sysvlong')));
    put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
    length memsize $32;
    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
Author	SHA1	Message	Date
Allan Bowe	c94c334c4b	Merge pull request #219 from sasjs/issue218 fix: check 4 special chars b4 replacing. Closes #218	2022-04-16 22:44:35 +03:00
munja	3bf44405f8	chore: alignment	2022-04-16 20:37:47 +01:00
munja	db68a256cb	fix: check 4 special chars b4 replacing. Closes #218	2022-04-16 20:33:21 +01:00
Allan Bowe	611fac6338	Merge pull request #217 from sasjs/bom fix: avoid sending BOM marker to SASjs API	2022-04-14 21:32:25 +03:00
munja	ddd120bb75	fix: avoid sending BOM marker to SASjs API	2022-04-14 19:31:36 +01:00
Allan Bowe	0d75e0bad8	Merge pull request #216 from sasjs/cli-issue-1113 chore(dep): removed sasjs/core and bumped sasjs/cli	2022-04-14 18:25:17 +03:00
Yury Shkoda	ba8c4ac844	chore: Merge branch 'main' of https://github.com/sasjs/core into cli-issue-1113	2022-04-14 18:22:57 +03:00
Allan Bowe	6938a42896	Merge pull request #215 from sasjs/serverconfig Change test server & update mx_webout() macros	2022-04-14 15:41:44 +03:00
munja	efff77c94e	fix: mx_webout	2022-04-14 13:40:14 +01:00
Yury Shkoda	2b10cf6192	chore(dep): removed sasjs/core and bumped sasjs/cli	2022-04-14 14:55:43 +03:00
Allan Bowe	134b91f266	fix: avoiding running tests on viya	2022-04-14 11:14:35 +00:00
Allan Bowe	969f551e10	Merge pull request #214 from sasjs/cond fix: adding cond/endcond to mp_ds2cards	2022-04-13 23:34:52 +03:00
munja	26623ba085	fix: adding cond/endcond to mp_ds2cards	2022-04-13 21:32:47 +01:00
Allan Bowe	8eb495890d	Merge pull request #213 from sasjs/issue212 fix: ensuring indexes are picked up in mp_getpk(). Closes #212	2022-04-13 15:59:22 +03:00
munja	c1a30977f1	chore: removing serverurl	2022-04-13 13:58:55 +01:00
munja	9a6be61651	fix: ensuring indexes are picked up in mp_getpk(). Closes #212	2022-04-13 13:57:43 +01:00
Allan Bowe	388839039e	Merge pull request #211 from sasjs/streamhtmlasfile fix: send html as attachment rather than streamed content	2022-04-11 12:56:02 +03:00
Allan Bowe	e760a89a6a	fix: send html as attachment rather than streamed content when mp_streamfile is used on all platforms	2022-04-11 09:54:07 +00:00
Allan Bowe	d2e30267e8	Merge pull request #210 from sasjs/nulfixes fix: using regex special chars instead of hex constants in mp_jsonout	2022-04-07 11:53:37 +03:00
munja	190dbddfe3	fix: using regex special chars instead of hex constants in mp_jsonout	2022-04-07 09:52:56 +01:00
Allan Bowe	05e769794e	Merge pull request #209 from sasjs/missref fix: avoid: ERROR: Variable "fref" may not be initialized	2022-04-06 18:26:45 +03:00
munja	558ebaf6f2	fix: avoid: ERROR: Variable "fref" may not be initialized	2022-04-06 16:25:50 +01:00
Allan Bowe	970b56fe5a	Merge pull request #208 from sasjs/issue207 fix: removing LUA dependency from mv_webout to enable Viya 4 compatibility	2022-04-01 20:42:22 +03:00
munja	c2597bd07b	fix: missing dependency in mp_hashdataset.test.sas	2022-04-01 17:29:06 +01:00
munja	c4baca477b	fix: removing LUA dependency from mv_webout to enable Viya 4 compatibility	2022-04-01 16:56:49 +01:00