Merge pull request #220 from sasjs/perf

fix: using findc instead of regex for faster parsing
2026-01-02 07:10:06 +00:00 · 2022-04-16 23:31:20 +03:00 · 2022-04-16 21:30:18 +01:00 · 2022-04-16 22:44:35 +03:00 · 2022-04-16 20:37:47 +01:00 · 2022-04-16 20:33:21 +01:00
13 changed files with 557 additions and 2220 deletions
--- a/all.sas
+++ b/all.sas
@@ -8239,7 +8239,8 @@ options
 %end;
 ;

-%mend mp_init;/**
+%mend mp_init;
+/**
  @file mp_jsonout.sas
  @brief Writes JSON in SASjs format to a fileref
  @details PROC JSON is faster but will produce errs like the ones below if
@@ -8440,19 +8441,23 @@ options
      format _numeric_ bart.;
    %do i=1 %to &numcols;
      %if &&typelong&i=char or &fmt=Y %then %do;
-        &&name&i='"'!!trim(prxchange('s/"/\"/',-1,
-                    prxchange('s/'!!'0A'x!!'/\n/',-1,
-                    prxchange('s/'!!'0D'x!!'/\r/',-1,
-                    prxchange('s/'!!'09'x!!'/\t/',-1,
-                    prxchange('s/\x00/\\u0000/',-1, /* NUL */
-                    prxchange('s/\x0E/\\u000E/',-1, /* SS  */
-                    prxchange('s/\x0F/\\u000F/',-1, /* SF  */
-                    prxchange('s/\x01/\\u0001/',-1, /* SOH */
-                    prxchange('s/\x02/\\u0002/',-1, /* STX */
-                    prxchange('s/\x10/\\u0010/',-1, /* DLE */
-                    prxchange('s/\x11/\\u0011/',-1, /* DC1 */
-                    prxchange('s/\\/\\\\/',-1,&&name&i)
-        ))))))))))))!!'"';
+        if findc(&&name&i,'"\'!!'0A0D09000E0F01021011'x) then do;
+          &&name&i='"'!!trim(
+            prxchange('s/"/\\"/',-1,        /* double quote */
+            prxchange('s/\x0A/\n/',-1,      /* new line */
+            prxchange('s/\x0D/\r/',-1,      /* carriage return */
+            prxchange('s/\x09/\\t/',-1,     /* tab */
+            prxchange('s/\x00/\\u0000/',-1, /* NUL */
+            prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+            prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+            prxchange('s/\x01/\\u0001/',-1, /* SOH */
+            prxchange('s/\x02/\\u0002/',-1, /* STX */
+            prxchange('s/\x10/\\u0010/',-1, /* DLE */
+            prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+            prxchange('s/\\/\\\\/',-1,&&name&i)
+          ))))))))))))!!'"';
+        end;
+        else &&name&i=quote(cats(&&name&i));
      %end;
    %end;
    run;
@@ -14965,19 +14970,23 @@ data _null_;
  put '      format _numeric_ bart.; ';
  put '    %do i=1 %to &numcols; ';
  put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
-  put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
-  put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
-  put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
-  put '                    prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
-  put '                    prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
-  put '                    prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
-  put '                    prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
-  put '                    prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
-  put '                    prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
-  put '                    prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
-  put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        ))))))))))))!!''"''; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '          &&name&i=''"''!!trim( ';
+  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '          ))))))))))))!!''"''; ';
+  put '        end; ';
+  put '        else &&name&i=quote(cats(&&name&i)); ';
  put '      %end; ';
  put '    %end; ';
  put '    run; ';
@@ -15161,14 +15170,16 @@ data _null_;
  put '    put '',"_METAPERSON": '' _METAPERSON; ';
  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
  put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
+  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
  put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
  put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
  put '    put ",""SYSSITE"" : ""&syssite"" "; ';
  put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
  put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
+  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
  put '    length memsize $32; ';
  put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
@@ -18698,14 +18709,16 @@ run;
    put ',"_METAPERSON": ' _METAPERSON;
    put ',"_PROGRAM" : ' _PROGRAM ;
    put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
    put ",""SYSJOBID"" : ""&sysjobid"" ";
    put ",""SYSSCPL"" : ""&sysscpl"" ";
    put ",""SYSSITE"" : ""&syssite"" ";
    sysvlong=quote(trim(symget('sysvlong')));
    put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
    length memsize $32;
    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
@@ -18933,12 +18946,16 @@ run;
    ,mdebug=0
  );

-%local fname0 fname1 fname2 boundary fname statcd msg;
+%local fname0 fname1 fname2 boundary fname statcd msg optval;
 %let fname0=%mf_getuniquefileref();
 %let fname1=%mf_getuniquefileref();
 %let fname2=%mf_getuniquefileref();
 %let boundary=%mf_getuniquename();

+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
 data _null_;
  file &fname0 termstr=crlf;
  infile &inref end=eof;
@@ -18999,6 +19016,9 @@ run;
  ,msg=%superq(msg)
 )

+/* reset options */
+options &optval;
+
 %mend ms_createfile;
 /**
  @file
@@ -19301,7 +19321,8 @@ run;
    put ",""_DEBUG"" : ""&_debug"" ";
    put ',"_PROGRAM" : ' _PROGRAM ;
    put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
    SYSHOSTINFOLONG=quote(trim(symget('SYSHOSTINFOLONG')));
    put ',"SYSHOSTINFOLONG" : ' SYSHOSTINFOLONG;
    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
@@ -19316,7 +19337,8 @@ run;
    put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" ";
    sysvlong=quote(trim(symget('sysvlong')));
    put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
    length autoexec $512;
    autoexec=quote(urlencode(trim(getoption('autoexec'))));
@@ -20443,19 +20465,23 @@ data _null_;
  put '      format _numeric_ bart.; ';
  put '    %do i=1 %to &numcols; ';
  put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
-  put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
-  put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
-  put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
-  put '                    prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
-  put '                    prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
-  put '                    prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
-  put '                    prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
-  put '                    prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
-  put '                    prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
-  put '                    prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
-  put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        ))))))))))))!!''"''; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '          &&name&i=''"''!!trim( ';
+  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '          ))))))))))))!!''"''; ';
+  put '        end; ';
+  put '        else &&name&i=quote(cats(&&name&i)); ';
  put '      %end; ';
  put '    %end; ';
  put '    run; ';
@@ -20672,13 +20698,15 @@ data _null_;
  put '    put ",""_DEBUG"" : ""&_debug"" "; ';
  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
  put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
+  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
  put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
  put '    put ",""SYSSITE"" : ""&syssite"" "; ';
  put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
  put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
+  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
  put '    length memsize $32; ';
  put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
@@ -24479,13 +24507,15 @@ filename &fref1 clear;
    put ",""_DEBUG"" : ""&_debug"" ";
    put ',"_PROGRAM" : ' _PROGRAM ;
    put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
    put ",""SYSSCPL"" : ""&sysscpl"" ";
    put ",""SYSSITE"" : ""&syssite"" ";
    sysvlong=quote(trim(symget('sysvlong')));
    put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
    length memsize $32;
    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
--- a/base/mp_init.sas
+++ b/base/mp_init.sas
@@ -73,4 +73,4 @@ options
 %end;
 ;

-%mend mp_init;
+%mend mp_init;
--- a/base/mp_jsonout.sas
+++ b/base/mp_jsonout.sas
@@ -199,19 +199,23 @@
      format _numeric_ bart.;
    %do i=1 %to &numcols;
      %if &&typelong&i=char or &fmt=Y %then %do;
-        &&name&i='"'!!trim(prxchange('s/"/\"/',-1,
-                    prxchange('s/'!!'0A'x!!'/\n/',-1,
-                    prxchange('s/'!!'0D'x!!'/\r/',-1,
-                    prxchange('s/'!!'09'x!!'/\t/',-1,
-                    prxchange('s/\x00/\\u0000/',-1, /* NUL */
-                    prxchange('s/\x0E/\\u000E/',-1, /* SS  */
-                    prxchange('s/\x0F/\\u000F/',-1, /* SF  */
-                    prxchange('s/\x01/\\u0001/',-1, /* SOH */
-                    prxchange('s/\x02/\\u0002/',-1, /* STX */
-                    prxchange('s/\x10/\\u0010/',-1, /* DLE */
-                    prxchange('s/\x11/\\u0011/',-1, /* DC1 */
-                    prxchange('s/\\/\\\\/',-1,&&name&i)
-        ))))))))))))!!'"';
+        if findc(&&name&i,'"\'!!'0A0D09000E0F01021011'x) then do;
+          &&name&i='"'!!trim(
+            prxchange('s/"/\\"/',-1,        /* double quote */
+            prxchange('s/\x0A/\n/',-1,      /* new line */
+            prxchange('s/\x0D/\r/',-1,      /* carriage return */
+            prxchange('s/\x09/\\t/',-1,     /* tab */
+            prxchange('s/\x00/\\u0000/',-1, /* NUL */
+            prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+            prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+            prxchange('s/\x01/\\u0001/',-1, /* SOH */
+            prxchange('s/\x02/\\u0002/',-1, /* STX */
+            prxchange('s/\x10/\\u0010/',-1, /* DLE */
+            prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+            prxchange('s/\\/\\\\/',-1,&&name&i)
+          ))))))))))))!!'"';
+        end;
+        else &&name&i=quote(cats(&&name&i));
      %end;
    %end;
    run;
--- a/meta/mm_createwebservice.sas
+++ b/meta/mm_createwebservice.sas
@@ -232,19 +232,23 @@ data _null_;
  put '      format _numeric_ bart.; ';
  put '    %do i=1 %to &numcols; ';
  put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
-  put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
-  put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
-  put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
-  put '                    prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
-  put '                    prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
-  put '                    prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
-  put '                    prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
-  put '                    prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
-  put '                    prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
-  put '                    prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
-  put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        ))))))))))))!!''"''; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '          &&name&i=''"''!!trim( ';
+  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '          ))))))))))))!!''"''; ';
+  put '        end; ';
+  put '        else &&name&i=quote(cats(&&name&i)); ';
  put '      %end; ';
  put '    %end; ';
  put '    run; ';
@@ -428,14 +432,16 @@ data _null_;
  put '    put '',"_METAPERSON": '' _METAPERSON; ';
  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
  put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
+  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
  put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
  put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
  put '    put ",""SYSSITE"" : ""&syssite"" "; ';
  put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
  put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
+  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
  put '    length memsize $32; ';
  put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
--- a/meta/mm_webout.sas
+++ b/meta/mm_webout.sas
@@ -157,14 +157,16 @@
    put ',"_METAPERSON": ' _METAPERSON;
    put ',"_PROGRAM" : ' _PROGRAM ;
    put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
    put ",""SYSJOBID"" : ""&sysjobid"" ";
    put ",""SYSSCPL"" : ""&sysscpl"" ";
    put ",""SYSSITE"" : ""&syssite"" ";
    sysvlong=quote(trim(symget('sysvlong')));
    put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
    length memsize $32;
    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
--- a/package-lock.json
+++ b/package-lock.json
--- a/package.json
+++ b/package.json
@@ -26,14 +26,13 @@
  "homepage": "https://core.sasjs.io",
  "main": "index.js",
  "scripts": {
-    "build": "sasjs cbd -t viya",
+    "build": "sasjs cbd -t server",
    "docs": "sasjs doc -t docsonly && ./sasjs/utils/build.sh",
-    "test": "sasjs test -t viya",
+    "test": "sasjs test -t server",
    "lint": "sasjs lint",
    "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
  },
  "devDependencies": {
-    "@sasjs/cli": "3.6.0",
-    "@sasjs/core": "4.4.7"
+    "@sasjs/cli": "3.13.0"
  }
 }
--- a/sasjs/sasjsconfig.json
+++ b/sasjs/sasjsconfig.json
@@ -25,16 +25,16 @@
      "mcTestAppLoc": "/Public/temp/macrocore"
    }
  },
-  "defaultTarget": "viya",
+  "defaultTarget": "server",
  "targets": [
    {
      "name": "viya",
-      "serverUrl": "https://sas.analytium.co.uk",
+      "serverUrl": "",
      "serverType": "SASVIYA",
      "httpsAgentOptions": {
        "allowInsecureRequests": false
      },
-      "appLoc": "/Public/temp/macrocore",
+      "appLoc": "/Public/app/macrocore",
      "macroFolders": [
        "viya",
        "tests/viyaonly"
@@ -65,7 +65,7 @@
    },
    {
      "name": "server",
-      "serverUrl": "",
+      "serverUrl": "https://sas.analytium.co.uk:5000",
      "serverType": "SASJS",
      "httpsAgentOptions": {
        "allowInsecureRequests": false
--- a/server/ms_createfile.sas
+++ b/server/ms_createfile.sas
@@ -29,12 +29,16 @@
    ,mdebug=0
  );

-%local fname0 fname1 fname2 boundary fname statcd msg;
+%local fname0 fname1 fname2 boundary fname statcd msg optval;
 %let fname0=%mf_getuniquefileref();
 %let fname1=%mf_getuniquefileref();
 %let fname2=%mf_getuniquefileref();
 %let boundary=%mf_getuniquename();

+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
 data _null_;
  file &fname0 termstr=crlf;
  infile &inref end=eof;
@@ -95,4 +99,7 @@ run;
  ,msg=%superq(msg)
 )

+/* reset options */
+options &optval;
+
 %mend ms_createfile;
--- a/server/ms_webout.sas
+++ b/server/ms_webout.sas
@@ -147,7 +147,8 @@
    put ",""_DEBUG"" : ""&_debug"" ";
    put ',"_PROGRAM" : ' _PROGRAM ;
    put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
    SYSHOSTINFOLONG=quote(trim(symget('SYSHOSTINFOLONG')));
    put ',"SYSHOSTINFOLONG" : ' SYSHOSTINFOLONG;
    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
@@ -162,7 +163,8 @@
    put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" ";
    sysvlong=quote(trim(symget('sysvlong')));
    put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
    length autoexec $512;
    autoexec=quote(urlencode(trim(getoption('autoexec'))));
--- a/tests/crossplatform/mp_jsonout.test.3.sas
+++ b/tests/crossplatform/mp_jsonout.test.3.sas
@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Testing mp_jsonout.sas macro with non-standard chars
+
+  <h4> SAS Macros </h4>
+  @li mp_jsonout.sas
+  @li mp_assert.sas
+
+**/
+
+filename webref temp;
+
+data demo;
+  do x='"','0A'x,'0D'x,'09'x,'00'x,'0E'x,'0F'x,'01'x,'02'x,'10'x,'11'x,'\';
+    output;
+  end;
+run;
+%mp_jsonout(OPEN,jref=webref)
+%mp_jsonout(OBJ,demo,jref=webref)
+%mp_jsonout(CLOSE,jref=webref)
+
+data _null_;
+  infile webref;
+  input;
+  putlog _infile_;
+run;
+
+libname web JSON fileref=webref;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking for error condition with special chars export,
+  outds=work.test_results
+)
+
+/*
+data _null_;
+  set work.demo (in=start) web.demo (in=end);
+  put (_all_)(=);
+run;
+proc sql;
+describe table work.demo;
+describe table web.demo;
+*/
+
+proc compare base=work.demo compare=web.demo(keep=x);
+quit;
+
+%mp_assert(
+  iftrue=(&sysinfo=0),
+  desc=Returned json is identical to input table for all special chars,
+  outds=work.test_results
+)
--- a/viya/mv_createwebservice.sas
+++ b/viya/mv_createwebservice.sas
@@ -376,19 +376,23 @@ data _null_;
  put '      format _numeric_ bart.; ';
  put '    %do i=1 %to &numcols; ';
  put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
-  put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
-  put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
-  put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
-  put '                    prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
-  put '                    prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
-  put '                    prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
-  put '                    prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
-  put '                    prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
-  put '                    prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
-  put '                    prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
-  put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        ))))))))))))!!''"''; ';
+  put '        if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; ';
+  put '          &&name&i=''"''!!trim( ';
+  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
+  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
+  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
+  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
+  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '          ))))))))))))!!''"''; ';
+  put '        end; ';
+  put '        else &&name&i=quote(cats(&&name&i)); ';
  put '      %end; ';
  put '    %end; ';
  put '    run; ';
@@ -605,13 +609,15 @@ data _null_;
  put '    put ",""_DEBUG"" : ""&_debug"" "; ';
  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
  put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
+  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
  put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
  put '    put ",""SYSSITE"" : ""&syssite"" "; ';
  put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
  put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
+  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
  put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
  put '    length memsize $32; ';
  put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
--- a/viya/mv_webout.sas
+++ b/viya/mv_webout.sas
@@ -193,13 +193,15 @@
    put ",""_DEBUG"" : ""&_debug"" ";
    put ',"_PROGRAM" : ' _PROGRAM ;
    put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
    put ",""SYSHOSTNAME"" : ""&syshostname"" ";
    put ",""SYSSCPL"" : ""&sysscpl"" ";
    put ",""SYSSITE"" : ""&syssite"" ";
    sysvlong=quote(trim(symget('sysvlong')));
    put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
    put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
    length memsize $32;
    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
Author	SHA1	Message	Date
Allan Bowe	c4cb0b2395	Merge pull request #220 from sasjs/perf fix: using findc instead of regex for faster parsing	2022-04-16 23:31:20 +03:00
munja	58614e9a3d	fix: using findc instead of regex for faster parsing	2022-04-16 21:30:18 +01:00
Allan Bowe	c94c334c4b	Merge pull request #219 from sasjs/issue218 fix: check 4 special chars b4 replacing. Closes #218	2022-04-16 22:44:35 +03:00
munja	3bf44405f8	chore: alignment	2022-04-16 20:37:47 +01:00
munja	db68a256cb	fix: check 4 special chars b4 replacing. Closes #218	2022-04-16 20:33:21 +01:00
Allan Bowe	611fac6338	Merge pull request #217 from sasjs/bom fix: avoid sending BOM marker to SASjs API	2022-04-14 21:32:25 +03:00
munja	ddd120bb75	fix: avoid sending BOM marker to SASjs API	2022-04-14 19:31:36 +01:00
Allan Bowe	0d75e0bad8	Merge pull request #216 from sasjs/cli-issue-1113 chore(dep): removed sasjs/core and bumped sasjs/cli	2022-04-14 18:25:17 +03:00
Yury Shkoda	ba8c4ac844	chore: Merge branch 'main' of https://github.com/sasjs/core into cli-issue-1113	2022-04-14 18:22:57 +03:00
Allan Bowe	6938a42896	Merge pull request #215 from sasjs/serverconfig Change test server & update mx_webout() macros	2022-04-14 15:41:44 +03:00
munja	efff77c94e	fix: mx_webout	2022-04-14 13:40:14 +01:00
Yury Shkoda	2b10cf6192	chore(dep): removed sasjs/core and bumped sasjs/cli	2022-04-14 14:55:43 +03:00
Allan Bowe	134b91f266	fix: avoiding running tests on viya	2022-04-14 11:14:35 +00:00