Merge pull request #219 from sasjs/issue218

fix: check 4 special chars b4 replacing. Closes #218

all.sas

@@ -8239,7 +8239,8 @@ options
 %end;
 ;
 
-%mend mp_init;/**
+%mend mp_init;
+/**
   @file mp_jsonout.sas
   @brief Writes JSON in SASjs format to a fileref
   @details PROC JSON is faster but will produce errs like the ones below if
@@ -8440,19 +8441,27 @@ options
       format _numeric_ bart.;
     %do i=1 %to &numcols;
       %if &&typelong&i=char or &fmt=Y %then %do;
-        &&name&i='"'!!trim(prxchange('s/"/\"/',-1,
+        if prxmatch(
-                    prxchange('s/'!!'0A'x!!'/\n/',-1,
+            'm/\"|\x0A|\x0D|\x09|\x00|\x0E|\x0F|\x01|\x02|\x10|\x11|\\/o',
-                    prxchange('s/'!!'0D'x!!'/\r/',-1,
+            &&name&i
-                    prxchange('s/'!!'09'x!!'/\t/',-1,
+          )>0
-                    prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        then do;
-                    prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+          &&name&i='"'!!trim(
-                    prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+            prxchange('s/"/\\"/',-1,        /* double quote */
-                    prxchange('s/\x01/\\u0001/',-1, /* SOH */
+            prxchange('s/\x0A/\n/',-1,      /* new line */
-                    prxchange('s/\x02/\\u0002/',-1, /* STX */
+            prxchange('s/\x0D/\r/',-1,      /* carriage return */
-                    prxchange('s/\x10/\\u0010/',-1, /* DLE */
+            prxchange('s/\x09/\\t/',-1,     /* tab */
-                    prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+            prxchange('s/\x00/\\u0000/',-1, /* NUL */
-                    prxchange('s/\\/\\\\/',-1,&&name&i)
+            prxchange('s/\x0E/\\u000E/',-1, /* SS  */
-        ))))))))))))!!'"';
+            prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+            prxchange('s/\x01/\\u0001/',-1, /* SOH */
+            prxchange('s/\x02/\\u0002/',-1, /* STX */
+            prxchange('s/\x10/\\u0010/',-1, /* DLE */
+            prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+            prxchange('s/\\/\\\\/',-1,&&name&i)
+          ))))))))))))!!'"';
+        end;
+        else &&name&i=quote(cats(&&name&i));
       %end;
     %end;
     run;
@@ -14965,19 +14974,27 @@ data _null_;
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
+  put '        if prxmatch( ';
-  put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
+  put '            ''m/\"|\x0A|\x0D|\x09|\x00|\x0E|\x0F|\x01|\x02|\x10|\x11|\\/o'', ';
-  put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
+  put '            &&name&i ';
-  put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
+  put '          )>0 ';
-  put '                    prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '        then do; ';
-  put '                    prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '          &&name&i=''"''!!trim( ';
-  put '                    prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
-  put '                    prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
-  put '                    prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
-  put '                    prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
-  put '                    prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
-  put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
-  put '        ))))))))))))!!''"''; ';
+  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '          ))))))))))))!!''"''; ';
+  put '        end; ';
+  put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
@@ -15161,14 +15178,16 @@ data _null_;
   put '    put '',"_METAPERSON": '' _METAPERSON; ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
+  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
+  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
@@ -18698,14 +18717,16 @@ run;
     put ',"_METAPERSON": ' _METAPERSON;
     put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
@@ -18933,12 +18954,16 @@ run;
     ,mdebug=0
   );
 
-%local fname0 fname1 fname2 boundary fname statcd msg;
+%local fname0 fname1 fname2 boundary fname statcd msg optval;
 %let fname0=%mf_getuniquefileref();
 %let fname1=%mf_getuniquefileref();
 %let fname2=%mf_getuniquefileref();
 %let boundary=%mf_getuniquename();
 
+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
 data _null_;
   file &fname0 termstr=crlf;
   infile &inref end=eof;
@@ -18999,6 +19024,9 @@ run;
   ,msg=%superq(msg)
 )
 
+/* reset options */
+options &optval;
+
 %mend ms_createfile;
 /**
   @file
@@ -19301,7 +19329,8 @@ run;
     put ",""_DEBUG"" : ""&_debug"" ";
     put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
     SYSHOSTINFOLONG=quote(trim(symget('SYSHOSTINFOLONG')));
     put ',"SYSHOSTINFOLONG" : ' SYSHOSTINFOLONG;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
@@ -19316,7 +19345,8 @@ run;
     put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length autoexec $512;
     autoexec=quote(urlencode(trim(getoption('autoexec'))));
@@ -20443,19 +20473,27 @@ data _null_;
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
+  put '        if prxmatch( ';
-  put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
+  put '            ''m/\"|\x0A|\x0D|\x09|\x00|\x0E|\x0F|\x01|\x02|\x10|\x11|\\/o'', ';
-  put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
+  put '            &&name&i ';
-  put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
+  put '          )>0 ';
-  put '                    prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '        then do; ';
-  put '                    prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '          &&name&i=''"''!!trim( ';
-  put '                    prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
-  put '                    prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
-  put '                    prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
-  put '                    prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
-  put '                    prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
-  put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
-  put '        ))))))))))))!!''"''; ';
+  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '          ))))))))))))!!''"''; ';
+  put '        end; ';
+  put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
@@ -20672,13 +20710,15 @@ data _null_;
   put '    put ",""_DEBUG"" : ""&_debug"" "; ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
+  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
+  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
@@ -24479,13 +24519,15 @@ filename &fref1 clear;
     put ",""_DEBUG"" : ""&_debug"" ";
     put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";

base/mp_init.sas

@@ -73,4 +73,4 @@ options
 %end;
 ;
 
-%mend mp_init;
+%mend mp_init;

base/mp_jsonout.sas

@@ -199,19 +199,27 @@
       format _numeric_ bart.;
     %do i=1 %to &numcols;
       %if &&typelong&i=char or &fmt=Y %then %do;
-        &&name&i='"'!!trim(prxchange('s/"/\"/',-1,
+        if prxmatch(
-                    prxchange('s/'!!'0A'x!!'/\n/',-1,
+            'm/\"|\x0A|\x0D|\x09|\x00|\x0E|\x0F|\x01|\x02|\x10|\x11|\\/o',
-                    prxchange('s/'!!'0D'x!!'/\r/',-1,
+            &&name&i
-                    prxchange('s/'!!'09'x!!'/\t/',-1,
+          )>0
-                    prxchange('s/\x00/\\u0000/',-1, /* NUL */
+        then do;
-                    prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+          &&name&i='"'!!trim(
-                    prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+            prxchange('s/"/\\"/',-1,        /* double quote */
-                    prxchange('s/\x01/\\u0001/',-1, /* SOH */
+            prxchange('s/\x0A/\n/',-1,      /* new line */
-                    prxchange('s/\x02/\\u0002/',-1, /* STX */
+            prxchange('s/\x0D/\r/',-1,      /* carriage return */
-                    prxchange('s/\x10/\\u0010/',-1, /* DLE */
+            prxchange('s/\x09/\\t/',-1,     /* tab */
-                    prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+            prxchange('s/\x00/\\u0000/',-1, /* NUL */
-                    prxchange('s/\\/\\\\/',-1,&&name&i)
+            prxchange('s/\x0E/\\u000E/',-1, /* SS  */
-        ))))))))))))!!'"';
+            prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+            prxchange('s/\x01/\\u0001/',-1, /* SOH */
+            prxchange('s/\x02/\\u0002/',-1, /* STX */
+            prxchange('s/\x10/\\u0010/',-1, /* DLE */
+            prxchange('s/\x11/\\u0011/',-1, /* DC1 */
+            prxchange('s/\\/\\\\/',-1,&&name&i)
+          ))))))))))))!!'"';
+        end;
+        else &&name&i=quote(cats(&&name&i));
       %end;
     %end;
     run;

meta/mm_createwebservice.sas

@@ -232,19 +232,27 @@ data _null_;
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
+  put '        if prxmatch( ';
-  put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
+  put '            ''m/\"|\x0A|\x0D|\x09|\x00|\x0E|\x0F|\x01|\x02|\x10|\x11|\\/o'', ';
-  put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
+  put '            &&name&i ';
-  put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
+  put '          )>0 ';
-  put '                    prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '        then do; ';
-  put '                    prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '          &&name&i=''"''!!trim( ';
-  put '                    prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
-  put '                    prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
-  put '                    prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
-  put '                    prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
-  put '                    prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
-  put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
-  put '        ))))))))))))!!''"''; ';
+  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '          ))))))))))))!!''"''; ';
+  put '        end; ';
+  put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
@@ -428,14 +436,16 @@ data _null_;
   put '    put '',"_METAPERSON": '' _METAPERSON; ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
+  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
+  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';

meta/mm_webout.sas

@@ -157,14 +157,16 @@
     put ',"_METAPERSON": ' _METAPERSON;
     put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";

package.json

@@ -26,14 +26,13 @@
   "homepage": "https://core.sasjs.io",
   "main": "index.js",
   "scripts": {
-    "build": "sasjs cbd -t viya",
+    "build": "sasjs cbd -t server",
     "docs": "sasjs doc -t docsonly && ./sasjs/utils/build.sh",
-    "test": "sasjs test -t viya",
+    "test": "sasjs test -t server",
     "lint": "sasjs lint",
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
   },
   "devDependencies": {
-    "@sasjs/cli": "3.6.0",
+    "@sasjs/cli": "3.13.0"
-    "@sasjs/core": "4.4.7"
   }
 }

sasjs/sasjsconfig.json

@@ -25,16 +25,16 @@
       "mcTestAppLoc": "/Public/temp/macrocore"
     }
   },
-  "defaultTarget": "viya",
+  "defaultTarget": "server",
   "targets": [
     {
       "name": "viya",
-      "serverUrl": "https://sas.analytium.co.uk",
+      "serverUrl": "",
       "serverType": "SASVIYA",
       "httpsAgentOptions": {
         "allowInsecureRequests": false
       },
-      "appLoc": "/Public/temp/macrocore",
+      "appLoc": "/Public/app/macrocore",
       "macroFolders": [
         "viya",
         "tests/viyaonly"
@@ -65,7 +65,7 @@
     },
     {
       "name": "server",
-      "serverUrl": "",
+      "serverUrl": "https://sas.analytium.co.uk:5000",
       "serverType": "SASJS",
       "httpsAgentOptions": {
         "allowInsecureRequests": false

server/ms_createfile.sas

@@ -29,12 +29,16 @@
     ,mdebug=0
   );
 
-%local fname0 fname1 fname2 boundary fname statcd msg;
+%local fname0 fname1 fname2 boundary fname statcd msg optval;
 %let fname0=%mf_getuniquefileref();
 %let fname1=%mf_getuniquefileref();
 %let fname2=%mf_getuniquefileref();
 %let boundary=%mf_getuniquename();
 
+/* avoid sending bom marker to API */
+%let optval=%sysfunc(getoption(bomfile));
+options nobomfile;
+
 data _null_;
   file &fname0 termstr=crlf;
   infile &inref end=eof;
@@ -95,4 +99,7 @@ run;
   ,msg=%superq(msg)
 )
 
+/* reset options */
+options &optval;
+
 %mend ms_createfile;

server/ms_webout.sas

@@ -147,7 +147,8 @@
     put ",""_DEBUG"" : ""&_debug"" ";
     put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
     SYSHOSTINFOLONG=quote(trim(symget('SYSHOSTINFOLONG')));
     put ',"SYSHOSTINFOLONG" : ' SYSHOSTINFOLONG;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
@@ -162,7 +163,8 @@
     put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length autoexec $512;
     autoexec=quote(urlencode(trim(getoption('autoexec'))));

tests/crossplatform/mp_jsonout.test.3.sas

@@ -0,0 +1,53 @@
+/**
+  @file
+  @brief Testing mp_jsonout.sas macro with non-standard chars
+
+  <h4> SAS Macros </h4>
+  @li mp_jsonout.sas
+  @li mp_assert.sas
+
+**/
+
+filename webref temp;
+
+data demo;
+  do x='"','0A'x,'0D'x,'09'x,'00'x,'0E'x,'0F'x,'01'x,'02'x,'10'x,'11'x,'\';
+    output;
+  end;
+run;
+%mp_jsonout(OPEN,jref=webref)
+%mp_jsonout(OBJ,demo,jref=webref)
+%mp_jsonout(CLOSE,jref=webref)
+
+data _null_;
+  infile webref;
+  input;
+  putlog _infile_;
+run;
+
+libname web JSON fileref=webref;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking for error condition with special chars export,
+  outds=work.test_results
+)
+
+/*
+data _null_;
+  set work.demo (in=start) web.demo (in=end);
+  put (_all_)(=);
+run;
+proc sql;
+describe table work.demo;
+describe table web.demo;
+*/
+
+proc compare base=work.demo compare=web.demo(keep=x);
+quit;
+
+%mp_assert(
+  iftrue=(&sysinfo=0),
+  desc=Returned json is identical to input table for all special chars,
+  outds=work.test_results
+)

viya/mv_createwebservice.sas

@@ -376,19 +376,27 @@ data _null_;
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
-  put '        &&name&i=''"''!!trim(prxchange(''s/"/\"/'',-1, ';
+  put '        if prxmatch( ';
-  put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
+  put '            ''m/\"|\x0A|\x0D|\x09|\x00|\x0E|\x0F|\x01|\x02|\x10|\x11|\\/o'', ';
-  put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
+  put '            &&name&i ';
-  put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
+  put '          )>0 ';
-  put '                    prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '        then do; ';
-  put '                    prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '          &&name&i=''"''!!trim( ';
-  put '                    prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/"/\\"/'',-1,        /* double quote */ ';
-  put '                    prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x0A/\n/'',-1,      /* new line */ ';
-  put '                    prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x0D/\r/'',-1,      /* carriage return */ ';
-  put '                    prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x09/\\t/'',-1,     /* tab */ ';
-  put '                    prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
-  put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '            prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
-  put '        ))))))))))))!!''"''; ';
+  put '            prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '            prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '            prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '            prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '            prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
+  put '            prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
+  put '          ))))))))))))!!''"''; ';
+  put '        end; ';
+  put '        else &&name&i=quote(cats(&&name&i)); ';
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
@@ -605,13 +613,15 @@ data _null_;
   put '    put ",""_DEBUG"" : ""&_debug"" "; ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
+  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" "; ';
+  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';

viya/mv_webout.sas

@@ -193,13 +193,15 @@
     put ",""_DEBUG"" : ""&_debug"" ";
     put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
-    put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
+    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
+    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";