Merge pull request #210 from sasjs/nulfixes

fix: using regex special chars instead of hex constants in mp_jsonout

all.sas

@@ -2194,7 +2194,7 @@ Usage:
   %end;
 
   /* Stored Process Server web app context */
-  %if %symexist(_metaport)
+  %if %symexist(_METAFOLDER)
     or "&SYSPROCESSNAME "="Compute Server "
     or &mode=INCLUDE
   %then %do;
@@ -2270,12 +2270,14 @@ Usage:
     /* send response in SASjs JSON format */
     data _null_;
       file _webout mod lrecl=32000 encoding='utf-8';
-      length msg $32767 ;
+      length msg syswarningtext syserrortext $32767 ;
       sasdatetime=datetime();
       msg=symget('msg');
     %if &logline>0 %then %do;
       msg=cats(msg,'\n\nLog Extract:\n',symget('logmsg'));
     %end;
+      /* escape the escapes */
+      msg=tranwrd(msg,'\','\\');
       /* escape the quotes */
       msg=tranwrd(msg,'"','\"');
       /* ditch the CRLFs as chrome complains */
@@ -3200,7 +3202,7 @@ run;
 
   proc compare
     base=&scopeds(where=(upcase(name) not in (%mf_getquotedstr(&ilist))))
-    compare=&ds;
+    compare=&ds noprint;
   run;
 
   %if &sysinfo=0 %then %do;
@@ -4306,6 +4308,7 @@ run;
   data &out_ds;
     set &out_ds;
     length infoname infoval $60 fref $8;
+    if _n_=1 then call missing(fref);
     rc=filename(fref,filepath);
     drop rc infoname fid i close fref;
     if file_or_folder='file' then do;
@@ -7952,8 +7955,8 @@ run;
 %mend mp_guesspk;/**
   @file
   @brief Returns a unique hash for a dataset
-  @details Ignores metadata attributes, used only to hash values. Compared
+  @details Ignores metadata attributes, used only to hash values. If used to
-  datasets must be in the same order.
+  compare datasets, they must have their columns and rows in the same order.
 
       %mp_hashdataset(sashelp.class,outds=myhash)
 
@@ -7968,7 +7971,10 @@ run;
   @li mf_getattrn.sas
   @li mf_getuniquename.sas
   @li mf_getvarlist.sas
-  @li mf_getvartype.sas
+  @li mp_md5.sas
+
+  <h4> Related Files </h4>
+  @li mp_hashdataset.test.sas
 
   @param [in] libds dataset to hash
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
@@ -7986,51 +7992,52 @@ run;
 
 %macro mp_hashdataset(
   libds,
-  outds=,
+  outds=work._data_,
   salt=,
   iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
-  %if not(%eval(%unquote(&iftrue))) %then %return;
+%local keyvar /* roll up the md5 */
+  prevkeyvar /* retain prev record md5 */
+  lastvar /* last var in input ds */
+  cvars nvars;
 
-  %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
+%if not(%eval(%unquote(&iftrue))) %then %return;
-    %put %str(WARN)ING: Dataset &libds is empty, or is not a dataset;
+
-  %end;
+/* avoid naming conflict for hash key vars */
-  %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
+%let keyvar=%mf_getuniquename();
-    %put %str(ERR)OR: Dataset &libds is not a dataset;
+%let prevkeyvar=%mf_getuniquename();
-  %end;
+%let lastvar=%mf_getuniquename();
-  %else %do;
+
-    %local keyvar /* roll up the md5 */
+%if %mf_getattrn(&libds,NLOBS)=0 %then %do;
-      prevkeyvar /* retain prev record md5 */
+  data &outds;
-      lastvar /* last var in input ds */
+    length hashkey $32;
-      varlist var i;
+    retain hashkey "%sysfunc(md5(%str(&salt)),$hex32.)";
-    /* avoid naming conflict for hash key vars */
+    output;
-    %let keyvar=%mf_getuniquename();
+    stop;
-    %let prevkeyvar=%mf_getuniquename();
+  run;
-    %let lastvar=%mf_getuniquename();
+  %put &sysmacroname: Dataset &libds is empty, or is not a dataset;
-    %let varlist=%mf_getvarlist(&libds);
+  %put &sysmacroname: hashkey of &outds is based on salt (&salt) only;
-    data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
+%end;
-      length &prevkeyvar &keyvar $32;
+%else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
-      retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
+  %put %str(ERR)OR: Dataset &libds is not a dataset;
-      set &libds end=&lastvar;
+%end;
-      /* hash should include previous row */
+%else %do;
-      &keyvar=put(md5(&prevkeyvar
+  data &outds(rename=(&keyvar=hashkey) keep=&keyvar)/nonote2err;
-      /* loop every column, hashing every individual value */
+    length &prevkeyvar &keyvar $32;
-    %do i=1 %to %sysfunc(countw(&varlist));
+    retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
-      %let var=%scan(&varlist,&i,%str( ));
+    set &libds end=&lastvar;
-      %if %mf_getvartype(&libds,&var)=C %then %do;
+    /* hash should include previous row */
-          !!put(md5(trim(&var)),$hex32.)
+    &keyvar=%mp_md5(
-      %end;
+      cvars=%mf_getvarlist(&libds,typefilter=C) &prevkeyvar,
-      %else %do;
+      nvars=%mf_getvarlist(&libds,typefilter=N)
-          !!put(md5(trim(put(&var*1,binary64.))),$hex32.)
+    );
-      %end;
+    &prevkeyvar=&keyvar;
-    %end;
+    if &lastvar then output;
-      ),$hex32.);
+  run;
-      &prevkeyvar=&keyvar;
+%end;
-      if &lastvar then output;
+%mend mp_hashdataset;
-    run;
+/**
-  %end;
-%mend mp_hashdataset;/**
   @file
   @brief Performs a wrapped \%include
   @details This macro wrapper is necessary if you need your included code to
@@ -8415,8 +8422,15 @@ options
                     prxchange('s/'!!'0A'x!!'/\n/',-1,
                     prxchange('s/'!!'0D'x!!'/\r/',-1,
                     prxchange('s/'!!'09'x!!'/\t/',-1,
+                    prxchange('s/\x00/\\u0000/',-1, /* NUL */
+                    prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+                    prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+                    prxchange('s/\x01/\\u0001/',-1, /* SOH */
+                    prxchange('s/\x02/\\u0002/',-1, /* STX */
+                    prxchange('s/\x10/\\u0010/',-1, /* DLE */
+                    prxchange('s/\x11/\\u0011/',-1, /* DC1 */
                     prxchange('s/\\/\\\\/',-1,&&name&i)
-        )))))!!'"';
+        ))))))))))))!!'"';
       %end;
     %end;
     run;
@@ -13358,8 +13372,8 @@ run;
   <h4> SAS Macros </h4>
   @li mp_abort.sas
 
-  @param libref the libref (not name) of the metadata library
+  @param [in] libref The libref (not name) of the metadata library
-  @param mAbort= If not assigned, HARD will call %mp_abort(), SOFT will
+  @param [in] mAbort= If not assigned, HARD will call %mp_abort(), SOFT will
     silently return
 
   @returns libname statement
@@ -13373,11 +13387,11 @@ run;
     libref
     ,mAbort=HARD
 )/*/STORE SOURCE*/;
-
+%local mp_abort msg;
+%let mp_abort=0;
 %if %sysfunc(libref(&libref)) %then %do;
-  %local mp_abort msg; %let mp_abort=0;
   data _null_;
-    length liburi LibName $200;
+    length liburi LibName msg $200;
     call missing(of _all_);
     nobj=metadata_getnobj("omsobj:SASLibrary?@Libref='&libref'",1,liburi);
     if nobj=1 then do;
@@ -13391,7 +13405,24 @@ run;
           * not always helpful though.  One example, previously received:
           * NOTE: Libref XX refers to the same library metadata as libref XX.
           */
-        call symputx('msg',sysmsg(),'l');
+        msg=sysmsg();
+        if msg=:'ERROR: Libref SAVE is not assigned.' then do;
+          msg=catx(" ",
+            "Could not assign %upcase(&libref).",
+            "Please check metadata permissions!  Libname:",libname,
+            "Liburi:",liburi
+          );
+        end;
+        else if msg="ERROR: User does not have appropriate authorization "!!
+          "level for library SAVE."
+        then do;
+          msg=catx(" ",
+            "ERROR: User does not have appropriate authorization level",
+            "for library %upcase(&libref), libname:",libname,
+            "Liburi:",liburi
+          );
+        end;
+        call symputx('msg',msg,'l');
         if "&mabort"='HARD' then call symputx('mp_abort',1,'l');
       end;
       else do;
@@ -13410,22 +13441,18 @@ run;
     end;
   run;
 
-  %if &mp_abort=1 %then %do;
+  %put NOTE: &msg;
-    %mp_abort(iftrue= (&mp_abort=1)
-      ,mac=mm_assignlib.sas
-      ,msg=&msg
-    )
-    %return;
-  %end;
-  %else %if %length(&msg)>2 %then %do;
-    %put NOTE: &msg;
-    %return;
-  %end;
 
 %end;
 %else %do;
   %put NOTE: Library &libref is already assigned;
 %end;
+
+%mp_abort(iftrue= (&mp_abort=1)
+  ,mac=mm_assignlib.sas
+  ,msg=%superq(msg)
+)
+
 %mend mm_assignlib;
 /**
   @file
@@ -14912,8 +14939,15 @@ data _null_;
   put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
   put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
   put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
+  put '                    prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '                    prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '                    prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '                    prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '                    prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '                    prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '                    prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
   put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        )))))!!''"''; ';
+  put '        ))))))))))))!!''"''; ';
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
@@ -18984,6 +19018,9 @@ run;
   @param [out] outref= (msgetfil) The fileref to contain the file.
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
 
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
 
 **/
 
@@ -18992,15 +19029,21 @@ run;
     ,mdebug=0
   );
 
-filename &outref temp;
+/* use the recfm in a separate fileref to avoid issues with subsequent reads */
+%local binaryfref floc;
+%let binaryfref=%mf_getuniquefileref();
+%let floc=%sysfunc(pathname(work))/%mf_getuniquename().txt;
+filename &outref "&floc";
+filename &binaryfref "&floc" recfm=n;
 
-proc http method='GET' out=&outref
+proc http method='GET' out=&binaryfref
   url="&_sasjs_apiserverurl/SASjsApi/drive/file?_filePath=&driveloc";
 %if &mdebug=1 %then %do;
   debug level=2;
 %end;
 run;
 
+filename &binaryfref clear;
 
 %mend ms_getfile;
 /**
@@ -19195,6 +19238,7 @@ run;
     data _null_;
       set &tempds;
       if not (upcase(name) =:"DATA"); /* ignore temp datasets */
+      if not (upcase(name)=:"_DATA_");
       i+1;
       call symputx(cats('wt',i),name,'l');
       call symputx('wtcnt',i,'l');
@@ -20373,8 +20417,15 @@ data _null_;
   put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
   put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
   put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
+  put '                    prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '                    prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '                    prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '                    prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '                    prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '                    prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '                    prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
   put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        )))))!!''"''; ';
+  put '        ))))))))))))!!''"''; ';
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
@@ -20447,7 +20498,7 @@ data _null_;
   put '  sasjs_tables SYS_JES_JOB_URI; ';
   put '%if %index("&_debug",log) %then %let _debug=131; ';
   put ' ';
-  put '%local i tempds; ';
+  put '%local i tempds table; ';
   put '%let action=%upcase(&action); ';
   put ' ';
   put '%if &action=FETCH %then %do; ';
@@ -20462,60 +20513,35 @@ data _null_;
   put '  %end; ';
   put ' ';
   put '  /* if the sasjs_tables param is passed, we expect param based upload */ ';
-  put '  %if %length(&sasjs_tables.XX)>2 %then %do; ';
+  put '  %if %length(&sasjs_tables.X)>1 %then %do; ';
-  put '    filename _sasjs "%sysfunc(pathname(work))/sasjs.lua"; ';
-  put '    data _null_; ';
-  put '      file _sasjs; ';
-  put '      put ''s=sas.symget("sasjs_tables")''; ';
-  put '      put ''if(s:sub(1,7) == "%nrstr(")''; ';
-  put '      put ''then''; ';
-  put '      put '' tablist=s:sub(8,s:len()-1)''; ';
-  put '      put ''else''; ';
-  put '      put '' tablist=s''; ';
-  put '      put ''end''; ';
-  put '      put ''for i = 1,sas.countw(tablist) ''; ';
-  put '      put ''do ''; ';
-  put '      put ''  tab=sas.scan(tablist,i)''; ';
-  put '      put ''  sasdata=""''; ';
-  put '      put ''  if (sas.symexist("sasjs"..i.."data0")==0)''; ';
-  put '      put ''  then''; ';
-  put '      /* TODO - condense this logic */ ';
-  put '      put ''    s=sas.symget("sasjs"..i.."data")''; ';
-  put '      put ''    if(s:sub(1,7) == "%nrstr(")''; ';
-  put '      put ''    then''; ';
-  put '      put ''      sasdata=s:sub(8,s:len()-1)''; ';
-  put '      put ''    else''; ';
-  put '      put ''      sasdata=s''; ';
-  put '      put ''    end''; ';
-  put '      put ''  else''; ';
-  put '      put ''    for d = 1, sas.symget("sasjs"..i.."data0")''; ';
-  put '      put ''    do''; ';
-  put '      put ''      s=sas.symget("sasjs"..i.."data"..d)''; ';
-  put '      put ''      if(s:sub(1,7) == "%nrstr(")''; ';
-  put '      put ''      then''; ';
-  put '      put ''        sasdata=sasdata..s:sub(8,s:len()-1)''; ';
-  put '      put ''      else''; ';
-  put '      put ''        sasdata=sasdata..s''; ';
-  put '      put ''      end''; ';
-  put '      put ''    end''; ';
-  put '      put ''  end''; ';
-  put '      put ''  file = io.open(sas.pathname("work").."/"..tab..".csv", "a")''; ';
-  put '      put ''  io.output(file)''; ';
-  put '      put ''  io.write(sasdata)''; ';
-  put '      put ''  io.close(file)''; ';
-  put '      put ''end''; ';
-  put '    run; ';
-  put '    %inc _sasjs; ';
   put ' ';
-  put '    /* now read in the data */ ';
+  put '    /* convert data from macro variables to datasets */ ';
   put '    %do i=1 %to %sysfunc(countw(&sasjs_tables)); ';
-  put '      %local table; %let table=%scan(&sasjs_tables,&i); ';
+  put '      %let table=%scan(&sasjs_tables,&i,%str( )); ';
+  put '      %if %symexist(sasjs&i.data0)=0 %then %let sasjs&i.data0=1; ';
+  put '      data _null_; ';
+  put '        file "%sysfunc(pathname(work))/&table..csv" recfm=n; ';
+  put '        retain nrflg 0; ';
+  put '        length line $32767; ';
+  put '        do i=1 to &&sasjs&i.data0; ';
+  put '          if &&sasjs&i.data0=1 then line=symget("sasjs&i.data"); ';
+  put '          else line=symget(cats("sasjs&i.data",i)); ';
+  put '          if i=1 and substr(line,1,7)=''%nrstr('' then do; ';
+  put '            nrflg=1; ';
+  put '            line=substr(line,8); ';
+  put '          end; ';
+  put '          if i=&&sasjs&i.data0 and nrflg=1 then do; ';
+  put '            line=substr(line,1,length(line)-1); ';
+  put '          end; ';
+  put '          put line +(-1) @; ';
+  put '        end; ';
+  put '      run; ';
   put '      data _null_; ';
   put '        infile "%sysfunc(pathname(work))/&table..csv" termstr=crlf ; ';
   put '        input; ';
   put '        if _n_=1 then call symputx(''input_statement'',_infile_); ';
   put '        list; ';
-  put '      data &table; ';
+  put '      data work.&table; ';
   put '        infile "%sysfunc(pathname(work))/&table..csv" firstobs=2 dsd ';
   put '          termstr=crlf; ';
   put '        input &input_statement; ';
@@ -24279,7 +24305,7 @@ filename &fref1 clear;
   sasjs_tables SYS_JES_JOB_URI;
 %if %index("&_debug",log) %then %let _debug=131;
 
-%local i tempds;
+%local i tempds table;
 %let action=%upcase(&action);
 
 %if &action=FETCH %then %do;
@@ -24294,60 +24320,35 @@ filename &fref1 clear;
   %end;
 
   /* if the sasjs_tables param is passed, we expect param based upload */
-  %if %length(&sasjs_tables.XX)>2 %then %do;
+  %if %length(&sasjs_tables.X)>1 %then %do;
-    filename _sasjs "%sysfunc(pathname(work))/sasjs.lua";
-    data _null_;
-      file _sasjs;
-      put 's=sas.symget("sasjs_tables")';
-      put 'if(s:sub(1,7) == "%nrstr(")';
-      put 'then';
-      put ' tablist=s:sub(8,s:len()-1)';
-      put 'else';
-      put ' tablist=s';
-      put 'end';
-      put 'for i = 1,sas.countw(tablist) ';
-      put 'do ';
-      put '  tab=sas.scan(tablist,i)';
-      put '  sasdata=""';
-      put '  if (sas.symexist("sasjs"..i.."data0")==0)';
-      put '  then';
-      /* TODO - condense this logic */
-      put '    s=sas.symget("sasjs"..i.."data")';
-      put '    if(s:sub(1,7) == "%nrstr(")';
-      put '    then';
-      put '      sasdata=s:sub(8,s:len()-1)';
-      put '    else';
-      put '      sasdata=s';
-      put '    end';
-      put '  else';
-      put '    for d = 1, sas.symget("sasjs"..i.."data0")';
-      put '    do';
-      put '      s=sas.symget("sasjs"..i.."data"..d)';
-      put '      if(s:sub(1,7) == "%nrstr(")';
-      put '      then';
-      put '        sasdata=sasdata..s:sub(8,s:len()-1)';
-      put '      else';
-      put '        sasdata=sasdata..s';
-      put '      end';
-      put '    end';
-      put '  end';
-      put '  file = io.open(sas.pathname("work").."/"..tab..".csv", "a")';
-      put '  io.output(file)';
-      put '  io.write(sasdata)';
-      put '  io.close(file)';
-      put 'end';
-    run;
-    %inc _sasjs;
 
-    /* now read in the data */
+    /* convert data from macro variables to datasets */
     %do i=1 %to %sysfunc(countw(&sasjs_tables));
-      %local table; %let table=%scan(&sasjs_tables,&i);
+      %let table=%scan(&sasjs_tables,&i,%str( ));
+      %if %symexist(sasjs&i.data0)=0 %then %let sasjs&i.data0=1;
+      data _null_;
+        file "%sysfunc(pathname(work))/&table..csv" recfm=n;
+        retain nrflg 0;
+        length line $32767;
+        do i=1 to &&sasjs&i.data0;
+          if &&sasjs&i.data0=1 then line=symget("sasjs&i.data");
+          else line=symget(cats("sasjs&i.data",i));
+          if i=1 and substr(line,1,7)='%nrstr(' then do;
+            nrflg=1;
+            line=substr(line,8);
+          end;
+          if i=&&sasjs&i.data0 and nrflg=1 then do;
+            line=substr(line,1,length(line)-1);
+          end;
+          put line +(-1) @;
+        end;
+      run;
       data _null_;
         infile "%sysfunc(pathname(work))/&table..csv" termstr=crlf ;
         input;
         if _n_=1 then call symputx('input_statement',_infile_);
         list;
-      data &table;
+      data work.&table;
         infile "%sysfunc(pathname(work))/&table..csv" firstobs=2 dsd
           termstr=crlf;
         input &input_statement;

base/mp_abort.sas

@@ -85,7 +85,7 @@
   %end;
 
   /* Stored Process Server web app context */
-  %if %symexist(_metaport)
+  %if %symexist(_METAFOLDER)
     or "&SYSPROCESSNAME "="Compute Server "
     or &mode=INCLUDE
   %then %do;
@@ -161,12 +161,14 @@
     /* send response in SASjs JSON format */
     data _null_;
       file _webout mod lrecl=32000 encoding='utf-8';
-      length msg $32767 ;
+      length msg syswarningtext syserrortext $32767 ;
       sasdatetime=datetime();
       msg=symget('msg');
     %if &logline>0 %then %do;
       msg=cats(msg,'\n\nLog Extract:\n',symget('logmsg'));
     %end;
+      /* escape the escapes */
+      msg=tranwrd(msg,'\','\\');
       /* escape the quotes */
       msg=tranwrd(msg,'"','\"');
       /* ditch the CRLFs as chrome complains */

base/mp_assertscope.sas

@@ -106,7 +106,7 @@
 
   proc compare
     base=&scopeds(where=(upcase(name) not in (%mf_getquotedstr(&ilist))))
-    compare=&ds;
+    compare=&ds noprint;
   run;
 
   %if &sysinfo=0 %then %do;

base/mp_dirlist.sas

@@ -152,6 +152,7 @@ run;
   data &out_ds;
     set &out_ds;
     length infoname infoval $60 fref $8;
+    if _n_=1 then call missing(fref);
     rc=filename(fref,filepath);
     drop rc infoname fid i close fref;
     if file_or_folder='file' then do;

base/mp_hashdataset.sas

@@ -1,8 +1,8 @@
 /**
   @file
   @brief Returns a unique hash for a dataset
-  @details Ignores metadata attributes, used only to hash values. Compared
+  @details Ignores metadata attributes, used only to hash values. If used to
-  datasets must be in the same order.
+  compare datasets, they must have their columns and rows in the same order.
 
       %mp_hashdataset(sashelp.class,outds=myhash)
 
@@ -17,7 +17,10 @@
   @li mf_getattrn.sas
   @li mf_getuniquename.sas
   @li mf_getvarlist.sas
-  @li mf_getvartype.sas
+  @li mp_md5.sas
+
+  <h4> Related Files </h4>
+  @li mp_hashdataset.test.sas
 
   @param [in] libds dataset to hash
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
@@ -35,48 +38,48 @@
 
 %macro mp_hashdataset(
   libds,
-  outds=,
+  outds=work._data_,
   salt=,
   iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
 
-  %if not(%eval(%unquote(&iftrue))) %then %return;
+%local keyvar /* roll up the md5 */
+  prevkeyvar /* retain prev record md5 */
+  lastvar /* last var in input ds */
+  cvars nvars;
 
-  %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
+%if not(%eval(%unquote(&iftrue))) %then %return;
-    %put %str(WARN)ING: Dataset &libds is empty, or is not a dataset;
+
-  %end;
+/* avoid naming conflict for hash key vars */
-  %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
+%let keyvar=%mf_getuniquename();
-    %put %str(ERR)OR: Dataset &libds is not a dataset;
+%let prevkeyvar=%mf_getuniquename();
-  %end;
+%let lastvar=%mf_getuniquename();
-  %else %do;
+
-    %local keyvar /* roll up the md5 */
+%if %mf_getattrn(&libds,NLOBS)=0 %then %do;
-      prevkeyvar /* retain prev record md5 */
+  data &outds;
-      lastvar /* last var in input ds */
+    length hashkey $32;
-      varlist var i;
+    retain hashkey "%sysfunc(md5(%str(&salt)),$hex32.)";
-    /* avoid naming conflict for hash key vars */
+    output;
-    %let keyvar=%mf_getuniquename();
+    stop;
-    %let prevkeyvar=%mf_getuniquename();
+  run;
-    %let lastvar=%mf_getuniquename();
+  %put &sysmacroname: Dataset &libds is empty, or is not a dataset;
-    %let varlist=%mf_getvarlist(&libds);
+  %put &sysmacroname: hashkey of &outds is based on salt (&salt) only;
-    data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
+%end;
-      length &prevkeyvar &keyvar $32;
+%else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
-      retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
+  %put %str(ERR)OR: Dataset &libds is not a dataset;
-      set &libds end=&lastvar;
+%end;
-      /* hash should include previous row */
+%else %do;
-      &keyvar=put(md5(&prevkeyvar
+  data &outds(rename=(&keyvar=hashkey) keep=&keyvar)/nonote2err;
-      /* loop every column, hashing every individual value */
+    length &prevkeyvar &keyvar $32;
-    %do i=1 %to %sysfunc(countw(&varlist));
+    retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
-      %let var=%scan(&varlist,&i,%str( ));
+    set &libds end=&lastvar;
-      %if %mf_getvartype(&libds,&var)=C %then %do;
+    /* hash should include previous row */
-          !!put(md5(trim(&var)),$hex32.)
+    &keyvar=%mp_md5(
-      %end;
+      cvars=%mf_getvarlist(&libds,typefilter=C) &prevkeyvar,
-      %else %do;
+      nvars=%mf_getvarlist(&libds,typefilter=N)
-          !!put(md5(trim(put(&var*1,binary64.))),$hex32.)
+    );
-      %end;
+    &prevkeyvar=&keyvar;
-    %end;
+    if &lastvar then output;
-      ),$hex32.);
+  run;
-      &prevkeyvar=&keyvar;
+%end;
-      if &lastvar then output;
+%mend mp_hashdataset;
-    run;
-  %end;
-%mend mp_hashdataset;

base/mp_jsonout.sas

@@ -203,8 +203,15 @@
                     prxchange('s/'!!'0A'x!!'/\n/',-1,
                     prxchange('s/'!!'0D'x!!'/\r/',-1,
                     prxchange('s/'!!'09'x!!'/\t/',-1,
+                    prxchange('s/\x00/\\u0000/',-1, /* NUL */
+                    prxchange('s/\x0E/\\u000E/',-1, /* SS  */
+                    prxchange('s/\x0F/\\u000F/',-1, /* SF  */
+                    prxchange('s/\x01/\\u0001/',-1, /* SOH */
+                    prxchange('s/\x02/\\u0002/',-1, /* STX */
+                    prxchange('s/\x10/\\u0010/',-1, /* DLE */
+                    prxchange('s/\x11/\\u0011/',-1, /* DC1 */
                     prxchange('s/\\/\\\\/',-1,&&name&i)
-        )))))!!'"';
+        ))))))))))))!!'"';
       %end;
     %end;
     run;

meta/mm_assignlib.sas

@@ -13,8 +13,8 @@
   <h4> SAS Macros </h4>
   @li mp_abort.sas
 
-  @param libref the libref (not name) of the metadata library
+  @param [in] libref The libref (not name) of the metadata library
-  @param mAbort= If not assigned, HARD will call %mp_abort(), SOFT will
+  @param [in] mAbort= If not assigned, HARD will call %mp_abort(), SOFT will
     silently return
 
   @returns libname statement
@@ -28,11 +28,11 @@
     libref
     ,mAbort=HARD
 )/*/STORE SOURCE*/;
-
+%local mp_abort msg;
+%let mp_abort=0;
 %if %sysfunc(libref(&libref)) %then %do;
-  %local mp_abort msg; %let mp_abort=0;
   data _null_;
-    length liburi LibName $200;
+    length liburi LibName msg $200;
     call missing(of _all_);
     nobj=metadata_getnobj("omsobj:SASLibrary?@Libref='&libref'",1,liburi);
     if nobj=1 then do;
@@ -46,7 +46,24 @@
           * not always helpful though.  One example, previously received:
           * NOTE: Libref XX refers to the same library metadata as libref XX.
           */
-        call symputx('msg',sysmsg(),'l');
+        msg=sysmsg();
+        if msg=:'ERROR: Libref SAVE is not assigned.' then do;
+          msg=catx(" ",
+            "Could not assign %upcase(&libref).",
+            "Please check metadata permissions!  Libname:",libname,
+            "Liburi:",liburi
+          );
+        end;
+        else if msg="ERROR: User does not have appropriate authorization "!!
+          "level for library SAVE."
+        then do;
+          msg=catx(" ",
+            "ERROR: User does not have appropriate authorization level",
+            "for library %upcase(&libref), libname:",libname,
+            "Liburi:",liburi
+          );
+        end;
+        call symputx('msg',msg,'l');
         if "&mabort"='HARD' then call symputx('mp_abort',1,'l');
       end;
       else do;
@@ -65,20 +82,16 @@
     end;
   run;
 
-  %if &mp_abort=1 %then %do;
+  %put NOTE: &msg;
-    %mp_abort(iftrue= (&mp_abort=1)
-      ,mac=mm_assignlib.sas
-      ,msg=&msg
-    )
-    %return;
-  %end;
-  %else %if %length(&msg)>2 %then %do;
-    %put NOTE: &msg;
-    %return;
-  %end;
 
 %end;
 %else %do;
   %put NOTE: Library &libref is already assigned;
 %end;
+
+%mp_abort(iftrue= (&mp_abort=1)
+  ,mac=mm_assignlib.sas
+  ,msg=%superq(msg)
+)
+
 %mend mm_assignlib;

meta/mm_createwebservice.sas

@@ -236,8 +236,15 @@ data _null_;
   put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
   put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
   put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
+  put '                    prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '                    prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '                    prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '                    prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '                    prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '                    prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '                    prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
   put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        )))))!!''"''; ';
+  put '        ))))))))))))!!''"''; ';
   put '      %end; ';
   put '    %end; ';
   put '    run; ';

sasjs/sasjsconfig.json

@@ -4,11 +4,8 @@
     "base",
     "ddl",
     "fcmp",
-    "meta",
-    "metax",
-    "server",
-    "viya",
     "lua",
+    "server",
     "tests/crossplatform",
     "tests/ddl"
   ],
@@ -39,14 +36,9 @@
       },
       "appLoc": "/Public/temp/macrocore",
       "macroFolders": [
+        "viya",
         "tests/viyaonly"
       ],
-      "programFolders": [],
-      "binaryFolders": [],
-      "deployConfig": {
-        "deployServicePack": true,
-        "deployScripts": []
-      },
       "contextName": "SAS Job Execution compute context"
     },
     {
@@ -58,6 +50,8 @@
       },
       "appLoc": "/Shared Data/temp/macrocore",
       "macroFolders": [
+        "meta",
+        "metax",
         "tests/sas9only"
       ],
       "programFolders": [],
@@ -71,13 +65,14 @@
     },
     {
       "name": "server",
-      "serverUrl": "https://sas.analytium.co.uk:5000",
+      "serverUrl": "",
       "serverType": "SASJS",
       "httpsAgentOptions": {
         "allowInsecureRequests": false
       },
       "appLoc": "/sasjs/core",
       "macroFolders": [
+        "server",
         "tests/serveronly"
       ],
       "programFolders": [],
@@ -92,6 +87,10 @@
       "serverType": "SAS9",
       "appLoc": "dummy",
       "macroFolders": [
+        "meta",
+        "metax",
+        "server",
+        "viya",
         "tests/sas9only",
         "tests/viyaonly"
       ]
@@ -102,6 +101,7 @@
       "serverType": "SASVIYA",
       "appLoc": "/Public/temp/macrocore",
       "macroFolders": [
+        "viya",
         "tests/viyaonly"
       ],
       "deployConfig": {
@@ -110,4 +110,4 @@
       "contextName": "SAS Job Execution compute context"
     }
   ]
-}
+}

server/ms_getfile.sas

@@ -11,6 +11,9 @@
   @param [out] outref= (msgetfil) The fileref to contain the file.
   @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
 
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
 
 **/
 
@@ -19,14 +22,20 @@
     ,mdebug=0
   );
 
-filename &outref temp;
+/* use the recfm in a separate fileref to avoid issues with subsequent reads */
+%local binaryfref floc;
+%let binaryfref=%mf_getuniquefileref();
+%let floc=%sysfunc(pathname(work))/%mf_getuniquename().txt;
+filename &outref "&floc";
+filename &binaryfref "&floc" recfm=n;
 
-proc http method='GET' out=&outref
+proc http method='GET' out=&binaryfref
   url="&_sasjs_apiserverurl/SASjsApi/drive/file?_filePath=&driveloc";
 %if &mdebug=1 %then %do;
   debug level=2;
 %end;
 run;
 
+filename &binaryfref clear;
 
 %mend ms_getfile;

server/ms_webout.sas

@@ -114,6 +114,7 @@
     data _null_;
       set &tempds;
       if not (upcase(name) =:"DATA"); /* ignore temp datasets */
+      if not (upcase(name)=:"_DATA_");
       i+1;
       call symputx(cats('wt',i),name,'l');
       call symputx('wtcnt',i,'l');

tests/crossplatform/mp_hashdataset.test.sas

@@ -0,0 +1,61 @@
+/**
+  @file
+  @brief Testing mp_hashdataset.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_hashdataset.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+/* test 1 - regular DS */
+data work.test;
+  set sashelp.vextfl;
+  missval=.;
+  misscval='';
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mp_hashdataset(test)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Regular test works,
+  outds=work.test_results
+)
+
+%mp_hashdataset(test,outds=work.test2)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=hash with output runs without errors,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.test2)=1),
+  desc=output has 1 row,
+  outds=work.test_results
+)
+
+
+data work.test3a;
+  set work.test;
+  stop;
+run;
+%mp_hashdataset(test3a,outds=work.test3b)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=hash with zero-row input runs without errors,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.test3b)=1),
+  desc=test 3 output has 1 row,
+  outds=work.test_results
+)

tests/viyaonly/mv_webout.test.sas

@@ -1,15 +1,33 @@
 /**
   @file
-  @brief Testing mm_webout macro
+  @brief Testing mv_webout macro
 
   <h4> SAS Macros </h4>
   @li mf_getuniquefileref.sas
   @li mv_webout.sas
   @li mp_assert.sas
+  @li mp_assertdsobs.sas
 
 **/
 
 
+/* testing FETCHing (WEB approach) */
+
+data _null_;
+  call symputx('sasjs1data','area:$char4.'!!'0d0a'x!!'Adak');
+  call symputx('sasjs_tables','areas');
+run;
+%put &=sasjs1data;
+
+%mv_webout(FETCH)
+
+%mp_assertdsobs(work.areas,
+  desc=Test input table has 1 row,
+  test=EQUALS 1,
+  outds=work.test_results
+)
+
+
 %let fref=%mf_getuniquefileref();
 %global _metaperson;
 data some datasets;
@@ -38,4 +56,4 @@ run;
 %mp_assert(
   iftrue=(%str(&checkval)=%str(&sysvlong)),
   desc=Check if the sysvlong value was created
-)
+)

viya/mv_createwebservice.sas

@@ -380,8 +380,15 @@ data _null_;
   put '                    prxchange(''s/''!!''0A''x!!''/\n/'',-1, ';
   put '                    prxchange(''s/''!!''0D''x!!''/\r/'',-1, ';
   put '                    prxchange(''s/''!!''09''x!!''/\t/'',-1, ';
+  put '                    prxchange(''s/\x00/\\u0000/'',-1, /* NUL */ ';
+  put '                    prxchange(''s/\x0E/\\u000E/'',-1, /* SS  */ ';
+  put '                    prxchange(''s/\x0F/\\u000F/'',-1, /* SF  */ ';
+  put '                    prxchange(''s/\x01/\\u0001/'',-1, /* SOH */ ';
+  put '                    prxchange(''s/\x02/\\u0002/'',-1, /* STX */ ';
+  put '                    prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ ';
+  put '                    prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ ';
   put '                    prxchange(''s/\\/\\\\/'',-1,&&name&i) ';
-  put '        )))))!!''"''; ';
+  put '        ))))))))))))!!''"''; ';
   put '      %end; ';
   put '    %end; ';
   put '    run; ';
@@ -454,7 +461,7 @@ data _null_;
   put '  sasjs_tables SYS_JES_JOB_URI; ';
   put '%if %index("&_debug",log) %then %let _debug=131; ';
   put ' ';
-  put '%local i tempds; ';
+  put '%local i tempds table; ';
   put '%let action=%upcase(&action); ';
   put ' ';
   put '%if &action=FETCH %then %do; ';
@@ -469,60 +476,35 @@ data _null_;
   put '  %end; ';
   put ' ';
   put '  /* if the sasjs_tables param is passed, we expect param based upload */ ';
-  put '  %if %length(&sasjs_tables.XX)>2 %then %do; ';
+  put '  %if %length(&sasjs_tables.X)>1 %then %do; ';
-  put '    filename _sasjs "%sysfunc(pathname(work))/sasjs.lua"; ';
-  put '    data _null_; ';
-  put '      file _sasjs; ';
-  put '      put ''s=sas.symget("sasjs_tables")''; ';
-  put '      put ''if(s:sub(1,7) == "%nrstr(")''; ';
-  put '      put ''then''; ';
-  put '      put '' tablist=s:sub(8,s:len()-1)''; ';
-  put '      put ''else''; ';
-  put '      put '' tablist=s''; ';
-  put '      put ''end''; ';
-  put '      put ''for i = 1,sas.countw(tablist) ''; ';
-  put '      put ''do ''; ';
-  put '      put ''  tab=sas.scan(tablist,i)''; ';
-  put '      put ''  sasdata=""''; ';
-  put '      put ''  if (sas.symexist("sasjs"..i.."data0")==0)''; ';
-  put '      put ''  then''; ';
-  put '      /* TODO - condense this logic */ ';
-  put '      put ''    s=sas.symget("sasjs"..i.."data")''; ';
-  put '      put ''    if(s:sub(1,7) == "%nrstr(")''; ';
-  put '      put ''    then''; ';
-  put '      put ''      sasdata=s:sub(8,s:len()-1)''; ';
-  put '      put ''    else''; ';
-  put '      put ''      sasdata=s''; ';
-  put '      put ''    end''; ';
-  put '      put ''  else''; ';
-  put '      put ''    for d = 1, sas.symget("sasjs"..i.."data0")''; ';
-  put '      put ''    do''; ';
-  put '      put ''      s=sas.symget("sasjs"..i.."data"..d)''; ';
-  put '      put ''      if(s:sub(1,7) == "%nrstr(")''; ';
-  put '      put ''      then''; ';
-  put '      put ''        sasdata=sasdata..s:sub(8,s:len()-1)''; ';
-  put '      put ''      else''; ';
-  put '      put ''        sasdata=sasdata..s''; ';
-  put '      put ''      end''; ';
-  put '      put ''    end''; ';
-  put '      put ''  end''; ';
-  put '      put ''  file = io.open(sas.pathname("work").."/"..tab..".csv", "a")''; ';
-  put '      put ''  io.output(file)''; ';
-  put '      put ''  io.write(sasdata)''; ';
-  put '      put ''  io.close(file)''; ';
-  put '      put ''end''; ';
-  put '    run; ';
-  put '    %inc _sasjs; ';
   put ' ';
-  put '    /* now read in the data */ ';
+  put '    /* convert data from macro variables to datasets */ ';
   put '    %do i=1 %to %sysfunc(countw(&sasjs_tables)); ';
-  put '      %local table; %let table=%scan(&sasjs_tables,&i); ';
+  put '      %let table=%scan(&sasjs_tables,&i,%str( )); ';
+  put '      %if %symexist(sasjs&i.data0)=0 %then %let sasjs&i.data0=1; ';
+  put '      data _null_; ';
+  put '        file "%sysfunc(pathname(work))/&table..csv" recfm=n; ';
+  put '        retain nrflg 0; ';
+  put '        length line $32767; ';
+  put '        do i=1 to &&sasjs&i.data0; ';
+  put '          if &&sasjs&i.data0=1 then line=symget("sasjs&i.data"); ';
+  put '          else line=symget(cats("sasjs&i.data",i)); ';
+  put '          if i=1 and substr(line,1,7)=''%nrstr('' then do; ';
+  put '            nrflg=1; ';
+  put '            line=substr(line,8); ';
+  put '          end; ';
+  put '          if i=&&sasjs&i.data0 and nrflg=1 then do; ';
+  put '            line=substr(line,1,length(line)-1); ';
+  put '          end; ';
+  put '          put line +(-1) @; ';
+  put '        end; ';
+  put '      run; ';
   put '      data _null_; ';
   put '        infile "%sysfunc(pathname(work))/&table..csv" termstr=crlf ; ';
   put '        input; ';
   put '        if _n_=1 then call symputx(''input_statement'',_infile_); ';
   put '        list; ';
-  put '      data &table; ';
+  put '      data work.&table; ';
   put '        infile "%sysfunc(pathname(work))/&table..csv" firstobs=2 dsd ';
   put '          termstr=crlf; ';
   put '        input &input_statement; ';

viya/mv_webout.sas

@@ -49,7 +49,7 @@
   sasjs_tables SYS_JES_JOB_URI;
 %if %index("&_debug",log) %then %let _debug=131;
 
-%local i tempds;
+%local i tempds table;
 %let action=%upcase(&action);
 
 %if &action=FETCH %then %do;
@@ -64,60 +64,35 @@
   %end;
 
   /* if the sasjs_tables param is passed, we expect param based upload */
-  %if %length(&sasjs_tables.XX)>2 %then %do;
+  %if %length(&sasjs_tables.X)>1 %then %do;
-    filename _sasjs "%sysfunc(pathname(work))/sasjs.lua";
-    data _null_;
-      file _sasjs;
-      put 's=sas.symget("sasjs_tables")';
-      put 'if(s:sub(1,7) == "%nrstr(")';
-      put 'then';
-      put ' tablist=s:sub(8,s:len()-1)';
-      put 'else';
-      put ' tablist=s';
-      put 'end';
-      put 'for i = 1,sas.countw(tablist) ';
-      put 'do ';
-      put '  tab=sas.scan(tablist,i)';
-      put '  sasdata=""';
-      put '  if (sas.symexist("sasjs"..i.."data0")==0)';
-      put '  then';
-      /* TODO - condense this logic */
-      put '    s=sas.symget("sasjs"..i.."data")';
-      put '    if(s:sub(1,7) == "%nrstr(")';
-      put '    then';
-      put '      sasdata=s:sub(8,s:len()-1)';
-      put '    else';
-      put '      sasdata=s';
-      put '    end';
-      put '  else';
-      put '    for d = 1, sas.symget("sasjs"..i.."data0")';
-      put '    do';
-      put '      s=sas.symget("sasjs"..i.."data"..d)';
-      put '      if(s:sub(1,7) == "%nrstr(")';
-      put '      then';
-      put '        sasdata=sasdata..s:sub(8,s:len()-1)';
-      put '      else';
-      put '        sasdata=sasdata..s';
-      put '      end';
-      put '    end';
-      put '  end';
-      put '  file = io.open(sas.pathname("work").."/"..tab..".csv", "a")';
-      put '  io.output(file)';
-      put '  io.write(sasdata)';
-      put '  io.close(file)';
-      put 'end';
-    run;
-    %inc _sasjs;
 
-    /* now read in the data */
+    /* convert data from macro variables to datasets */
     %do i=1 %to %sysfunc(countw(&sasjs_tables));
-      %local table; %let table=%scan(&sasjs_tables,&i);
+      %let table=%scan(&sasjs_tables,&i,%str( ));
+      %if %symexist(sasjs&i.data0)=0 %then %let sasjs&i.data0=1;
+      data _null_;
+        file "%sysfunc(pathname(work))/&table..csv" recfm=n;
+        retain nrflg 0;
+        length line $32767;
+        do i=1 to &&sasjs&i.data0;
+          if &&sasjs&i.data0=1 then line=symget("sasjs&i.data");
+          else line=symget(cats("sasjs&i.data",i));
+          if i=1 and substr(line,1,7)='%nrstr(' then do;
+            nrflg=1;
+            line=substr(line,8);
+          end;
+          if i=&&sasjs&i.data0 and nrflg=1 then do;
+            line=substr(line,1,length(line)-1);
+          end;
+          put line +(-1) @;
+        end;
+      run;
       data _null_;
         infile "%sysfunc(pathname(work))/&table..csv" termstr=crlf ;
         input;
         if _n_=1 then call symputx('input_statement',_infile_);
         list;
-      data &table;
+      data work.&table;
         infile "%sysfunc(pathname(work))/&table..csv" firstobs=2 dsd
           termstr=crlf;
         input &input_statement;