fix: compatibility with compute sessions

README.md

@@ -1,6 +1,6 @@
 # Macro Core
 
-Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of Application Development on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/master/CONTRIBUTING.md) are welcomed.
+Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of Application Development on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/CONTRIBUTING.md) are welcomed.
 
 You can download and compile them all in just two lines of SAS code:
 

all.sas

@@ -3080,7 +3080,9 @@ create table &outds (rename=(
   * PROCJSON (default)
   * DATASTEP 
 
-  @param dbg= Typically used with an _debug (numeric) option
+  @param dbg= DEPRECATED - was used to conditionally add PRETTY to
+    proc json but this can cause line truncation in large files.
+    
 
   @version 9.2
   @author Allan Bowe
@@ -3106,9 +3108,8 @@ create table &outds (rename=(
     proc sql;drop table &tempds;
     data &tempds /view=&tempds;set &ds; 
     %if &fmt=N %then format _numeric_ best32.;;
-    proc json out=&jref
+    proc json out=&jref pretty
         %if &action=ARR %then nokeys ;
-        %if &dbg ge 131  %then pretty ;
         ;export &tempds / nosastags fmtnumeric;
     run;
     proc sql;drop view &tempds;
@@ -3862,6 +3863,64 @@ proc sql
 
 %mp_binarycopy(inloc="&inloc",outref=_webout)
 
+%mend;/**
+  @file mp_testwritespeedlibrary.sas
+  @brief Tests the write speed of a new table in a SAS library
+  @details Will create a new table of a certain size in an 
+  existing SAS library.  The table will have one column,
+  and will be subsequently deleted.
+      
+      %mp_testwritespeedlibrary(
+        lib=work
+        ,size=0.5
+        ,outds=work.results
+      )
+
+  @param lib= (WORK) The library in which to create the table
+  @param size= (0.1) The size in GB of the table to create
+  @param outds= (WORK.RESULTS) The output dataset to be created.
+
+  <h4> Dependencies </h4>
+  @li mf_getuniquename.sas
+  @li mf_existds.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mp_testwritespeedlibrary(lib=WORK
+  ,outds=work.results
+  ,size=0.1
+)/*/STORE SOURCE*/;
+%local ds start;
+
+/* find an unused, unique name for the new table */
+%let ds=%mf_getuniquename();
+%do %until(%mf_existds(&lib..&ds)=0);
+  %let ds=%mf_getuniquename();
+%end;
+
+%let start=%sysfunc(datetime());
+
+data &lib..&ds(compress=no keep=x);
+  header=128*1024;
+  size=(1073741824/8 * &size) - header;
+  do x=1 to size;
+    output;
+  end;
+run;
+
+proc sql;
+drop table &lib..&ds;
+
+data &outds;
+  lib="&lib";
+  start_dttm=put(&start,datetime19.);
+  end_dttm=put(datetime(),datetime19.);
+  duration_seconds=end_dttm-start_dttm;
+run;
+
 %mend;/**
   @file mp_unzip.sas
   @brief Unzips a zip file
@@ -6053,9 +6112,8 @@ data _null_;
   put '    proc sql;drop table &tempds; ';
   put '    data &tempds /view=&tempds;set &ds; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
-  put '    proc json out=&jref ';
+  put '    proc json out=&jref pretty ';
   put '        %if &action=ARR %then nokeys ; ';
-  put '        %if &dbg ge 131  %then pretty ; ';
   put '        ;export &tempds / nosastags fmtnumeric; ';
   put '    run; ';
   put '    proc sql;drop view &tempds; ';
@@ -6150,7 +6208,8 @@ data _null_;
   put '%end; ';
   put '%mend; ';
   put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y); ';
-  put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug; ';
+  put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
+  put '  sasjs_tables; ';
   put '%local i tempds; ';
   put ' ';
   put '%if &action=FETCH %then %do; ';
@@ -6177,6 +6236,7 @@ data _null_;
   put '        if _n_<20 then putlog _infile_; ';
   put '      %end; ';
   put '    run; ';
+  put '    %let sasjs_tables=&sasjs_tables &&_webin_name&i; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
@@ -9188,7 +9248,8 @@ run;
 
 **/
 %macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y);
-%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug;
+%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug 
+  sasjs_tables;
 %local i tempds;
 
 %if &action=FETCH %then %do;
@@ -9215,6 +9276,7 @@ run;
         if _n_<20 then putlog _infile_;
       %end;
     run;
+    %let sasjs_tables=&sasjs_tables &&_webin_name&i;
   %end;
 %end;
 
@@ -9836,9 +9898,8 @@ data _null_;
   put '    proc sql;drop table &tempds; ';
   put '    data &tempds /view=&tempds;set &ds; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
-  put '    proc json out=&jref ';
+  put '    proc json out=&jref pretty ';
   put '        %if &action=ARR %then nokeys ; ';
-  put '        %if &dbg ge 131  %then pretty ; ';
   put '        ;export &tempds / nosastags fmtnumeric; ';
   put '    run; ';
   put '    proc sql;drop view &tempds; ';
@@ -9933,7 +9994,8 @@ data _null_;
   put '%end; ';
   put '%mend; ';
   put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y); ';
-  put '%global _webin_file_count _webin_fileuri _debug _omittextlog ; ';
+  put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
+  put '  sasjs_tables SYS_JES_JOB_URI; ';
   put '%if %index("&_debug",log) %then %let _debug=131; ';
   put ' ';
   put '%local i tempds; ';
@@ -9947,11 +10009,11 @@ data _null_;
   put '  %if not %symexist(_webin_fileuri1) %then %do; ';
   put '    %let _webin_file_count=%eval(&_webin_file_count+0); ';
   put '    %let _webin_fileuri1=&_webin_fileuri; ';
+  put '    %let _webin_name1=&_webin_name; ';
   put '  %end; ';
   put ' ';
-  put '  %if %symexist(sasjs_tables) %then %do; ';
-  put '    /* small volumes of non-special data are sent as params for responsiveness */ ';
-  put '    /* to do - deal with escaped values  */ ';
+  put '  /* if the sasjs_tables param is passed, we expect param based upload */ ';
+  put '  %if %length(&sasjs_tables.XX)>2 %then %do; ';
   put '    filename _sasjs "%sysfunc(pathname(work))/sasjs.lua"; ';
   put '    data _null_; ';
   put '      file _sasjs; ';
@@ -10030,13 +10092,19 @@ data _null_;
   put '      infile indata firstobs=2 dsd termstr=crlf ; ';
   put '      input &input_statement; ';
   put '    run; ';
+  put '    %let sasjs_tables=&sasjs_tables &&_webin_name&i; ';
   put '  %end; ';
   put '%end; ';
   put '%else %if &action=OPEN %then %do; ';
   put '  /* setup webout */ ';
   put '  OPTIONS NOBOMFILE; ';
+  put '  %if "X&SYS_JES_JOB_URI.X"="XX" %then %do; ';
+  put '     filename _webout temp lrecl=999999 mod; ';
+  put '  %end; ';
+  put '  %else %do; ';
   put '    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" ';
   put '      name="_webout.json" lrecl=999999 mod; ';
+  put '  %end; ';
   put ' ';
   put '  /* setup temp ref */ ';
   put '  %if %upcase(&fref) ne _WEBOUT %then %do; ';
@@ -10066,13 +10134,13 @@ data _null_;
   put '      i+1; ';
   put '      call symputx(''wt''!!left(i),name); ';
   put '      call symputx(''wtcnt'',i); ';
-  put '    data _null_; file &fref; put ",""WORK"":{"; ';
+  put '    data _null_; file &fref mod; put ",""WORK"":{"; ';
   put '    %do i=1 %to &wtcnt; ';
   put '      %let wt=&&wt&i; ';
   put '      proc contents noprint data=&wt ';
   put '        out=_data_ (keep=name type length format:); ';
   put '      run;%let tempds=%scan(&syslast,2,.); ';
-  put '      data _null_; file &fref; ';
+  put '      data _null_; file &fref mod; ';
   put '        dsid=open("WORK.&wt",''is''); ';
   put '        nlobs=attrn(dsid,''NLOBS''); ';
   put '        nvars=attrn(dsid,''NVARS''); ';
@@ -10083,9 +10151,9 @@ data _null_;
   put '        put '',"nvars":'' nvars; ';
   put '      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP) ';
   put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP) ';
-  put '      data _null_; file &fref;put "}"; ';
+  put '      data _null_; file &fref mod;put "}"; ';
   put '    %end; ';
-  put '    data _null_; file &fref;put "}";run; ';
+  put '    data _null_; file &fref mod;put "}";run; ';
   put '  %end; ';
   put ' ';
   put '  /* close off json */ ';
@@ -11954,7 +12022,8 @@ filename &fref1 clear;
 
 **/
 %macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y);
-%global _webin_file_count _webin_fileuri _debug _omittextlog ;
+%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name
+  sasjs_tables SYS_JES_JOB_URI;
 %if %index("&_debug",log) %then %let _debug=131; 
 
 %local i tempds;
@@ -11968,11 +12037,11 @@ filename &fref1 clear;
   %if not %symexist(_webin_fileuri1) %then %do;
     %let _webin_file_count=%eval(&_webin_file_count+0);
     %let _webin_fileuri1=&_webin_fileuri;
+    %let _webin_name1=&_webin_name;
   %end;
 
-  %if %symexist(sasjs_tables) %then %do;
-    /* small volumes of non-special data are sent as params for responsiveness */
-    /* to do - deal with escaped values  */
+  /* if the sasjs_tables param is passed, we expect param based upload */
+  %if %length(&sasjs_tables.XX)>2 %then %do;
     filename _sasjs "%sysfunc(pathname(work))/sasjs.lua";
     data _null_;
       file _sasjs;
@@ -12051,13 +12120,19 @@ filename &fref1 clear;
       infile indata firstobs=2 dsd termstr=crlf ;
       input &input_statement;
     run;
+    %let sasjs_tables=&sasjs_tables &&_webin_name&i;
   %end;
 %end;
 %else %if &action=OPEN %then %do;
   /* setup webout */
   OPTIONS NOBOMFILE;
+  %if "X&SYS_JES_JOB_URI.X"="XX" %then %do;
+     filename _webout temp lrecl=999999 mod;
+  %end;
+  %else %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" 
       name="_webout.json" lrecl=999999 mod; 
+  %end;
 
   /* setup temp ref */
   %if %upcase(&fref) ne _WEBOUT %then %do;
@@ -12087,13 +12162,13 @@ filename &fref1 clear;
       i+1;
       call symputx('wt'!!left(i),name);
       call symputx('wtcnt',i);
-    data _null_; file &fref; put ",""WORK"":{";
+    data _null_; file &fref mod; put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
       proc contents noprint data=&wt
         out=_data_ (keep=name type length format:);
       run;%let tempds=%scan(&syslast,2,.);
-      data _null_; file &fref;
+      data _null_; file &fref mod;
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
         nvars=attrn(dsid,'NVARS');
@@ -12104,9 +12179,9 @@ filename &fref1 clear;
         put ',"nvars":' nvars;
       %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
       %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
-      data _null_; file &fref;put "}";
+      data _null_; file &fref mod;put "}";
     %end;
-    data _null_; file &fref;put "}";run;
+    data _null_; file &fref mod;put "}";run;
   %end;
 
   /* close off json */

base/mp_jsonout.sas

@@ -40,7 +40,9 @@
   * PROCJSON (default)
   * DATASTEP 
 
-  @param dbg= Typically used with an _debug (numeric) option
+  @param dbg= DEPRECATED - was used to conditionally add PRETTY to
+    proc json but this can cause line truncation in large files.
+    
 
   @version 9.2
   @author Allan Bowe
@@ -66,9 +68,8 @@
     proc sql;drop table &tempds;
     data &tempds /view=&tempds;set &ds; 
     %if &fmt=N %then format _numeric_ best32.;;
-    proc json out=&jref
+    proc json out=&jref pretty
         %if &action=ARR %then nokeys ;
-        %if &dbg ge 131  %then pretty ;
         ;export &tempds / nosastags fmtnumeric;
     run;
     proc sql;drop view &tempds;

make_singlefile.sh

@@ -13,7 +13,7 @@ cat > $OUTFILE <<'EOL'
     'included' in SAS with just 2 lines of code:
 
       filename mc url
-        "https://raw.githubusercontent.com/sasjs/core/master/macrocore.sas";
+        "https://raw.githubusercontent.com/sasjs/core/main/macrocore.sas";
       %inc mc;
 
     The `build.sh` file in the https://github.com/sasjs/core repo

meta/mm_createwebservice.sas

@@ -104,9 +104,8 @@ data _null_;
   put '    proc sql;drop table &tempds; ';
   put '    data &tempds /view=&tempds;set &ds; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
-  put '    proc json out=&jref ';
+  put '    proc json out=&jref pretty ';
   put '        %if &action=ARR %then nokeys ; ';
-  put '        %if &dbg ge 131  %then pretty ; ';
   put '        ;export &tempds / nosastags fmtnumeric; ';
   put '    run; ';
   put '    proc sql;drop view &tempds; ';
@@ -201,7 +200,8 @@ data _null_;
   put '%end; ';
   put '%mend; ';
   put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y); ';
-  put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug; ';
+  put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
+  put '  sasjs_tables; ';
   put '%local i tempds; ';
   put ' ';
   put '%if &action=FETCH %then %do; ';
@@ -228,6 +228,7 @@ data _null_;
   put '        if _n_<20 then putlog _infile_; ';
   put '      %end; ';
   put '    run; ';
+  put '    %let sasjs_tables=&sasjs_tables &&_webin_name&i; ';
   put '  %end; ';
   put '%end; ';
   put ' ';

meta/mm_webout.sas

@@ -33,7 +33,8 @@
 
 **/
 %macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y);
-%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug;
+%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug 
+  sasjs_tables;
 %local i tempds;
 
 %if &action=FETCH %then %do;
@@ -60,6 +61,7 @@
         if _n_<20 then putlog _infile_;
       %end;
     run;
+    %let sasjs_tables=&sasjs_tables &&_webin_name&i;
   %end;
 %end;
 

viya/mv_createwebservice.sas

@@ -241,9 +241,8 @@ data _null_;
   put '    proc sql;drop table &tempds; ';
   put '    data &tempds /view=&tempds;set &ds; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
-  put '    proc json out=&jref ';
+  put '    proc json out=&jref pretty ';
   put '        %if &action=ARR %then nokeys ; ';
-  put '        %if &dbg ge 131  %then pretty ; ';
   put '        ;export &tempds / nosastags fmtnumeric; ';
   put '    run; ';
   put '    proc sql;drop view &tempds; ';
@@ -338,7 +337,8 @@ data _null_;
   put '%end; ';
   put '%mend; ';
   put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y); ';
-  put '%global _webin_file_count _webin_fileuri _debug _omittextlog ; ';
+  put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
+  put '  sasjs_tables SYS_JES_JOB_URI; ';
   put '%if %index("&_debug",log) %then %let _debug=131; ';
   put ' ';
   put '%local i tempds; ';
@@ -352,11 +352,11 @@ data _null_;
   put '  %if not %symexist(_webin_fileuri1) %then %do; ';
   put '    %let _webin_file_count=%eval(&_webin_file_count+0); ';
   put '    %let _webin_fileuri1=&_webin_fileuri; ';
+  put '    %let _webin_name1=&_webin_name; ';
   put '  %end; ';
   put ' ';
-  put '  %if %symexist(sasjs_tables) %then %do; ';
-  put '    /* small volumes of non-special data are sent as params for responsiveness */ ';
-  put '    /* to do - deal with escaped values  */ ';
+  put '  /* if the sasjs_tables param is passed, we expect param based upload */ ';
+  put '  %if %length(&sasjs_tables.XX)>2 %then %do; ';
   put '    filename _sasjs "%sysfunc(pathname(work))/sasjs.lua"; ';
   put '    data _null_; ';
   put '      file _sasjs; ';
@@ -435,13 +435,19 @@ data _null_;
   put '      infile indata firstobs=2 dsd termstr=crlf ; ';
   put '      input &input_statement; ';
   put '    run; ';
+  put '    %let sasjs_tables=&sasjs_tables &&_webin_name&i; ';
   put '  %end; ';
   put '%end; ';
   put '%else %if &action=OPEN %then %do; ';
   put '  /* setup webout */ ';
   put '  OPTIONS NOBOMFILE; ';
+  put '  %if "X&SYS_JES_JOB_URI.X"="XX" %then %do; ';
+  put '     filename _webout temp lrecl=999999 mod; ';
+  put '  %end; ';
+  put '  %else %do; ';
   put '    filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" ';
   put '      name="_webout.json" lrecl=999999 mod; ';
+  put '  %end; ';
   put ' ';
   put '  /* setup temp ref */ ';
   put '  %if %upcase(&fref) ne _WEBOUT %then %do; ';
@@ -471,13 +477,13 @@ data _null_;
   put '      i+1; ';
   put '      call symputx(''wt''!!left(i),name); ';
   put '      call symputx(''wtcnt'',i); ';
-  put '    data _null_; file &fref; put ",""WORK"":{"; ';
+  put '    data _null_; file &fref mod; put ",""WORK"":{"; ';
   put '    %do i=1 %to &wtcnt; ';
   put '      %let wt=&&wt&i; ';
   put '      proc contents noprint data=&wt ';
   put '        out=_data_ (keep=name type length format:); ';
   put '      run;%let tempds=%scan(&syslast,2,.); ';
-  put '      data _null_; file &fref; ';
+  put '      data _null_; file &fref mod; ';
   put '        dsid=open("WORK.&wt",''is''); ';
   put '        nlobs=attrn(dsid,''NLOBS''); ';
   put '        nvars=attrn(dsid,''NVARS''); ';
@@ -488,9 +494,9 @@ data _null_;
   put '        put '',"nvars":'' nvars; ';
   put '      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP) ';
   put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP) ';
-  put '      data _null_; file &fref;put "}"; ';
+  put '      data _null_; file &fref mod;put "}"; ';
   put '    %end; ';
-  put '    data _null_; file &fref;put "}";run; ';
+  put '    data _null_; file &fref mod;put "}";run; ';
   put '  %end; ';
   put ' ';
   put '  /* close off json */ ';

viya/mv_webout.sas

@@ -36,7 +36,8 @@
 
 **/
 %macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y);
-%global _webin_file_count _webin_fileuri _debug _omittextlog ;
+%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name
+  sasjs_tables SYS_JES_JOB_URI;
 %if %index("&_debug",log) %then %let _debug=131; 
 
 %local i tempds;
@@ -50,11 +51,11 @@
   %if not %symexist(_webin_fileuri1) %then %do;
     %let _webin_file_count=%eval(&_webin_file_count+0);
     %let _webin_fileuri1=&_webin_fileuri;
+    %let _webin_name1=&_webin_name;
   %end;
 
-  %if %symexist(sasjs_tables) %then %do;
-    /* small volumes of non-special data are sent as params for responsiveness */
-    /* to do - deal with escaped values  */
+  /* if the sasjs_tables param is passed, we expect param based upload */
+  %if %length(&sasjs_tables.XX)>2 %then %do;
     filename _sasjs "%sysfunc(pathname(work))/sasjs.lua";
     data _null_;
       file _sasjs;
@@ -133,13 +134,19 @@
       infile indata firstobs=2 dsd termstr=crlf ;
       input &input_statement;
     run;
+    %let sasjs_tables=&sasjs_tables &&_webin_name&i;
   %end;
 %end;
 %else %if &action=OPEN %then %do;
   /* setup webout */
   OPTIONS NOBOMFILE;
+  %if "X&SYS_JES_JOB_URI.X"="XX" %then %do;
+     filename _webout temp lrecl=999999 mod;
+  %end;
+  %else %do;
     filename _webout filesrvc parenturi="&SYS_JES_JOB_URI" 
       name="_webout.json" lrecl=999999 mod; 
+  %end;
 
   /* setup temp ref */
   %if %upcase(&fref) ne _WEBOUT %then %do;
@@ -169,13 +176,13 @@
       i+1;
       call symputx('wt'!!left(i),name);
       call symputx('wtcnt',i);
-    data _null_; file &fref; put ",""WORK"":{";
+    data _null_; file &fref mod; put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
       proc contents noprint data=&wt
         out=_data_ (keep=name type length format:);
       run;%let tempds=%scan(&syslast,2,.);
-      data _null_; file &fref;
+      data _null_; file &fref mod;
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
         nvars=attrn(dsid,'NVARS');
@@ -186,9 +193,9 @@
         put ',"nvars":' nvars;
       %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
       %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
-      data _null_; file &fref;put "}";
+      data _null_; file &fref mod;put "}";
     %end;
-    data _null_; file &fref;put "}";run;
+    data _null_; file &fref mod;put "}";run;
   %end;
 
   /* close off json */