Merge pull request #292 from sasjs/allanbowe/increase-length-for-syswarningtext-291

fix: adding length statement for SYSWARNINGTEXT. Closes #291

README.md

@@ -237,6 +237,7 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 
 The following repositories are also worth checking out:
 
+* [SASJedi/sas-macros](https://github.com/SASJedi/sas-macros)
 * [chris-swenson/sasmacros](https://github.com/chris-swenson/sasmacros)
 * [greg-wotton/sas-programs](https://github.com/greg-wootton/sas-programs)
 * [KatjaGlassConsulting/SMILE-SmartSASMacros](https://github.com/KatjaGlassConsulting/SMILE-SmartSASMacros)

base/mf_existfeature.sas

@@ -40,6 +40,11 @@
     %else %if "&SYSVLONG" < "9.04.01M3" %then 0;
     %else 1;
   %end;
+  %else %if &feature=DBMS_MEMTYPE %then %do;
+    /* does dbms_memtype exist in dictionary.tables? */
+    %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 0;
+    %else 1;
+  %end;
   %else %if &feature=EXPORTXLS %then %do;
     /* is it possible to PROC EXPORT an excel file? */
     %if "%substr(&sysver,1,1)"="4" or "%substr(&sysver,1,1)"="5" %then 1;

base/mf_increment.sas

@@ -0,0 +1,29 @@
+/**
+  @file
+  @brief Increments a macro variable
+  @details Useful outside of do-loops - will increment a macro variable every
+  time it is called.
+
+  Example:
+
+      %let cnt=1;
+      %put We have run %mf_increment(cnt) lines;
+      %put Now we have run %mf_increment(cnt) lines;
+      %put There are %mf_increment(cnt) lines in total;
+
+  @param [in] MACRO_NAME the name of the macro variable to increment
+  @param [in] ITER= The amount to add or subtract to the macro
+
+  <h4> Related Files </h4>
+  @li mf_increment.test.sas
+
+**/
+
+%macro mf_increment(macro_name,incr=1);
+
+  /* iterate the value */
+  %let &macro_name=%eval(&&&macro_name+&incr);
+  /* return the value */
+  &&&macro_name
+
+%mend mf_increment;

base/mp_abort.sas

@@ -225,7 +225,7 @@ and %superq(SYSPROCESSNAME) ne %str(Compute Server)
     _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
-    syserrortext=quote(trim(symget('syserrortext')));
+    syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
     put ",""SYSERRORTEXT"" : " syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSJOBID"" : ""&sysjobid"" ";
@@ -233,7 +233,7 @@ and %superq(SYSPROCESSNAME) ne %str(Compute Server)
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    syswarningtext=quote(trim(symget('syswarningtext')));
+    syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');
     put ",""SYSWARNINGTEXT"" : " syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     put "}" ;

base/mp_binarycopy.sas

@@ -31,6 +31,7 @@
   @param [in] mode (CREATE) Valid values:
     @li CREATE - Create the file (even if it already exists)
     @li APPEND - Append to the file (don't overwrite)
+  @param iftrue= (1=1) Supply a condition for which the macro should be executed
 
   @returns nothing
 
@@ -44,8 +45,12 @@
     ,inref=____in   /* override default to use own filerefs */
     ,outref=____out /* override default to use own filerefs */
     ,mode=CREATE
+    ,iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
   %local mod;
+
+  %if not(%eval(%unquote(&iftrue))) %then %return;
+
   %if &mode=APPEND %then %let mod=mod;
 
   /* these IN and OUT filerefs can point to anything */

base/mp_cleancsv.sas

@@ -1,5 +1,5 @@
 /**
-  @file mp_cleancsv.sas
+  @file
   @brief Fixes embedded cr / lf / crlf in CSV
   @details CSVs will sometimes contain lf or crlf within quotes (eg when
   saved by excel).  When the termstr is ALSO lf or crlf that can be tricky
@@ -7,14 +7,16 @@
   This macro converts any csv to follow the convention of a windows excel file,
   applying CRLF line endings and converting embedded cr and crlf to lf.
 
-  usage:
+  Usage:
 
       fileref mycsv "/path/your/csv";
       %mp_cleancsv(in=mycsv,out=/path/new.csv)
 
-  @param in= provide path or fileref to input csv
-  @param out= output path or fileref to output csv
-  @param qchar= quote char - hex code 22 is the double quote.
+  @param in= (NOTPROVIDED) Provide path or fileref to input csv. If a period is
+    found, it is assumed to be a file.
+  @param out= (NOTPROVIDED) Output path or fileref to output csv.  If a period
+    is found, it is assumed to be a file.
+  @param qchar= ('22'x) Quote char - hex code 22 is the double quote.
 
   @version 9.2
   @author Allan Bowe
@@ -56,9 +58,14 @@
     else do;
       /* outside a quote, change cr and lf to crlf */
       if inchar='0D'x then do;
+        crblank:
         put '0D0A'x;
         input inchar $char1.;
-        if inchar ne '0A'x then do;
+        if inchar='0D'x then do;
+          /* multiple CR indicates CR formatted file with blank lines */
+          goto crblank;
+        end;
+        else if inchar ne '0A'x then do;
           put inchar $char1.;
           if inchar=qchar then isq = mod(isq+1,2);
         end;

base/mp_filtercheck.sas

@@ -92,7 +92,8 @@ data &outds;
   /*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
     OPERATOR_NM $10 RAW_VALUE $4000;*/
   set &inds;
-  length reason_cd $4032 vtype $1 vnum dsid 8;
+  length reason_cd $4032 vtype $1 vnum dsid 8 tmp $4000;
+  drop tmp;
 
   /* quick check to ensure column exists */
   if upcase(VARIABLE_NM) not in
@@ -168,18 +169,32 @@ data &outds;
   end;
 
   /* special logic */
-  if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ','');
-  else if OPERATOR_NM in ('IN','NOT IN') then do;
-    if substr(raw_value,1,1) ne '('
-    or substr(cats(reverse(raw_value)),1,1) ne ')'
-    then do;
-      REASON_CD='Missing start/end bracket in RAW_VALUE';
-      putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
-      call symputx('reason_cd',reason_cd,'l');
-      call symputx('nobs',_n_,'l');
-      output;
+  if OPERATOR_NM in ('IN','NOT IN','BETWEEN') then do;
+    if OPERATOR_NM='BETWEEN' then raw_value1=tranwrd(raw_value,' AND ',',');
+    else do;
+      if substr(raw_value,1,1) ne '('
+      or substr(cats(reverse(raw_value)),1,1) ne ')'
+      then do;
+        REASON_CD='Missing start/end bracket in RAW_VALUE';
+        putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+        call symputx('reason_cd',reason_cd,'l');
+        call symputx('nobs',_n_,'l');
+        output;
+      end;
+      else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
+    end;
+    /* we now have a comma seperated list of values */
+    if vtype='N' then do i=1 to countc(raw_value1, ',')+1;
+      tmp=scan(raw_value1,i,',');
+      if cats(tmp) ne '.' and input(tmp, ?? 8.) eq . then do;
+        REASON_CD='Non Numeric value provided';
+        putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
+        call symputx('reason_cd',reason_cd,'l');
+        call symputx('nobs',_n_,'l');
+        output;
+      end;
+      return;
     end;
-    else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
   end;
   else raw_value1=raw_value;
 

base/mp_getconstraints.sas

@@ -94,8 +94,11 @@ create table &outds as
 /**
   * We cannot apply this clause to the underlying dictionary table.  See:
   * https://communities.sas.com/t5/SAS-Programming/Unexpected-Where-Clause-behaviour-in-dictionary-TABLE/m-p/771554#M244867
+  * cannot use`where calculated libref="&lib"` either as it will STILL execute
+  * all the underlying constraint queries, causing exception errors in some
+  * cases: https://github.com/sasjs/core/issues/283
   */
-  where calculated libref="&lib"
+  where a.TABLE_CATALOG="&lib"
   %if "&ds" ne "" %then %do;
     and upcase(a.TABLE_NAME)="&ds"
     and upcase(b.TABLE_NAME)="&ds"

base/mp_getddl.sas

@@ -130,13 +130,13 @@ run;
 
 %local x curds;
 %if &flavour=SAS %then %do;
-  data _null_;
-    file &fref mod;
-    put "/* SAS Flavour DDL for %upcase(&libref).&curds */";
-    put "proc sql;";
-  run;
   %do x=1 %to %sysfunc(countw(&dsnlist));
     %let curds=%scan(&dsnlist,&x);
+    data _null_;
+      file &fref mod;
+      put "/* SAS Flavour DDL for %upcase(&libref).&curds */";
+      put "proc sql;";
+    run;
     data _null_;
       file &fref mod;
       length lab $1024 typ $20;

base/mp_getpk.sas

@@ -34,6 +34,7 @@
   @param [out] outds= (work.mp_getpk) The name of the output table to create.
 
   <h4> SAS Macros </h4>
+  @li mf_existfeature.sas
   @li mf_getengine.sas
   @li mf_getschema.sas
   @li mp_dropmembers.sas
@@ -229,7 +230,12 @@ create table work.&tabs1 as select
   libname as libref
   ,upcase(memname) as dsn
   ,memtype
+%if %mf_existfeature(DBMS_MEMTYPE)=1 %then %do;
   ,dbms_memtype
+%end;
+%else %do;
+  ,'n/a' as dbms_memtype format=$32.
+%end;
   ,typemem
   ,memlabel
   ,nvar

base/mp_jsonout.sas

@@ -28,7 +28,7 @@
 
         %mp_jsonout(OPEN,jref=tmp)
         %mp_jsonout(OBJ,class,jref=tmp)
-        %mp_jsonout(OBJ,class,dslabel=class2,jref=tmp,showmeta=YES)
+        %mp_jsonout(OBJ,class,dslabel=class2,jref=tmp,showmeta=Y)
         %mp_jsonout(CLOSE,jref=tmp)
 
         data _null_;
@@ -55,10 +55,12 @@
     @li DATASTEP (more reliable when data has non standard characters)
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
-  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+  @param [in] showmeta= (N) Set to Y to output metadata alongside each table,
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
+  @param [in] maxobs= (MAX) Provide an integer to limit the number of input rows
+    that should be converted to JSON
 
   <h4> Related Macros <h4>
   @li mp_ds2fmtds.sas
@@ -72,10 +74,12 @@
 %macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y
   ,engine=DATASTEP
   ,missing=NULL
-  ,showmeta=NO
+  ,showmeta=N
+  ,maxobs=MAX
 )/*/STORE SOURCE*/;
-%local tempds colinfo fmtds i numcols;
+%local tempds colinfo fmtds i numcols stmt_obs;
 %let numcols=0;
+%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;);
 
 %if &action=OPEN %then %do;
   options nobomfile;
@@ -122,7 +126,6 @@
       if format='' then fmt=cats('$',length,'.');
       else if formatl=0 then fmt=cats(format,'.');
       else fmt=cats(format,formatl,'.');
-      newlen=max(formatl,length);
     end;
     else do;
       typelong='num';
@@ -130,15 +133,12 @@
       else if formatl=0 then fmt=cats(format,'.');
       else if formatd=0 then fmt=cats(format,formatl,'.');
       else fmt=cats(format,formatl,'.',formatd);
-      /* needs to be wide, for datetimes etc */
-      newlen=max(length,formatl,24);
     end;
     /* 32 char unique name */
     newname='sasjs'!!substr(cats(put(md5(name),$hex32.)),1,27);
 
     call symputx(cats('name',_n_),name,'l');
     call symputx(cats('newname',_n_),newname,'l');
-    call symputx(cats('len',_n_),newlen,'l');
     call symputx(cats('length',_n_),length,'l');
     call symputx(cats('fmt',_n_),fmt,'l');
     call symputx(cats('type',_n_),type,'l');
@@ -154,7 +154,9 @@
       %put &sysmacroname: Switching to DATASTEP engine;
       %goto datastep;
     %end;
-    data &tempds;set &ds;
+    data &tempds;
+      set &ds;
+      &stmt_obs;
     %if &fmt=N %then format _numeric_ best32.;;
     /* PRETTY is necessary to avoid line truncation in large files */
     filename _sjs2 temp lrecl=131068 encoding='utf-8';
@@ -189,7 +191,8 @@
       %end;
         ));
       %do i=1 %to &numcols;
-        length &&name&i $&&len&i;
+        /* formatted values can be up to length 32767 */
+        length &&name&i $32767;
         %if &&typelong&i=num %then %do;
           &&name&i=left(put(&&newname&i,&&fmt&i));
         %end;
@@ -229,6 +232,7 @@
       %else %do;
         set &ds;
       %end;
+      &stmt_obs;
       format _numeric_ bart.;
     %do i=1 %to &numcols;
       %if &&typelong&i=char or &fmt=Y %then %do;
@@ -285,10 +289,11 @@
   proc sql;
   drop table &colinfo, &tempds;
 
-  %if &showmeta=YES %then %do;
+  %if %substr(&showmeta,1,1)=Y %then %do;
     filename _sjs4 temp lrecl=131068 encoding='utf-8';
     data _null_;
       file _sjs4;
+      length label $350;
       put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{";
       do i=1 to &numcols;
         name=quote(trim(symget(cats('name',i))));

base/mp_storediffs.sas

@@ -154,7 +154,9 @@ run;
 %else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
 
 data &ds4;
-  length &inds_keep $41 tgtvar_nm $32;
+  length &inds_keep $41 tgtvar_nm $32 _label_ $256;
+  if _n_=1 then call missing(_label_);
+  drop _label_;
   set &ds2 &ds3 indsname=&inds_auto;
 
   tgtvar_nm=upcase(tgtvar_nm);

meta/mm_createwebservice.sas

@@ -97,10 +97,12 @@ data _null_;
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
-  put '  ,showmeta=NO ';
+  put '  ,showmeta=N ';
+  put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols; ';
+  put '%local tempds colinfo fmtds i numcols stmt_obs; ';
   put '%let numcols=0; ';
+  put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
@@ -147,7 +149,6 @@ data _null_;
   put '      if format='''' then fmt=cats(''$'',length,''.''); ';
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else fmt=cats(format,formatl,''.''); ';
-  put '      newlen=max(formatl,length); ';
   put '    end; ';
   put '    else do; ';
   put '      typelong=''num''; ';
@@ -155,15 +156,12 @@ data _null_;
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else if formatd=0 then fmt=cats(format,formatl,''.''); ';
   put '      else fmt=cats(format,formatl,''.'',formatd); ';
-  put '      /* needs to be wide, for datetimes etc */ ';
-  put '      newlen=max(length,formatl,24); ';
   put '    end; ';
   put '    /* 32 char unique name */ ';
   put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
   put ' ';
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
-  put '    call symputx(cats(''len'',_n_),newlen,''l''); ';
   put '    call symputx(cats(''length'',_n_),length,''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
@@ -179,7 +177,9 @@ data _null_;
   put '      %put &sysmacroname: Switching to DATASTEP engine; ';
   put '      %goto datastep; ';
   put '    %end; ';
-  put '    data &tempds;set &ds; ';
+  put '    data &tempds; ';
+  put '      set &ds; ';
+  put '      &stmt_obs; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
   put '    /* PRETTY is necessary to avoid line truncation in large files */ ';
   put '    filename _sjs2 temp lrecl=131068 encoding=''utf-8''; ';
@@ -214,7 +214,8 @@ data _null_;
   put '      %end; ';
   put '        )); ';
   put '      %do i=1 %to &numcols; ';
-  put '        length &&name&i $&&len&i; ';
+  put '        /* formatted values can be up to length 32767 */ ';
+  put '        length &&name&i $32767; ';
   put '        %if &&typelong&i=num %then %do; ';
   put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
@@ -254,6 +255,7 @@ data _null_;
   put '      %else %do; ';
   put '        set &ds; ';
   put '      %end; ';
+  put '      &stmt_obs; ';
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
@@ -310,10 +312,11 @@ data _null_;
   put '  proc sql; ';
   put '  drop table &colinfo, &tempds; ';
   put ' ';
-  put '  %if &showmeta=YES %then %do; ';
+  put '  %if %substr(&showmeta,1,1)=Y %then %do; ';
   put '    filename _sjs4 temp lrecl=131068 encoding=''utf-8''; ';
   put '    data _null_; ';
   put '      file _sjs4; ';
+  put '      length label $350; ';
   put '      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; ';
   put '      do i=1 to &numcols; ';
   put '        name=quote(trim(symget(cats(''name'',i)))); ';
@@ -366,7 +369,7 @@ data _null_;
   put ' ';
   put '%mend mf_getuser; ';
   put '%macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
-  put '  ,showmeta=NO ';
+  put '  ,showmeta=N ';
   put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -441,7 +444,6 @@ data _null_;
   put '  filename _sjsref temp lrecl=131068; ';
   put '  %if %str(&_debug) ge 131 %then %do; ';
   put '    /* if debug mode, send back first 10 records of each work table also */ ';
-  put '    options obs=10; ';
   put '    data;run;%let tempds=%scan(&syslast,2,.); ';
   put '    ods output Members=&tempds; ';
   put '    proc datasets library=WORK memtype=data; ';
@@ -465,7 +467,7 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=YES) ';
+  put '      %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=Y,maxobs=10) ';
   put '      data _null_; file _sjsref mod encoding=''utf-8''; ';
   put '        put "}"; ';
   put '    %end; ';
@@ -475,26 +477,33 @@ data _null_;
   put '  %end; ';
   put '  /* close off json */ ';
   put '  data _null_;file _sjsref mod encoding=''utf-8''; ';
-  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
-  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
-  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
+  put '    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; ';
   put '    put ",""_DEBUG"" : ""&_debug"" "; ';
   put '    _METAUSER=quote(trim(symget(''_METAUSER''))); ';
   put '    put ",""_METAUSER"": " _METAUSER; ';
   put '    _METAPERSON=quote(trim(symget(''_METAPERSON''))); ';
   put '    put '',"_METAPERSON": '' _METAPERSON; ';
+  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
+  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
+  put '    put '',"AUTOEXEC" : '' autoexec; ';
+  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    put ",""SYSENCODING"" : ""&sysencoding"" "; ';
-  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    syserrortext=cats(''"'',tranwrd(symget(''syserrortext''),''"'',''\"''),''"''); ';
   put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
+  put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
+  put '    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; ';
+  put '    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); ';
+  put '    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
+  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    syswarningtext=cats(''"'',tranwrd(symget(''syswarningtext''),''"'',''\"''),''"''); ';
   put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';

meta/mm_getstpcode.sas

@@ -58,7 +58,7 @@ data _null_;
     cnt=1;
     do while (metadata_getnasn(uri,"Notes",cnt,tsuri)>0);
       rc=metadata_getattr(tsuri,"Name",value);
-      put tsuri= value=;
+      &mD.put tsuri= value=;
       if value="SourceCode" then do;
         /* found it! */
         rc=metadata_getattr(tsuri,"Id",value);

meta/mm_updatestpsourcecode.sas

@@ -43,7 +43,9 @@ data _null_;
     cnt=1;
     do while (metadata_getnasn(uri,"Notes",cnt,tsuri)>0);
       rc=metadata_getattr(tsuri,"Name",value);
+    %if &mdebug=1 %then %do;
       put tsuri= value=;
+    %end;
       if value="SourceCode" then do;
         /* found it! */
         rc=metadata_getattr(tsuri,"Id",value);
@@ -126,4 +128,4 @@ run;
   filename &frefout clear;
 %end;
 
-%mend mm_updatestpsourcecode;
+%mend mm_updatestpsourcecode;

meta/mm_webout.sas

@@ -30,7 +30,7 @@
   @param [out] fref= (_webout) The fileref to which to write the JSON
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
-  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+  @param [in] showmeta= (N) Set to Y to output metadata alongside each table,
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
@@ -44,7 +44,7 @@
 
 **/
 %macro mm_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
-  ,showmeta=NO
+  ,showmeta=N
 );
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
@@ -119,7 +119,6 @@
   filename _sjsref temp lrecl=131068;
   %if %str(&_debug) ge 131 %then %do;
     /* if debug mode, send back first 10 records of each work table also */
-    options obs=10;
     data;run;%let tempds=%scan(&syslast,2,.);
     ods output Members=&tempds;
     proc datasets library=WORK memtype=data;
@@ -143,7 +142,7 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=YES)
+      %mp_jsonout(OBJ,&wt,jref=_sjsref,dslabel=first10rows,showmeta=Y,maxobs=10)
       data _null_; file _sjsref mod encoding='utf-8';
         put "}";
     %end;
@@ -153,26 +152,33 @@
   %end;
   /* close off json */
   data _null_;file _sjsref mod encoding='utf-8';
-    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
-    put ",""SYSUSERID"" : ""&sysuserid"" ";
-    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
+    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512;
     put ",""_DEBUG"" : ""&_debug"" ";
     _METAUSER=quote(trim(symget('_METAUSER')));
     put ",""_METAUSER"": " _METAUSER;
     _METAPERSON=quote(trim(symget('_METAPERSON')));
     put ',"_METAPERSON": ' _METAPERSON;
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ',"_PROGRAM" : ' _PROGRAM ;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
+    put ',"AUTOEXEC" : ' autoexec;
+    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""SYSCC"" : ""&syscc"" ";
     put ",""SYSENCODING"" : ""&sysencoding"" ";
-    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
     put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
+    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
+    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
+    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');
     put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;

metax/mmx_createmetafolder.sas

@@ -0,0 +1,50 @@
+/**
+  @file
+  @brief Creates a metadata folder
+  @details Creates a metadata folder using the batch tools
+
+  Usage:
+
+      %mmx_createmetafolder(loc=/some/meta/folder,user=sasdemo,pass=mars345)
+
+  <h4> SAS Macros </h4>
+  @li mf_loc.sas
+  @li mp_abort.sas
+
+  @param loc= the metadata folder to delete
+  @param user= username
+  @param pass= password
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mmx_createmetafolder(loc=,user=,pass=);
+
+%local host port path connx_string msg;
+%let host=%sysfunc(getoption(metaserver));
+%let port=%sysfunc(getoption(metaport));
+%let path=%mf_loc(POF)/tools;
+
+%let connx_string= -host &host -port &port -user '&user' -password '&pass';
+/* remove directory */
+data _null_;
+  infile " &path/sas-make-folder &connx_string ""&loc"" -makeFullPath 2>&1"
+    pipe lrecl=10000;
+  input;
+  putlog _infile_;
+run;
+
+data _null_; /* check tree exists */
+  length type uri $256;
+  rc=metadata_pathobj("","&loc","Folder",type,uri);
+  call symputx('foldertype',type,'l');
+run;
+%let msg=Location (&loc) was not created!!;
+%mp_abort(iftrue= (&foldertype ne Tree)
+  ,mac=&_program..sas
+  ,msg=%superq(msg)
+)
+
+%mend mmx_createmetafolder;

metax/mmx_deletemetafolder.sas

@@ -7,7 +7,7 @@
 
   Usage:
 
-    %mmx_deletemetafolder(loc=/some/meta/folder,user=sasdemo,pass=mars345)
+      %mmx_deletemetafolder(loc=/some/meta/folder,user=sasdemo,pass=mars345)
 
   <h4> SAS Macros </h4>
   @li mf_loc.sas
@@ -37,4 +37,4 @@ data _null_;
   putlog _infile_;
 run;
 
-%mend mmx_deletemetafolder;
+%mend mmx_deletemetafolder;

sasjs/doxy/new_header.html

@@ -25,6 +25,21 @@
   <link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
   <link rel="shortcut icon" href="$relpath^favicon.ico" type="image/x-icon" />
   $extrastylesheet
+  <!-- Matomo -->
+  <script>
+    var _paq = window._paq = window._paq || [];
+    /* tracker methods like "setCustomDimension" should be called before "trackPageView" */
+    _paq.push(['trackPageView']);
+    _paq.push(['enableLinkTracking']);
+    (function () {
+      var u = "https://analytics.4gl.io/";
+      _paq.push(['setTrackerUrl', u + 'matomo.php']);
+      _paq.push(['setSiteId', '6']);
+      var d = document, g = d.createElement('script'), s = d.getElementsByTagName('script')[0];
+      g.async = true; g.src = u + 'matomo.js'; s.parentNode.insertBefore(g, s);
+    })();
+  </script>
+  <!-- End Matomo Code -->
 </head>
 
 <body>
@@ -68,4 +83,4 @@
   </div>
 </body>
 
-</html>
+</html>

sasjs/sasjsconfig.json

@@ -67,7 +67,7 @@
     },
     {
       "name": "server",
-      "serverUrl": "",
+      "serverUrl": "https://sas.4gl.io",
       "serverType": "SASJS",
       "httpsAgentOptions": {
         "allowInsecureRequests": false
@@ -107,4 +107,4 @@
       "contextName": "SAS Job Execution compute context"
     }
   ]
-}
+}

server/ms_createwebservice.sas

@@ -98,10 +98,12 @@ data _null_;
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
-  put '  ,showmeta=NO ';
+  put '  ,showmeta=N ';
+  put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols; ';
+  put '%local tempds colinfo fmtds i numcols stmt_obs; ';
   put '%let numcols=0; ';
+  put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
@@ -148,7 +150,6 @@ data _null_;
   put '      if format='''' then fmt=cats(''$'',length,''.''); ';
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else fmt=cats(format,formatl,''.''); ';
-  put '      newlen=max(formatl,length); ';
   put '    end; ';
   put '    else do; ';
   put '      typelong=''num''; ';
@@ -156,15 +157,12 @@ data _null_;
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else if formatd=0 then fmt=cats(format,formatl,''.''); ';
   put '      else fmt=cats(format,formatl,''.'',formatd); ';
-  put '      /* needs to be wide, for datetimes etc */ ';
-  put '      newlen=max(length,formatl,24); ';
   put '    end; ';
   put '    /* 32 char unique name */ ';
   put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
   put ' ';
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
-  put '    call symputx(cats(''len'',_n_),newlen,''l''); ';
   put '    call symputx(cats(''length'',_n_),length,''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
@@ -180,7 +178,9 @@ data _null_;
   put '      %put &sysmacroname: Switching to DATASTEP engine; ';
   put '      %goto datastep; ';
   put '    %end; ';
-  put '    data &tempds;set &ds; ';
+  put '    data &tempds; ';
+  put '      set &ds; ';
+  put '      &stmt_obs; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
   put '    /* PRETTY is necessary to avoid line truncation in large files */ ';
   put '    filename _sjs2 temp lrecl=131068 encoding=''utf-8''; ';
@@ -215,7 +215,8 @@ data _null_;
   put '      %end; ';
   put '        )); ';
   put '      %do i=1 %to &numcols; ';
-  put '        length &&name&i $&&len&i; ';
+  put '        /* formatted values can be up to length 32767 */ ';
+  put '        length &&name&i $32767; ';
   put '        %if &&typelong&i=num %then %do; ';
   put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
@@ -255,6 +256,7 @@ data _null_;
   put '      %else %do; ';
   put '        set &ds; ';
   put '      %end; ';
+  put '      &stmt_obs; ';
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
@@ -311,10 +313,11 @@ data _null_;
   put '  proc sql; ';
   put '  drop table &colinfo, &tempds; ';
   put ' ';
-  put '  %if &showmeta=YES %then %do; ';
+  put '  %if %substr(&showmeta,1,1)=Y %then %do; ';
   put '    filename _sjs4 temp lrecl=131068 encoding=''utf-8''; ';
   put '    data _null_; ';
   put '      file _sjs4; ';
+  put '      length label $350; ';
   put '      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; ';
   put '      do i=1 to &numcols; ';
   put '        name=quote(trim(symget(cats(''name'',i)))); ';
@@ -368,7 +371,7 @@ data _null_;
   put '%mend mf_getuser; ';
   put ' ';
   put '%macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL ';
-  put '  ,showmeta=NO ';
+  put '  ,showmeta=N ';
   put '); ';
   put '%global _webin_file_count _webin_fileref1 _webin_name1 _program _debug ';
   put '  sasjs_tables; ';
@@ -433,7 +436,6 @@ data _null_;
   put '%else %if &action=CLOSE %then %do; ';
   put '  %if %str(&_debug) ge 131 %then %do; ';
   put '    /* if debug mode, send back first 10 records of each work table also */ ';
-  put '    options obs=10; ';
   put '    data;run;%let tempds=%scan(&syslast,2,.); ';
   put '    ods output Members=&tempds; ';
   put '    proc datasets library=WORK memtype=data; ';
@@ -458,7 +460,7 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) ';
+  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y,maxobs=10) ';
   put '      data _null_; file &fref mod encoding=''utf-8'' termstr=lf; ';
   put '        put "}"; ';
   put '    %end; ';
@@ -468,35 +470,34 @@ data _null_;
   put '  %end; ';
   put '  /* close off json */ ';
   put '  data _null_;file &fref mod encoding=''utf-8'' termstr=lf lrecl=32767; ';
-  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
-  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
-  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
+  put '    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; ';
   put '    put ",""_DEBUG"" : ""&_debug"" "; ';
+  put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
   put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
+  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
+  put '    put '',"AUTOEXEC" : '' autoexec; ';
+  put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    put ",""SYSENCODING"" : ""&sysencoding"" "; ';
-  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    syserrortext=cats(''"'',tranwrd(symget(''syserrortext''),''"'',''\"''),''"''); ';
   put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    SYSHOSTINFOLONG=quote(trim(symget(''SYSHOSTINFOLONG''))); ';
   put '    put '',"SYSHOSTINFOLONG" : '' SYSHOSTINFOLONG; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
   put '    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; ';
-  put '    length SYSPROCESSNAME $512; ';
   put '    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); ';
   put '    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" "; ';
+  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    syswarningtext=cats(''"'',tranwrd(symget(''syswarningtext''),''"'',''\"''),''"''); ';
   put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
-  put '    length autoexec $512; ';
-  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
-  put '    put '',"AUTOEXEC" : '' autoexec; ';
   put '    length memsize $32; ';
   put '    memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)"; ';
   put '    memsize=quote(cats(memsize)); ';

server/ms_getusers.sas

@@ -116,14 +116,16 @@ libname &libref JSON fileref=&fref1;
 %if "&group"="0" and "&gid"="0" %then %do;
   data &outds;
     length DISPLAYNAME $60 USERNAME:$30 ID 8;
-    set &libref..root;
+    if nobs=0 then call missing(of _all_);
+    set &libref..root nobs=nobs;
     drop ordinal_root;
   run;
 %end;
 %else %do;
   data &outds;
     length DISPLAYNAME $60 USERNAME:$30 ID 8;
-    set &libref..users;
+    if nobs=0 then call missing(of _all_);
+    set &libref..users nobs=nobs;
     drop ordinal_root ordinal_users;
   run;
 %end;

server/ms_webout.sas

@@ -27,7 +27,7 @@
   @param [out] fref= (_webout) The fileref to which to write the JSON
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
-  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+  @param [in] showmeta= (N) Set to Y to output metadata alongside each table,
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
@@ -47,7 +47,7 @@
 **/
 
 %macro ms_webout(action,ds,dslabel=,fref=_webout,fmt=Y,missing=NULL
-  ,showmeta=NO
+  ,showmeta=N
 );
 %global _webin_file_count _webin_fileref1 _webin_name1 _program _debug
   sasjs_tables;
@@ -112,7 +112,6 @@
 %else %if &action=CLOSE %then %do;
   %if %str(&_debug) ge 131 %then %do;
     /* if debug mode, send back first 10 records of each work table also */
-    options obs=10;
     data;run;%let tempds=%scan(&syslast,2,.);
     ods output Members=&tempds;
     proc datasets library=WORK memtype=data;
@@ -137,7 +136,7 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES)
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y,maxobs=10)
       data _null_; file &fref mod encoding='utf-8' termstr=lf;
         put "}";
     %end;
@@ -147,35 +146,34 @@
   %end;
   /* close off json */
   data _null_;file &fref mod encoding='utf-8' termstr=lf lrecl=32767;
-    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
-    put ",""SYSUSERID"" : ""&sysuserid"" ";
-    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
+    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512;
     put ",""_DEBUG"" : ""&_debug"" ";
+    _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ',"_PROGRAM" : ' _PROGRAM ;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
+    put ',"AUTOEXEC" : ' autoexec;
+    put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     put ",""SYSCC"" : ""&syscc"" ";
     put ",""SYSENCODING"" : ""&sysencoding"" ";
-    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
     put ',"SYSERRORTEXT" : ' syserrortext;
     SYSHOSTINFOLONG=quote(trim(symget('SYSHOSTINFOLONG')));
     put ',"SYSHOSTINFOLONG" : ' SYSHOSTINFOLONG;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
     put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
-    length SYSPROCESSNAME $512;
     SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
     put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     put ",""SYSTCPIPHOSTNAME"" : ""&SYSTCPIPHOSTNAME"" ";
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');
     put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
-    length autoexec $512;
-    autoexec=quote(urlencode(trim(getoption('autoexec'))));
-    put ',"AUTOEXEC" : ' autoexec;
     length memsize $32;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));

tests/base/mf_increment.test.sas

@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Testing mf_increment macro
+
+  <h4> SAS Macros </h4>
+  @li mf_increment.sas
+  @li mp_assert.sas
+
+**/
+
+%let var=0;
+
+%mp_assert(
+  iftrue=(
+    "%mf_increment(var)"="1"
+  ),
+  desc=Checking basic mf_increment usage 1,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_increment(var)"="2"
+  ),
+  desc=Checking basic mf_increment usage 2,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    "%mf_increment(var,incr=2)"="4"
+  ),
+  desc=Checking incr option,
+  outds=work.test_results
+)

tests/base/mp_cleancsv.test.sas

@@ -0,0 +1,43 @@
+/**
+  @file
+  @brief Testing mp_cleancsv.sas macro
+  @details Credit for test 1 goes to
+  [Tom](https://communities.sas.com/t5/user/viewprofilepage/user-id/159) from
+  SAS Communities:
+https://communities.sas.com/t5/SAS-Programming/Removing-embedded-carriage-returns/m-p/824790#M325761
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_cleancsv.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+/* test 1 - cope with empty rows on CR formatted file */
+
+filename crlf "%sysfunc(pathname(work))/crlf";
+filename cr "%sysfunc(pathname(work))/cr";
+data _null_;
+  file cr termstr=cr ;
+  put 'line 1'///'line 4'/'line 5';
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mp_cleancsv(in=cr,out=crlf)
+%mp_assertscope(COMPARE)
+
+/* 5 rows as all converted to OD0A */
+data test1;
+  infile "%sysfunc(pathname(work))/crlf" lrecl=100 termstr=crlf;
+  input;
+  list;
+run;
+
+%put test1=%mf_nobs(test1);
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.test1)=5),
+  desc=Checking blank rows on CR formatted file,
+  outds=work.test_results
+)

tests/base/mp_filtercheck.test.sas

@@ -38,7 +38,7 @@ William,M,15,66.5,112
 ;;;;
 run;
 
-/* valid filter conditions */
+/* VALID filter conditions */
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
@@ -51,6 +51,9 @@ AND,OR,2,Weight,>=,77.7
 AND,OR,2,Weight,NE,77.7
 AND,AND,1,age,=,.A
 AND,AND,1,height,<,.B
+AND,AND,1,age,IN,"(.a,.b,.)"
+AND,AND,1,age,IN,"(.A)"
+
 ;;;;
 run;
 
@@ -115,6 +118,28 @@ run;
   outds=work.test_results
 )
 
+/* invalid IN value */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,OR,2,age,IN,"(.,.a,X)"
+;;;;
+run;
+
+%mp_filtercheck(work.inds,
+  targetds=work.class,
+  outds=work.badrecords,
+  abort=NO
+)
+%let syscc=0;
+%mp_assertdsobs(work.badrecords,
+  desc=Invalid IN value,
+  test=HASOBS,
+  outds=work.test_results
+)
+
 /* Code injection - column name */
 data work.inds;
   infile datalines4 dsd;
@@ -163,7 +188,7 @@ run;
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
-    OPERATOR_NM:$10. RAW_VALUE:8;
+    OPERATOR_NM:$10. RAW_VALUE:8.;
 datalines4;
 AND,AND,1,age,=,0
 ;;;;

tests/base/mp_filterstore.test.1.sas

@@ -56,6 +56,9 @@ AND,AND,1,SEX,<=,"'M'"
 AND,OR,2,Name,NOT IN,"('Jane','Alfred')"
 AND,OR,2,Weight,>=,77.7
 AND,OR,2,Weight,NE,77.7
+AND,AND,3,age,NOT IN,"(.a,.b,.)"
+AND,AND,3,age,NOT IN,"(.A)"
+AND,AND,4,Name,=,"'Jeremiah'"
 ;;;;
 run;
 

viya/mv_createwebservice.sas

@@ -240,10 +240,12 @@ data _null_;
   put '%macro mp_jsonout(action,ds,jref=_webout,dslabel=,fmt=Y ';
   put '  ,engine=DATASTEP ';
   put '  ,missing=NULL ';
-  put '  ,showmeta=NO ';
+  put '  ,showmeta=N ';
+  put '  ,maxobs=MAX ';
   put ')/*/STORE SOURCE*/; ';
-  put '%local tempds colinfo fmtds i numcols; ';
+  put '%local tempds colinfo fmtds i numcols stmt_obs; ';
   put '%let numcols=0; ';
+  put '%if &maxobs ne MAX %then %let stmt_obs=%str(if _n_>&maxobs then stop;); ';
   put ' ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
@@ -290,7 +292,6 @@ data _null_;
   put '      if format='''' then fmt=cats(''$'',length,''.''); ';
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else fmt=cats(format,formatl,''.''); ';
-  put '      newlen=max(formatl,length); ';
   put '    end; ';
   put '    else do; ';
   put '      typelong=''num''; ';
@@ -298,15 +299,12 @@ data _null_;
   put '      else if formatl=0 then fmt=cats(format,''.''); ';
   put '      else if formatd=0 then fmt=cats(format,formatl,''.''); ';
   put '      else fmt=cats(format,formatl,''.'',formatd); ';
-  put '      /* needs to be wide, for datetimes etc */ ';
-  put '      newlen=max(length,formatl,24); ';
   put '    end; ';
   put '    /* 32 char unique name */ ';
   put '    newname=''sasjs''!!substr(cats(put(md5(name),$hex32.)),1,27); ';
   put ' ';
   put '    call symputx(cats(''name'',_n_),name,''l''); ';
   put '    call symputx(cats(''newname'',_n_),newname,''l''); ';
-  put '    call symputx(cats(''len'',_n_),newlen,''l''); ';
   put '    call symputx(cats(''length'',_n_),length,''l''); ';
   put '    call symputx(cats(''fmt'',_n_),fmt,''l''); ';
   put '    call symputx(cats(''type'',_n_),type,''l''); ';
@@ -322,7 +320,9 @@ data _null_;
   put '      %put &sysmacroname: Switching to DATASTEP engine; ';
   put '      %goto datastep; ';
   put '    %end; ';
-  put '    data &tempds;set &ds; ';
+  put '    data &tempds; ';
+  put '      set &ds; ';
+  put '      &stmt_obs; ';
   put '    %if &fmt=N %then format _numeric_ best32.;; ';
   put '    /* PRETTY is necessary to avoid line truncation in large files */ ';
   put '    filename _sjs2 temp lrecl=131068 encoding=''utf-8''; ';
@@ -357,7 +357,8 @@ data _null_;
   put '      %end; ';
   put '        )); ';
   put '      %do i=1 %to &numcols; ';
-  put '        length &&name&i $&&len&i; ';
+  put '        /* formatted values can be up to length 32767 */ ';
+  put '        length &&name&i $32767; ';
   put '        %if &&typelong&i=num %then %do; ';
   put '          &&name&i=left(put(&&newname&i,&&fmt&i)); ';
   put '        %end; ';
@@ -397,6 +398,7 @@ data _null_;
   put '      %else %do; ';
   put '        set &ds; ';
   put '      %end; ';
+  put '      &stmt_obs; ';
   put '      format _numeric_ bart.; ';
   put '    %do i=1 %to &numcols; ';
   put '      %if &&typelong&i=char or &fmt=Y %then %do; ';
@@ -453,10 +455,11 @@ data _null_;
   put '  proc sql; ';
   put '  drop table &colinfo, &tempds; ';
   put ' ';
-  put '  %if &showmeta=YES %then %do; ';
+  put '  %if %substr(&showmeta,1,1)=Y %then %do; ';
   put '    filename _sjs4 temp lrecl=131068 encoding=''utf-8''; ';
   put '    data _null_; ';
   put '      file _sjs4; ';
+  put '      length label $350; ';
   put '      put ", ""$%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":{""vars"":{"; ';
   put '      do i=1 to &numcols; ';
   put '        name=quote(trim(symget(cats(''name'',i)))); ';
@@ -509,7 +512,7 @@ data _null_;
   put ' ';
   put '%mend mf_getuser; ';
   put '%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL ';
-  put '  ,showmeta=NO ';
+  put '  ,showmeta=N ';
   put '); ';
   put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
   put '  sasjs_tables SYS_JES_JOB_URI; ';
@@ -601,8 +604,7 @@ data _null_;
   put ' ';
   put '  /* setup temp ref */ ';
   put '  %if %upcase(&fref) ne _WEBOUT %then %do; ';
-  put '    filename &fref temp lrecl=999999 permission=''A::u::rwx,A::g::rw-,A::o::---'' ';
-  put '      mod; ';
+  put '    filename &fref temp lrecl=999999 permission=''A::u::rwx,A::g::rw-,A::o::---''; ';
   put '  %end; ';
   put ' ';
   put '  /* setup json */ ';
@@ -619,7 +621,6 @@ data _null_;
   put '%else %if &action=CLOSE %then %do; ';
   put '  %if %str(&_debug) ge 131 %then %do; ';
   put '    /* send back first 10 records of each work table for debugging */ ';
-  put '    options obs=10; ';
   put '    data;run;%let tempds=%scan(&syslast,2,.); ';
   put '    ods output Members=&tempds; ';
   put '    proc datasets library=WORK memtype=data; ';
@@ -642,7 +643,7 @@ data _null_;
   put '        put " ""&wt"" : {"; ';
   put '        put ''"nlobs":'' nlobs; ';
   put '        put '',"nvars":'' nvars; ';
-  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES) ';
+  put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y,maxobs=10) ';
   put '      data _null_; file &fref mod;put "}"; ';
   put '    %end; ';
   put '    data _null_; file &fref mod;put "}";run; ';
@@ -650,23 +651,31 @@ data _null_;
   put ' ';
   put '  /* close off json */ ';
   put '  data _null_;file &fref mod; ';
+  put '    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512; ';
+  put '    put ",""_DEBUG"" : ""&_debug"" "; ';
   put '    _PROGRAM=quote(trim(resolve(symget(''_PROGRAM'')))); ';
-  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
+  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
+  put '    autoexec=quote(urlencode(trim(getoption(''autoexec'')))); ';
+  put '    put '',"AUTOEXEC" : '' autoexec; ';
   put '    put ",""MF_GETUSER"" : ""%mf_getuser()"" "; ';
   put '    SYS_JES_JOB_URI=quote(trim(resolve(symget(''SYS_JES_JOB_URI'')))); ';
   put '    put '',"SYS_JES_JOB_URI" : '' SYS_JES_JOB_URI ; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
-  put '    put ",""_DEBUG"" : ""&_debug"" "; ';
-  put '    put '',"_PROGRAM" : '' _PROGRAM ; ';
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
-  put '    syserrortext=quote(cats(symget(''SYSERRORTEXT''))); ';
+  put '    syserrortext=cats(''"'',tranwrd(symget(''syserrortext''),''"'',''\"''),''"''); ';
   put '    put '',"SYSERRORTEXT" : '' syserrortext; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
+  put '    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" "; ';
+  put '    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" "; ';
+  put '    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME))); ';
+  put '    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME; ';
+  put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
   put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
+  put '    put ",""SYSUSERID"" : ""&sysuserid"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
-  put '    syswarningtext=quote(cats(symget(''SYSWARNINGTEXT''))); ';
+  put '    syswarningtext=cats(''"'',tranwrd(symget(''syswarningtext''),''"'',''\"''),''"''); ';
   put '    put '',"SYSWARNINGTEXT" : '' syswarningtext; ';
   put '    put '',"END_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''" ''; ';
   put '    length memsize $32; ';

viya/mv_webout.sas

@@ -29,7 +29,7 @@
   @param [in] stream=(Y) Change to N if not streaming to _webout
   @param [in] missing= (NULL) Special numeric missing values can be sent as NULL
     (eg `null`) or as STRING values (eg `".a"` or `".b"`)
-  @param [in] showmeta= (NO) Set to YES to output metadata alongside each table,
+  @param [in] showmeta= (N) Set to Y to output metadata alongside each table,
     such as the column formats and types.  The metadata is contained inside an
     object with the same name as the table but prefixed with a dollar sign - ie,
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
@@ -43,7 +43,7 @@
 
 **/
 %macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y,missing=NULL
-  ,showmeta=NO
+  ,showmeta=N
 );
 %global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name
   sasjs_tables SYS_JES_JOB_URI;
@@ -135,8 +135,7 @@
 
   /* setup temp ref */
   %if %upcase(&fref) ne _WEBOUT %then %do;
-    filename &fref temp lrecl=999999 permission='A::u::rwx,A::g::rw-,A::o::---'
-      mod;
+    filename &fref temp lrecl=999999 permission='A::u::rwx,A::g::rw-,A::o::---';
   %end;
 
   /* setup json */
@@ -153,7 +152,6 @@
 %else %if &action=CLOSE %then %do;
   %if %str(&_debug) ge 131 %then %do;
     /* send back first 10 records of each work table for debugging */
-    options obs=10;
     data;run;%let tempds=%scan(&syslast,2,.);
     ods output Members=&tempds;
     proc datasets library=WORK memtype=data;
@@ -176,7 +174,7 @@
         put " ""&wt"" : {";
         put '"nlobs":' nlobs;
         put ',"nvars":' nvars;
-      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=YES)
+      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,showmeta=Y,maxobs=10)
       data _null_; file &fref mod;put "}";
     %end;
     data _null_; file &fref mod;put "}";run;
@@ -184,23 +182,31 @@
 
   /* close off json */
   data _null_;file &fref mod;
+    length SYSPROCESSNAME syserrortext syswarningtext autoexec $512;
+    put ",""_DEBUG"" : ""&_debug"" ";
     _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
-    put ",""SYSUSERID"" : ""&sysuserid"" ";
+    put ',"_PROGRAM" : ' _PROGRAM ;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
+    put ',"AUTOEXEC" : ' autoexec;
     put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
     SYS_JES_JOB_URI=quote(trim(resolve(symget('SYS_JES_JOB_URI'))));
     put ',"SYS_JES_JOB_URI" : ' SYS_JES_JOB_URI ;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
-    put ",""_DEBUG"" : ""&_debug"" ";
-    put ',"_PROGRAM" : ' _PROGRAM ;
     put ",""SYSCC"" : ""&syscc"" ";
-    syserrortext=quote(cats(symget('SYSERRORTEXT')));
+    syserrortext=cats('"',tranwrd(symget('syserrortext'),'"','\"'),'"');
     put ',"SYSERRORTEXT" : ' syserrortext;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
+    put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
+    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
+    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
+    put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
+    put ",""SYSUSERID"" : ""&sysuserid"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
-    syswarningtext=quote(cats(symget('SYSWARNINGTEXT')));
+    syswarningtext=cats('"',tranwrd(symget('syswarningtext'),'"','\"'),'"');
     put ',"SYSWARNINGTEXT" : ' syswarningtext;
     put ',"END_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '" ';
     length memsize $32;