Merge pull request #124 from sasjs/updates

feat: mf_islibds() macro to test if a library.dataset reference is valid

all.sas

@@ -1532,6 +1532,45 @@ Usage:
   %else %do;1%end;
 
 %mend mf_isint;/**
+  @file
+  @brief Checks whether a string follows correct library.dataset format
+  @details Many macros in the core library accept a library.dataset parameter
+  referred to as 'libds'.  This macro validates the structure of that parameter,
+  eg:
+
+    @li 8 character libref?
+    @li 32 character dataset?
+    @li contains a period?
+
+  It does NOT check whether the dataset exists, or if the library is assigned.
+
+  Usage:
+
+      %put %mf_islibds(work.something)=1;
+      %put %mf_islibds(nolib)=0;
+      %put %mf_islibds(badlibref.ds)=0;
+      %put %mf_islibds(w.t.f)=0;
+
+  @param [in] libds The string to be checked
+
+  @return output Returns 1 if libds is valid, 0 if it is not
+
+  <h4> Related Macros </h4>
+  @li mf_islibds.test.sas
+  @li mp_validatecol.sas
+
+  @version 9.2
+**/
+
+%macro mf_islibds(libds
+)/*/STORE SOURCE*/;
+
+%local regex;
+%let regex=%sysfunc(prxparse(%str(/^[_a-z]\w{0,7}\.[_a-z]\w{0,31}$/i)));
+
+%sysfunc(prxmatch(&regex,&libds))
+
+%mend mf_islibds;/**
   @file
   @brief Returns physical location of various SAS items
   @details Returns location of the PlatformObjectFramework tools
@@ -1805,6 +1844,60 @@ Usage:
   %exit_success:
 
 %mend mf_verifymacvars;
+/**
+  @file
+  @brief Returns words that are in both string 1 and string 2
+  @details  Compares two space separated strings and returns the words that are
+  in both.
+  Usage:
+
+      %put %mf_wordsInStr1andStr2(
+        Str1=blah sss blaaah brah bram boo
+        ,Str2=   blah blaaah brah ssss
+      );
+
+  returns:
+  > blah blaaah brah
+
+  @param str1= string containing words to extract
+  @param str2= used to compare with the extract string
+
+  @warning CASE SENSITIVE!
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_wordsInStr1andStr2(
+  Str1= /* string containing words to extract */
+  ,Str2= /* used to compare with the extract string */
+)/*/STORE SOURCE*/;
+
+%local count_base count_extr i i2 extr_word base_word match outvar;
+%if %length(&str1)=0 or %length(&str2)=0 %then %do;
+  %put %str(WARN)ING: empty string provided!;
+  %put base string (str1)= &str1;
+  %put compare string (str2) = &str2;
+  %return;
+%end;
+%let count_base=%sysfunc(countw(&Str2));
+%let count_extr=%sysfunc(countw(&Str1));
+
+%do i=1 %to &count_extr;
+  %let extr_word=%scan(&Str1,&i,%str( ));
+  %let match=0;
+  %do i2=1 %to &count_base;
+    %let base_word=%scan(&Str2,&i2,%str( ));
+    %if &extr_word=&base_word %then %let match=1;
+  %end;
+  %if &match=1 %then %let outvar=&outvar &extr_word;
+%end;
+
+  &outvar
+
+%mend mf_wordsInStr1andStr2;
+
 /**
   @file
   @brief Returns words that are in string 1 but not in string 2
@@ -6901,13 +6994,13 @@ filename &tempref clear;
 %put output location=&jref;
 %if &action=OPEN %then %do;
   options nobomfile;
-  data _null_;file &jref encoding='utf-8';
+  data _null_;file &jref encoding='utf-8' ;
     put '{"PROCESSED_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '"';
   run;
 %end;
 %else %if (&action=ARR or &action=OBJ) %then %do;
   options validvarname=upcase;
-  data _null_;file &jref mod encoding='utf-8';
+  data _null_;file &jref mod encoding='utf-8' ;
     put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":";
 
   %if &engine=PROCJSON %then %do;
@@ -6986,7 +7079,7 @@ filename &tempref clear;
       run;
       %let ds=&fmtds;
     %end; /* &fmt=Y */
-    data _null_;file &jref mod encoding='utf-8';
+    data _null_;file &jref mod encoding='utf-8' ;
       put "["; call symputx('cols',0,'l');
     proc sort
       data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
@@ -7031,7 +7124,7 @@ filename &tempref clear;
     /* write to temp loc to avoid _webout truncation
       - https://support.sas.com/kb/49/325.html */
     filename _sjs temp lrecl=131068 encoding='utf-8';
-    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod;
+    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod ;
       set &tempds;
       if _n_>1 then put "," @; put
       %if &action=ARR %then "[" ; %else "{" ;
@@ -7058,14 +7151,14 @@ filename &tempref clear;
       rc = fclose(fileid);
     run;
     filename _sjs clear;
-    data _null_; file &jref mod encoding='utf-8';
+    data _null_; file &jref mod encoding='utf-8' ;
       put "]";
     run;
   %end;
 %end;
 
 %else %if &action=CLOSE %then %do;
-  data _null_;file &jref encoding='utf-8' mod;
+  data _null_;file &jref encoding='utf-8' mod ;
     put "}";
   run;
 %end;
@@ -7586,8 +7679,6 @@ lock &libds clear;
     according to the variable types and formats.
 
     TODO:
-      @li Respect PKs
-      @li Respect NOT NULLs
       @li Consider dates, datetimes, times, integers etc
 
   Usage:
@@ -7603,12 +7694,15 @@ lock &libds clear;
       );
       %mp_makedata(work.example)
 
-  @param [in] libds The empty table in which to create data
-  @param [out] obs= (500) The number of records to create.
+  @param [in] libds The empty table (libref.dataset) in which to create data
+  @param [out] obs= (500) The maximum number of records to create.  The table
+    is sorted with nodup on the primary key, so the actual number of records may
+    be lower than this.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas
   @li mf_getvarlen.sas
+  @li mf_islibds.sas
   @li mf_nobs.sas
   @li mp_getcols.sas
   @li mp_getpk.sas
@@ -7620,45 +7714,59 @@ lock &libds clear;
 
 %macro mp_makedata(libds
   ,obs=500
+  ,seed=1
 )/*/STORE SOURCE*/;
 
-%local ds1 c1 n1 i col charvars numvars;
+%local ds1 ds2 lib ds pk_fields i col charvars numvars ispk;
 
-%if %mf_nobs(&libds)>0 %then %do;
+%if %mf_islibds(&libds)=0 %then %do;
+  %put &sysmacroname: Invalid libds (&libds) - should be library.dataset format;
+  %return;
+%end;
+%else %if %mf_nobs(&libds)>0 %then %do;
   %put &sysmacroname: &libds has data, it will not be recreated;
   %return;
 %end;
 
-%local ds1 c1 n1;
-%let ds1=%mf_getuniquename(prefix=mp_makedata);
-%let c1=%mf_getuniquename(prefix=mp_makedatacol);
-%let n1=%mf_getuniquename(prefix=mp_makedatacol);
-data &ds1;
+/* set up temporary vars */
+%let ds1=%mf_getuniquename(prefix=mp_makedatads1);
+%let ds2=%mf_getuniquename(prefix=mp_makedatads2);
+%let lib=%scan(&libds,1,.);
+%let ds=%scan(&libds,2,.);
+
+/* grab the primary key vars */
+%mp_getpk(&lib,ds=&ds,outds=&ds1)
+
+proc sql noprint;
+select pk_fields into: pk_fields from &ds1;
+
+data &ds2;
   if 0 then set &libds;
   do _n_=1 to &obs;
-    &c1=repeat(uuidgen(),10);
-    &n1=ranuni(1)*5000000;
-    drop &c1 &n1;
     %let charvars=%mf_getvarlist(&libds,typefilter=C);
     %if &charvars ^= %then %do i=1 %to %sysfunc(countw(&charvars));
       %let col=%scan(&charvars,&i);
-      &col=subpad(&c1,1,%mf_getvarlen(&libds,&col));
+      /* create random value based on observation number and colum length */
+      &col=substr(put(md5(_n_),$hex32.),1,%mf_getvarlen(&libds,&col));
     %end;
 
     %let numvars=%mf_getvarlist(&libds,typefilter=N);
     %if &numvars ^= %then %do i=1 %to %sysfunc(countw(&numvars));
       %let col=%scan(&numvars,&i);
-      &col=&n1;
+      &col=_n_;
     %end;
     output;
   end;
 run;
+proc sort data=&ds2 nodupkey;
+  by &pk_fields;
+run;
 
-proc append base=&libds data=&ds1;
+proc append base=&libds data=&ds2;
 run;
 
 proc sql;
-drop table &ds1;
+drop table &ds1, &ds2;
 
 %mend mp_makedata;/**
   @file
@@ -7909,13 +8017,15 @@ options obs=max replace nosyntaxcheck;
   @brief Reset an option to original value
   @details Inspired by the SAS Jedi -
 https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options
-    Called as follows:
 
-    options obs=30;
-    %mp_resetoption(OBS)
+  Called as follows:
+
+      options obs=30 ps=max;
+      %mp_resetoption(OBS)
+      %mp_resetoption(PS)
 
 
-  @param option the option to reset
+  @param [in] option the option to reset
 
   @version 9.2
   @author Allan Bowe
@@ -8284,9 +8394,7 @@ select distinct memname into: table_list separated by ' '
 
   This macro will only work for BASE (V9) engine libraries.  It works by
   creating a copy of the dataset (without data, WITH constraints) in the same
-  library, appending a sorted view into it, and finally - renaming it.  By
-  default, COMPRESS=CHAR and REUSE=YES will be applied, this behaviour can
-  be adjusted using the `dsoptions=` parameter.
+  library, appending a sorted view into it, and finally - renaming it.
 
   Example usage:
 
@@ -8314,7 +8422,6 @@ select distinct memname into: table_list separated by ' '
 
   @version 9.2
   @author Allan Bowe
-  @source https://github.com/sasjs/core
 
 **/
 
@@ -11711,13 +11818,13 @@ data _null_;
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
-  put '  data _null_;file &jref encoding=''utf-8''; ';
+  put '  data _null_;file &jref encoding=''utf-8'' ; ';
   put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
   put '  options validvarname=upcase; ';
-  put '  data _null_;file &jref mod encoding=''utf-8''; ';
+  put '  data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
@@ -11796,7 +11903,7 @@ data _null_;
   put '      run; ';
   put '      %let ds=&fmtds; ';
   put '    %end; /* &fmt=Y */ ';
-  put '    data _null_;file &jref mod encoding=''utf-8''; ';
+  put '    data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
   put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
@@ -11841,7 +11948,7 @@ data _null_;
   put '    /* write to temp loc to avoid _webout truncation ';
   put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
-  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod; ';
+  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
   put '      %if &action=ARR %then "[" ; %else "{" ; ';
@@ -11868,14 +11975,14 @@ data _null_;
   put '      rc = fclose(fileid); ';
   put '    run; ';
   put '    filename _sjs clear; ';
-  put '    data _null_; file &jref mod encoding=''utf-8''; ';
+  put '    data _null_; file &jref mod encoding=''utf-8'' ; ';
   put '      put "]"; ';
   put '    run; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8'' mod; ';
+  put '  data _null_;file &jref encoding=''utf-8'' mod ; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
@@ -15723,7 +15830,7 @@ run;
   OPTIONS NOBOMFILE;
 
   /* setup json */
-  data _null_;file &fref encoding='utf-8';
+  data _null_;file &fref encoding='utf-8' termstr=lf;
   %if %str(&_debug) ge 131 %then %do;
     put '>>weboutBEGIN<<';
   %end;
@@ -15752,14 +15859,14 @@ run;
       i+1;
       call symputx('wt'!!left(i),name,'l');
       call symputx('wtcnt',i,'l');
-    data _null_; file &fref mod encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8' termstr=lf;
       put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
       proc contents noprint data=&wt
         out=_data_ (keep=name type length format:);
       run;%let tempds=%scan(&syslast,2,.);
-      data _null_; file &fref mod encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8' termstr=lf;
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
         nvars=attrn(dsid,'NVARS');
@@ -15770,15 +15877,15 @@ run;
         put ',"nvars":' nvars;
       %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
       %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
-      data _null_; file &fref mod encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8' termstr=lf;
         put "}";
     %end;
-    data _null_; file &fref mod encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8' termstr=lf termstr=lf;
       put "}";
     run;
   %end;
   /* close off json */
-  data _null_;file &fref mod encoding='utf-8';
+  data _null_;file &fref mod encoding='utf-8' termstr=lf;
     _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ",""SYSUSERID"" : ""&sysuserid"" ";
     put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
@@ -15791,7 +15898,9 @@ run;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
     put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
-    put ",""SYSPROCESSNAME"" : ""&SYSPROCESSNAME"" ";
+    length SYSPROCESSNAME $512;
+    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
+    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
@@ -15800,7 +15909,8 @@ run;
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
     put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
-    autoexec=quote(trim(getoption('autoexec')));
+    length autoexec $512;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
     put ',"AUTOEXEC" : ' autoexec;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));
@@ -16792,13 +16902,13 @@ data _null_;
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
-  put '  data _null_;file &jref encoding=''utf-8''; ';
+  put '  data _null_;file &jref encoding=''utf-8'' ; ';
   put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
   put '  options validvarname=upcase; ';
-  put '  data _null_;file &jref mod encoding=''utf-8''; ';
+  put '  data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
@@ -16877,7 +16987,7 @@ data _null_;
   put '      run; ';
   put '      %let ds=&fmtds; ';
   put '    %end; /* &fmt=Y */ ';
-  put '    data _null_;file &jref mod encoding=''utf-8''; ';
+  put '    data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
   put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
@@ -16922,7 +17032,7 @@ data _null_;
   put '    /* write to temp loc to avoid _webout truncation ';
   put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
-  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod; ';
+  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
   put '      %if &action=ARR %then "[" ; %else "{" ; ';
@@ -16949,14 +17059,14 @@ data _null_;
   put '      rc = fclose(fileid); ';
   put '    run; ';
   put '    filename _sjs clear; ';
-  put '    data _null_; file &jref mod encoding=''utf-8''; ';
+  put '    data _null_; file &jref mod encoding=''utf-8'' ; ';
   put '      put "]"; ';
   put '    run; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8'' mod; ';
+  put '  data _null_;file &jref encoding=''utf-8'' mod ; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';

base/mf_islibds.sas

@@ -0,0 +1,40 @@
+/**
+  @file
+  @brief Checks whether a string follows correct library.dataset format
+  @details Many macros in the core library accept a library.dataset parameter
+  referred to as 'libds'.  This macro validates the structure of that parameter,
+  eg:
+
+    @li 8 character libref?
+    @li 32 character dataset?
+    @li contains a period?
+
+  It does NOT check whether the dataset exists, or if the library is assigned.
+
+  Usage:
+
+      %put %mf_islibds(work.something)=1;
+      %put %mf_islibds(nolib)=0;
+      %put %mf_islibds(badlibref.ds)=0;
+      %put %mf_islibds(w.t.f)=0;
+
+  @param [in] libds The string to be checked
+
+  @return output Returns 1 if libds is valid, 0 if it is not
+
+  <h4> Related Macros </h4>
+  @li mf_islibds.test.sas
+  @li mp_validatecol.sas
+
+  @version 9.2
+**/
+
+%macro mf_islibds(libds
+)/*/STORE SOURCE*/;
+
+%local regex;
+%let regex=%sysfunc(prxparse(%str(/^[_a-z]\w{0,7}\.[_a-z]\w{0,31}$/i)));
+
+%sysfunc(prxmatch(&regex,&libds))
+
+%mend mf_islibds;

base/mf_wordsinstr1andstr2.sas

@@ -0,0 +1,54 @@
+/**
+  @file
+  @brief Returns words that are in both string 1 and string 2
+  @details  Compares two space separated strings and returns the words that are
+  in both.
+  Usage:
+
+      %put %mf_wordsInStr1andStr2(
+        Str1=blah sss blaaah brah bram boo
+        ,Str2=   blah blaaah brah ssss
+      );
+
+  returns:
+  > blah blaaah brah
+
+  @param str1= string containing words to extract
+  @param str2= used to compare with the extract string
+
+  @warning CASE SENSITIVE!
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_wordsInStr1andStr2(
+  Str1= /* string containing words to extract */
+  ,Str2= /* used to compare with the extract string */
+)/*/STORE SOURCE*/;
+
+%local count_base count_extr i i2 extr_word base_word match outvar;
+%if %length(&str1)=0 or %length(&str2)=0 %then %do;
+  %put %str(WARN)ING: empty string provided!;
+  %put base string (str1)= &str1;
+  %put compare string (str2) = &str2;
+  %return;
+%end;
+%let count_base=%sysfunc(countw(&Str2));
+%let count_extr=%sysfunc(countw(&Str1));
+
+%do i=1 %to &count_extr;
+  %let extr_word=%scan(&Str1,&i,%str( ));
+  %let match=0;
+  %do i2=1 %to &count_base;
+    %let base_word=%scan(&Str2,&i2,%str( ));
+    %if &extr_word=&base_word %then %let match=1;
+  %end;
+  %if &match=1 %then %let outvar=&outvar &extr_word;
+%end;
+
+  &outvar
+
+%mend mf_wordsInStr1andStr2;
+

base/mp_jsonout.sas

@@ -62,13 +62,13 @@
 %put output location=&jref;
 %if &action=OPEN %then %do;
   options nobomfile;
-  data _null_;file &jref encoding='utf-8';
+  data _null_;file &jref encoding='utf-8' ;
     put '{"PROCESSED_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '"';
   run;
 %end;
 %else %if (&action=ARR or &action=OBJ) %then %do;
   options validvarname=upcase;
-  data _null_;file &jref mod encoding='utf-8';
+  data _null_;file &jref mod encoding='utf-8' ;
     put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":";
 
   %if &engine=PROCJSON %then %do;
@@ -147,7 +147,7 @@
       run;
       %let ds=&fmtds;
     %end; /* &fmt=Y */
-    data _null_;file &jref mod encoding='utf-8';
+    data _null_;file &jref mod encoding='utf-8' ;
       put "["; call symputx('cols',0,'l');
     proc sort
       data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
@@ -192,7 +192,7 @@
     /* write to temp loc to avoid _webout truncation
       - https://support.sas.com/kb/49/325.html */
     filename _sjs temp lrecl=131068 encoding='utf-8';
-    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod;
+    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod ;
       set &tempds;
       if _n_>1 then put "," @; put
       %if &action=ARR %then "[" ; %else "{" ;
@@ -219,14 +219,14 @@
       rc = fclose(fileid);
     run;
     filename _sjs clear;
-    data _null_; file &jref mod encoding='utf-8';
+    data _null_; file &jref mod encoding='utf-8' ;
       put "]";
     run;
   %end;
 %end;
 
 %else %if &action=CLOSE %then %do;
-  data _null_;file &jref encoding='utf-8' mod;
+  data _null_;file &jref encoding='utf-8' mod ;
     put "}";
   run;
 %end;

base/mp_makedata.sas

@@ -10,8 +10,6 @@
     according to the variable types and formats.
 
     TODO:
-      @li Respect PKs
-      @li Respect NOT NULLs
       @li Consider dates, datetimes, times, integers etc
 
   Usage:
@@ -27,12 +25,15 @@
       );
       %mp_makedata(work.example)
 
-  @param [in] libds The empty table in which to create data
-  @param [out] obs= (500) The number of records to create.
+  @param [in] libds The empty table (libref.dataset) in which to create data
+  @param [out] obs= (500) The maximum number of records to create.  The table
+    is sorted with nodup on the primary key, so the actual number of records may
+    be lower than this.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas
   @li mf_getvarlen.sas
+  @li mf_islibds.sas
   @li mf_nobs.sas
   @li mp_getcols.sas
   @li mp_getpk.sas
@@ -44,44 +45,58 @@
 
 %macro mp_makedata(libds
   ,obs=500
+  ,seed=1
 )/*/STORE SOURCE*/;
 
-%local ds1 c1 n1 i col charvars numvars;
+%local ds1 ds2 lib ds pk_fields i col charvars numvars ispk;
 
-%if %mf_nobs(&libds)>0 %then %do;
+%if %mf_islibds(&libds)=0 %then %do;
+  %put &sysmacroname: Invalid libds (&libds) - should be library.dataset format;
+  %return;
+%end;
+%else %if %mf_nobs(&libds)>0 %then %do;
   %put &sysmacroname: &libds has data, it will not be recreated;
   %return;
 %end;
 
-%local ds1 c1 n1;
-%let ds1=%mf_getuniquename(prefix=mp_makedata);
-%let c1=%mf_getuniquename(prefix=mp_makedatacol);
-%let n1=%mf_getuniquename(prefix=mp_makedatacol);
-data &ds1;
+/* set up temporary vars */
+%let ds1=%mf_getuniquename(prefix=mp_makedatads1);
+%let ds2=%mf_getuniquename(prefix=mp_makedatads2);
+%let lib=%scan(&libds,1,.);
+%let ds=%scan(&libds,2,.);
+
+/* grab the primary key vars */
+%mp_getpk(&lib,ds=&ds,outds=&ds1)
+
+proc sql noprint;
+select pk_fields into: pk_fields from &ds1;
+
+data &ds2;
   if 0 then set &libds;
   do _n_=1 to &obs;
-    &c1=repeat(uuidgen(),10);
-    &n1=ranuni(1)*5000000;
-    drop &c1 &n1;
     %let charvars=%mf_getvarlist(&libds,typefilter=C);
     %if &charvars ^= %then %do i=1 %to %sysfunc(countw(&charvars));
       %let col=%scan(&charvars,&i);
-      &col=subpad(&c1,1,%mf_getvarlen(&libds,&col));
+      /* create random value based on observation number and colum length */
+      &col=substr(put(md5(_n_),$hex32.),1,%mf_getvarlen(&libds,&col));
     %end;
 
     %let numvars=%mf_getvarlist(&libds,typefilter=N);
     %if &numvars ^= %then %do i=1 %to %sysfunc(countw(&numvars));
       %let col=%scan(&numvars,&i);
-      &col=&n1;
+      &col=_n_;
     %end;
     output;
   end;
 run;
+proc sort data=&ds2 nodupkey;
+  by &pk_fields;
+run;
 
-proc append base=&libds data=&ds1;
+proc append base=&libds data=&ds2;
 run;
 
 proc sql;
-drop table &ds1;
+drop table &ds1, &ds2;
 
 %mend mp_makedata;

base/mp_resetoption.sas

@@ -3,13 +3,15 @@
   @brief Reset an option to original value
   @details Inspired by the SAS Jedi -
 https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options
-    Called as follows:
 
-    options obs=30;
-    %mp_resetoption(OBS)
+  Called as follows:
+
+      options obs=30 ps=max;
+      %mp_resetoption(OBS)
+      %mp_resetoption(PS)
 
 
-  @param option the option to reset
+  @param [in] option the option to reset
 
   @version 9.2
   @author Allan Bowe

base/mp_sortinplace.sas

@@ -11,9 +11,7 @@
 
   This macro will only work for BASE (V9) engine libraries.  It works by
   creating a copy of the dataset (without data, WITH constraints) in the same
-  library, appending a sorted view into it, and finally - renaming it.  By
-  default, COMPRESS=CHAR and REUSE=YES will be applied, this behaviour can
-  be adjusted using the `dsoptions=` parameter.
+  library, appending a sorted view into it, and finally - renaming it.
 
   Example usage:
 
@@ -41,7 +39,6 @@
 
   @version 9.2
   @author Allan Bowe
-  @source https://github.com/sasjs/core
 
 **/
 

meta/mm_createwebservice.sas

@@ -94,13 +94,13 @@ data _null_;
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
-  put '  data _null_;file &jref encoding=''utf-8''; ';
+  put '  data _null_;file &jref encoding=''utf-8'' ; ';
   put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
   put '  options validvarname=upcase; ';
-  put '  data _null_;file &jref mod encoding=''utf-8''; ';
+  put '  data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
@@ -179,7 +179,7 @@ data _null_;
   put '      run; ';
   put '      %let ds=&fmtds; ';
   put '    %end; /* &fmt=Y */ ';
-  put '    data _null_;file &jref mod encoding=''utf-8''; ';
+  put '    data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
   put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
@@ -224,7 +224,7 @@ data _null_;
   put '    /* write to temp loc to avoid _webout truncation ';
   put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
-  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod; ';
+  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
   put '      %if &action=ARR %then "[" ; %else "{" ; ';
@@ -251,14 +251,14 @@ data _null_;
   put '      rc = fclose(fileid); ';
   put '    run; ';
   put '    filename _sjs clear; ';
-  put '    data _null_; file &jref mod encoding=''utf-8''; ';
+  put '    data _null_; file &jref mod encoding=''utf-8'' ; ';
   put '      put "]"; ';
   put '    run; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8'' mod; ';
+  put '  data _null_;file &jref encoding=''utf-8'' mod ; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';

server/ms_webout.sas

@@ -79,7 +79,7 @@
   OPTIONS NOBOMFILE;
 
   /* setup json */
-  data _null_;file &fref encoding='utf-8';
+  data _null_;file &fref encoding='utf-8' termstr=lf;
   %if %str(&_debug) ge 131 %then %do;
     put '>>weboutBEGIN<<';
   %end;
@@ -108,14 +108,14 @@
       i+1;
       call symputx('wt'!!left(i),name,'l');
       call symputx('wtcnt',i,'l');
-    data _null_; file &fref mod encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8' termstr=lf;
       put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
       proc contents noprint data=&wt
         out=_data_ (keep=name type length format:);
       run;%let tempds=%scan(&syslast,2,.);
-      data _null_; file &fref mod encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8' termstr=lf;
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
         nvars=attrn(dsid,'NVARS');
@@ -126,15 +126,15 @@
         put ',"nvars":' nvars;
       %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
       %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
-      data _null_; file &fref mod encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8' termstr=lf;
         put "}";
     %end;
-    data _null_; file &fref mod encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8' termstr=lf termstr=lf;
       put "}";
     run;
   %end;
   /* close off json */
-  data _null_;file &fref mod encoding='utf-8';
+  data _null_;file &fref mod encoding='utf-8' termstr=lf;
     _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ",""SYSUSERID"" : ""&sysuserid"" ";
     put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
@@ -147,7 +147,9 @@
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
     put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
-    put ",""SYSPROCESSNAME"" : ""&SYSPROCESSNAME"" ";
+    length SYSPROCESSNAME $512;
+    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
+    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
@@ -156,7 +158,8 @@
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
     put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
-    autoexec=quote(trim(getoption('autoexec')));
+    length autoexec $512;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
     put ',"AUTOEXEC" : ' autoexec;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));

tests/crossplatform/mf_islibds.test.sas

@@ -0,0 +1,46 @@
+/**
+  @file
+  @brief Testing mf_islibds macro
+
+      %put %mf_islibds(work.something)=1;
+      %put %mf_islibds(nolib)=0;
+      %put %mf_islibds(badlibref.ds)=0;
+      %put %mf_islibds(w.t.f)=0;
+
+  <h4> SAS Macros </h4>
+  @li mf_islibds.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(
+    %mf_islibds(work.something)=1
+  ),
+  desc=%str(Checking mf_islibds(work.something)=1),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    %mf_islibds(nolib)=0
+  ),
+  desc=%str(Checking mf_islibds(nolib)=0),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    %mf_islibds(badlibref.ds)=0
+  ),
+  desc=%str(Checking mf_islibds(badlibref.ds)=0),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    %mf_islibds(w.t.f)=0
+  ),
+  desc=%str(Checking mf_islibds(w.t.f)=0),
+  outds=work.test_results
+)

tests/crossplatform/mf_wordsinstr1andstr2.test.sas

@@ -0,0 +1,20 @@
+/**
+  @file
+  @brief Testing mf_wordsinstr1andstr2 macro
+
+  <h4> SAS Macros </h4>
+  @li mf_wordsinstr1andstr2.sas
+  @li mp_assert.sas
+
+**/
+
+%let x=%mf_wordsinstr1andstr2(str1=xx DOLLAR x $CHAR xxx W MONNAME
+  ,str2=DOLLAR $CHAR W MONNAME xxxxxx
+);
+%mp_assert(
+  iftrue=(
+    "&x"="DOLLAR $CHAR W MONNAME"
+  ),
+  desc=Checking basic string,
+  outds=work.test_results
+)

tests/crossplatform/mf_wordsinstr1butnotstr2.test.sas

@@ -0,0 +1,20 @@
+/**
+  @file
+  @brief Testing mf_wordsinstr1butnotstr2 macro
+
+  <h4> SAS Macros </h4>
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_assert.sas
+
+**/
+
+%let x=%mf_wordsinstr1butnotstr2(str1=xx DOLLAR x $CHAR xxx W MONNAME
+  ,str2=ff xx x xxx xxxxxx
+);
+%mp_assert(
+  iftrue=(
+    "&x"="DOLLAR $CHAR W MONNAME"
+  ),
+  desc=Checking basic string,
+  outds=work.test_results
+)

viya/mv_createwebservice.sas

@@ -242,13 +242,13 @@ data _null_;
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
-  put '  data _null_;file &jref encoding=''utf-8''; ';
+  put '  data _null_;file &jref encoding=''utf-8'' ; ';
   put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
   put '  options validvarname=upcase; ';
-  put '  data _null_;file &jref mod encoding=''utf-8''; ';
+  put '  data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
@@ -327,7 +327,7 @@ data _null_;
   put '      run; ';
   put '      %let ds=&fmtds; ';
   put '    %end; /* &fmt=Y */ ';
-  put '    data _null_;file &jref mod encoding=''utf-8''; ';
+  put '    data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
   put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
@@ -372,7 +372,7 @@ data _null_;
   put '    /* write to temp loc to avoid _webout truncation ';
   put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
-  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod; ';
+  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
   put '      %if &action=ARR %then "[" ; %else "{" ; ';
@@ -399,14 +399,14 @@ data _null_;
   put '      rc = fclose(fileid); ';
   put '    run; ';
   put '    filename _sjs clear; ';
-  put '    data _null_; file &jref mod encoding=''utf-8''; ';
+  put '    data _null_; file &jref mod encoding=''utf-8'' ; ';
   put '      put "]"; ';
   put '    run; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8'' mod; ';
+  put '  data _null_;file &jref encoding=''utf-8'' mod ; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';