Merge pull request #128 from sasjs/writefile

Writefile

all.sas

@@ -249,8 +249,12 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
 
         %put %mf_existvar(work.someds, somevar)
 
-  @param libds (positional) - 2 part dataset or view reference
+  @param [in] libds 2 part dataset or view reference
-  @param var (positional) - variable name
+  @param [in] var variable name
+
+  <h4> Related Macros </h4>
+  @li mf_existvar.test.sas
+
   @version 9.2
   @author Allan Bowe
 **/
@@ -484,9 +488,10 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
   contributors of Chris Hemedingers blog [post](
   http://blogs.sas.com/content/sasdummy/2013/06/04/find-a-sas-library-engine/)
 
-  @param libref Library reference (also accepts a 2 level libds ref).
+  @param [in] libref Library reference (also accepts a 2 level libds ref).
 
-  @return output returns the library engine for the FIRST library encountered.
+  @return output returns the library engine (uppercase) for the FIRST library
+    encountered.
 
   @warning will only return the FIRST library engine - for concatenated
     libraries, with different engines, inconsistent results may be encountered.
@@ -518,7 +523,7 @@ https://github.com/yabwon/SAS_PACKAGES/blob/main/packages/baseplus.md#functionex
     %let rc= %sysfunc(close(&dsid));
   %end;
 
-  &engine
+  %upcase(&engine)
 
 %mend mf_getengine;
 
@@ -1532,6 +1537,45 @@ Usage:
   %else %do;1%end;
 
 %mend mf_isint;/**
+  @file
+  @brief Checks whether a string follows correct library.dataset format
+  @details Many macros in the core library accept a library.dataset parameter
+  referred to as 'libds'.  This macro validates the structure of that parameter,
+  eg:
+
+    @li 8 character libref?
+    @li 32 character dataset?
+    @li contains a period?
+
+  It does NOT check whether the dataset exists, or if the library is assigned.
+
+  Usage:
+
+      %put %mf_islibds(work.something)=1;
+      %put %mf_islibds(nolib)=0;
+      %put %mf_islibds(badlibref.ds)=0;
+      %put %mf_islibds(w.t.f)=0;
+
+  @param [in] libds The string to be checked
+
+  @return output Returns 1 if libds is valid, 0 if it is not
+
+  <h4> Related Macros </h4>
+  @li mf_islibds.test.sas
+  @li mp_validatecol.sas
+
+  @version 9.2
+**/
+
+%macro mf_islibds(libds
+)/*/STORE SOURCE*/;
+
+%local regex;
+%let regex=%sysfunc(prxparse(%str(/^[_a-z]\w{0,7}\.[_a-z]\w{0,31}$/i)));
+
+%sysfunc(prxmatch(&regex,&libds))
+
+%mend mf_islibds;/**
   @file
   @brief Returns physical location of various SAS items
   @details Returns location of the PlatformObjectFramework tools
@@ -1805,6 +1849,60 @@ Usage:
   %exit_success:
 
 %mend mf_verifymacvars;
+/**
+  @file
+  @brief Returns words that are in both string 1 and string 2
+  @details  Compares two space separated strings and returns the words that are
+  in both.
+  Usage:
+
+      %put %mf_wordsInStr1andStr2(
+        Str1=blah sss blaaah brah bram boo
+        ,Str2=   blah blaaah brah ssss
+      );
+
+  returns:
+  > blah blaaah brah
+
+  @param str1= string containing words to extract
+  @param str2= used to compare with the extract string
+
+  @warning CASE SENSITIVE!
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_wordsInStr1andStr2(
+  Str1= /* string containing words to extract */
+  ,Str2= /* used to compare with the extract string */
+)/*/STORE SOURCE*/;
+
+%local count_base count_extr i i2 extr_word base_word match outvar;
+%if %length(&str1)=0 or %length(&str2)=0 %then %do;
+  %put %str(WARN)ING: empty string provided!;
+  %put base string (str1)= &str1;
+  %put compare string (str2) = &str2;
+  %return;
+%end;
+%let count_base=%sysfunc(countw(&Str2));
+%let count_extr=%sysfunc(countw(&Str1));
+
+%do i=1 %to &count_extr;
+  %let extr_word=%scan(&Str1,&i,%str( ));
+  %let match=0;
+  %do i2=1 %to &count_base;
+    %let base_word=%scan(&Str2,&i2,%str( ));
+    %if &extr_word=&base_word %then %let match=1;
+  %end;
+  %if &match=1 %then %let outvar=&outvar &extr_word;
+%end;
+
+  &outvar
+
+%mend mf_wordsInStr1andStr2;
+
 /**
   @file
   @brief Returns words that are in string 1 but not in string 2
@@ -1860,6 +1958,72 @@ Usage:
 %mend mf_wordsInStr1ButNotStr2;
 
 /**
+  @file
+  @brief Creates a text file using pure macro
+  @details Creates a text file of up to 10 lines.  If further lines are
+    desired, feel free to [create an issue](
+    https://github.com/sasjs/core/issues/new), or make a pull request!
+
+    The use of PARMBUFF was considered for this macro, but it would have made
+    things problematic for writing lines containing commas.
+
+    Usage:
+
+        %mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content)
+        data _null_;
+          infile "&sasjswork/myfile.txt";
+          input;
+          list;
+        run;
+
+  @param [in] fpath Full path to file to be created or appended to
+  @param [in] mode= (O) Available options are A or O as follows:
+    @li A APPEND mode, writes new records after the current end of the file.
+    @li O OUTPUT mode, writes new records from the beginning of the file.
+  @param [in] l1= First line
+  @param [in] l2= Second line (etc through to l10)
+
+  <h4> Related Macros </h4>
+  @li mf_writefile.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mf_writefile(fpath,mode=O,l1=,l2=,l3=,l4=,l5=,l6=,l7=,l8=,l9=,l10=
+)/*/STORE SOURCE*/;
+%local fref rc fid i total_lines;
+
+/* find number of lines by reference to first non-blank param */
+%do i=10 %to 1 %by -1;
+  %if %str(&&l&i) ne %str() %then %goto continue;
+%end;
+%continue:
+%let total_lines=&i;
+
+%if %sysfunc(filename(fref,&fpath)) ne 0 %then %do;
+  %put &=fref &=fpath;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,&mode));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%do i=1 %to &total_lines;
+  %let rc=%sysfunc(fput(&fid, &&l&i));
+  %let rc=%sysfunc(fwrite(&fid));
+%end;
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mf_writefile;
+/** @endcond *//**
   @file
   @brief abort gracefully according to context
   @details Configures an abort mechanism according to site specific policies or
@@ -2171,6 +2335,186 @@ Usage:
 %end;
 
 %mend mp_appendfile;/**
+  @file
+  @brief Apply a set of formats to a table
+  @details Applies a set of formats to one or more SAS datasets.  Can be used
+    to migrate formats from one table to another.  The input table must contain
+    the following columns:
+
+    @li lib - the libref of the table to be updated
+    @li ds - the dataset to be updated
+    @li var - the variable to be updated
+    @li fmt - the format to apply.  Missing or default ($CHAR, 8.) formats are
+      ignored.
+
+  The macro will abort in the following scenarios:
+
+    @li Libref not assigned
+    @li Dataset does not exist
+    @li Input table contains null or invalid values
+
+  Example usage:
+
+      data work.example;
+        set sashelp.prdsale;
+        format _all_ clear;
+      run;
+
+      %mp_getcols(sashelp.prdsale,outds=work.cols)
+
+      data work.cols2;
+        set work.cols;
+        lib='WORK';
+        ds='EXAMPLE';
+        var=name;
+        fmt=format;
+        keep lib ds var fmt;
+      run;
+
+      %mp_applyformats(work.cols2)
+
+  @param [in] inds The input dataset containing the formats to apply (and where
+    to apply them).  Example structure:
+  |LIB:$8.|DS:$32.|VAR:$32.|FMT:$49.|
+  |---|---|---|---|
+  |`WORK `|`EXAMPLE `|`ACTUAL `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`COUNTRY `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`DIVISION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`MONTH `|`MONNAME3. `|
+  |`WORK `|`EXAMPLE `|`PREDICT `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`PRODTYPE `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`PRODUCT `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`QUARTER `|`8. `|
+  |`WORK `|`EXAMPLE `|`REGION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`YEAR `|`8. `|
+
+  @param [out] errds= (0) Provide a libds reference here to export the
+    error messages to a table.  In this case, they will not be printed to the
+    log.
+
+  <h4> SAS Macros </h4>
+  @li mf_getengine.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_validatecol.sas
+
+
+  <h4> Related Macros </h4>
+  @li mp_getformats.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_applyformats(inds,errds=0
+)/*/STORE SOURCE*/;
+%local outds liblist i engine lib msg ;
+
+/**
+  * Validations
+  */
+proc sort data=&inds;
+  by lib ds var fmt;
+run;
+
+%if &errds=0 %then %let outds=%mf_getuniquename(prefix=mp_applyformats);
+%else %let outds=&errds;
+
+data &outds;
+  set &inds;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  length msg $128;
+  by lib ds var fmt;
+  if libref(lib) ne 0 then do;
+    msg=catx(' ','libref',lib,'is not assigned!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  if exist(cats(lib,'.',ds)) ne 1 then do;
+    msg=catx(' ','libds',lib,'.',ds,'does not exist!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  %mp_validatecol(fmt,FORMAT,is_fmt)
+  if is_fmt=0 then do;
+    msg=catx(' ','format',fmt,'on libds',lib,'.',ds,'.',var,'is not valid!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+
+  if first.ds then do;
+    retain dsid;
+    dsid=open(cats(lib,'.',ds));
+    if dsid=0 then do;
+      msg=catx(' ','libds',lib,'.',ds,' could not be opened!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+      return;
+    end;
+    if varnum(dsid,var)<1 then do;
+      msg=catx(' ','Variable',lib,'.',ds,'.',var,' was not found!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+    end;
+  end;
+  if last.ds then rc=close(dsid);
+run;
+
+proc sql noprint;
+select distinct lib into: liblist separated by ' ' from &inds;
+%put &=liblist;
+%do i=1 %to %sysfunc(countw(&liblist));
+  %let lib=%scan(&liblist,1);
+  %let engine=%mf_getengine(&lib);
+  %if &engine ne V9 and &engine ne BASE %then %do;
+    %let msg=&lib has &engine engine - formats cannot be applied;
+    proc sql;
+    insert into &outds set lib="&lib",ds="_all_",var="_all", msg="&msg" ;
+    %if &errds=0 %then %put %str(ERR)OR: &msg;
+  %end;
+%end;
+
+%if %mf_nobs(&outds)>0 %then %return;
+
+/**
+  * Validations complete - now apply the actual formats!
+  */
+%let fref=%mf_getuniquefileref();
+data _null_;
+  set &inds;
+  by lib ds var fmt;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  file &fref;
+  if first.lib then put 'proc datasets nolist lib=' lib ';';
+  if first.ds then put '  modify ' ds ';';
+  put '    format ' var fmt ';';
+  if last.ds then put '  run;';
+  if last.lib then put 'quit;';
+run;
+
+%inc &fref/source2;
+
+%if &errds=0 %then %do;
+  proc sql;
+  drop table &outds;
+%end;
+
+%mend mp_applyformats;/**
   @file
   @brief Generic assertion
   @details Useful in the context of writing sasjs tests.  The results of the
@@ -5801,6 +6145,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 
 
   <h4> Related Macros </h4>
+  @li mp_applyformats.sas
   @li mp_getformats.test.sas
 
   @version 9.2
@@ -6901,13 +7246,13 @@ filename &tempref clear;
 %put output location=&jref;
 %if &action=OPEN %then %do;
   options nobomfile;
-  data _null_;file &jref encoding='utf-8';
+  data _null_;file &jref encoding='utf-8' ;
     put '{"PROCESSED_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '"';
   run;
 %end;
 %else %if (&action=ARR or &action=OBJ) %then %do;
   options validvarname=upcase;
-  data _null_;file &jref mod encoding='utf-8';
+  data _null_;file &jref mod encoding='utf-8' ;
     put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":";
 
   %if &engine=PROCJSON %then %do;
@@ -6986,7 +7331,7 @@ filename &tempref clear;
       run;
       %let ds=&fmtds;
     %end; /* &fmt=Y */
-    data _null_;file &jref mod encoding='utf-8';
+    data _null_;file &jref mod encoding='utf-8' ;
       put "["; call symputx('cols',0,'l');
     proc sort
       data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
@@ -7031,7 +7376,7 @@ filename &tempref clear;
     /* write to temp loc to avoid _webout truncation
       - https://support.sas.com/kb/49/325.html */
     filename _sjs temp lrecl=131068 encoding='utf-8';
-    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod;
+    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod ;
       set &tempds;
       if _n_>1 then put "," @; put
       %if &action=ARR %then "[" ; %else "{" ;
@@ -7058,14 +7403,14 @@ filename &tempref clear;
       rc = fclose(fileid);
     run;
     filename _sjs clear;
-    data _null_; file &jref mod encoding='utf-8';
+    data _null_; file &jref mod encoding='utf-8' ;
       put "]";
     run;
   %end;
 %end;
 
 %else %if &action=CLOSE %then %do;
-  data _null_;file &jref encoding='utf-8' mod;
+  data _null_;file &jref encoding='utf-8' mod ;
     put "}";
   run;
 %end;
@@ -7586,8 +7931,6 @@ lock &libds clear;
     according to the variable types and formats.
 
     TODO:
-      @li Respect PKs
-      @li Respect NOT NULLs
       @li Consider dates, datetimes, times, integers etc
 
   Usage:
@@ -7603,16 +7946,23 @@ lock &libds clear;
       );
       %mp_makedata(work.example)
 
-  @param [in] libds The empty table in which to create data
+  @param [in] libds The empty table (libref.dataset) in which to create data
-  @param [out] obs= (500) The number of records to create.
+  @param [out] obs= (500) The maximum number of records to create.  The table
+    is sorted with nodup on the primary key, so the actual number of records may
+    be lower than this.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas
   @li mf_getvarlen.sas
+  @li mf_getvarlist.sas
+  @li mf_islibds.sas
   @li mf_nobs.sas
   @li mp_getcols.sas
   @li mp_getpk.sas
 
+  <h4> Related Macros </h4>
+  @li mp_makedata.test.sas
+
   @version 9.2
   @author Allan Bowe
 
@@ -7620,45 +7970,60 @@ lock &libds clear;
 
 %macro mp_makedata(libds
   ,obs=500
+  ,seed=1
 )/*/STORE SOURCE*/;
 
-%local ds1 c1 n1 i col charvars numvars;
+%local ds1 ds2 lib ds pk_fields i col charvars numvars ispk;
 
-%if %mf_nobs(&libds)>0 %then %do;
+%if %mf_islibds(&libds)=0 %then %do;
+  %put &sysmacroname: Invalid libds (&libds) - should be library.dataset format;
+  %return;
+%end;
+%else %if %mf_nobs(&libds)>0 %then %do;
   %put &sysmacroname: &libds has data, it will not be recreated;
   %return;
 %end;
 
-%local ds1 c1 n1;
+/* set up temporary vars */
-%let ds1=%mf_getuniquename(prefix=mp_makedata);
+%let ds1=%mf_getuniquename(prefix=mp_makedatads1);
-%let c1=%mf_getuniquename(prefix=mp_makedatacol);
+%let ds2=%mf_getuniquename(prefix=mp_makedatads2);
-%let n1=%mf_getuniquename(prefix=mp_makedatacol);
+%let lib=%scan(&libds,1,.);
-data &ds1;
+%let ds=%scan(&libds,2,.);
+
+/* grab the primary key vars */
+%mp_getpk(&lib,ds=&ds,outds=&ds1)
+
+proc sql noprint;
+select coalescec(pk_fields,'_all_') into: pk_fields from &ds1;
+
+data &ds2;
   if 0 then set &libds;
   do _n_=1 to &obs;
-    &c1=repeat(uuidgen(),10);
-    &n1=ranuni(1)*5000000;
-    drop &c1 &n1;
     %let charvars=%mf_getvarlist(&libds,typefilter=C);
     %if &charvars ^= %then %do i=1 %to %sysfunc(countw(&charvars));
       %let col=%scan(&charvars,&i);
-      &col=subpad(&c1,1,%mf_getvarlen(&libds,&col));
+      /* create random value based on observation number and colum length */
+      &col=repeat(put(md5(cats(_n_)),$hex32.),%mf_getvarlen(&libds,&col)/32);
     %end;
 
     %let numvars=%mf_getvarlist(&libds,typefilter=N);
     %if &numvars ^= %then %do i=1 %to %sysfunc(countw(&numvars));
       %let col=%scan(&numvars,&i);
-      &col=&n1;
+      &col=_n_;
     %end;
     output;
   end;
+  stop;
+run;
+proc sort data=&ds2 nodupkey;
+  by &pk_fields;
 run;
 
-proc append base=&libds data=&ds1;
+proc append base=&libds data=&ds2;
 run;
 
 proc sql;
-drop table &ds1;
+drop table &ds1, &ds2;
 
 %mend mp_makedata;/**
   @file
@@ -7909,13 +8274,15 @@ options obs=max replace nosyntaxcheck;
   @brief Reset an option to original value
   @details Inspired by the SAS Jedi -
 https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options
-    Called as follows:
 
-    options obs=30;
+  Called as follows:
-    %mp_resetoption(OBS)
+
+      options obs=30 ps=max;
+      %mp_resetoption(OBS)
+      %mp_resetoption(PS)
 
 
-  @param option the option to reset
+  @param [in] option the option to reset
 
   @version 9.2
   @author Allan Bowe
@@ -8284,9 +8651,7 @@ select distinct memname into: table_list separated by ' '
 
   This macro will only work for BASE (V9) engine libraries.  It works by
   creating a copy of the dataset (without data, WITH constraints) in the same
-  library, appending a sorted view into it, and finally - renaming it.  By
+  library, appending a sorted view into it, and finally - renaming it.
-  default, COMPRESS=CHAR and REUSE=YES will be applied, this behaviour can
-  be adjusted using the `dsoptions=` parameter.
 
   Example usage:
 
@@ -8314,7 +8679,6 @@ select distinct memname into: table_list separated by ' '
 
   @version 9.2
   @author Allan Bowe
-  @source https://github.com/sasjs/core
 
 **/
 
@@ -9528,10 +9892,13 @@ alter table &libds modify &var char(&len);
       ;;;;
       run;
 
+  Tip - when contributing, use https://regex101.com to test the regex validity!
+
   @param [in] incol The column to be validated
   @param [in] rule The rule to apply.  Current rules:
     @li ISNUM - checks if the variable is numeric
     @li LIBDS - matches LIBREF.DATASET format
+    @li FORMAT - checks if the provided format is syntactically valid
   @param [out] outcol The variable to create, with the results of the match
 
   <h4> SAS Macros </h4>
@@ -9570,6 +9937,20 @@ alter table &libds modify &var char(&len);
   if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
   else &outcol=0;
 %end;
+%else %if &rule=FORMAT %then %do;
+  /* match valid format - regex could probably be improved */
+  if _n_=1 then do;
+    retain &tempcol;
+    &tempcol=prxparse('/^[_a-z\$]\w{0,31}\.[0-9]*$/i');
+    if missing(&tempcol) then do;
+      putlog "%str(ERR)OR: Invalid expression for FORMAT";
+      stop;
+    end;
+    drop &tempcol;
+  end;
+  if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
+  else &outcol=0;
+%end;
 
 %mend mp_validatecol;
 /**
@@ -11711,13 +12092,13 @@ data _null_;
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
-  put '  data _null_;file &jref encoding=''utf-8''; ';
+  put '  data _null_;file &jref encoding=''utf-8'' ; ';
   put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
   put '  options validvarname=upcase; ';
-  put '  data _null_;file &jref mod encoding=''utf-8''; ';
+  put '  data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
@@ -11796,7 +12177,7 @@ data _null_;
   put '      run; ';
   put '      %let ds=&fmtds; ';
   put '    %end; /* &fmt=Y */ ';
-  put '    data _null_;file &jref mod encoding=''utf-8''; ';
+  put '    data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
   put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
@@ -11841,7 +12222,7 @@ data _null_;
   put '    /* write to temp loc to avoid _webout truncation ';
   put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
-  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod; ';
+  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
   put '      %if &action=ARR %then "[" ; %else "{" ; ';
@@ -11868,14 +12249,14 @@ data _null_;
   put '      rc = fclose(fileid); ';
   put '    run; ';
   put '    filename _sjs clear; ';
-  put '    data _null_; file &jref mod encoding=''utf-8''; ';
+  put '    data _null_; file &jref mod encoding=''utf-8'' ; ';
   put '      put "]"; ';
   put '    run; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8'' mod; ';
+  put '  data _null_;file &jref encoding=''utf-8'' mod ; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
@@ -15723,7 +16104,7 @@ run;
   OPTIONS NOBOMFILE;
 
   /* setup json */
-  data _null_;file &fref encoding='utf-8';
+  data _null_;file &fref encoding='utf-8' termstr=lf;
   %if %str(&_debug) ge 131 %then %do;
     put '>>weboutBEGIN<<';
   %end;
@@ -15752,14 +16133,14 @@ run;
       i+1;
       call symputx('wt'!!left(i),name,'l');
       call symputx('wtcnt',i,'l');
-    data _null_; file &fref mod encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8' termstr=lf;
       put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
       proc contents noprint data=&wt
         out=_data_ (keep=name type length format:);
       run;%let tempds=%scan(&syslast,2,.);
-      data _null_; file &fref mod encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8' termstr=lf;
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
         nvars=attrn(dsid,'NVARS');
@@ -15770,15 +16151,15 @@ run;
         put ',"nvars":' nvars;
       %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
       %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
-      data _null_; file &fref mod encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8' termstr=lf;
         put "}";
     %end;
-    data _null_; file &fref mod encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8' termstr=lf termstr=lf;
       put "}";
     run;
   %end;
   /* close off json */
-  data _null_;file &fref mod encoding='utf-8';
+  data _null_;file &fref mod encoding='utf-8' termstr=lf;
     _PROGRAM=quote(trim(resolve(symget('_PROGRAM'))));
     put ",""SYSUSERID"" : ""&sysuserid"" ";
     put ",""MF_GETUSER"" : ""%mf_getuser()"" ";
@@ -15791,7 +16172,9 @@ run;
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSPROCESSID"" : ""&SYSPROCESSID"" ";
     put ",""SYSPROCESSMODE"" : ""&SYSPROCESSMODE"" ";
-    put ",""SYSPROCESSNAME"" : ""&SYSPROCESSNAME"" ";
+    length SYSPROCESSNAME $512;
+    SYSPROCESSNAME=quote(urlencode(cats(SYSPROCESSNAME)));
+    put ",""SYSPROCESSNAME"" : " SYSPROCESSNAME;
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
@@ -15800,7 +16183,8 @@ run;
     put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
     put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
-    autoexec=quote(trim(getoption('autoexec')));
+    length autoexec $512;
+    autoexec=quote(urlencode(trim(getoption('autoexec'))));
     put ',"AUTOEXEC" : ' autoexec;
     memsize="%sysfunc(INPUTN(%sysfunc(getoption(memsize)), best.),sizekmg.)";
     memsize=quote(cats(memsize));
@@ -16792,13 +17176,13 @@ data _null_;
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
-  put '  data _null_;file &jref encoding=''utf-8''; ';
+  put '  data _null_;file &jref encoding=''utf-8'' ; ';
   put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
   put '  options validvarname=upcase; ';
-  put '  data _null_;file &jref mod encoding=''utf-8''; ';
+  put '  data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
@@ -16877,7 +17261,7 @@ data _null_;
   put '      run; ';
   put '      %let ds=&fmtds; ';
   put '    %end; /* &fmt=Y */ ';
-  put '    data _null_;file &jref mod encoding=''utf-8''; ';
+  put '    data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
   put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
@@ -16922,7 +17306,7 @@ data _null_;
   put '    /* write to temp loc to avoid _webout truncation ';
   put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
-  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod; ';
+  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
   put '      %if &action=ARR %then "[" ; %else "{" ; ';
@@ -16949,14 +17333,14 @@ data _null_;
   put '      rc = fclose(fileid); ';
   put '    run; ';
   put '    filename _sjs clear; ';
-  put '    data _null_; file &jref mod encoding=''utf-8''; ';
+  put '    data _null_; file &jref mod encoding=''utf-8'' ; ';
   put '      put "]"; ';
   put '    run; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8'' mod; ';
+  put '  data _null_;file &jref encoding=''utf-8'' mod ; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';

base/mf_existvar.sas

@@ -7,8 +7,12 @@
 
         %put %mf_existvar(work.someds, somevar)
 
-  @param libds (positional) - 2 part dataset or view reference
+  @param [in] libds 2 part dataset or view reference
-  @param var (positional) - variable name
+  @param [in] var variable name
+
+  <h4> Related Macros </h4>
+  @li mf_existvar.test.sas
+
   @version 9.2
   @author Allan Bowe
 **/

base/mf_getengine.sas

@@ -12,9 +12,10 @@
   contributors of Chris Hemedingers blog [post](
   http://blogs.sas.com/content/sasdummy/2013/06/04/find-a-sas-library-engine/)
 
-  @param libref Library reference (also accepts a 2 level libds ref).
+  @param [in] libref Library reference (also accepts a 2 level libds ref).
 
-  @return output returns the library engine for the FIRST library encountered.
+  @return output returns the library engine (uppercase) for the FIRST library
+    encountered.
 
   @warning will only return the FIRST library engine - for concatenated
     libraries, with different engines, inconsistent results may be encountered.
@@ -46,7 +47,7 @@
     %let rc= %sysfunc(close(&dsid));
   %end;
 
-  &engine
+  %upcase(&engine)
 
 %mend mf_getengine;
 

base/mf_islibds.sas

@@ -0,0 +1,40 @@
+/**
+  @file
+  @brief Checks whether a string follows correct library.dataset format
+  @details Many macros in the core library accept a library.dataset parameter
+  referred to as 'libds'.  This macro validates the structure of that parameter,
+  eg:
+
+    @li 8 character libref?
+    @li 32 character dataset?
+    @li contains a period?
+
+  It does NOT check whether the dataset exists, or if the library is assigned.
+
+  Usage:
+
+      %put %mf_islibds(work.something)=1;
+      %put %mf_islibds(nolib)=0;
+      %put %mf_islibds(badlibref.ds)=0;
+      %put %mf_islibds(w.t.f)=0;
+
+  @param [in] libds The string to be checked
+
+  @return output Returns 1 if libds is valid, 0 if it is not
+
+  <h4> Related Macros </h4>
+  @li mf_islibds.test.sas
+  @li mp_validatecol.sas
+
+  @version 9.2
+**/
+
+%macro mf_islibds(libds
+)/*/STORE SOURCE*/;
+
+%local regex;
+%let regex=%sysfunc(prxparse(%str(/^[_a-z]\w{0,7}\.[_a-z]\w{0,31}$/i)));
+
+%sysfunc(prxmatch(&regex,&libds))
+
+%mend mf_islibds;

base/mf_wordsinstr1andstr2.sas

@@ -0,0 +1,54 @@
+/**
+  @file
+  @brief Returns words that are in both string 1 and string 2
+  @details  Compares two space separated strings and returns the words that are
+  in both.
+  Usage:
+
+      %put %mf_wordsInStr1andStr2(
+        Str1=blah sss blaaah brah bram boo
+        ,Str2=   blah blaaah brah ssss
+      );
+
+  returns:
+  > blah blaaah brah
+
+  @param str1= string containing words to extract
+  @param str2= used to compare with the extract string
+
+  @warning CASE SENSITIVE!
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mf_wordsInStr1andStr2(
+  Str1= /* string containing words to extract */
+  ,Str2= /* used to compare with the extract string */
+)/*/STORE SOURCE*/;
+
+%local count_base count_extr i i2 extr_word base_word match outvar;
+%if %length(&str1)=0 or %length(&str2)=0 %then %do;
+  %put %str(WARN)ING: empty string provided!;
+  %put base string (str1)= &str1;
+  %put compare string (str2) = &str2;
+  %return;
+%end;
+%let count_base=%sysfunc(countw(&Str2));
+%let count_extr=%sysfunc(countw(&Str1));
+
+%do i=1 %to &count_extr;
+  %let extr_word=%scan(&Str1,&i,%str( ));
+  %let match=0;
+  %do i2=1 %to &count_base;
+    %let base_word=%scan(&Str2,&i2,%str( ));
+    %if &extr_word=&base_word %then %let match=1;
+  %end;
+  %if &match=1 %then %let outvar=&outvar &extr_word;
+%end;
+
+  &outvar
+
+%mend mf_wordsInStr1andStr2;
+

base/mf_writefile.sas

@@ -0,0 +1,67 @@
+/**
+  @file
+  @brief Creates a text file using pure macro
+  @details Creates a text file of up to 10 lines.  If further lines are
+    desired, feel free to [create an issue](
+    https://github.com/sasjs/core/issues/new), or make a pull request!
+
+    The use of PARMBUFF was considered for this macro, but it would have made
+    things problematic for writing lines containing commas.
+
+    Usage:
+
+        %mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content)
+        data _null_;
+          infile "&sasjswork/myfile.txt";
+          input;
+          list;
+        run;
+
+  @param [in] fpath Full path to file to be created or appended to
+  @param [in] mode= (O) Available options are A or O as follows:
+    @li A APPEND mode, writes new records after the current end of the file.
+    @li O OUTPUT mode, writes new records from the beginning of the file.
+  @param [in] l1= First line
+  @param [in] l2= Second line (etc through to l10)
+
+  <h4> Related Macros </h4>
+  @li mf_writefile.test.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+/** @cond */
+
+%macro mf_writefile(fpath,mode=O,l1=,l2=,l3=,l4=,l5=,l6=,l7=,l8=,l9=,l10=
+)/*/STORE SOURCE*/;
+%local fref rc fid i total_lines;
+
+/* find number of lines by reference to first non-blank param */
+%do i=10 %to 1 %by -1;
+  %if %str(&&l&i) ne %str() %then %goto continue;
+%end;
+%continue:
+%let total_lines=&i;
+
+%if %sysfunc(filename(fref,&fpath)) ne 0 %then %do;
+  %put &=fref &=fpath;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,&mode));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%do i=1 %to &total_lines;
+  %let rc=%sysfunc(fput(&fid, &&l&i));
+  %let rc=%sysfunc(fwrite(&fid));
+%end;
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mf_writefile;
+/** @endcond */

base/mp_applyformats.sas

@@ -0,0 +1,181 @@
+/**
+  @file
+  @brief Apply a set of formats to a table
+  @details Applies a set of formats to one or more SAS datasets.  Can be used
+    to migrate formats from one table to another.  The input table must contain
+    the following columns:
+
+    @li lib - the libref of the table to be updated
+    @li ds - the dataset to be updated
+    @li var - the variable to be updated
+    @li fmt - the format to apply.  Missing or default ($CHAR, 8.) formats are
+      ignored.
+
+  The macro will abort in the following scenarios:
+
+    @li Libref not assigned
+    @li Dataset does not exist
+    @li Input table contains null or invalid values
+
+  Example usage:
+
+      data work.example;
+        set sashelp.prdsale;
+        format _all_ clear;
+      run;
+
+      %mp_getcols(sashelp.prdsale,outds=work.cols)
+
+      data work.cols2;
+        set work.cols;
+        lib='WORK';
+        ds='EXAMPLE';
+        var=name;
+        fmt=format;
+        keep lib ds var fmt;
+      run;
+
+      %mp_applyformats(work.cols2)
+
+  @param [in] inds The input dataset containing the formats to apply (and where
+    to apply them).  Example structure:
+  |LIB:$8.|DS:$32.|VAR:$32.|FMT:$49.|
+  |---|---|---|---|
+  |`WORK `|`EXAMPLE `|`ACTUAL `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`COUNTRY `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`DIVISION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`MONTH `|`MONNAME3. `|
+  |`WORK `|`EXAMPLE `|`PREDICT `|`DOLLAR12.2 `|
+  |`WORK `|`EXAMPLE `|`PRODTYPE `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`PRODUCT `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`QUARTER `|`8. `|
+  |`WORK `|`EXAMPLE `|`REGION `|`$CHAR10. `|
+  |`WORK `|`EXAMPLE `|`YEAR `|`8. `|
+
+  @param [out] errds= (0) Provide a libds reference here to export the
+    error messages to a table.  In this case, they will not be printed to the
+    log.
+
+  <h4> SAS Macros </h4>
+  @li mf_getengine.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_validatecol.sas
+
+
+  <h4> Related Macros </h4>
+  @li mp_getformats.sas
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_applyformats(inds,errds=0
+)/*/STORE SOURCE*/;
+%local outds liblist i engine lib msg ;
+
+/**
+  * Validations
+  */
+proc sort data=&inds;
+  by lib ds var fmt;
+run;
+
+%if &errds=0 %then %let outds=%mf_getuniquename(prefix=mp_applyformats);
+%else %let outds=&errds;
+
+data &outds;
+  set &inds;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  length msg $128;
+  by lib ds var fmt;
+  if libref(lib) ne 0 then do;
+    msg=catx(' ','libref',lib,'is not assigned!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  if exist(cats(lib,'.',ds)) ne 1 then do;
+    msg=catx(' ','libds',lib,'.',ds,'does not exist!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+  %mp_validatecol(fmt,FORMAT,is_fmt)
+  if is_fmt=0 then do;
+    msg=catx(' ','format',fmt,'on libds',lib,'.',ds,'.',var,'is not valid!');
+    %if &errds=0 %then %do;
+      putlog "%str(ERR)OR: " msg;
+    %end;
+    output;
+    return;
+  end;
+
+  if first.ds then do;
+    retain dsid;
+    dsid=open(cats(lib,'.',ds));
+    if dsid=0 then do;
+      msg=catx(' ','libds',lib,'.',ds,' could not be opened!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+      return;
+    end;
+    if varnum(dsid,var)<1 then do;
+      msg=catx(' ','Variable',lib,'.',ds,'.',var,' was not found!');
+      %if &errds=0 %then %do;
+        putlog "%str(ERR)OR: " msg;
+      %end;
+      output;
+    end;
+  end;
+  if last.ds then rc=close(dsid);
+run;
+
+proc sql noprint;
+select distinct lib into: liblist separated by ' ' from &inds;
+%put &=liblist;
+%do i=1 %to %sysfunc(countw(&liblist));
+  %let lib=%scan(&liblist,1);
+  %let engine=%mf_getengine(&lib);
+  %if &engine ne V9 and &engine ne BASE %then %do;
+    %let msg=&lib has &engine engine - formats cannot be applied;
+    proc sql;
+    insert into &outds set lib="&lib",ds="_all_",var="_all", msg="&msg" ;
+    %if &errds=0 %then %put %str(ERR)OR: &msg;
+  %end;
+%end;
+
+%if %mf_nobs(&outds)>0 %then %return;
+
+/**
+  * Validations complete - now apply the actual formats!
+  */
+%let fref=%mf_getuniquefileref();
+data _null_;
+  set &inds;
+  by lib ds var fmt;
+  where fmt not in ('','.', '$', '$CHAR.','8.');
+  file &fref;
+  if first.lib then put 'proc datasets nolist lib=' lib ';';
+  if first.ds then put '  modify ' ds ';';
+  put '    format ' var fmt ';';
+  if last.ds then put '  run;';
+  if last.lib then put 'quit;';
+run;
+
+%inc &fref/source2;
+
+%if &errds=0 %then %do;
+  proc sql;
+  drop table &outds;
+%end;
+
+%mend mp_applyformats;

base/mp_getformats.sas

@@ -48,6 +48,7 @@ https://support.sas.com/documentation/cdl/en/proc/61895/HTML/default/viewer.htm#
 
 
   <h4> Related Macros </h4>
+  @li mp_applyformats.sas
   @li mp_getformats.test.sas
 
   @version 9.2

base/mp_jsonout.sas

@@ -62,13 +62,13 @@
 %put output location=&jref;
 %if &action=OPEN %then %do;
   options nobomfile;
-  data _null_;file &jref encoding='utf-8' termstr=lf;
+  data _null_;file &jref encoding='utf-8' ;
     put '{"PROCESSED_DTTM" : "' "%sysfunc(datetime(),E8601DT26.6)" '"';
   run;
 %end;
 %else %if (&action=ARR or &action=OBJ) %then %do;
   options validvarname=upcase;
-  data _null_;file &jref mod encoding='utf-8' termstr=lf;
+  data _null_;file &jref mod encoding='utf-8' ;
     put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":";
 
   %if &engine=PROCJSON %then %do;
@@ -147,7 +147,7 @@
       run;
       %let ds=&fmtds;
     %end; /* &fmt=Y */
-    data _null_;file &jref mod encoding='utf-8' termstr=lf;
+    data _null_;file &jref mod encoding='utf-8' ;
       put "["; call symputx('cols',0,'l');
     proc sort
       data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))
@@ -192,7 +192,7 @@
     /* write to temp loc to avoid _webout truncation
       - https://support.sas.com/kb/49/325.html */
     filename _sjs temp lrecl=131068 encoding='utf-8';
-    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod termstr=lf;
+    data _null_; file _sjs lrecl=131068 encoding='utf-8' mod ;
       set &tempds;
       if _n_>1 then put "," @; put
       %if &action=ARR %then "[" ; %else "{" ;
@@ -219,14 +219,14 @@
       rc = fclose(fileid);
     run;
     filename _sjs clear;
-    data _null_; file &jref mod encoding='utf-8' termstr=lf;
+    data _null_; file &jref mod encoding='utf-8' ;
       put "]";
     run;
   %end;
 %end;
 
 %else %if &action=CLOSE %then %do;
-  data _null_;file &jref encoding='utf-8' mod termstr=lf;
+  data _null_;file &jref encoding='utf-8' mod ;
     put "}";
   run;
 %end;

base/mp_makedata.sas

@@ -10,8 +10,6 @@
     according to the variable types and formats.
 
     TODO:
-      @li Respect PKs
-      @li Respect NOT NULLs
       @li Consider dates, datetimes, times, integers etc
 
   Usage:
@@ -27,16 +25,23 @@
       );
       %mp_makedata(work.example)
 
-  @param [in] libds The empty table in which to create data
+  @param [in] libds The empty table (libref.dataset) in which to create data
-  @param [out] obs= (500) The number of records to create.
+  @param [out] obs= (500) The maximum number of records to create.  The table
+    is sorted with nodup on the primary key, so the actual number of records may
+    be lower than this.
 
   <h4> SAS Macros </h4>
   @li mf_getuniquename.sas
   @li mf_getvarlen.sas
+  @li mf_getvarlist.sas
+  @li mf_islibds.sas
   @li mf_nobs.sas
   @li mp_getcols.sas
   @li mp_getpk.sas
 
+  <h4> Related Macros </h4>
+  @li mp_makedata.test.sas
+
   @version 9.2
   @author Allan Bowe
 
@@ -44,44 +49,59 @@
 
 %macro mp_makedata(libds
   ,obs=500
+  ,seed=1
 )/*/STORE SOURCE*/;
 
-%local ds1 c1 n1 i col charvars numvars;
+%local ds1 ds2 lib ds pk_fields i col charvars numvars ispk;
 
-%if %mf_nobs(&libds)>0 %then %do;
+%if %mf_islibds(&libds)=0 %then %do;
+  %put &sysmacroname: Invalid libds (&libds) - should be library.dataset format;
+  %return;
+%end;
+%else %if %mf_nobs(&libds)>0 %then %do;
   %put &sysmacroname: &libds has data, it will not be recreated;
   %return;
 %end;
 
-%local ds1 c1 n1;
+/* set up temporary vars */
-%let ds1=%mf_getuniquename(prefix=mp_makedata);
+%let ds1=%mf_getuniquename(prefix=mp_makedatads1);
-%let c1=%mf_getuniquename(prefix=mp_makedatacol);
+%let ds2=%mf_getuniquename(prefix=mp_makedatads2);
-%let n1=%mf_getuniquename(prefix=mp_makedatacol);
+%let lib=%scan(&libds,1,.);
-data &ds1;
+%let ds=%scan(&libds,2,.);
+
+/* grab the primary key vars */
+%mp_getpk(&lib,ds=&ds,outds=&ds1)
+
+proc sql noprint;
+select coalescec(pk_fields,'_all_') into: pk_fields from &ds1;
+
+data &ds2;
   if 0 then set &libds;
   do _n_=1 to &obs;
-    &c1=repeat(uuidgen(),10);
-    &n1=ranuni(1)*5000000;
-    drop &c1 &n1;
     %let charvars=%mf_getvarlist(&libds,typefilter=C);
     %if &charvars ^= %then %do i=1 %to %sysfunc(countw(&charvars));
       %let col=%scan(&charvars,&i);
-      &col=subpad(&c1,1,%mf_getvarlen(&libds,&col));
+      /* create random value based on observation number and colum length */
+      &col=repeat(put(md5(cats(_n_)),$hex32.),%mf_getvarlen(&libds,&col)/32);
     %end;
 
     %let numvars=%mf_getvarlist(&libds,typefilter=N);
     %if &numvars ^= %then %do i=1 %to %sysfunc(countw(&numvars));
       %let col=%scan(&numvars,&i);
-      &col=&n1;
+      &col=_n_;
     %end;
     output;
   end;
+  stop;
+run;
+proc sort data=&ds2 nodupkey;
+  by &pk_fields;
 run;
 
-proc append base=&libds data=&ds1;
+proc append base=&libds data=&ds2;
 run;
 
 proc sql;
-drop table &ds1;
+drop table &ds1, &ds2;
 
 %mend mp_makedata;

base/mp_resetoption.sas

@@ -3,13 +3,15 @@
   @brief Reset an option to original value
   @details Inspired by the SAS Jedi -
 https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options
-    Called as follows:
 
-    options obs=30;
+  Called as follows:
-    %mp_resetoption(OBS)
+
+      options obs=30 ps=max;
+      %mp_resetoption(OBS)
+      %mp_resetoption(PS)
 
 
-  @param option the option to reset
+  @param [in] option the option to reset
 
   @version 9.2
   @author Allan Bowe

base/mp_sortinplace.sas

@@ -11,9 +11,7 @@
 
   This macro will only work for BASE (V9) engine libraries.  It works by
   creating a copy of the dataset (without data, WITH constraints) in the same
-  library, appending a sorted view into it, and finally - renaming it.  By
+  library, appending a sorted view into it, and finally - renaming it.
-  default, COMPRESS=CHAR and REUSE=YES will be applied, this behaviour can
-  be adjusted using the `dsoptions=` parameter.
 
   Example usage:
 
@@ -41,7 +39,6 @@
 
   @version 9.2
   @author Allan Bowe
-  @source https://github.com/sasjs/core
 
 **/
 

base/mp_validatecol.sas

@@ -20,10 +20,13 @@
       ;;;;
       run;
 
+  Tip - when contributing, use https://regex101.com to test the regex validity!
+
   @param [in] incol The column to be validated
   @param [in] rule The rule to apply.  Current rules:
     @li ISNUM - checks if the variable is numeric
     @li LIBDS - matches LIBREF.DATASET format
+    @li FORMAT - checks if the provided format is syntactically valid
   @param [out] outcol The variable to create, with the results of the match
 
   <h4> SAS Macros </h4>
@@ -62,5 +65,19 @@
   if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
   else &outcol=0;
 %end;
+%else %if &rule=FORMAT %then %do;
+  /* match valid format - regex could probably be improved */
+  if _n_=1 then do;
+    retain &tempcol;
+    &tempcol=prxparse('/^[_a-z\$]\w{0,31}\.[0-9]*$/i');
+    if missing(&tempcol) then do;
+      putlog "%str(ERR)OR: Invalid expression for FORMAT";
+      stop;
+    end;
+    drop &tempcol;
+  end;
+  if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
+  else &outcol=0;
+%end;
 
 %mend mp_validatecol;

meta/mm_createwebservice.sas

@@ -94,13 +94,13 @@ data _null_;
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
-  put '  data _null_;file &jref encoding=''utf-8''; ';
+  put '  data _null_;file &jref encoding=''utf-8'' ; ';
   put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
   put '  options validvarname=upcase; ';
-  put '  data _null_;file &jref mod encoding=''utf-8''; ';
+  put '  data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
@@ -179,7 +179,7 @@ data _null_;
   put '      run; ';
   put '      %let ds=&fmtds; ';
   put '    %end; /* &fmt=Y */ ';
-  put '    data _null_;file &jref mod encoding=''utf-8''; ';
+  put '    data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
   put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
@@ -224,7 +224,7 @@ data _null_;
   put '    /* write to temp loc to avoid _webout truncation ';
   put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
-  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod; ';
+  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
   put '      %if &action=ARR %then "[" ; %else "{" ; ';
@@ -251,14 +251,14 @@ data _null_;
   put '      rc = fclose(fileid); ';
   put '    run; ';
   put '    filename _sjs clear; ';
-  put '    data _null_; file &jref mod encoding=''utf-8''; ';
+  put '    data _null_; file &jref mod encoding=''utf-8'' ; ';
   put '      put "]"; ';
   put '    run; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8'' mod; ';
+  put '  data _null_;file &jref encoding=''utf-8'' mod ; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';

tests/crossplatform/mf_existvar.test.sas

@@ -0,0 +1,22 @@
+/**
+  @file
+  @brief Testing mf_existvar macro
+
+  <h4> SAS Macros </h4>
+  @li mf_existvar.sas
+  @li mp_assert.sas
+
+**/
+
+
+%mp_assert(
+  iftrue=(%mf_existvar(sashelp.class,age)=1),
+  desc=Checking existing var exists,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(%mf_existvar(sashelp.class,isjustanumber)=0),
+  desc=Checking non existing var does not exist,
+  outds=work.test_results
+)

tests/crossplatform/mf_islibds.test.sas

@@ -0,0 +1,46 @@
+/**
+  @file
+  @brief Testing mf_islibds macro
+
+      %put %mf_islibds(work.something)=1;
+      %put %mf_islibds(nolib)=0;
+      %put %mf_islibds(badlibref.ds)=0;
+      %put %mf_islibds(w.t.f)=0;
+
+  <h4> SAS Macros </h4>
+  @li mf_islibds.sas
+  @li mp_assert.sas
+
+**/
+
+%mp_assert(
+  iftrue=(
+    %mf_islibds(work.something)=1
+  ),
+  desc=%str(Checking mf_islibds(work.something)=1),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    %mf_islibds(nolib)=0
+  ),
+  desc=%str(Checking mf_islibds(nolib)=0),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    %mf_islibds(badlibref.ds)=0
+  ),
+  desc=%str(Checking mf_islibds(badlibref.ds)=0),
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(
+    %mf_islibds(w.t.f)=0
+  ),
+  desc=%str(Checking mf_islibds(w.t.f)=0),
+  outds=work.test_results
+)

tests/crossplatform/mf_wordsinstr1andstr2.test.sas

@@ -0,0 +1,20 @@
+/**
+  @file
+  @brief Testing mf_wordsinstr1andstr2 macro
+
+  <h4> SAS Macros </h4>
+  @li mf_wordsinstr1andstr2.sas
+  @li mp_assert.sas
+
+**/
+
+%let x=%mf_wordsinstr1andstr2(str1=xx DOLLAR x $CHAR xxx W MONNAME
+  ,str2=DOLLAR $CHAR W MONNAME xxxxxx
+);
+%mp_assert(
+  iftrue=(
+    "&x"="DOLLAR $CHAR W MONNAME"
+  ),
+  desc=Checking basic string,
+  outds=work.test_results
+)

tests/crossplatform/mf_wordsinstr1butnotstr2.test.sas

@@ -0,0 +1,20 @@
+/**
+  @file
+  @brief Testing mf_wordsinstr1butnotstr2 macro
+
+  <h4> SAS Macros </h4>
+  @li mf_wordsinstr1butnotstr2.sas
+  @li mp_assert.sas
+
+**/
+
+%let x=%mf_wordsinstr1butnotstr2(str1=xx DOLLAR x $CHAR xxx W MONNAME
+  ,str2=ff xx x xxx xxxxxx
+);
+%mp_assert(
+  iftrue=(
+    "&x"="DOLLAR $CHAR W MONNAME"
+  ),
+  desc=Checking basic string,
+  outds=work.test_results
+)

tests/crossplatform/mf_writefile.test.sas

@@ -0,0 +1,74 @@
+/**
+  @file
+  @brief Testing mf_writefile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_writefile.sas
+  @li mp_assert.sas
+
+**/
+
+%mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=more content)
+data _null_;
+  infile "&sasjswork/myfile.txt";
+  input;
+  if _n_=2 then call symputx('test1',_infile_);
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check code ran without errors,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&test1=more content),
+  desc=Checking line was created,
+  outds=work.test_results
+)
+
+%mf_writefile(&sasjswork/myfile.txt,l1=some content,l2=different content)
+data _null_;
+  infile "&sasjswork/myfile.txt";
+  input;
+  if _n_=2 then call symputx('test2',_infile_);
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check code ran without errors for test2,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&test2=different content),
+  desc=Checking second line was overwritten,
+  outds=work.test_results
+)
+
+%global test3 test4;
+%mf_writefile(&sasjswork/myfile.txt
+  ,mode=a
+  ,l1=%str(aah, content)
+  ,l2=append content
+)
+data _null_;
+  infile "&sasjswork/myfile.txt";
+  input;
+  if _n_=2 then call symputx('test3',_infile_);
+  if _n_=4 then call symputx('test4',_infile_);
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check code ran without errors for test2,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&test3=different content),
+  desc=Checking second line was not overwritten,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(&test4=append content),
+  desc=Checking fourth line was appended,
+  outds=work.test_results
+)

tests/crossplatform/mp_applyformats.test.sas

@@ -0,0 +1,45 @@
+/**
+  @file
+  @brief Testing mp_applyformats.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getvarformat.sas
+  @li mp_applyformats.sas
+  @li mp_assert.sas
+  @li mp_getcols.sas
+
+**/
+
+/**
+  * Test 1 Base case
+  */
+
+data work.example;
+  set sashelp.prdsale;
+  format _all_;
+run;
+%let origfmt=%mf_getvarformat(work.example,month);
+
+%mp_getcols(sashelp.prdsale,outds=work.cols)
+
+data work.cols2;
+  set work.cols;
+  lib='WORK';
+  ds='EXAMPLE';
+  var=name;
+  fmt=format;
+  keep lib ds var fmt;
+run;
+
+%mp_applyformats(work.cols2)
+
+%mp_assert(
+  iftrue=("&orig_fmt"=""),
+  desc=Check that formats were cleared,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("%mf_getvarformat(work.example,month)"="MONNAME3."),
+  desc=Check that formats were applied,
+  outds=work.test_results
+)

tests/crossplatform/mp_makedata.test.sas

@@ -0,0 +1,59 @@
+/**
+  @file
+  @brief Testing mp_makedata.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mf_nobs.sas
+  @li mp_makedata.sas
+  @li mp_assert.sas
+
+**/
+
+/**
+  * Test 1 - Regular makedata call
+  */
+
+proc sql;
+create table work.example(
+  TX_FROM float format=datetime19.,
+  DD_TYPE char(16),
+  DD_SOURCE char(2048),
+  DD_SHORTDESC char(256),
+  constraint pk primary key(tx_from, dd_type,dd_source),
+  constraint nnn not null(DD_SHORTDESC)
+);
+%mp_makedata(work.example,obs=500)
+
+%mp_assert(
+  iftrue=("%mf_nobs(work.example)"="500"),
+  desc=Check that 500 rows were created,
+  outds=work.test_results
+)
+
+data _null_;
+  set work.example;
+  call symputx('lenvar',length(dd_source));
+  stop;
+run;
+%mp_assert(
+  iftrue=("&lenvar"="2048"),
+  desc=Check that entire length of variable is populated,
+  outds=work.test_results
+)
+
+
+proc sql;
+create table work.example2(
+  TX_FROM float format=datetime19.,
+  DD_TYPE char(16),
+  DD_SOURCE char(2048),
+  DD_SHORTDESC char(256),
+  some_num num
+);
+%mp_makedata(work.example2)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Ensure tables without keys still generate,
+  outds=work.test_results
+)

tests/crossplatform/mp_validatecol.test.sas

@@ -59,4 +59,37 @@ run;
   desc=Test2 - ISNUM,
   test=EQUALS 4,
   outds=work.test_results
+)
+
+/**
+  * Test 3 - FORMAT
+  */
+data test3;
+  infile datalines4 dsd;
+  input;
+  infile=_infile_;
+  %mp_validatecol(infile,FORMAT,is_format)
+  if is_format=1;
+datalines4;
+$.
+$format.
+$format12.2
+somenum.
+somenum12.4
+above are good
+the rest are bad
+%abort
+1&somethingverybad.
+&
++-1
+.
+a.A
+$format12.1b
+$format12.1b1
+;;;;
+run;
+%mp_assertdsobs(work.test3,
+  desc=Test3 - ISFORMAT,
+  test=EQUALS 5,
+  outds=work.test_results
 )

viya/mv_createwebservice.sas

@@ -242,13 +242,13 @@ data _null_;
   put '%put output location=&jref; ';
   put '%if &action=OPEN %then %do; ';
   put '  options nobomfile; ';
-  put '  data _null_;file &jref encoding=''utf-8''; ';
+  put '  data _null_;file &jref encoding=''utf-8'' ; ';
   put '    put ''{"PROCESSED_DTTM" : "'' "%sysfunc(datetime(),E8601DT26.6)" ''"''; ';
   put '  run; ';
   put '%end; ';
   put '%else %if (&action=ARR or &action=OBJ) %then %do; ';
   put '  options validvarname=upcase; ';
-  put '  data _null_;file &jref mod encoding=''utf-8''; ';
+  put '  data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '    put ", ""%lowcase(%sysfunc(coalescec(&dslabel,&ds)))"":"; ';
   put ' ';
   put '  %if &engine=PROCJSON %then %do; ';
@@ -327,7 +327,7 @@ data _null_;
   put '      run; ';
   put '      %let ds=&fmtds; ';
   put '    %end; /* &fmt=Y */ ';
-  put '    data _null_;file &jref mod encoding=''utf-8''; ';
+  put '    data _null_;file &jref mod encoding=''utf-8'' ; ';
   put '      put "["; call symputx(''cols'',0,''l''); ';
   put '    proc sort ';
   put '      data=sashelp.vcolumn(where=(libname=''WORK'' & memname="%upcase(&ds)")) ';
@@ -372,7 +372,7 @@ data _null_;
   put '    /* write to temp loc to avoid _webout truncation ';
   put '      - https://support.sas.com/kb/49/325.html */ ';
   put '    filename _sjs temp lrecl=131068 encoding=''utf-8''; ';
-  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod; ';
+  put '    data _null_; file _sjs lrecl=131068 encoding=''utf-8'' mod ; ';
   put '      set &tempds; ';
   put '      if _n_>1 then put "," @; put ';
   put '      %if &action=ARR %then "[" ; %else "{" ; ';
@@ -399,14 +399,14 @@ data _null_;
   put '      rc = fclose(fileid); ';
   put '    run; ';
   put '    filename _sjs clear; ';
-  put '    data _null_; file &jref mod encoding=''utf-8''; ';
+  put '    data _null_; file &jref mod encoding=''utf-8'' ; ';
   put '      put "]"; ';
   put '    run; ';
   put '  %end; ';
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8'' mod; ';
+  put '  data _null_;file &jref encoding=''utf-8'' mod ; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';