feat: mp_prevobs.sas macro

all.sas

@@ -39,8 +39,8 @@ options noquotelenmax;
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
-/** @cond */
 
 %macro mf_abort(mac=mf_abort.sas, type=, msg=, iftrue=%str(1=1)
 )/*/STORE SOURCE*/;
@@ -278,6 +278,7 @@ options noquotelenmax;
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
 %macro mf_existvarlist(libds, varlist
@@ -317,7 +318,9 @@ options noquotelenmax;
     0
     %put Vars not found: &found;
   %end;
-%mend;/**
+%mend;
+
+/** @endcond *//**
   @file
   @brief Returns a character attribute of a dataset.
   @details Can be used in open code, eg as follows:
@@ -633,6 +636,7 @@ options noquotelenmax;
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
 %macro mf_getschema(libref
@@ -654,6 +658,8 @@ options noquotelenmax;
   &schema
 
 %mend;
+
+/** @endcond */
 /**
   @file
   @brief Assigns and returns an unused fileref
@@ -1529,11 +1535,11 @@ Usage:
     the particulars of an environment.  For instance, can stream custom
     results back to the client in an STP Web App context, or completely stop
     in the case of a batch run.
-  
+
   Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored Process
   environments.  This macro takes a unique approach - we set the SAS syscc to 0,
-  run `stpsrvset('program error', 0)` (if SAS 9) and then - we open a macro 
-  but don't close it!  This provides a graceful abort for SAS web services in all 
+  run `stpsrvset('program error', 0)` (if SAS 9) and then - we open a macro
+  but don't close it!  This provides a graceful abort for SAS web services in all
   web enabled environments.
 
   @param mac= to contain the name of the calling macro
@@ -1542,6 +1548,7 @@ Usage:
 
   @version 9.4M3
   @author Allan Bowe
+  @cond
 **/
 
 %macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1)
@@ -1554,7 +1561,7 @@ Usage:
   %put NOTE - &msg;
 
   /* Stored Process Server web app context */
-  %if %symexist(_metaperson) 
+  %if %symexist(_metaperson)
   or (%symexist(SYSPROCESSNAME) and "&SYSPROCESSNAME"="Compute Server" )
   %then %do;
     options obs=max replace nosyntaxcheck mprint;
@@ -1675,7 +1682,7 @@ Usage:
   %end;
 %mend;
 
-/**
+/** @endcond *//**
   @file
   @brief Copy any file using binary input / output streams
   @details Reads in a file byte by byte and writes it back out.  Is an
@@ -1739,6 +1746,7 @@ Usage:
     applying CRLF line endings and converting embedded cr and crlf to lf.
 
   usage:
+
       fileref mycsv "/path/your/csv";
       %mp_cleancsv(in=mycsv,out=/path/new.csv)
 
@@ -1748,6 +1756,7 @@ Usage:
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
 %macro mp_cleancsv(in=NOTPROVIDED,out=NOTPROVIDED,qchar='22'x);
@@ -1796,7 +1805,8 @@ Usage:
       else put inchar $char1.;
     end;
   run;
-%mend;/**
+%mend;
+/** @endcond *//**
   @file mp_createconstraints.sas
   @brief Creates constraints
   @details Takes the output from mp_getconstraints.sas as input
@@ -2635,6 +2645,63 @@ quit;
 %put NOTE-;%put NOTE-;
 %put NOTE- %sysfunc(dequote(&cards_file.));
 %put NOTE-;%put NOTE-;
+%mend;/**
+  @file
+  @brief Export a dataset to a CSV file
+  @details Export to a file or a fileref
+  Usage:
+
+      %mp_ds2csv(sashelp.class,outref="%sysfunc(pathname(work))/file.csv")
+
+  @param ds The dataset to be exported
+  @param outfile= The output filename - should be quoted.
+  @param outref= The output fileref (takes precedence if provided)
+  @param outencoding= The output encoding to use (unquoted)
+
+  @version 9.2
+  @author Allan Bowe (credit mjsq)
+**/
+
+%macro mp_ds2csv(ds, outref=0, outfile=, outencoding=0
+)/*/STORE SOURCE*/;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+   %put WARNING:  &ds does not exist;
+   %return;
+%end;
+
+%if %index(&ds,.)=0 %then %let ds=WORK.&ds;
+
+%if &outencoding=0 %then %let outencoding=;
+%else %let outencoding=encoding="&outencoding";
+
+%local outloc;
+%if &outref=0 %then %let outloc=&outfile;
+%else %let outloc=&outref;
+
+/* credit to mjsq - https://stackoverflow.com/a/55642267 */
+
+/* first get headers */
+data _null_;
+   file &outloc dlm=',' dsd &outencoding lrecl=32767;
+   length header $ 2000;
+   dsid=open("&ds.","i");
+   num=attrn(dsid,"nvars");
+   do i=1 to num;
+      header = trim(left(coalescec(varlabel(dsid,i),varname(dsid,i))));
+      put header @;
+   end;
+   rc=close(dsid);
+run;
+
+/* next, export data */
+data _null_;
+   set &ds.;
+   file &outloc mod dlm=',' dsd &outencoding lrecl=32767;
+   put (_all_) (+0);
+run;
+
+
 %mend;/**
   @file mp_getconstraints.sas
   @brief Get constraint details at column level
@@ -2712,6 +2779,8 @@ create table &outds as
 
   You may need to adjust the rendered DBML to suit your needs.
 
+  ![dbml for sas](https://i.imgur.com/8T1tIZp.gif)
+
 
   <h4> SAS Macros </h4>
   @li mf_getquotedstr.sas
@@ -3997,6 +4066,93 @@ select distinct lowcase(memname)
         ,dttm=%sysfunc(datetime());
   quit;
 
+%mend;/**
+  @file
+  @brief Enables previous observations to be re-instated
+  @details Remembers the last X observations by storing them in a hash table.
+  Is a convenience over the use of lag() or retain, when an entire observation
+  needs to be restored.
+
+  This macro will also restore automatic variables (such as _n_ and _error_).
+
+  Example Usage:
+
+      data example;
+        set sashelp.class;
+        calc_var=_n_*3;
+        * initialise hash and save from PDV ;
+        %mp_prevobs(INIT,history=2)
+        if _n_ =10 then do;
+          * fetch previous but 1 record;
+          %mp_prevobs(FETCH,-2) 
+          put _n_= name= age= calc_var=; 
+          * fetch previous record;
+          %mp_prevobs(FETCH,-1) 
+          put _n_= name= age= calc_var=; 
+          * reinstate current record ;
+          %mp_prevobs(FETCH,0) 
+          put _n_= name= age= calc_var=;
+        end;
+      run;
+
+  Result:
+
+  <img src="https://imgur.com/PSjHoET.png" alt="mp_prevobs sas" width="400"/>
+
+  Credit is made to `data _null_` for authoring this very helpful paper:
+  https://www.lexjansen.com/pharmasug/2008/cc/CC08.pdf
+
+  @param action Either FETCH a current or previous record, or INITialise.
+  @param record The relative (to current) position of the previous observation 
+   to return.  
+  @param history= The number of records to retain in the hash table. Default=5
+  @param prefix= the prefix to give to the variables used to store the hash name
+   and index. Default=mp_prevobs
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_prevobs(action,record,history=5,prefix=mp_prevobs
+)/*/STORE SOURCE*/;
+%let action=%upcase(&action);
+%let prefix=%upcase(&prefix);
+%let record=%eval((&record+0) * -1);
+
+%if &action=INIT %then %do;
+    
+  if _n_ eq 1 then do; 
+    attrib &prefix._VAR length=$64; 
+    dcl hash &prefix._HASH(ordered:'Y');
+    &prefix._KEY=0;
+    &prefix._HASH.defineKey("&prefix._KEY"); 
+    do while(1); 
+      call vnext(&prefix._VAR); 
+      if &prefix._VAR='' then leave;
+      if &prefix._VAR eq "&prefix._VAR" then continue; 
+      else if &prefix._VAR eq "&prefix._KEY" then continue; 
+      &prefix._HASH.defineData(&prefix._VAR);
+    end; 
+    &prefix._HASH.defineDone(); 
+  end;
+  /* this part has to happen before FETCHing */
+  &prefix._KEY+1;
+  &prefix._rc=&prefix._HASH.add();
+  if &prefix._rc then putlog 'adding' &prefix._rc=;
+  %if &history>0 %then %do;
+    if &prefix._key>&history+1 then 
+      &prefix._HASH.remove(key: &prefix._KEY - &history - 1);
+    if &prefix._rc then putlog 'removing' &prefix._rc=;
+  %end;
+%end;
+%else %if &action=FETCH %then %do;
+  if &record > &prefix._key then putlog "Not enough records in &Prefix._hash yet";
+  else &prefix._rc=&prefix._HASH.find(key: &prefix._KEY - &record);
+  if &prefix._rc then putlog &prefix._rc= " when fetching " &prefix._KEY=
+    "with record &record and " _n_=;
+%end;
+
 %mend;/**
   @file
   @brief Returns all children from a hierarchy table for a specified parent
@@ -4598,6 +4754,64 @@ proc sql
  %mp_binarycopy(inloc="&inloc",outref=_webout)
 %end;
 
+%mend;/**
+  @file mp_testwritespeedlibrary.sas
+  @brief Tests the write speed of a new table in a SAS library
+  @details Will create a new table of a certain size in an 
+  existing SAS library.  The table will have one column,
+  and will be subsequently deleted.
+      
+      %mp_testwritespeedlibrary(
+        lib=work
+        ,size=0.5
+        ,outds=work.results
+      )
+
+  @param lib= (WORK) The library in which to create the table
+  @param size= (0.1) The size in GB of the table to create
+  @param outds= (WORK.RESULTS) The output dataset to be created.
+
+  <h4> Dependencies </h4>
+  @li mf_getuniquename.sas
+  @li mf_existds.sas
+
+  @version 9.4
+  @author Allan Bowe
+
+**/
+
+%macro mp_testwritespeedlibrary(lib=WORK
+  ,outds=work.results
+  ,size=0.1
+)/*/STORE SOURCE*/;
+%local ds start;
+
+/* find an unused, unique name for the new table */
+%let ds=%mf_getuniquename();
+%do %until(%mf_existds(&lib..&ds)=0);
+  %let ds=%mf_getuniquename();
+%end;
+
+%let start=%sysfunc(datetime());
+
+data &lib..&ds(compress=no keep=x);
+  header=128*1024;
+  size=(1073741824/8 * &size) - header;
+  do x=1 to size;
+    output;
+  end;
+run;
+
+proc sql;
+drop table &lib..&ds;
+
+data &outds;
+  lib="&lib";
+  start_dttm=put(&start,datetime19.);
+  end_dttm=put(datetime(),datetime19.);
+  duration_seconds=end_dttm-start_dttm;
+run;
+
 %mend;/**
   @file
   @brief Recursively scans a directory tree to get all subfolders and content

base/mf_abort.sas

@@ -21,8 +21,8 @@
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
-/** @cond */
 
 %macro mf_abort(mac=mf_abort.sas, type=, msg=, iftrue=%str(1=1)
 )/*/STORE SOURCE*/;

base/mf_existvarlist.sas

@@ -14,6 +14,7 @@
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
 %macro mf_existvarlist(libds, varlist
@@ -53,4 +54,6 @@
     0
     %put Vars not found: &found;
   %end;
-%mend;
+%mend;
+
+/** @endcond */

base/mf_getschema.sas

@@ -17,6 +17,7 @@
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
 %macro mf_getschema(libref
@@ -38,3 +39,5 @@
   &schema
 
 %mend;
+
+/** @endcond */

base/mp_abort.sas

@@ -5,11 +5,11 @@
     the particulars of an environment.  For instance, can stream custom
     results back to the client in an STP Web App context, or completely stop
     in the case of a batch run.
-  
+
   Using SAS Abort Cancel mechanisms can cause hung sessions in some Stored Process
   environments.  This macro takes a unique approach - we set the SAS syscc to 0,
-  run `stpsrvset('program error', 0)` (if SAS 9) and then - we open a macro 
-  but don't close it!  This provides a graceful abort for SAS web services in all 
+  run `stpsrvset('program error', 0)` (if SAS 9) and then - we open a macro
+  but don't close it!  This provides a graceful abort for SAS web services in all
   web enabled environments.
 
   @param mac= to contain the name of the calling macro
@@ -18,6 +18,7 @@
 
   @version 9.4M3
   @author Allan Bowe
+  @cond
 **/
 
 %macro mp_abort(mac=mp_abort.sas, type=, msg=, iftrue=%str(1=1)
@@ -30,7 +31,7 @@
   %put NOTE - &msg;
 
   /* Stored Process Server web app context */
-  %if %symexist(_metaperson) 
+  %if %symexist(_metaperson)
   or (%symexist(SYSPROCESSNAME) and "&SYSPROCESSNAME"="Compute Server" )
   %then %do;
     options obs=max replace nosyntaxcheck mprint;
@@ -151,3 +152,4 @@
   %end;
 %mend;
 
+/** @endcond */

base/mp_cleancsv.sas

@@ -8,6 +8,7 @@
     applying CRLF line endings and converting embedded cr and crlf to lf.
 
   usage:
+
       fileref mycsv "/path/your/csv";
       %mp_cleancsv(in=mycsv,out=/path/new.csv)
 
@@ -17,6 +18,7 @@
 
   @version 9.2
   @author Allan Bowe
+  @cond
 **/
 
 %macro mp_cleancsv(in=NOTPROVIDED,out=NOTPROVIDED,qchar='22'x);
@@ -65,4 +67,5 @@
       else put inchar $char1.;
     end;
   run;
-%mend;
+%mend;
+/** @endcond */

base/mp_ds2csv.sas

@@ -0,0 +1,58 @@
+/**
+  @file
+  @brief Export a dataset to a CSV file
+  @details Export to a file or a fileref
+  Usage:
+
+      %mp_ds2csv(sashelp.class,outref="%sysfunc(pathname(work))/file.csv")
+
+  @param ds The dataset to be exported
+  @param outfile= The output filename - should be quoted.
+  @param outref= The output fileref (takes precedence if provided)
+  @param outencoding= The output encoding to use (unquoted)
+
+  @version 9.2
+  @author Allan Bowe (credit mjsq)
+**/
+
+%macro mp_ds2csv(ds, outref=0, outfile=, outencoding=0
+)/*/STORE SOURCE*/;
+
+%if not %sysfunc(exist(&ds)) %then %do;
+   %put WARNING:  &ds does not exist;
+   %return;
+%end;
+
+%if %index(&ds,.)=0 %then %let ds=WORK.&ds;
+
+%if &outencoding=0 %then %let outencoding=;
+%else %let outencoding=encoding="&outencoding";
+
+%local outloc;
+%if &outref=0 %then %let outloc=&outfile;
+%else %let outloc=&outref;
+
+/* credit to mjsq - https://stackoverflow.com/a/55642267 */
+
+/* first get headers */
+data _null_;
+   file &outloc dlm=',' dsd &outencoding lrecl=32767;
+   length header $ 2000;
+   dsid=open("&ds.","i");
+   num=attrn(dsid,"nvars");
+   do i=1 to num;
+      header = trim(left(coalescec(varlabel(dsid,i),varname(dsid,i))));
+      put header @;
+   end;
+   rc=close(dsid);
+run;
+
+/* next, export data */
+data _null_;
+   set &ds.;
+   file &outloc mod dlm=',' dsd &outencoding lrecl=32767;
+   put (_all_) (+0);
+run;
+
+
+%mend;

base/mp_getdbml.sas

@@ -16,6 +16,8 @@
 
   You may need to adjust the rendered DBML to suit your needs.
 
+  ![dbml for sas](https://i.imgur.com/8T1tIZp.gif)
+
 
   <h4> SAS Macros </h4>
   @li mf_getquotedstr.sas

base/mp_prevobs.sas

@@ -0,0 +1,88 @@
+/**
+  @file
+  @brief Enables previous observations to be re-instated
+  @details Remembers the last X observations by storing them in a hash table.
+  Is a convenience over the use of lag() or retain, when an entire observation
+  needs to be restored.
+
+  This macro will also restore automatic variables (such as _n_ and _error_).
+
+  Example Usage:
+
+      data example;
+        set sashelp.class;
+        calc_var=_n_*3;
+        * initialise hash and save from PDV ;
+        %mp_prevobs(INIT,history=2)
+        if _n_ =10 then do;
+          * fetch previous but 1 record;
+          %mp_prevobs(FETCH,-2) 
+          put _n_= name= age= calc_var=; 
+          * fetch previous record;
+          %mp_prevobs(FETCH,-1) 
+          put _n_= name= age= calc_var=; 
+          * reinstate current record ;
+          %mp_prevobs(FETCH,0) 
+          put _n_= name= age= calc_var=;
+        end;
+      run;
+
+  Result:
+
+  <img src="https://imgur.com/PSjHoET.png" alt="mp_prevobs sas" width="400"/>
+
+  Credit is made to `data _null_` for authoring this very helpful paper:
+  https://www.lexjansen.com/pharmasug/2008/cc/CC08.pdf
+
+  @param action Either FETCH a current or previous record, or INITialise.
+  @param record The relative (to current) position of the previous observation 
+   to return.  
+  @param history= The number of records to retain in the hash table. Default=5
+  @param prefix= the prefix to give to the variables used to store the hash name
+   and index. Default=mp_prevobs
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_prevobs(action,record,history=5,prefix=mp_prevobs
+)/*/STORE SOURCE*/;
+%let action=%upcase(&action);
+%let prefix=%upcase(&prefix);
+%let record=%eval((&record+0) * -1);
+
+%if &action=INIT %then %do;
+    
+  if _n_ eq 1 then do; 
+    attrib &prefix._VAR length=$64; 
+    dcl hash &prefix._HASH(ordered:'Y');
+    &prefix._KEY=0;
+    &prefix._HASH.defineKey("&prefix._KEY"); 
+    do while(1); 
+      call vnext(&prefix._VAR); 
+      if &prefix._VAR='' then leave;
+      if &prefix._VAR eq "&prefix._VAR" then continue; 
+      else if &prefix._VAR eq "&prefix._KEY" then continue; 
+      &prefix._HASH.defineData(&prefix._VAR);
+    end; 
+    &prefix._HASH.defineDone(); 
+  end;
+  /* this part has to happen before FETCHing */
+  &prefix._KEY+1;
+  &prefix._rc=&prefix._HASH.add();
+  if &prefix._rc then putlog 'adding' &prefix._rc=;
+  %if &history>0 %then %do;
+    if &prefix._key>&history+1 then 
+      &prefix._HASH.remove(key: &prefix._KEY - &history - 1);
+    if &prefix._rc then putlog 'removing' &prefix._rc=;
+  %end;
+%end;
+%else %if &action=FETCH %then %do;
+  if &record > &prefix._key then putlog "Not enough records in &Prefix._hash yet";
+  else &prefix._rc=&prefix._HASH.find(key: &prefix._KEY - &record);
+  if &prefix._rc then putlog &prefix._rc= " when fetching " &prefix._KEY=
+    "with record &record and " _n_=;
+%end;
+
+%mend;

main.dox

@@ -13,6 +13,11 @@
         * Not metadata aware
         * No X command
         * Prefixes:  _mf_, _mp_
+
+    Macros starting `mf_` are macro _functions_ and can be used in assignment
+    statements.  Those starting `mp_` are macro _procedures_, which generate
+    SAS statements, and must therefore be applied accordingly.
+
  */
 
 /*! \dir meta