From 5207a77591dc244093e7bdf6595cb3ffd28b1c8d Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Fri, 30 Jul 2021 00:14:29 +0300 Subject: [PATCH] feat: new mp_lib2inserts macro. In addition, modified mp_getddl to ignore views, closing #5. Created a test, which highlighted another issue in mp_getddl (labels were being double quoted which caused macro resolution attempts when %including). Changed to single quotes. Switched 'outlib' to 'outschema' in mp_ds2inserts to harmonise with mp_getddl. Added a maxobs option, to speed up testing. --- all.sas | 111 ++++++++++++++++++++++++++--- base/mp_ds2inserts.sas | 30 +++++--- base/mp_getddl.sas | 9 ++- base/mp_lib2inserts.sas | 73 +++++++++++++++++++ tests/base/mp_lib2inserts.test.sas | 42 +++++++++++ 5 files changed, 247 insertions(+), 18 deletions(-) create mode 100644 base/mp_lib2inserts.sas create mode 100644 tests/base/mp_lib2inserts.test.sas diff --git a/all.sas b/all.sas index 7962e64..8538b5a 100644 --- a/all.sas +++ b/all.sas @@ -3438,9 +3438,10 @@ run; %inc myref; @param [in] ds The dataset to be exported + @param [in] maxobs= (max) The max number of inserts to create @param [out] outref= (0) The output fileref. If it does not exist, it is created. If it does exist, new records are APPENDED. - @param [out] outlib= (0) The library (or schema) in which the target table is + @param [out] schema= (0) The library (or schema) in which the target table is located. If not provided, is ignored. @param [out] outds= (0) The output table to load. If not provided, will default to the table in the &ds parameter. @@ -3459,7 +3460,7 @@ run; @author Allan Bowe (credit mjsq) **/ -%macro mp_ds2inserts(ds, outref=0,outlib=0,outds=0,flavour=SAS +%macro mp_ds2inserts(ds, outref=0,schema=0,outds=0,flavour=SAS,maxobs=max )/*/STORE SOURCE*/; %if not %sysfunc(exist(&ds)) %then %do; @@ -3488,8 +3489,8 @@ run; filename &outref temp lrecl=66000; %end; -%if &outlib=0 %then %let outlib=; -%else %let outlib=&outlib..; +%if &schema=0 %then %let schema=; +%else %let schema=&schema..; %if &outds=0 %then %let outds=%scan(&ds,2,.); @@ -3508,8 +3509,18 @@ select count(*) into: nobs TRIMMED from &ds; %if &vars=0 %then %do; data _null_; file &outref mod; - put "/* No columns found in &ds */"; + put "/* No columns found in &schema.&ds */"; run; + %return; +%end; +%else %if &vars>1600 and &flavour=PGSQL %then %do; + data _null_; + file &fref mod; + put "/* &schema.&ds contains &vars vars */"; + put "/* Postgres cannot handle tables with over 1600 vars */"; + put "/* No inserts will be generated for this table"; + run; + %return; %end; %local varlist varlistcomma; @@ -3519,8 +3530,11 @@ select count(*) into: nobs TRIMMED from &ds; /* next, export data */ data _null_; file &outref mod ; - if _n_=1 then put "/* &outlib.&outds (&nobs rows, &vars columns) */"; + if _n_=1 then put "/* &schema.&outds (&nobs rows, &vars columns) */"; set &ds; + %if &maxobs ne max %then %do; + if _n_>&maxobs then stop; + %end; length _____str $32767; format _numeric_ best.; format _character_ ; @@ -3530,12 +3544,12 @@ data _null_; %let vtype=%mf_getvartype(&ds,&var); %if &i=1 %then %do; %if &flavour=SAS %then %do; - put "insert into &outlib.&outds set "; + put "insert into &schema.&outds set "; put " &var="@; %end; %else %if &flavour=PGSQL %then %do; _____str=cats( - "INSERT INTO &outlib.&outds (" + "INSERT INTO &schema.&outds (" ,symget('varlistcomma') ,") VALUES (" ); @@ -4386,6 +4400,8 @@ run; to create tables in SAS or a database. The macro can be used at table or library level. The default behaviour is to create DDL in SAS format. + Note - views are not currently supported. + Usage: data test(index=(pk=(x y)/unique /nomiss)); @@ -4431,6 +4447,7 @@ proc sql noprint; create table _data_ as select * from dictionary.tables where upcase(libname)="%upcase(&libref)" + and memtype='DATA' /* views not currently supported */ %if %length(&ds)>0 %then %do; and upcase(memname)="%upcase(&ds)" %end; @@ -4525,13 +4542,15 @@ run; put "create table &libref..&curds("; end; else do; + /* just a placeholder - we filter out views at the top */ put "create view &libref..&curds("; end; put " "@@; end; else put " ,"@@; if length(format)>1 then fmt=" format="!!cats(format); - if length(label)>1 then lab=" label="!!quote(trim(label)); + if length(label)>1 then + lab=" label="!!cats("'",tranwrd(label,"'","''"),"'"); if notnull='yes' then notnul=' not null'; if type='char' then typ=cats('char(',length,')'); else if length ne 8 then typ='num length='!!left(length); @@ -4603,6 +4622,7 @@ run; put "create table [&schema].[&curds]("; end; else do; + /* just a placeholder - we filter out views at the top */ put "create view [&schema].[&curds]("; end; put " "@@; @@ -4709,6 +4729,7 @@ run; put "CREATE TABLE &schema..&curds ("; end; else do; + /* just a placeholder - we filter out views at the top */ put "CREATE VIEW &schema..&curds ("; end; put " "@@; @@ -5533,6 +5554,78 @@ select distinct lowcase(memname) %end; %mend mp_lib2cards;/** + @file + @brief Convert all data in a library to SQL insert statements + @details Gets list of members then calls the %mp_ds2inserts() + macro. + Usage: + + %mp_getddl(sashelp, schema=work, fref=tempref) + + %mp_lib2inserts(sashelp, schema=work, outref=tempref) + + %inc tempref; + + + The output will be one file in the outref fileref. + + +

SAS Macros

+ @li mp_ds2inserts.sas + + + @param [in] lib Library in which to convert all datasets to inserts + @param [in] flavour= (SAS) The SQL flavour to be applied to the output. Valid + options: + @li SAS (default) - suitable for regular proc sql + @li PGSQL - Used for Postgres databases + @param [in] maxobs= (max) The max number of observations (per table) to create + @param [out] outref= Output fileref in which to create the insert statements. + If it exists, it will be appended to, otherwise it will be created. + @param [out] schema= (0) The schema of the target database, or the libref. + + @version 9.2 + @author Allan Bowe +**/ + +%macro mp_lib2inserts(lib + ,flavour=SAS + ,outref=0 + ,schema=0 + ,maxobs=max +)/*/STORE SOURCE*/; + +/* Find the tables */ +%local x ds memlist; +proc sql noprint; +select distinct lowcase(memname) + into: memlist + separated by ' ' + from dictionary.tables + where upcase(libname)="%upcase(&lib)" + and memtype='DATA'; /* exclude views */ + + +%let flavour=%upcase(&flavour); +%if &flavour ne SAS and &flavour ne PGSQL %then %do; + %put %str(WAR)NING: &flavour is not supported; + %return; +%end; + + +/* create the inserts */ +%do x=1 %to %sysfunc(countw(&memlist)); + %let ds=%scan(&memlist,&x); + %mp_ds2inserts(&lib..&ds + ,outref=&outref + ,schema=&schema + ,outds=&ds + ,flavour=&flavour + ,maxobs=&maxobs + ) +%end; + +%mend mp_lib2inserts;/** @file @brief Create a Markdown Table from a dataset @details A markdown table is a simple table representation for use in diff --git a/base/mp_ds2inserts.sas b/base/mp_ds2inserts.sas index dbdbff4..a2a2301 100644 --- a/base/mp_ds2inserts.sas +++ b/base/mp_ds2inserts.sas @@ -14,9 +14,10 @@ %inc myref; @param [in] ds The dataset to be exported + @param [in] maxobs= (max) The max number of inserts to create @param [out] outref= (0) The output fileref. If it does not exist, it is created. If it does exist, new records are APPENDED. - @param [out] outlib= (0) The library (or schema) in which the target table is + @param [out] schema= (0) The library (or schema) in which the target table is located. If not provided, is ignored. @param [out] outds= (0) The output table to load. If not provided, will default to the table in the &ds parameter. @@ -35,7 +36,7 @@ @author Allan Bowe (credit mjsq) **/ -%macro mp_ds2inserts(ds, outref=0,outlib=0,outds=0,flavour=SAS +%macro mp_ds2inserts(ds, outref=0,schema=0,outds=0,flavour=SAS,maxobs=max )/*/STORE SOURCE*/; %if not %sysfunc(exist(&ds)) %then %do; @@ -64,8 +65,8 @@ filename &outref temp lrecl=66000; %end; -%if &outlib=0 %then %let outlib=; -%else %let outlib=&outlib..; +%if &schema=0 %then %let schema=; +%else %let schema=&schema..; %if &outds=0 %then %let outds=%scan(&ds,2,.); @@ -84,8 +85,18 @@ select count(*) into: nobs TRIMMED from &ds; %if &vars=0 %then %do; data _null_; file &outref mod; - put "/* No columns found in &ds */"; + put "/* No columns found in &schema.&ds */"; run; + %return; +%end; +%else %if &vars>1600 and &flavour=PGSQL %then %do; + data _null_; + file &fref mod; + put "/* &schema.&ds contains &vars vars */"; + put "/* Postgres cannot handle tables with over 1600 vars */"; + put "/* No inserts will be generated for this table"; + run; + %return; %end; %local varlist varlistcomma; @@ -95,8 +106,11 @@ select count(*) into: nobs TRIMMED from &ds; /* next, export data */ data _null_; file &outref mod ; - if _n_=1 then put "/* &outlib.&outds (&nobs rows, &vars columns) */"; + if _n_=1 then put "/* &schema.&outds (&nobs rows, &vars columns) */"; set &ds; + %if &maxobs ne max %then %do; + if _n_>&maxobs then stop; + %end; length _____str $32767; format _numeric_ best.; format _character_ ; @@ -106,12 +120,12 @@ data _null_; %let vtype=%mf_getvartype(&ds,&var); %if &i=1 %then %do; %if &flavour=SAS %then %do; - put "insert into &outlib.&outds set "; + put "insert into &schema.&outds set "; put " &var="@; %end; %else %if &flavour=PGSQL %then %do; _____str=cats( - "INSERT INTO &outlib.&outds (" + "INSERT INTO &schema.&outds (" ,symget('varlistcomma') ,") VALUES (" ); diff --git a/base/mp_getddl.sas b/base/mp_getddl.sas index 658fd65..92da0c0 100644 --- a/base/mp_getddl.sas +++ b/base/mp_getddl.sas @@ -5,6 +5,8 @@ to create tables in SAS or a database. The macro can be used at table or library level. The default behaviour is to create DDL in SAS format. + Note - views are not currently supported. + Usage: data test(index=(pk=(x y)/unique /nomiss)); @@ -50,6 +52,7 @@ proc sql noprint; create table _data_ as select * from dictionary.tables where upcase(libname)="%upcase(&libref)" + and memtype='DATA' /* views not currently supported */ %if %length(&ds)>0 %then %do; and upcase(memname)="%upcase(&ds)" %end; @@ -144,13 +147,15 @@ run; put "create table &libref..&curds("; end; else do; + /* just a placeholder - we filter out views at the top */ put "create view &libref..&curds("; end; put " "@@; end; else put " ,"@@; if length(format)>1 then fmt=" format="!!cats(format); - if length(label)>1 then lab=" label="!!quote(trim(label)); + if length(label)>1 then + lab=" label="!!cats("'",tranwrd(label,"'","''"),"'"); if notnull='yes' then notnul=' not null'; if type='char' then typ=cats('char(',length,')'); else if length ne 8 then typ='num length='!!left(length); @@ -222,6 +227,7 @@ run; put "create table [&schema].[&curds]("; end; else do; + /* just a placeholder - we filter out views at the top */ put "create view [&schema].[&curds]("; end; put " "@@; @@ -328,6 +334,7 @@ run; put "CREATE TABLE &schema..&curds ("; end; else do; + /* just a placeholder - we filter out views at the top */ put "CREATE VIEW &schema..&curds ("; end; put " "@@; diff --git a/base/mp_lib2inserts.sas b/base/mp_lib2inserts.sas new file mode 100644 index 0000000..86ef6d2 --- /dev/null +++ b/base/mp_lib2inserts.sas @@ -0,0 +1,73 @@ +/** + @file + @brief Convert all data in a library to SQL insert statements + @details Gets list of members then calls the %mp_ds2inserts() + macro. + Usage: + + %mp_getddl(sashelp, schema=work, fref=tempref) + + %mp_lib2inserts(sashelp, schema=work, outref=tempref) + + %inc tempref; + + + The output will be one file in the outref fileref. + + +

SAS Macros

+ @li mp_ds2inserts.sas + + + @param [in] lib Library in which to convert all datasets to inserts + @param [in] flavour= (SAS) The SQL flavour to be applied to the output. Valid + options: + @li SAS (default) - suitable for regular proc sql + @li PGSQL - Used for Postgres databases + @param [in] maxobs= (max) The max number of observations (per table) to create + @param [out] outref= Output fileref in which to create the insert statements. + If it exists, it will be appended to, otherwise it will be created. + @param [out] schema= (0) The schema of the target database, or the libref. + + @version 9.2 + @author Allan Bowe +**/ + +%macro mp_lib2inserts(lib + ,flavour=SAS + ,outref=0 + ,schema=0 + ,maxobs=max +)/*/STORE SOURCE*/; + +/* Find the tables */ +%local x ds memlist; +proc sql noprint; +select distinct lowcase(memname) + into: memlist + separated by ' ' + from dictionary.tables + where upcase(libname)="%upcase(&lib)" + and memtype='DATA'; /* exclude views */ + + +%let flavour=%upcase(&flavour); +%if &flavour ne SAS and &flavour ne PGSQL %then %do; + %put %str(WAR)NING: &flavour is not supported; + %return; +%end; + + +/* create the inserts */ +%do x=1 %to %sysfunc(countw(&memlist)); + %let ds=%scan(&memlist,&x); + %mp_ds2inserts(&lib..&ds + ,outref=&outref + ,schema=&schema + ,outds=&ds + ,flavour=&flavour + ,maxobs=&maxobs + ) +%end; + +%mend mp_lib2inserts; \ No newline at end of file diff --git a/tests/base/mp_lib2inserts.test.sas b/tests/base/mp_lib2inserts.test.sas new file mode 100644 index 0000000..e28d79f --- /dev/null +++ b/tests/base/mp_lib2inserts.test.sas @@ -0,0 +1,42 @@ +/** + @file + @brief Testing mp_ds2inserts.sas macro + +

SAS Macros

+ @li mf_mkdir.sas + @li mp_getddl.sas + @li mp_lib2inserts.sas + @li mp_assert.sas + +**/ + +/* grab 20 datasets from SASHELP */ +%let path=%sysfunc(pathname(work)); +%mf_mkdir(&path) +libname sashlp "&path"; +proc sql noprint; +create table members as + select distinct lowcase(memname) as memname + from dictionary.tables + where upcase(libname)="SASHELP" + and memtype='DATA'; /* exclude views */ +data _null_; + set work.members; + call execute(cats('data sashlp.',memname,';set sashelp.',memname,';run;')); + if _n_>20 then stop; +run; + +/* export DDL and inserts */ +%mp_getddl(sashlp, schema=work, fref=tempref) +%mp_lib2inserts(sashlp, schema=work, outref=tempref,maxobs=50) + +/* check if it actually runs */ +options source2; +%inc tempref; + +/* without errors.. */ +%mp_assert( + iftrue=(&syscc=0), + desc=Able to export 20 tables from sashelp using mp_lib2inserts, + outds=work.test_results +) \ No newline at end of file