From 2031a5b0c06b916703adc0d45eef54bbf5ecb23e Mon Sep 17 00:00:00 2001 From: munja Date: Sun, 21 Aug 2022 19:01:01 +0100 Subject: [PATCH 1/7] feat: new mp_dsmeta macro --- .github/workflows/run-tests.yml | 2 - base/mp_dsmeta.sas | 98 +++++++++++++++++++++++++++++++++ sasjs/sasjsconfig.json | 2 +- tests/base/mp_dsmeta.test.sas | 32 +++++++++++ 4 files changed, 131 insertions(+), 3 deletions(-) create mode 100644 base/mp_dsmeta.sas create mode 100644 tests/base/mp_dsmeta.test.sas diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 18a2a81..628e169 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -78,7 +78,5 @@ jobs: SECRET: ${{secrets.SECRET}} SAS_USERNAME: ${{secrets.SAS_USERNAME}} SAS_PASSWORD: ${{secrets.SAS_PASSWORD}} - SERVER_URL: ${{secrets.SERVER_URL}} - SERVER_TYPE: ${{secrets.SERVER_TYPE}} ACCESS_TOKEN: ${{secrets.ACCESS_TOKEN}} REFRESH_TOKEN: ${{secrets.REFRESH_TOKEN}} diff --git a/base/mp_dsmeta.sas b/base/mp_dsmeta.sas new file mode 100644 index 0000000..2c3c54e --- /dev/null +++ b/base/mp_dsmeta.sas @@ -0,0 +1,98 @@ +/** + @file + @brief Export dataset metadata to a single output table + @details Exports the dataset attributes and enginehost information, then + converts the datasets into a single output table in the following format: + +|ODS_TABLE:$10.|NAME:$100.|VALUE:$1000.| +|---|---|---| +|`ATTRIBUTES `|`Data Set Name `|`SASHELP.CLASS `| +|`ATTRIBUTES `|`Observations `|`19 `| +|`ATTRIBUTES `|`Member Type `|`DATA `| +|`ATTRIBUTES `|`Variables `|`5 `| +|`ATTRIBUTES `|`Engine `|`V9 `| +|`ATTRIBUTES `|`Indexes `|`0 `| +|`ATTRIBUTES `|`Created `|`06/08/2020 00:59:14 `| +|`ATTRIBUTES `|`Observation Length `|`40 `| +|`ATTRIBUTES `|`Last Modified `|`06/08/2020 00:59:14 `| +|`ATTRIBUTES `|`Deleted Observations `|`0 `| +|`ATTRIBUTES `|`Protection `|`. `| +|`ATTRIBUTES `|`Compressed `|`NO `| +|`ATTRIBUTES `|`Data Set Type `|`. `| +|`ATTRIBUTES `|`Sorted `|`NO `| +|`ATTRIBUTES `|`Label `|`Student Data `| +|`ATTRIBUTES `|`Data Representation `|`SOLARIS_X86_64, LINUX_X86_64, ALPHA_TRU64, LINUX_IA64 `| +|`ATTRIBUTES `|`Encoding `|`us-ascii ASCII (ANSI) `| +|`ENGINEHOST `|`Data Set Page Size `|`65536 `| +|`ENGINEHOST `|`Number of Data Set Pages `|`1 `| +|`ENGINEHOST `|`First Data Page `|`1 `| +|`ENGINEHOST `|`Max Obs per Page `|`1632 `| +|`ENGINEHOST `|`Obs in First Data Page `|`19 `| +|`ENGINEHOST `|`Number of Data Set Repairs `|`0 `| +|`ENGINEHOST `|`Filename `|`/opt/sas/sas9/SASHome/SASFoundation/9.4/sashelp/class.sas7bdat `| +|`ENGINEHOST `|`Release Created `|`9.0401M7 `| +|`ENGINEHOST `|`Host Created `|`Linux `| +|`ENGINEHOST `|`Inode Number `|`28314616 `| +|`ENGINEHOST `|`Access Permission `|`rw-r--r-- `| +|`ENGINEHOST `|`Owner Name `|`sas `| +|`ENGINEHOST `|`File Size `|`128KB `| +|`ENGINEHOST `|`File Size (bytes) `|`131072 `| + + Example usage: + + %mp_dsmeta(work.sashelp,outds=work.mymeta) + proc print data=work.mymeta; + run; + + @param libds The library.dataset to export the metadata for + @param outds= (work.dsmeta) The output table to contain the metadata + +

Related Files

+ @li mp_dsmeta.test.sas + +**/ + +%macro mp_dsmeta(libds,outds=work.dsmeta); + +%local ds1 ds2; +data;run; %let ds1=&syslast; +data;run; %let ds2=&syslast; + +/* setup the ODS capture */ +ods output attributes=&ds1 enginehost=&ds2; + +/* export the metadata */ +proc contents data=&libds; +run; + +/* load it into a single table */ +data &outds (keep=ods_table name value); + length ods_table $10 name label1 label $100 value cvalue1 cvalue $1000 + nvalue nvalue1 nvalue2 8; + if _n_=1 then call missing (of _all_); + * putlog (_all_)(=); + set &ds1 (in=atrs) &ds2 (in=eng); + if atrs then do; + ods_table='ATTRIBUTES'; + name=coalescec(label1,label); + value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.)); + output; + if label2 ne '' then do; + name=label2; + value=coalescec(cvalue2,put(nvalue2,best.)); + output; + end; + end; + else if eng then do; + ods_table='ENGINEHOST'; + name=coalescec(label1,label); + value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.)); + output; + end; +run; + +proc sql; +drop table &ds1, &ds2; + +%mend mp_dsmeta; + diff --git a/sasjs/sasjsconfig.json b/sasjs/sasjsconfig.json index ae89c0e..bcf3911 100644 --- a/sasjs/sasjsconfig.json +++ b/sasjs/sasjsconfig.json @@ -107,4 +107,4 @@ "contextName": "SAS Job Execution compute context" } ] -} \ No newline at end of file +} diff --git a/tests/base/mp_dsmeta.test.sas b/tests/base/mp_dsmeta.test.sas new file mode 100644 index 0000000..052b19e --- /dev/null +++ b/tests/base/mp_dsmeta.test.sas @@ -0,0 +1,32 @@ +/** + @file + @brief Testing mp_dsmeta.sas macro + +

SAS Macros

+ @li mp_assert.sas + @li mp_assertscope.sas + @li mp_dsmeta.sas + +**/ + +data work.Example; + set sashelp.vmacro; +run; + +%mp_assertscope(SNAPSHOT) +%mp_dsmeta(work.example,outds=work.test) +%mp_assertscope(COMPARE) + +proc sql noprint; +select count(*) into nobs from work.test; +select count(distinct ods_table) into: tnobs from work.test; + +%mp_assert( + iftrue=(&tnobs=2), + desc=Check that both ATTRIBUTES and ENGINEHOST are provided +) +%mp_assert( + iftrue=(&nobs>10), + desc=Check that sufficient details are provided +) + From 6d15465bac8e164f1bcb3dc238fa935321e9eb27 Mon Sep 17 00:00:00 2001 From: munja Date: Sun, 21 Aug 2022 19:17:56 +0100 Subject: [PATCH 2/7] fix: generating all.sas and fixing failing test --- all.sas | 99 +++++++++++++++++++++++++++++++++++ base/mp_dsmeta.sas | 3 +- sasjs/sasjsconfig.json | 2 +- tests/base/mp_dsmeta.test.sas | 2 +- 4 files changed, 103 insertions(+), 3 deletions(-) diff --git a/all.sas b/all.sas index 66d2c2b..f885910 100644 --- a/all.sas +++ b/all.sas @@ -5826,6 +5826,105 @@ options varlenchk=&optval; %put &sysmacroname: &outds is %mf_getfilesize(libds=&outds,format=yes); %mend mp_ds2squeeze;/** + @file + @brief Export dataset metadata to a single output table + @details Exports the dataset attributes and enginehost information, then + converts the datasets into a single output table in the following format: + +|ODS_TABLE:$10.|NAME:$100.|VALUE:$1000.| +|---|---|---| +|`ATTRIBUTES `|`Data Set Name `|`SASHELP.CLASS `| +|`ATTRIBUTES `|`Observations `|`19 `| +|`ATTRIBUTES `|`Member Type `|`DATA `| +|`ATTRIBUTES `|`Variables `|`5 `| +|`ATTRIBUTES `|`Engine `|`V9 `| +|`ATTRIBUTES `|`Indexes `|`0 `| +|`ATTRIBUTES `|`Created `|`06/08/2020 00:59:14 `| +|`ATTRIBUTES `|`Observation Length `|`40 `| +|`ATTRIBUTES `|`Last Modified `|`06/08/2020 00:59:14 `| +|`ATTRIBUTES `|`Deleted Observations `|`0 `| +|`ATTRIBUTES `|`Protection `|`. `| +|`ATTRIBUTES `|`Compressed `|`NO `| +|`ATTRIBUTES `|`Data Set Type `|`. `| +|`ATTRIBUTES `|`Sorted `|`NO `| +|`ATTRIBUTES `|`Label `|`Student Data `| +|`ATTRIBUTES `|`Data Representation `|`SOLARIS_X86_64, LINUX_X86_64, ALPHA_TRU64, LINUX_IA64 `| +|`ATTRIBUTES `|`Encoding `|`us-ascii ASCII (ANSI) `| +|`ENGINEHOST `|`Data Set Page Size `|`65536 `| +|`ENGINEHOST `|`Number of Data Set Pages `|`1 `| +|`ENGINEHOST `|`First Data Page `|`1 `| +|`ENGINEHOST `|`Max Obs per Page `|`1632 `| +|`ENGINEHOST `|`Obs in First Data Page `|`19 `| +|`ENGINEHOST `|`Number of Data Set Repairs `|`0 `| +|`ENGINEHOST `|`Filename `|`/opt/sas/sas9/SASHome/SASFoundation/9.4/sashelp/class.sas7bdat `| +|`ENGINEHOST `|`Release Created `|`9.0401M7 `| +|`ENGINEHOST `|`Host Created `|`Linux `| +|`ENGINEHOST `|`Inode Number `|`28314616 `| +|`ENGINEHOST `|`Access Permission `|`rw-r--r-- `| +|`ENGINEHOST `|`Owner Name `|`sas `| +|`ENGINEHOST `|`File Size `|`128KB `| +|`ENGINEHOST `|`File Size (bytes) `|`131072 `| + + Example usage: + + %mp_dsmeta(work.sashelp,outds=work.mymeta) + proc print data=work.mymeta; + run; + + @param libds The library.dataset to export the metadata for + @param outds= (work.dsmeta) The output table to contain the metadata + +

Related Files

+ @li mp_dsmeta.test.sas + +**/ + +%macro mp_dsmeta(libds,outds=work.dsmeta); + +%local ds1 ds2; +data;run; %let ds1=&syslast; +data;run; %let ds2=&syslast; + +/* setup the ODS capture */ +ods output attributes=&ds1 enginehost=&ds2; + +/* export the metadata */ +proc contents data=&libds; +run; + +/* load it into a single table */ +data &outds (keep=ods_table name value); + length ods_table $10 name label2 label1 label $100 + value cvalue cvalue1 cvalue2 $1000 + nvalue nvalue1 nvalue2 8; + if _n_=1 then call missing (of _all_); + * putlog (_all_)(=); + set &ds1 (in=atrs) &ds2 (in=eng); + if atrs then do; + ods_table='ATTRIBUTES'; + name=coalescec(label1,label); + value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.)); + output; + if label2 ne '' then do; + name=label2; + value=coalescec(cvalue2,put(nvalue2,best.)); + output; + end; + end; + else if eng then do; + ods_table='ENGINEHOST'; + name=coalescec(label1,label); + value=coalescec(cvalue1,cvalue,put(coalesce(nvalue1,nvalue),best.)); + output; + end; +run; + +proc sql; +drop table &ds1, &ds2; + +%mend mp_dsmeta; + +/** @file @brief Checks an input filter table for validity @details Performs checks on the input table to ensure it arrives in the diff --git a/base/mp_dsmeta.sas b/base/mp_dsmeta.sas index 2c3c54e..33206f1 100644 --- a/base/mp_dsmeta.sas +++ b/base/mp_dsmeta.sas @@ -67,7 +67,8 @@ run; /* load it into a single table */ data &outds (keep=ods_table name value); - length ods_table $10 name label1 label $100 value cvalue1 cvalue $1000 + length ods_table $10 name label2 label1 label $100 + value cvalue cvalue1 cvalue2 $1000 nvalue nvalue1 nvalue2 8; if _n_=1 then call missing (of _all_); * putlog (_all_)(=); diff --git a/sasjs/sasjsconfig.json b/sasjs/sasjsconfig.json index bcf3911..ae89c0e 100644 --- a/sasjs/sasjsconfig.json +++ b/sasjs/sasjsconfig.json @@ -107,4 +107,4 @@ "contextName": "SAS Job Execution compute context" } ] -} +} \ No newline at end of file diff --git a/tests/base/mp_dsmeta.test.sas b/tests/base/mp_dsmeta.test.sas index 052b19e..7a9fc6c 100644 --- a/tests/base/mp_dsmeta.test.sas +++ b/tests/base/mp_dsmeta.test.sas @@ -18,7 +18,7 @@ run; %mp_assertscope(COMPARE) proc sql noprint; -select count(*) into nobs from work.test; +select count(*) into: nobs from work.test; select count(distinct ods_table) into: tnobs from work.test; %mp_assert( From ca045e3ebf4092cbc5c42238fceea6284959bc68 Mon Sep 17 00:00:00 2001 From: munja Date: Sun, 21 Aug 2022 19:27:52 +0100 Subject: [PATCH 3/7] chore(docs): typo --- all.sas | 2 +- base/mp_dsmeta.sas | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/all.sas b/all.sas index f885910..0007767 100644 --- a/all.sas +++ b/all.sas @@ -5867,7 +5867,7 @@ options varlenchk=&optval; Example usage: - %mp_dsmeta(work.sashelp,outds=work.mymeta) + %mp_dsmeta(sashelp.class,outds=work.mymeta) proc print data=work.mymeta; run; diff --git a/base/mp_dsmeta.sas b/base/mp_dsmeta.sas index 33206f1..8557ac3 100644 --- a/base/mp_dsmeta.sas +++ b/base/mp_dsmeta.sas @@ -40,7 +40,7 @@ Example usage: - %mp_dsmeta(work.sashelp,outds=work.mymeta) + %mp_dsmeta(sashelp.class,outds=work.mymeta) proc print data=work.mymeta; run; From 093dc87aadab649b6589d92096cf4f5f957d5d41 Mon Sep 17 00:00:00 2001 From: munja Date: Sun, 21 Aug 2022 19:55:02 +0100 Subject: [PATCH 4/7] chore(docs): crediting louise --- all.sas | 5 +++++ base/mp_dsmeta.sas | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/all.sas b/all.sas index 0007767..6f3addd 100644 --- a/all.sas +++ b/all.sas @@ -5871,6 +5871,11 @@ options varlenchk=&optval; proc print data=work.mymeta; run; + For more details on creating datasets from PROC CONTENTS check out this + excellent [paper]( + https://support.sas.com/resources/papers/proceedings14/1549-2014.pdf) by + [Louise Hadden](https://www.linkedin.com/in/louisehadden/). + @param libds The library.dataset to export the metadata for @param outds= (work.dsmeta) The output table to contain the metadata diff --git a/base/mp_dsmeta.sas b/base/mp_dsmeta.sas index 8557ac3..f472242 100644 --- a/base/mp_dsmeta.sas +++ b/base/mp_dsmeta.sas @@ -44,6 +44,11 @@ proc print data=work.mymeta; run; + For more details on creating datasets from PROC CONTENTS check out this + excellent [paper]( + https://support.sas.com/resources/papers/proceedings14/1549-2014.pdf) by + [Louise Hadden](https://www.linkedin.com/in/louisehadden/). + @param libds The library.dataset to export the metadata for @param outds= (work.dsmeta) The output table to contain the metadata From 36faa194a8c2b11e05447c31daee5486396ee21c Mon Sep 17 00:00:00 2001 From: munja Date: Sun, 21 Aug 2022 21:15:24 +0100 Subject: [PATCH 5/7] chore(docs): more related files in mp_dsmeta.sas --- all.sas | 6 ++++++ base/mp_dsmeta.sas | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/all.sas b/all.sas index 6f3addd..9d3010f 100644 --- a/all.sas +++ b/all.sas @@ -5881,6 +5881,12 @@ options varlenchk=&optval;

Related Files

@li mp_dsmeta.test.sas + @li mp_getcols.sas + @li mp_getdbml.sas + @li mp_getddl.sas + @li mp_getformats.sas + @li mp_getpk.sas + @li mp_guesspk.sas **/ diff --git a/base/mp_dsmeta.sas b/base/mp_dsmeta.sas index f472242..4c16cc6 100644 --- a/base/mp_dsmeta.sas +++ b/base/mp_dsmeta.sas @@ -54,6 +54,12 @@

Related Files

@li mp_dsmeta.test.sas + @li mp_getcols.sas + @li mp_getdbml.sas + @li mp_getddl.sas + @li mp_getformats.sas + @li mp_getpk.sas + @li mp_guesspk.sas **/ From 6456c2f6e2437f654cd6c321f5e4c984e114763e Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Mon, 22 Aug 2022 13:14:20 +0000 Subject: [PATCH 6/7] fix: support for SUB (1A) hex char in DATASTEP generated JSON. Closes #307 --- all.sas | 60 ++++++++++++++++++---------------- base/mp_jsonout.sas | 15 +++++---- meta/mm_createwebservice.sas | 15 +++++---- server/ms_createwebservice.sas | 15 +++++---- viya/mv_createwebservice.sas | 15 +++++---- 5 files changed, 64 insertions(+), 56 deletions(-) diff --git a/all.sas b/all.sas index 9d3010f..3750893 100644 --- a/all.sas +++ b/all.sas @@ -8903,7 +8903,7 @@ options call symputx(cats('label',_n_),coalescec(label,name),'l'); /* overwritten when fmt=Y and a custom format exists in catalog */ if typelong='num' then call symputx(cats('fmtlen',_n_),200,'l'); - else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+3)*1.5)),'l'); + else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+10)*1.5)),'l'); run; %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); @@ -8959,8 +8959,8 @@ options %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); proc sql noprint; create table &tmpds1 as - select cats(libname,'.',memname) as fmtcat, - fmtname + select cats(libname,'.',memname) as FMTCAT, + FMTNAME from dictionary.formats where fmttype='F' and libname is not null and fmtname in (select format from &colinfo where format is not null) @@ -8985,7 +8985,7 @@ options proc sql; create table &tmpds4 as - select a.*, b.length as maxw + select a.*, b.length as MAXW from &colinfo a left join &tmpds2 b on cats(a.format)=cats(upcase(b.fmtname)) @@ -8996,7 +8996,7 @@ options call symputx( cats('fmtlen',_n_), /* vars need extra padding due to JSON escaping of special chars */ - min(32767,ceil((max(length,maxw)+3)*1.5)) + min(32767,ceil((max(length,maxw)+10)*1.5)) ,'l' ); run; @@ -9071,7 +9071,7 @@ options format _numeric_ bart.; %do i=1 %to &numcols; %if &&typelong&i=char or &fmt=Y %then %do; - if findc(&&name&i,'"\'!!'0A0D09000E0F01021011'x) then do; + if findc(&&name&i,'"\'!!'0A0D09000E0F010210111A'x) then do; &&name&i='"'!!trim( prxchange('s/"/\\"/',-1, /* double quote */ prxchange('s/\x0A/\n/',-1, /* new line */ @@ -9084,8 +9084,9 @@ options prxchange('s/\x02/\\u0002/',-1, /* STX */ prxchange('s/\x10/\\u0010/',-1, /* DLE */ prxchange('s/\x11/\\u0011/',-1, /* DC1 */ + prxchange('s/\x1A/\\u001A/',-1, /* SUB */ prxchange('s/\\/\\\\/',-1,&&name&i) - ))))))))))))!!'"'; + )))))))))))))!!'"'; end; else &&name&i=quote(cats(&&name&i)); %end; @@ -15398,7 +15399,7 @@ data _null_; put ' call symputx(cats(''label'',_n_),coalescec(label,name),''l''); '; put ' /* overwritten when fmt=Y and a custom format exists in catalog */ '; put ' if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); '; - put ' else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); '; + put ' else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); '; put ' run; '; put ' '; put ' %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; @@ -15454,8 +15455,8 @@ data _null_; put ' %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; put ' proc sql noprint; '; put ' create table &tmpds1 as '; - put ' select cats(libname,''.'',memname) as fmtcat, '; - put ' fmtname '; + put ' select cats(libname,''.'',memname) as FMTCAT, '; + put ' FMTNAME '; put ' from dictionary.formats '; put ' where fmttype=''F'' and libname is not null '; put ' and fmtname in (select format from &colinfo where format is not null) '; @@ -15480,7 +15481,7 @@ data _null_; put ' '; put ' proc sql; '; put ' create table &tmpds4 as '; - put ' select a.*, b.length as maxw '; + put ' select a.*, b.length as MAXW '; put ' from &colinfo a '; put ' left join &tmpds2 b '; put ' on cats(a.format)=cats(upcase(b.fmtname)) '; @@ -15491,7 +15492,7 @@ data _null_; put ' call symputx( '; put ' cats(''fmtlen'',_n_), '; put ' /* vars need extra padding due to JSON escaping of special chars */ '; - put ' min(32767,ceil((max(length,maxw)+3)*1.5)) '; + put ' min(32767,ceil((max(length,maxw)+10)*1.5)) '; put ' ,''l'' '; put ' ); '; put ' run; '; @@ -15566,7 +15567,7 @@ data _null_; put ' format _numeric_ bart.; '; put ' %do i=1 %to &numcols; '; put ' %if &&typelong&i=char or &fmt=Y %then %do; '; - put ' if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; '; + put ' if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; '; put ' &&name&i=''"''!!trim( '; put ' prxchange(''s/"/\\"/'',-1, /* double quote */ '; put ' prxchange(''s/\x0A/\n/'',-1, /* new line */ '; @@ -15579,8 +15580,9 @@ data _null_; put ' prxchange(''s/\x02/\\u0002/'',-1, /* STX */ '; put ' prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ '; put ' prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ '; + put ' prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ '; put ' prxchange(''s/\\/\\\\/'',-1,&&name&i) '; - put ' ))))))))))))!!''"''; '; + put ' )))))))))))))!!''"''; '; put ' end; '; put ' else &&name&i=quote(cats(&&name&i)); '; put ' %end; '; @@ -20419,7 +20421,7 @@ data _null_; put ' call symputx(cats(''label'',_n_),coalescec(label,name),''l''); '; put ' /* overwritten when fmt=Y and a custom format exists in catalog */ '; put ' if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); '; - put ' else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); '; + put ' else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); '; put ' run; '; put ' '; put ' %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; @@ -20475,8 +20477,8 @@ data _null_; put ' %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; put ' proc sql noprint; '; put ' create table &tmpds1 as '; - put ' select cats(libname,''.'',memname) as fmtcat, '; - put ' fmtname '; + put ' select cats(libname,''.'',memname) as FMTCAT, '; + put ' FMTNAME '; put ' from dictionary.formats '; put ' where fmttype=''F'' and libname is not null '; put ' and fmtname in (select format from &colinfo where format is not null) '; @@ -20501,7 +20503,7 @@ data _null_; put ' '; put ' proc sql; '; put ' create table &tmpds4 as '; - put ' select a.*, b.length as maxw '; + put ' select a.*, b.length as MAXW '; put ' from &colinfo a '; put ' left join &tmpds2 b '; put ' on cats(a.format)=cats(upcase(b.fmtname)) '; @@ -20512,7 +20514,7 @@ data _null_; put ' call symputx( '; put ' cats(''fmtlen'',_n_), '; put ' /* vars need extra padding due to JSON escaping of special chars */ '; - put ' min(32767,ceil((max(length,maxw)+3)*1.5)) '; + put ' min(32767,ceil((max(length,maxw)+10)*1.5)) '; put ' ,''l'' '; put ' ); '; put ' run; '; @@ -20587,7 +20589,7 @@ data _null_; put ' format _numeric_ bart.; '; put ' %do i=1 %to &numcols; '; put ' %if &&typelong&i=char or &fmt=Y %then %do; '; - put ' if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; '; + put ' if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; '; put ' &&name&i=''"''!!trim( '; put ' prxchange(''s/"/\\"/'',-1, /* double quote */ '; put ' prxchange(''s/\x0A/\n/'',-1, /* new line */ '; @@ -20600,8 +20602,9 @@ data _null_; put ' prxchange(''s/\x02/\\u0002/'',-1, /* STX */ '; put ' prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ '; put ' prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ '; + put ' prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ '; put ' prxchange(''s/\\/\\\\/'',-1,&&name&i) '; - put ' ))))))))))))!!''"''; '; + put ' )))))))))))))!!''"''; '; put ' end; '; put ' else &&name&i=quote(cats(&&name&i)); '; put ' %end; '; @@ -22882,7 +22885,7 @@ data _null_; put ' call symputx(cats(''label'',_n_),coalescec(label,name),''l''); '; put ' /* overwritten when fmt=Y and a custom format exists in catalog */ '; put ' if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); '; - put ' else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); '; + put ' else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); '; put ' run; '; put ' '; put ' %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; @@ -22938,8 +22941,8 @@ data _null_; put ' %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; put ' proc sql noprint; '; put ' create table &tmpds1 as '; - put ' select cats(libname,''.'',memname) as fmtcat, '; - put ' fmtname '; + put ' select cats(libname,''.'',memname) as FMTCAT, '; + put ' FMTNAME '; put ' from dictionary.formats '; put ' where fmttype=''F'' and libname is not null '; put ' and fmtname in (select format from &colinfo where format is not null) '; @@ -22964,7 +22967,7 @@ data _null_; put ' '; put ' proc sql; '; put ' create table &tmpds4 as '; - put ' select a.*, b.length as maxw '; + put ' select a.*, b.length as MAXW '; put ' from &colinfo a '; put ' left join &tmpds2 b '; put ' on cats(a.format)=cats(upcase(b.fmtname)) '; @@ -22975,7 +22978,7 @@ data _null_; put ' call symputx( '; put ' cats(''fmtlen'',_n_), '; put ' /* vars need extra padding due to JSON escaping of special chars */ '; - put ' min(32767,ceil((max(length,maxw)+3)*1.5)) '; + put ' min(32767,ceil((max(length,maxw)+10)*1.5)) '; put ' ,''l'' '; put ' ); '; put ' run; '; @@ -23050,7 +23053,7 @@ data _null_; put ' format _numeric_ bart.; '; put ' %do i=1 %to &numcols; '; put ' %if &&typelong&i=char or &fmt=Y %then %do; '; - put ' if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; '; + put ' if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; '; put ' &&name&i=''"''!!trim( '; put ' prxchange(''s/"/\\"/'',-1, /* double quote */ '; put ' prxchange(''s/\x0A/\n/'',-1, /* new line */ '; @@ -23063,8 +23066,9 @@ data _null_; put ' prxchange(''s/\x02/\\u0002/'',-1, /* STX */ '; put ' prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ '; put ' prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ '; + put ' prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ '; put ' prxchange(''s/\\/\\\\/'',-1,&&name&i) '; - put ' ))))))))))))!!''"''; '; + put ' )))))))))))))!!''"''; '; put ' end; '; put ' else &&name&i=quote(cats(&&name&i)); '; put ' %end; '; diff --git a/base/mp_jsonout.sas b/base/mp_jsonout.sas index 45c38f3..f4bb74b 100644 --- a/base/mp_jsonout.sas +++ b/base/mp_jsonout.sas @@ -146,7 +146,7 @@ call symputx(cats('label',_n_),coalescec(label,name),'l'); /* overwritten when fmt=Y and a custom format exists in catalog */ if typelong='num' then call symputx(cats('fmtlen',_n_),200,'l'); - else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+3)*1.5)),'l'); + else call symputx(cats('fmtlen',_n_),min(32767,ceil((length+10)*1.5)),'l'); run; %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); @@ -202,8 +202,8 @@ %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); proc sql noprint; create table &tmpds1 as - select cats(libname,'.',memname) as fmtcat, - fmtname + select cats(libname,'.',memname) as FMTCAT, + FMTNAME from dictionary.formats where fmttype='F' and libname is not null and fmtname in (select format from &colinfo where format is not null) @@ -228,7 +228,7 @@ proc sql; create table &tmpds4 as - select a.*, b.length as maxw + select a.*, b.length as MAXW from &colinfo a left join &tmpds2 b on cats(a.format)=cats(upcase(b.fmtname)) @@ -239,7 +239,7 @@ call symputx( cats('fmtlen',_n_), /* vars need extra padding due to JSON escaping of special chars */ - min(32767,ceil((max(length,maxw)+3)*1.5)) + min(32767,ceil((max(length,maxw)+10)*1.5)) ,'l' ); run; @@ -314,7 +314,7 @@ format _numeric_ bart.; %do i=1 %to &numcols; %if &&typelong&i=char or &fmt=Y %then %do; - if findc(&&name&i,'"\'!!'0A0D09000E0F01021011'x) then do; + if findc(&&name&i,'"\'!!'0A0D09000E0F010210111A'x) then do; &&name&i='"'!!trim( prxchange('s/"/\\"/',-1, /* double quote */ prxchange('s/\x0A/\n/',-1, /* new line */ @@ -327,8 +327,9 @@ prxchange('s/\x02/\\u0002/',-1, /* STX */ prxchange('s/\x10/\\u0010/',-1, /* DLE */ prxchange('s/\x11/\\u0011/',-1, /* DC1 */ + prxchange('s/\x1A/\\u001A/',-1, /* SUB */ prxchange('s/\\/\\\\/',-1,&&name&i) - ))))))))))))!!'"'; + )))))))))))))!!'"'; end; else &&name&i=quote(cats(&&name&i)); %end; diff --git a/meta/mm_createwebservice.sas b/meta/mm_createwebservice.sas index 4590686..ba3adf0 100644 --- a/meta/mm_createwebservice.sas +++ b/meta/mm_createwebservice.sas @@ -169,7 +169,7 @@ data _null_; put ' call symputx(cats(''label'',_n_),coalescec(label,name),''l''); '; put ' /* overwritten when fmt=Y and a custom format exists in catalog */ '; put ' if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); '; - put ' else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); '; + put ' else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); '; put ' run; '; put ' '; put ' %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; @@ -225,8 +225,8 @@ data _null_; put ' %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; put ' proc sql noprint; '; put ' create table &tmpds1 as '; - put ' select cats(libname,''.'',memname) as fmtcat, '; - put ' fmtname '; + put ' select cats(libname,''.'',memname) as FMTCAT, '; + put ' FMTNAME '; put ' from dictionary.formats '; put ' where fmttype=''F'' and libname is not null '; put ' and fmtname in (select format from &colinfo where format is not null) '; @@ -251,7 +251,7 @@ data _null_; put ' '; put ' proc sql; '; put ' create table &tmpds4 as '; - put ' select a.*, b.length as maxw '; + put ' select a.*, b.length as MAXW '; put ' from &colinfo a '; put ' left join &tmpds2 b '; put ' on cats(a.format)=cats(upcase(b.fmtname)) '; @@ -262,7 +262,7 @@ data _null_; put ' call symputx( '; put ' cats(''fmtlen'',_n_), '; put ' /* vars need extra padding due to JSON escaping of special chars */ '; - put ' min(32767,ceil((max(length,maxw)+3)*1.5)) '; + put ' min(32767,ceil((max(length,maxw)+10)*1.5)) '; put ' ,''l'' '; put ' ); '; put ' run; '; @@ -337,7 +337,7 @@ data _null_; put ' format _numeric_ bart.; '; put ' %do i=1 %to &numcols; '; put ' %if &&typelong&i=char or &fmt=Y %then %do; '; - put ' if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; '; + put ' if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; '; put ' &&name&i=''"''!!trim( '; put ' prxchange(''s/"/\\"/'',-1, /* double quote */ '; put ' prxchange(''s/\x0A/\n/'',-1, /* new line */ '; @@ -350,8 +350,9 @@ data _null_; put ' prxchange(''s/\x02/\\u0002/'',-1, /* STX */ '; put ' prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ '; put ' prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ '; + put ' prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ '; put ' prxchange(''s/\\/\\\\/'',-1,&&name&i) '; - put ' ))))))))))))!!''"''; '; + put ' )))))))))))))!!''"''; '; put ' end; '; put ' else &&name&i=quote(cats(&&name&i)); '; put ' %end; '; diff --git a/server/ms_createwebservice.sas b/server/ms_createwebservice.sas index f864d45..a8dcc44 100644 --- a/server/ms_createwebservice.sas +++ b/server/ms_createwebservice.sas @@ -170,7 +170,7 @@ data _null_; put ' call symputx(cats(''label'',_n_),coalescec(label,name),''l''); '; put ' /* overwritten when fmt=Y and a custom format exists in catalog */ '; put ' if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); '; - put ' else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); '; + put ' else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); '; put ' run; '; put ' '; put ' %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; @@ -226,8 +226,8 @@ data _null_; put ' %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; put ' proc sql noprint; '; put ' create table &tmpds1 as '; - put ' select cats(libname,''.'',memname) as fmtcat, '; - put ' fmtname '; + put ' select cats(libname,''.'',memname) as FMTCAT, '; + put ' FMTNAME '; put ' from dictionary.formats '; put ' where fmttype=''F'' and libname is not null '; put ' and fmtname in (select format from &colinfo where format is not null) '; @@ -252,7 +252,7 @@ data _null_; put ' '; put ' proc sql; '; put ' create table &tmpds4 as '; - put ' select a.*, b.length as maxw '; + put ' select a.*, b.length as MAXW '; put ' from &colinfo a '; put ' left join &tmpds2 b '; put ' on cats(a.format)=cats(upcase(b.fmtname)) '; @@ -263,7 +263,7 @@ data _null_; put ' call symputx( '; put ' cats(''fmtlen'',_n_), '; put ' /* vars need extra padding due to JSON escaping of special chars */ '; - put ' min(32767,ceil((max(length,maxw)+3)*1.5)) '; + put ' min(32767,ceil((max(length,maxw)+10)*1.5)) '; put ' ,''l'' '; put ' ); '; put ' run; '; @@ -338,7 +338,7 @@ data _null_; put ' format _numeric_ bart.; '; put ' %do i=1 %to &numcols; '; put ' %if &&typelong&i=char or &fmt=Y %then %do; '; - put ' if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; '; + put ' if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; '; put ' &&name&i=''"''!!trim( '; put ' prxchange(''s/"/\\"/'',-1, /* double quote */ '; put ' prxchange(''s/\x0A/\n/'',-1, /* new line */ '; @@ -351,8 +351,9 @@ data _null_; put ' prxchange(''s/\x02/\\u0002/'',-1, /* STX */ '; put ' prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ '; put ' prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ '; + put ' prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ '; put ' prxchange(''s/\\/\\\\/'',-1,&&name&i) '; - put ' ))))))))))))!!''"''; '; + put ' )))))))))))))!!''"''; '; put ' end; '; put ' else &&name&i=quote(cats(&&name&i)); '; put ' %end; '; diff --git a/viya/mv_createwebservice.sas b/viya/mv_createwebservice.sas index 8405226..e7573ec 100644 --- a/viya/mv_createwebservice.sas +++ b/viya/mv_createwebservice.sas @@ -312,7 +312,7 @@ data _null_; put ' call symputx(cats(''label'',_n_),coalescec(label,name),''l''); '; put ' /* overwritten when fmt=Y and a custom format exists in catalog */ '; put ' if typelong=''num'' then call symputx(cats(''fmtlen'',_n_),200,''l''); '; - put ' else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+3)*1.5)),''l''); '; + put ' else call symputx(cats(''fmtlen'',_n_),min(32767,ceil((length+10)*1.5)),''l''); '; put ' run; '; put ' '; put ' %let tempds=%substr(_%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; @@ -368,8 +368,8 @@ data _null_; put ' %let tmpds4=%substr(col%sysfunc(compress(%sysfunc(uuidgen()),-)),1,32); '; put ' proc sql noprint; '; put ' create table &tmpds1 as '; - put ' select cats(libname,''.'',memname) as fmtcat, '; - put ' fmtname '; + put ' select cats(libname,''.'',memname) as FMTCAT, '; + put ' FMTNAME '; put ' from dictionary.formats '; put ' where fmttype=''F'' and libname is not null '; put ' and fmtname in (select format from &colinfo where format is not null) '; @@ -394,7 +394,7 @@ data _null_; put ' '; put ' proc sql; '; put ' create table &tmpds4 as '; - put ' select a.*, b.length as maxw '; + put ' select a.*, b.length as MAXW '; put ' from &colinfo a '; put ' left join &tmpds2 b '; put ' on cats(a.format)=cats(upcase(b.fmtname)) '; @@ -405,7 +405,7 @@ data _null_; put ' call symputx( '; put ' cats(''fmtlen'',_n_), '; put ' /* vars need extra padding due to JSON escaping of special chars */ '; - put ' min(32767,ceil((max(length,maxw)+3)*1.5)) '; + put ' min(32767,ceil((max(length,maxw)+10)*1.5)) '; put ' ,''l'' '; put ' ); '; put ' run; '; @@ -480,7 +480,7 @@ data _null_; put ' format _numeric_ bart.; '; put ' %do i=1 %to &numcols; '; put ' %if &&typelong&i=char or &fmt=Y %then %do; '; - put ' if findc(&&name&i,''"\''!!''0A0D09000E0F01021011''x) then do; '; + put ' if findc(&&name&i,''"\''!!''0A0D09000E0F010210111A''x) then do; '; put ' &&name&i=''"''!!trim( '; put ' prxchange(''s/"/\\"/'',-1, /* double quote */ '; put ' prxchange(''s/\x0A/\n/'',-1, /* new line */ '; @@ -493,8 +493,9 @@ data _null_; put ' prxchange(''s/\x02/\\u0002/'',-1, /* STX */ '; put ' prxchange(''s/\x10/\\u0010/'',-1, /* DLE */ '; put ' prxchange(''s/\x11/\\u0011/'',-1, /* DC1 */ '; + put ' prxchange(''s/\x1A/\\u001A/'',-1, /* SUB */ '; put ' prxchange(''s/\\/\\\\/'',-1,&&name&i) '; - put ' ))))))))))))!!''"''; '; + put ' )))))))))))))!!''"''; '; put ' end; '; put ' else &&name&i=quote(cats(&&name&i)); '; put ' %end; '; From b75369b28d27e352f3f9a4863cde14571407f3f7 Mon Sep 17 00:00:00 2001 From: munja Date: Tue, 23 Aug 2022 16:00:42 +0100 Subject: [PATCH 7/7] fix: pgm uninitialised in mm_getstpinfo --- all.sas | 2 +- meta/mm_getstpinfo.sas | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/all.sas b/all.sas index 3750893..87ca8a9 100644 --- a/all.sas +++ b/all.sas @@ -17831,7 +17831,7 @@ data &outds; rc5=metadata_getattr(tsuri,"Name",servercontext); end; else do; - put "%str(ERR)OR: could not find " pgm; + put "%str(ERR)OR: could not find " path; put (_all_)(=); end; &md.put (_all_)(=); diff --git a/meta/mm_getstpinfo.sas b/meta/mm_getstpinfo.sas index 5a2c9e0..d9b0f2d 100644 --- a/meta/mm_getstpinfo.sas +++ b/meta/mm_getstpinfo.sas @@ -58,7 +58,7 @@ data &outds; rc5=metadata_getattr(tsuri,"Name",servercontext); end; else do; - put "%str(ERR)OR: could not find " pgm; + put "%str(ERR)OR: could not find " path; put (_all_)(=); end; &md.put (_all_)(=);