Merge pull request #77 from sasjs/dictfix

fix: ensuring upcase comparisons for dictionary tables

all.sas

@@ -1841,6 +1841,7 @@ Usage:
       put ",""SYSERRORTEXT"" : " syserrortext;
       put ",""SYSHOSTNAME"" : ""&syshostname"" ";
       put ",""SYSJOBID"" : ""&sysjobid"" ";
+      put ",""SYSSCPL"" : ""&sysscpl"" ";
       put ",""SYSSITE"" : ""&syssite"" ";
       sysvlong=quote(trim(symget('sysvlong')));
       put ',"SYSVLONG" : ' sysvlong;
@@ -1855,7 +1856,7 @@ Usage:
 
     %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
       data _null_;
-        putlog 'stpsrvset program error and syscc';
+        putlog 'stpsrvset program err and syscc';
         rc=stpsrvset('program error', 0);
         call symputx("syscc",0,"g");
       run;
@@ -1901,6 +1902,62 @@ Usage:
 %mend mp_abort;
 
 /** @endcond *//**
+  @file
+  @brief Append (concatenate) two or more files.
+  @details Will append one more more `appendrefs` (filerefs) to a `baseref`.
+  Uses a binary mechanism, so will work with any file type.  For that reason -
+  use with care!   And supply your own trailing carriage returns in each file..
+
+  Usage:
+
+        filename tmp1 temp;
+        filename tmp2 temp;
+        filename tmp3 temp;
+        data _null_; file tmp1; put 'base file';
+        data _null_; file tmp2; put 'append1';
+        data _null_; file tmp3; put 'append2';
+        run;
+        %mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
+
+
+  @param [in] baseref= Fileref of the base file (should exist)
+  @param [in] appendrefs= One or more filerefs to be appended to the base
+    fileref.  Space separated.
+
+  @version 9.2
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+
+
+**/
+
+%macro mp_appendfile(
+  baseref=0,
+  appendrefs=0
+)/*/STORE SOURCE*/;
+
+%mp_abort(iftrue= (&baseref=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Baseref NOT specified!)
+)
+%mp_abort(iftrue= (&appendrefs=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Appendrefs NOT specified!)
+)
+
+%local i;
+%do i=1 %to %sysfunc(countw(&appendrefs));
+  %mp_abort(iftrue= (&syscc>0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc)
+  )
+  %mp_binarycopy(inref=%scan(&appendrefs,&i), outref=&baseref, mode=APPEND)
+%end;
+
+%mend mp_appendfile;/**
   @file
   @brief Generic assertion
   @details Useful in the context of writing sasjs tests.  The results of the
@@ -2131,6 +2188,7 @@ Usage:
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
+  @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
 
@@ -2216,13 +2274,25 @@ Usage:
   select count(*) into: orig from &lib..&ds;
   quit;
 
-  %local notfound;
-  proc sql outobs=10 noprint;
-  select distinct &col  into: notfound separated by ' '
+  %local notfound tmp1 tmp2;
+  %let tmp1=%mf_getuniquename();
+  %let tmp2=%mf_getuniquename();
+
+  /* this is a bit convoluted - but using sql outobs=10 throws warnings */
+  proc sql noprint;
+  create view &tmp1 as
+    select distinct &col
     from &lib..&ds
     where &col not in (
       select &ccol from &clib..&cds
     );
+  data &tmp2;
+    set &tmp1;
+    if _n_>10 then stop;
+  run;
+  proc sql;
+  select distinct &col  into: notfound separated by ' ' from &tmp2;
+
 
   %mp_abort(iftrue= (&syscc ne 0)
     ,mac=&sysmacroname
@@ -3301,8 +3371,8 @@ run;
 %let nvars=0;
 proc sql noprint;
 select count(*) into: nvars from dictionary.columns
-  where libname="%scan(%upcase(&base_ds),1)"
-    and memname="%scan(%upcase(&base_ds),2)";
+  where upcase(libname)="%scan(%upcase(&base_ds),1)"
+    and upcase(memname)="%scan(%upcase(&base_ds),2)";
 %if &nvars=0 %then %do;
   %put %str(WARN)ING: Dataset &base_ds has no variables, will not be converted.;
   %return;
@@ -3358,8 +3428,8 @@ proc sql
 reset outobs=max;
 create table datalines1 as
   select name,type,length,varnum,format,label from dictionary.columns
-  where libname="%upcase(%scan(&base_ds,1))"
-    and memname="%upcase(%scan(&base_ds,2))";
+  where upcase(libname)="%upcase(%scan(&base_ds,1))"
+    and upcase(memname)="%upcase(%scan(&base_ds,2))";
 
 /**
   Due to long decimals cannot use best. format
@@ -4265,8 +4335,6 @@ filename &fref1 clear;
 
   @version 9.2
   @author Allan Bowe
-  @copyright Macro People Ltd - this is a licensed product and
-    NOT FOR RESALE OR DISTRIBUTION.
 
 **/
 
@@ -4344,21 +4412,21 @@ run;
 /* must use SQL as proc datasets does not support length changes */
 proc sql noprint;
 create table &outds as
-  select a.TABLE_CATALOG as libref
-    ,a.TABLE_NAME
+  select upcase(a.TABLE_CATALOG) as libref
+    ,upcase(a.TABLE_NAME) as TABLE_NAME
     ,a.constraint_type
     ,a.constraint_name
     ,b.column_name
   from dictionary.TABLE_CONSTRAINTS a
   left join dictionary.constraint_column_usage  b
-  on a.TABLE_CATALOG=b.TABLE_CATALOG
-    and a.TABLE_NAME=b.TABLE_NAME
+  on upcase(a.TABLE_CATALOG)=upcase(b.TABLE_CATALOG)
+    and upcase(a.TABLE_NAME)=upcase(b.TABLE_NAME)
     and a.constraint_name=b.constraint_name
-  where a.TABLE_CATALOG="&lib"
-    and b.TABLE_CATALOG="&lib"
+  where upcase(a.TABLE_CATALOG)="&lib"
+    and upcase(b.TABLE_CATALOG)="&lib"
   %if "&ds" ne "" %then %do;
-    and a.TABLE_NAME="&ds"
-    and b.TABLE_NAME="&ds"
+    and upcase(a.TABLE_NAME)="&ds"
+    and upcase(b.TABLE_NAME)="&ds"
   %end;
   ;
 
@@ -4909,7 +4977,7 @@ run;
   proc sql noprint;
   select sysvalue into: schemaactual
     from dictionary.libnames
-    where libname="&libref" and engine='SQLSVR';
+    where upcase(libname)="&libref" and engine='SQLSVR';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
 
   %do x=1 %to %sysfunc(countw(&dsnlist));
@@ -5002,7 +5070,7 @@ run;
   proc sql noprint;
   select sysvalue into: schemaactual
     from dictionary.libnames
-    where libname="&libref" and engine='POSTGRES';
+    where upcase(libname)="&libref" and engine='POSTGRES';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
   data _null_;
     file &fref mod;
@@ -6104,14 +6172,14 @@ select distinct lowcase(memname)
   We take the standard definition one step further by embedding the informat
   in the table header row, like so:
 
-      |var1:$|var2:best.|var3:date9.|
+      |var1:$32|var2:best.|var3:date9.|
       |---|---|---|
       |some text|42|01JAN1960|
       |blah|1|31DEC1999|
 
   Which resolves to:
 
-  |var1:$|var2:best.|var3:date9.|
+  |var1:$32|var2:best.|var3:date9.|
   |---|---|---|
   |some text|42|01JAN1960|
   |blah|1|31DEC1999|
@@ -10060,6 +10128,7 @@ data _null_;
   put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
+  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
@@ -13555,6 +13624,7 @@ run;
     put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSJOBID"" : ""&sysjobid"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;
@@ -15026,6 +15096,7 @@ data _null_;
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
+  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';
@@ -18865,6 +18936,7 @@ filename &fref1 clear;
     put ",""SYSCC"" : ""&syscc"" ";
     put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;

base/mp_abort.sas

@@ -193,6 +193,7 @@
       put ",""SYSERRORTEXT"" : " syserrortext;
       put ",""SYSHOSTNAME"" : ""&syshostname"" ";
       put ",""SYSJOBID"" : ""&sysjobid"" ";
+      put ",""SYSSCPL"" : ""&sysscpl"" ";
       put ",""SYSSITE"" : ""&syssite"" ";
       sysvlong=quote(trim(symget('sysvlong')));
       put ',"SYSVLONG" : ' sysvlong;
@@ -207,7 +208,7 @@
 
     %if "&sysprocessmode " = "SAS Stored Process Server " %then %do;
       data _null_;
-        putlog 'stpsrvset program error and syscc';
+        putlog 'stpsrvset program err and syscc';
         rc=stpsrvset('program error', 0);
         call symputx("syscc",0,"g");
       run;

base/mp_appendfile.sas

@@ -0,0 +1,57 @@
+/**
+  @file
+  @brief Append (concatenate) two or more files.
+  @details Will append one more more `appendrefs` (filerefs) to a `baseref`.
+  Uses a binary mechanism, so will work with any file type.  For that reason -
+  use with care!   And supply your own trailing carriage returns in each file..
+
+  Usage:
+
+        filename tmp1 temp;
+        filename tmp2 temp;
+        filename tmp3 temp;
+        data _null_; file tmp1; put 'base file';
+        data _null_; file tmp2; put 'append1';
+        data _null_; file tmp3; put 'append2';
+        run;
+        %mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
+
+
+  @param [in] baseref= Fileref of the base file (should exist)
+  @param [in] appendrefs= One or more filerefs to be appended to the base
+    fileref.  Space separated.
+
+  @version 9.2
+  @author Allan Bowe, source: https://github.com/sasjs/core
+
+  <h4> SAS Macros </h4>
+  @li mp_abort.sas
+  @li mp_binarycopy.sas
+
+
+**/
+
+%macro mp_appendfile(
+  baseref=0,
+  appendrefs=0
+)/*/STORE SOURCE*/;
+
+%mp_abort(iftrue= (&baseref=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Baseref NOT specified!)
+)
+%mp_abort(iftrue= (&appendrefs=0)
+  ,mac=&sysmacroname
+  ,msg=%str(Appendrefs NOT specified!)
+)
+
+%local i;
+%do i=1 %to %sysfunc(countw(&appendrefs));
+  %mp_abort(iftrue= (&syscc>0)
+    ,mac=&sysmacroname
+    ,msg=%str(syscc=&syscc)
+  )
+  %mp_binarycopy(inref=%scan(&appendrefs,&i), outref=&baseref, mode=APPEND)
+%end;
+
+%mend mp_appendfile;

base/mp_assertcolvals.sas

@@ -30,6 +30,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_existds.sas
+  @li mf_getuniquename.sas
   @li mf_nobs.sas
   @li mp_abort.sas
 
@@ -115,13 +116,25 @@
   select count(*) into: orig from &lib..&ds;
   quit;
 
-  %local notfound;
-  proc sql outobs=10 noprint;
-  select distinct &col  into: notfound separated by ' '
+  %local notfound tmp1 tmp2;
+  %let tmp1=%mf_getuniquename();
+  %let tmp2=%mf_getuniquename();
+
+  /* this is a bit convoluted - but using sql outobs=10 throws warnings */
+  proc sql noprint;
+  create view &tmp1 as
+    select distinct &col
     from &lib..&ds
     where &col not in (
       select &ccol from &clib..&cds
     );
+  data &tmp2;
+    set &tmp1;
+    if _n_>10 then stop;
+  run;
+  proc sql;
+  select distinct &col  into: notfound separated by ' ' from &tmp2;
+
 
   %mp_abort(iftrue= (&syscc ne 0)
     ,mac=&sysmacroname

base/mp_ds2cards.sas

@@ -58,8 +58,8 @@
 %let nvars=0;
 proc sql noprint;
 select count(*) into: nvars from dictionary.columns
-  where libname="%scan(%upcase(&base_ds),1)"
-    and memname="%scan(%upcase(&base_ds),2)";
+  where upcase(libname)="%scan(%upcase(&base_ds),1)"
+    and upcase(memname)="%scan(%upcase(&base_ds),2)";
 %if &nvars=0 %then %do;
   %put %str(WARN)ING: Dataset &base_ds has no variables, will not be converted.;
   %return;
@@ -115,8 +115,8 @@ proc sql
 reset outobs=max;
 create table datalines1 as
   select name,type,length,varnum,format,label from dictionary.columns
-  where libname="%upcase(%scan(&base_ds,1))"
-    and memname="%upcase(%scan(&base_ds,2))";
+  where upcase(libname)="%upcase(%scan(&base_ds,1))"
+    and upcase(memname)="%upcase(%scan(&base_ds,2))";
 
 /**
   Due to long decimals cannot use best. format

base/mp_getcols.sas

@@ -26,8 +26,6 @@
 
   @version 9.2
   @author Allan Bowe
-  @copyright Macro People Ltd - this is a licensed product and
-    NOT FOR RESALE OR DISTRIBUTION.
 
 **/
 

base/mp_getconstraints.sas

@@ -39,21 +39,21 @@
 /* must use SQL as proc datasets does not support length changes */
 proc sql noprint;
 create table &outds as
-  select a.TABLE_CATALOG as libref
-    ,a.TABLE_NAME
+  select upcase(a.TABLE_CATALOG) as libref
+    ,upcase(a.TABLE_NAME) as TABLE_NAME
     ,a.constraint_type
     ,a.constraint_name
     ,b.column_name
   from dictionary.TABLE_CONSTRAINTS a
   left join dictionary.constraint_column_usage  b
-  on a.TABLE_CATALOG=b.TABLE_CATALOG
-    and a.TABLE_NAME=b.TABLE_NAME
+  on upcase(a.TABLE_CATALOG)=upcase(b.TABLE_CATALOG)
+    and upcase(a.TABLE_NAME)=upcase(b.TABLE_NAME)
     and a.constraint_name=b.constraint_name
-  where a.TABLE_CATALOG="&lib"
-    and b.TABLE_CATALOG="&lib"
+  where upcase(a.TABLE_CATALOG)="&lib"
+    and upcase(b.TABLE_CATALOG)="&lib"
   %if "&ds" ne "" %then %do;
-    and a.TABLE_NAME="&ds"
-    and b.TABLE_NAME="&ds"
+    and upcase(a.TABLE_NAME)="&ds"
+    and upcase(b.TABLE_NAME)="&ds"
   %end;
   ;
 

base/mp_getddl.sas

@@ -211,7 +211,7 @@ run;
   proc sql noprint;
   select sysvalue into: schemaactual
     from dictionary.libnames
-    where libname="&libref" and engine='SQLSVR';
+    where upcase(libname)="&libref" and engine='SQLSVR';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
 
   %do x=1 %to %sysfunc(countw(&dsnlist));
@@ -304,7 +304,7 @@ run;
   proc sql noprint;
   select sysvalue into: schemaactual
     from dictionary.libnames
-    where libname="&libref" and engine='POSTGRES';
+    where upcase(libname)="&libref" and engine='POSTGRES';
   %let schema=%sysfunc(coalescec(&schemaactual,&schema,&libref));
   data _null_;
     file &fref mod;

base/mp_mdtablewrite.sas

@@ -14,14 +14,14 @@
   We take the standard definition one step further by embedding the informat
   in the table header row, like so:
 
-      |var1:$|var2:best.|var3:date9.|
+      |var1:$32|var2:best.|var3:date9.|
       |---|---|---|
       |some text|42|01JAN1960|
       |blah|1|31DEC1999|
 
   Which resolves to:
 
-  |var1:$|var2:best.|var3:date9.|
+  |var1:$32|var2:best.|var3:date9.|
   |---|---|---|
   |some text|42|01JAN1960|
   |blah|1|31DEC1999|

meta/mm_createwebservice.sas

@@ -385,6 +385,7 @@ data _null_;
   put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
   put '    put ",""SYSJOBID"" : ""&sysjobid"" "; ';
+  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';

meta/mm_webout.sas

@@ -155,6 +155,7 @@
     put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSJOBID"" : ""&sysjobid"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;

tests/crossplatform/mp_appendfile.test.sas

@@ -0,0 +1,41 @@
+/**
+  @file
+  @brief Testing mp_appendfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_appendfile.sas
+  @li mp_assert.sas
+
+**/
+
+
+filename tmp1 temp;
+filename tmp2 temp;
+filename tmp3 temp;
+data _null_; file tmp1; put 'base file';
+data _null_; file tmp2; put 'append1';
+data _null_; file tmp3; put 'append2';
+run;
+%mp_appendfile(baseref=tmp1, appendrefs=tmp2 tmp3)
+data _null_;
+  infile tmp1;
+  input;
+  put _infile_;
+  call symputx(cats('check',_n_),_infile_);
+run;
+%global check1 check2 check3;
+%mp_assert(
+  iftrue=("&check1"="base file"),
+  desc=Line 1 of file tmp1 is correct,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&check2"="append1"),
+  desc=Line 2 of file tmp1 is correct,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&check3"="append2"),
+  desc=Line 3 of file tmp1 is correct,
+  outds=work.test_results
+)

viya/mv_createwebservice.sas

@@ -590,6 +590,7 @@ data _null_;
   put '    put ",""SYSCC"" : ""&syscc"" "; ';
   put '    put ",""SYSERRORTEXT"" : ""&syserrortext"" "; ';
   put '    put ",""SYSHOSTNAME"" : ""&syshostname"" "; ';
+  put '    put ",""SYSSCPL"" : ""&sysscpl"" "; ';
   put '    put ",""SYSSITE"" : ""&syssite"" "; ';
   put '    sysvlong=quote(trim(symget(''sysvlong''))); ';
   put '    put '',"SYSVLONG" : '' sysvlong; ';

viya/mv_webout.sas

@@ -215,6 +215,7 @@
     put ",""SYSCC"" : ""&syscc"" ";
     put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
+    put ",""SYSSCPL"" : ""&sysscpl"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
     sysvlong=quote(trim(symget('sysvlong')));
     put ',"SYSVLONG" : ' sysvlong;