Merge pull request #419 from sasjs/islib_validation

Islib validation

.github/workflows/main.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Install dependencies
         run: |
@@ -54,7 +54,7 @@ jobs:
           echo "REFRESH_TOKEN=${{secrets.SAS9_4GL_IO_REFRESH_TOKEN}}" >> .env.server
 
       - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v4
+        uses: cycjimmy/semantic-release-action@v6
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/notmain.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Install dependencies
         run: |

.github/workflows/run-tests.yml

@@ -15,7 +15,7 @@ jobs:
         node-version: [lts/iron]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v6
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v2
         with:

all.sas

@@ -1018,8 +1018,9 @@ or %index(&pgm,/tests/testteardown)
   %local dsid vnum rc schema;
   /* in case the parameter is a libref.tablename, pull off just the libref */
   %let libref = %upcase(%scan(&libref, 1, %str(.)));
+  /* sysname can be 'Schema/Owner' or just 'Schema' (eg snowflake) */
   %let dsid=%sysfunc(open(sashelp.vlibnam(where=(
-    libname="%upcase(&libref)" and sysname='Schema/Owner'
+    libname="%upcase(&libref)" and sysname=:'Schema'
   )),i));
   %if (&dsid ^= 0) %then %do;
     %let vnum=%sysfunc(varnum(&dsid,SYSVALUE));
@@ -1214,8 +1215,9 @@ or %index(&pgm,/tests/testteardown)
 %mend mf_getuser;
 /**
   @file
-  @brief Retrieves a value from a dataset.  If no filter supplied, then first
+  @brief Retrieves a value from a dataset. Returned value is fetched from the
-    record is used.
+  'fetchobs=' record (row 1 by default), after applying the optional filter.
+
   @details Be sure to <code>%quote()</code> your where clause.  Example usage:
 
       %put %mf_getvalue(sashelp.class,name,filter=%quote(age=15));
@@ -1230,24 +1232,43 @@ or %index(&pgm,/tests/testteardown)
   @param [in] libds dataset to query
   @param [in] variable the variable which contains the value to return.
   @param [in] filter= (1) contents of where clause
+  @param [in] fetchobs= (1) observation to fetch. NB: Filter applies first.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-%macro mf_getvalue(libds,variable,filter=1
+%macro mf_getvalue(libds,variable,filter=1,fetchobs=1
 )/*/STORE SOURCE*/;
-  %if %mf_getattrn(&libds,NLOBS)>0 %then %do;
+  %local dsid;
-    %local dsid rc &variable;
+
-    %let dsid=%sysfunc(open(&libds(where=(&filter))));
+  %let dsid=%sysfunc(open(&libds(where=(&filter))));
+  %if (&dsid) %then %do;
+    %local rc &variable;
     %syscall set(dsid);
-    %let rc = %sysfunc(fetch(&dsid));
+    %let rc = %sysfunc(fetchobs(&dsid,&fetchobs));
+    %if (&rc ne 0) %then %do;
+      %put NOTE: Problem reading obs &fetchobs from &libds..;
+      %put %sysfunc(sysmsg());
+      /* Coerce an rc value of -1 (read past end of data) to a 4
+      that, in SAS condition code terms, represents the sysmsg
+      w@rning it generates. */
+      %if &rc eq -1 %then %let rc = 4;
+      /* And update SYSCC if the &rc value is higher */
+      %let syscc = %sysfunc(max(&syscc,&rc));
+    %end;
     %let rc = %sysfunc(close(&dsid));
 
     %trim(&&&variable)
 
   %end;
-%mend mf_getvalue;/**
+  %else %do;
+    %put %sysfunc(sysmsg());
+    %let syscc = %sysfunc(max(&syscc,%sysfunc(sysrc())));
+  %end;
+
+%mend mf_getvalue;
+/**
   @file
   @brief Returns number of variables in a dataset
   @details Useful to identify those renagade datasets that have no columns!
@@ -14520,7 +14541,7 @@ alter table &libds modify &var char(&len);
 %mend mp_updatevarlength;
 /**
   @file
-  @brief Used to validate variables in a dataset
+  @brief Used to validate values in a data step
   @details Useful when sanitising inputs, to ensure that they arrive with a
   certain pattern.
   Usage:
@@ -14547,6 +14568,7 @@ alter table &libds modify &var char(&len);
   @param [in] incol The column to be validated
   @param [in] rule The rule to apply.  Current rules:
     @li ISINT - checks if the variable is an integer
+    @li ISLIB - checks if the value is a valid libref (NOT whether it exists)
     @li ISNUM - checks if the variable is numeric
     @li LIBDS - matches LIBREF.DATASET format
     @li FORMAT - checks if the provided format is syntactically valid
@@ -14585,6 +14607,19 @@ alter table &libds modify &var char(&len);
   else &outcol=1;
   drop &tempcol;
 %end;
+%else %if &rule=ISLIB %then %do;
+  if _n_=1 then do;
+    retain &tempcol;
+    &tempcol=prxparse('/^[_a-z]\w{0,7}$/i');
+    if missing(&tempcol) then do;
+      putlog 'ERR' +(-1) "OR: Invalid expression for ISLIB";
+      stop;
+    end;
+    drop &tempcol;
+  end;
+  if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
+  else &outcol=0;
+%end;
 %else %if &rule=LIBDS %then %do;
   /* match libref.dataset */
   if _n_=1 then do;
@@ -15372,7 +15407,8 @@ run;
 %else %if &engine=ODBC %then %do;
   %&mD.put NOTE: Retrieving ODBC connection details;
   data _null_;
-    length connx_uri conprop_uri value datasource up_uri schema domprop_uri authdomain $256.;
+    length connx_uri conprop_uri value datasource up_uri schema domprop_uri
+      authdomain $256.;
     call missing (of _all_);
     /* get source connection ID */
     rc=metadata_getnasn("&liburi",'LibraryConnection',1,connx_uri);
@@ -15572,6 +15608,55 @@ run;
 
   libname &libref SQLSVR datasrc=&path schema=&schema user="&user" pass="&pass";
 %end;
+%else %if &engine=SASIOSNF or &engine=SNOW %then %do;
+  %&mD.put NOTE: Retrieving SNOW connection details;
+  data _null_;
+    length connx_uri conprop_uri value server up_uri schema domprop_uri
+      authdomain database $256.;
+    call missing (of _all_);
+    /* get source connection ID */
+    rc=metadata_getnasn("&liburi",'LibraryConnection',1,connx_uri);
+    /* get connection properties */
+    i=0;
+    do until (rc2<0);
+      i+1;
+      rc2=metadata_getnasn(connx_uri,'Properties',i,conprop_uri);
+      rc3=metadata_getattr(conprop_uri,'Name',value);
+      if value='Connection.DBMS.Property.SERVER.Name.xmlKey.txt' then do;
+        rc4=metadata_getattr(conprop_uri,'DefaultValue',server);
+        rc2=-1;
+      end;
+    end;
+
+    /* get auth domain */
+    autrc=metadata_getnasn(connx_uri,"Domain",1,domprop_uri);
+    arc=metadata_getattr(domprop_uri,"Name",authdomain);
+    if not missing(authdomain) then authdomain=cats('AUTHDOMAIN=',authdomain);
+    call symputx('authdomain',authdomain,'l');
+
+    /* get SCHEMA */
+    rc6=metadata_getnasn("&liburi",'UsingPackages',1,up_uri);
+    rc7=metadata_getattr(up_uri,'SchemaName',schema);
+    &mD.put rc= connx_uri= rc2= conprop_uri= rc3= value= rc4= server=
+      rc6= up_uri= rc7= schema=;
+
+    /* get database value */
+    prop='Connection.DBMS.Property.DB.Name.xmlKey.txt';
+    rc=metadata_getprop("&liburi",prop,database,"");
+    if database^='' then database='database='!!quote(trim(database));
+    call symputx('database',database,'l');
+
+    call symputx('snow_schema',schema,'l');
+    call symputx('snow_server',server,'l');
+  run;
+
+  libname &libref SNOW SERVER="&snow_server" SCHEMA=&snow_schema &authdomain
+    &database;
+  %if %length(&open_passthrough)>0 %then %do;
+    proc sql;
+    connect using &libref as &open_passthrough;
+  %end;
+%end;
 %else %if &engine=TERADATA %then %do;
   %put NOTE: Obtaining &engine library details;
   data _null;

base/mf_getschema.sas

@@ -25,8 +25,9 @@
   %local dsid vnum rc schema;
   /* in case the parameter is a libref.tablename, pull off just the libref */
   %let libref = %upcase(%scan(&libref, 1, %str(.)));
+  /* sysname can be 'Schema/Owner' or just 'Schema' (eg snowflake) */
   %let dsid=%sysfunc(open(sashelp.vlibnam(where=(
-    libname="%upcase(&libref)" and sysname='Schema/Owner'
+    libname="%upcase(&libref)" and sysname=:'Schema'
   )),i));
   %if (&dsid ^= 0) %then %do;
     %let vnum=%sysfunc(varnum(&dsid,SYSVALUE));

base/mf_getvalue.sas

@@ -1,7 +1,8 @@
 /**
   @file
-  @brief Retrieves a value from a dataset.  If no filter supplied, then first
+  @brief Retrieves a value from a dataset. Returned value is fetched from the
-    record is used.
+  'fetchobs=' record (row 1 by default), after applying the optional filter.
+
   @details Be sure to <code>%quote()</code> your where clause.  Example usage:
 
       %put %mf_getvalue(sashelp.class,name,filter=%quote(age=15));
@@ -16,21 +17,39 @@
   @param [in] libds dataset to query
   @param [in] variable the variable which contains the value to return.
   @param [in] filter= (1) contents of where clause
+  @param [in] fetchobs= (1) observation to fetch. NB: Filter applies first.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-%macro mf_getvalue(libds,variable,filter=1
+%macro mf_getvalue(libds,variable,filter=1,fetchobs=1
 )/*/STORE SOURCE*/;
-  %if %mf_getattrn(&libds,NLOBS)>0 %then %do;
+  %local dsid;
-    %local dsid rc &variable;
+
-    %let dsid=%sysfunc(open(&libds(where=(&filter))));
+  %let dsid=%sysfunc(open(&libds(where=(&filter))));
+  %if (&dsid) %then %do;
+    %local rc &variable;
     %syscall set(dsid);
-    %let rc = %sysfunc(fetch(&dsid));
+    %let rc = %sysfunc(fetchobs(&dsid,&fetchobs));
+    %if (&rc ne 0) %then %do;
+      %put NOTE: Problem reading obs &fetchobs from &libds..;
+      %put %sysfunc(sysmsg());
+      /* Coerce an rc value of -1 (read past end of data) to a 4
+      that, in SAS condition code terms, represents the sysmsg
+      w@rning it generates. */
+      %if &rc eq -1 %then %let rc = 4;
+      /* And update SYSCC if the &rc value is higher */
+      %let syscc = %sysfunc(max(&syscc,&rc));
+    %end;
     %let rc = %sysfunc(close(&dsid));
 
     %trim(&&&variable)
 
   %end;
-%mend mf_getvalue;
+  %else %do;
+    %put %sysfunc(sysmsg());
+    %let syscc = %sysfunc(max(&syscc,%sysfunc(sysrc())));
+  %end;
+
+%mend mf_getvalue;

base/mp_validatecol.sas

@@ -1,6 +1,6 @@
 /**
   @file
-  @brief Used to validate variables in a dataset
+  @brief Used to validate values in a data step
   @details Useful when sanitising inputs, to ensure that they arrive with a
   certain pattern.
   Usage:
@@ -27,6 +27,7 @@
   @param [in] incol The column to be validated
   @param [in] rule The rule to apply.  Current rules:
     @li ISINT - checks if the variable is an integer
+    @li ISLIB - checks if the value is a valid libref (NOT whether it exists)
     @li ISNUM - checks if the variable is numeric
     @li LIBDS - matches LIBREF.DATASET format
     @li FORMAT - checks if the provided format is syntactically valid
@@ -65,6 +66,19 @@
   else &outcol=1;
   drop &tempcol;
 %end;
+%else %if &rule=ISLIB %then %do;
+  if _n_=1 then do;
+    retain &tempcol;
+    &tempcol=prxparse('/^[_a-z]\w{0,7}$/i');
+    if missing(&tempcol) then do;
+      putlog 'ERR' +(-1) "OR: Invalid expression for ISLIB";
+      stop;
+    end;
+    drop &tempcol;
+  end;
+  if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
+  else &outcol=0;
+%end;
 %else %if &rule=LIBDS %then %do;
   /* match libref.dataset */
   if _n_=1 then do;

meta/mm_assigndirectlib.sas

@@ -213,7 +213,8 @@ run;
 %else %if &engine=ODBC %then %do;
   %&mD.put NOTE: Retrieving ODBC connection details;
   data _null_;
-    length connx_uri conprop_uri value datasource up_uri schema domprop_uri authdomain $256.;
+    length connx_uri conprop_uri value datasource up_uri schema domprop_uri
+      authdomain $256.;
     call missing (of _all_);
     /* get source connection ID */
     rc=metadata_getnasn("&liburi",'LibraryConnection',1,connx_uri);
@@ -413,6 +414,55 @@ run;
 
   libname &libref SQLSVR datasrc=&path schema=&schema user="&user" pass="&pass";
 %end;
+%else %if &engine=SASIOSNF or &engine=SNOW %then %do;
+  %&mD.put NOTE: Retrieving SNOW connection details;
+  data _null_;
+    length connx_uri conprop_uri value server up_uri schema domprop_uri
+      authdomain database $256.;
+    call missing (of _all_);
+    /* get source connection ID */
+    rc=metadata_getnasn("&liburi",'LibraryConnection',1,connx_uri);
+    /* get connection properties */
+    i=0;
+    do until (rc2<0);
+      i+1;
+      rc2=metadata_getnasn(connx_uri,'Properties',i,conprop_uri);
+      rc3=metadata_getattr(conprop_uri,'Name',value);
+      if value='Connection.DBMS.Property.SERVER.Name.xmlKey.txt' then do;
+        rc4=metadata_getattr(conprop_uri,'DefaultValue',server);
+        rc2=-1;
+      end;
+    end;
+
+    /* get auth domain */
+    autrc=metadata_getnasn(connx_uri,"Domain",1,domprop_uri);
+    arc=metadata_getattr(domprop_uri,"Name",authdomain);
+    if not missing(authdomain) then authdomain=cats('AUTHDOMAIN=',authdomain);
+    call symputx('authdomain',authdomain,'l');
+
+    /* get SCHEMA */
+    rc6=metadata_getnasn("&liburi",'UsingPackages',1,up_uri);
+    rc7=metadata_getattr(up_uri,'SchemaName',schema);
+    &mD.put rc= connx_uri= rc2= conprop_uri= rc3= value= rc4= server=
+      rc6= up_uri= rc7= schema=;
+
+    /* get database value */
+    prop='Connection.DBMS.Property.DB.Name.xmlKey.txt';
+    rc=metadata_getprop("&liburi",prop,database,"");
+    if database^='' then database='database='!!quote(trim(database));
+    call symputx('database',database,'l');
+
+    call symputx('snow_schema',schema,'l');
+    call symputx('snow_server',server,'l');
+  run;
+
+  libname &libref SNOW SERVER="&snow_server" SCHEMA=&snow_schema &authdomain
+    &database;
+  %if %length(&open_passthrough)>0 %then %do;
+    proc sql;
+    connect using &libref as &open_passthrough;
+  %end;
+%end;
 %else %if &engine=TERADATA %then %do;
   %put NOTE: Obtaining &engine library details;
   data _null;

tests/base/mf_getvalue.test.sas

@@ -0,0 +1,91 @@
+/**
+  @file
+  @brief Testing mf_getvalue macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getvalue.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+data work.test_data;
+  do i = 1 to 10;
+    output;
+  end;
+  stop;
+run;
+
+/* - Test 1 -
+  Get value from default first observation.
+  No filter.
+*/
+%mp_assertscope(SNAPSHOT)
+%let test_value=%mf_getvalue(work.test_data,i);
+%mp_assertscope(COMPARE,ignorelist=test_value)
+
+%mp_assert(
+  iftrue=(&test_value=1 and &syscc eq 0),
+  desc=Basic test fetching value from default first obs,
+  outds=work.test_results
+)
+
+/* - Test 2 -
+  Get value from 10th observation.
+  No filter.
+*/
+%let test_value=%mf_getvalue(work.test_data,i,fetchobs=10);
+%mp_assert(
+  iftrue=(&test_value=10 and &syscc eq 0),
+  desc=Test fetching value from specifically the 10th row,
+  outds=work.test_results
+)
+
+/* - Test 3 -
+  Get value from default first observation.
+  With filter.
+*/
+%let test_value=%mf_getvalue(work.test_data,i,filter=(i>4));
+%mp_assert(
+  iftrue=(&test_value=5 and &syscc eq 0),
+  desc=Test fetching value from default row of filtered data,
+  outds=work.test_results
+)
+
+/* - Test 4 -
+  Get value from specified observation.
+  With filter.
+*/
+%let test_value=%mf_getvalue(work.test_data,i,filter=(i>4),fetchobs=5);
+%mp_assert(
+  iftrue=(&test_value=9 and &syscc eq 0),
+  desc=Test fetching value from 5th row of filtered data,
+  outds=work.test_results
+)
+
+/* - Test 5 -
+  Get value from default observation.
+  Filter removes all rows. This simulates providing an empty dataset
+  or specifying an observation number beyond the set returned by the filter.
+*/
+%let test_value=%mf_getvalue(work.test_data,i,filter=(i>10));
+%mp_assert(
+  iftrue=(&test_value=%str() and &syscc eq 4),
+  desc=Test fetching value from 1st row of empty (filtered) data,
+  outds=work.test_results
+)
+
+%let syscc = 0; /* Reset w@rning To ensure confidence in next test */
+
+/* - Test 6 -
+  Get value from default observation.
+  Dataset does not exist.
+*/
+%let test_value=%mf_getvalue(work.test_data_x,i);
+%mp_assert(
+  iftrue=(&test_value=%str() and &syscc gt 0),
+  desc=Test fetching value from 1st row of non-existent data,
+  outds=work.test_results
+)
+
+%let syscc = 0; /* To reset expected error and allow test job to exit clean. */

tests/base/mp_validatecol.test.sas

@@ -129,4 +129,30 @@ run;
   desc=Test4 - ISFORMAT,
   test=EQUALS 6,
   outds=work.test_results
+)
+
+/**
+  * Test 5 - ISLIB
+  */
+data test5;
+  infile datalines4 dsd;
+  input;
+  inf=_infile_;
+  %mp_validatecol(inf,ISLIB,islib)
+  if islib=1;
+datalines4;
+some
+!lib
+%abort
+definite
+2fail
+nineletrs
+.failalso
+_valid
+;;;;
+run;
+%mp_assertdsobs(work.test5,
+  desc=Testing ISLIB,
+  test=EQUALS 3,
+  outds=work.test_results
 )

tests/sas9only/mm_assigndirectlib.test.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief Testing mm_assigndirectlib macro
+  @details  A valid library must first be configured in metadata.
+  To test success, also define a table for which we can test the existence.
+  This is a unit test - not part of the full test run, as it would be a
+  lot of overhead to create an external DB and metadata setup each time.
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mm_assigndirectlib.sas
+
+**/
+
+%let runtest=0;
+%let libref=XXX;
+%let ds=XXXX;
+
+
+%mp_assertscope(SNAPSHOT)
+%mm_assigndirectlib(&libref)
+%mp_assertscope(COMPARE)
+
+
+%mp_assert(
+  iftrue=(&runtest=1 and %mf_existds(&libref..&ds)),
+  desc=Check if &libref..&ds exists
+)