Merge pull request #419 from sasjs/islib_validation

Islib validation

.github/workflows/main.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Install dependencies
         run: |
@@ -54,7 +54,7 @@ jobs:
           echo "REFRESH_TOKEN=${{secrets.SAS9_4GL_IO_REFRESH_TOKEN}}" >> .env.server
 
       - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v4
+        uses: cycjimmy/semantic-release-action@v6
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

.github/workflows/notmain.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Install dependencies
         run: |

.github/workflows/run-tests.yml

@@ -12,10 +12,10 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [lts/hydrogen]
+        node-version: [lts/iron]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v6
       - name: Use Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v2
         with:
@@ -73,6 +73,8 @@ jobs:
 
       - name: Build & Deploy Project to SAS server
         run: npx @sasjs/cli cbd -t server
+        env:
+          macroCorePath: .
 
       - name: Run all tests
         run: npx @sasjs/cli test -t server
@@ -84,3 +86,4 @@ jobs:
           SAS_PASSWORD: ${{secrets.SAS_PASSWORD}}
           ACCESS_TOKEN: ${{secrets.ACCESS_TOKEN}}
           REFRESH_TOKEN: ${{secrets.REFRESH_TOKEN}}
+

.gitpod.dockerfile

@@ -1,6 +0,0 @@
-FROM gitpod/workspace-full
-
-RUN sudo apt-get update \
- && sudo apt-get install -y doxygen \
- && sudo apt-get install -y graphviz \
- && sudo rm -rf /var/lib/apt/lists/*

.gitpod.yml

@@ -1,27 +0,0 @@
-tasks:
-  - init: npm install -g npm
-  - command: npm i
-  - command: npm i -g @sasjs/cli
-
-image:
-  file: .gitpod.dockerfile
-vscode:
-  extensions:
-    - sasjs.sasjs-for-vscode
-
-github:
-  prebuilds:
-    # enable for the master/default branch (defaults to true)
-    master: true
-    # enable for all branches in this repo (defaults to false)
-    branches: false
-    # enable for pull requests coming from this repo (defaults to true)
-    pullRequests: true
-    # enable for pull requests coming from forks (defaults to false)
-    pullRequestsFromForks: true
-    # add a "Review in Gitpod" button as a comment to pull requests (defaults to true)
-    addComment: true
-    # add a "Review in Gitpod" button to pull requests (defaults to false)
-    addBadge: false
-    # add a label once the prebuild is ready to pull requests (defaults to false)
-    addLabel: prebuilt-in-gitpod

.npmignore

@@ -1,6 +1,5 @@
 all.sas
 build.py
-.gitpod*
 tests/
 sasjs/
 .github/

.npmrc

@@ -0,0 +1 @@
+ignore-scripts=true

all.sas

@@ -1018,8 +1018,9 @@ or %index(&pgm,/tests/testteardown)
   %local dsid vnum rc schema;
   /* in case the parameter is a libref.tablename, pull off just the libref */
   %let libref = %upcase(%scan(&libref, 1, %str(.)));
+  /* sysname can be 'Schema/Owner' or just 'Schema' (eg snowflake) */
   %let dsid=%sysfunc(open(sashelp.vlibnam(where=(
-    libname="%upcase(&libref)" and sysname='Schema/Owner'
+    libname="%upcase(&libref)" and sysname=:'Schema'
   )),i));
   %if (&dsid ^= 0) %then %do;
     %let vnum=%sysfunc(varnum(&dsid,SYSVALUE));
@@ -1214,8 +1215,9 @@ or %index(&pgm,/tests/testteardown)
 %mend mf_getuser;
 /**
   @file
-  @brief Retrieves a value from a dataset.  If no filter supplied, then first
+  @brief Retrieves a value from a dataset. Returned value is fetched from the
-    record is used.
+  'fetchobs=' record (row 1 by default), after applying the optional filter.
+
   @details Be sure to <code>%quote()</code> your where clause.  Example usage:
 
       %put %mf_getvalue(sashelp.class,name,filter=%quote(age=15));
@@ -1230,24 +1232,43 @@ or %index(&pgm,/tests/testteardown)
   @param [in] libds dataset to query
   @param [in] variable the variable which contains the value to return.
   @param [in] filter= (1) contents of where clause
+  @param [in] fetchobs= (1) observation to fetch. NB: Filter applies first.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-%macro mf_getvalue(libds,variable,filter=1
+%macro mf_getvalue(libds,variable,filter=1,fetchobs=1
 )/*/STORE SOURCE*/;
-  %if %mf_getattrn(&libds,NLOBS)>0 %then %do;
+  %local dsid;
-    %local dsid rc &variable;
+
-    %let dsid=%sysfunc(open(&libds(where=(&filter))));
+  %let dsid=%sysfunc(open(&libds(where=(&filter))));
+  %if (&dsid) %then %do;
+    %local rc &variable;
     %syscall set(dsid);
-    %let rc = %sysfunc(fetch(&dsid));
+    %let rc = %sysfunc(fetchobs(&dsid,&fetchobs));
+    %if (&rc ne 0) %then %do;
+      %put NOTE: Problem reading obs &fetchobs from &libds..;
+      %put %sysfunc(sysmsg());
+      /* Coerce an rc value of -1 (read past end of data) to a 4
+      that, in SAS condition code terms, represents the sysmsg
+      w@rning it generates. */
+      %if &rc eq -1 %then %let rc = 4;
+      /* And update SYSCC if the &rc value is higher */
+      %let syscc = %sysfunc(max(&syscc,&rc));
+    %end;
     %let rc = %sysfunc(close(&dsid));
 
     %trim(&&&variable)
 
   %end;
-%mend mf_getvalue;/**
+  %else %do;
+    %put %sysfunc(sysmsg());
+    %let syscc = %sysfunc(max(&syscc,%sysfunc(sysrc())));
+  %end;
+
+%mend mf_getvalue;
+/**
   @file
   @brief Returns number of variables in a dataset
   @details Useful to identify those renagade datasets that have no columns!
@@ -13859,6 +13880,7 @@ run;
   @li mf_islibds.sas
   @li mf_wordsinstr1butnotstr2.sas
   @li mp_abort.sas
+  @li mp_ds2squeeze.sas
 
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas
@@ -14519,7 +14541,7 @@ alter table &libds modify &var char(&len);
 %mend mp_updatevarlength;
 /**
   @file
-  @brief Used to validate variables in a dataset
+  @brief Used to validate values in a data step
   @details Useful when sanitising inputs, to ensure that they arrive with a
   certain pattern.
   Usage:
@@ -14546,6 +14568,7 @@ alter table &libds modify &var char(&len);
   @param [in] incol The column to be validated
   @param [in] rule The rule to apply.  Current rules:
     @li ISINT - checks if the variable is an integer
+    @li ISLIB - checks if the value is a valid libref (NOT whether it exists)
     @li ISNUM - checks if the variable is numeric
     @li LIBDS - matches LIBREF.DATASET format
     @li FORMAT - checks if the provided format is syntactically valid
@@ -14584,6 +14607,19 @@ alter table &libds modify &var char(&len);
   else &outcol=1;
   drop &tempcol;
 %end;
+%else %if &rule=ISLIB %then %do;
+  if _n_=1 then do;
+    retain &tempcol;
+    &tempcol=prxparse('/^[_a-z]\w{0,7}$/i');
+    if missing(&tempcol) then do;
+      putlog 'ERR' +(-1) "OR: Invalid expression for ISLIB";
+      stop;
+    end;
+    drop &tempcol;
+  end;
+  if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
+  else &outcol=0;
+%end;
 %else %if &rule=LIBDS %then %do;
   /* match libref.dataset */
   if _n_=1 then do;
@@ -15371,7 +15407,8 @@ run;
 %else %if &engine=ODBC %then %do;
   %&mD.put NOTE: Retrieving ODBC connection details;
   data _null_;
-    length connx_uri conprop_uri value datasource up_uri schema domprop_uri authdomain $256.;
+    length connx_uri conprop_uri value datasource up_uri schema domprop_uri
+      authdomain $256.;
     call missing (of _all_);
     /* get source connection ID */
     rc=metadata_getnasn("&liburi",'LibraryConnection',1,connx_uri);
@@ -15571,6 +15608,55 @@ run;
 
   libname &libref SQLSVR datasrc=&path schema=&schema user="&user" pass="&pass";
 %end;
+%else %if &engine=SASIOSNF or &engine=SNOW %then %do;
+  %&mD.put NOTE: Retrieving SNOW connection details;
+  data _null_;
+    length connx_uri conprop_uri value server up_uri schema domprop_uri
+      authdomain database $256.;
+    call missing (of _all_);
+    /* get source connection ID */
+    rc=metadata_getnasn("&liburi",'LibraryConnection',1,connx_uri);
+    /* get connection properties */
+    i=0;
+    do until (rc2<0);
+      i+1;
+      rc2=metadata_getnasn(connx_uri,'Properties',i,conprop_uri);
+      rc3=metadata_getattr(conprop_uri,'Name',value);
+      if value='Connection.DBMS.Property.SERVER.Name.xmlKey.txt' then do;
+        rc4=metadata_getattr(conprop_uri,'DefaultValue',server);
+        rc2=-1;
+      end;
+    end;
+
+    /* get auth domain */
+    autrc=metadata_getnasn(connx_uri,"Domain",1,domprop_uri);
+    arc=metadata_getattr(domprop_uri,"Name",authdomain);
+    if not missing(authdomain) then authdomain=cats('AUTHDOMAIN=',authdomain);
+    call symputx('authdomain',authdomain,'l');
+
+    /* get SCHEMA */
+    rc6=metadata_getnasn("&liburi",'UsingPackages',1,up_uri);
+    rc7=metadata_getattr(up_uri,'SchemaName',schema);
+    &mD.put rc= connx_uri= rc2= conprop_uri= rc3= value= rc4= server=
+      rc6= up_uri= rc7= schema=;
+
+    /* get database value */
+    prop='Connection.DBMS.Property.DB.Name.xmlKey.txt';
+    rc=metadata_getprop("&liburi",prop,database,"");
+    if database^='' then database='database='!!quote(trim(database));
+    call symputx('database',database,'l');
+
+    call symputx('snow_schema',schema,'l');
+    call symputx('snow_server',server,'l');
+  run;
+
+  libname &libref SNOW SERVER="&snow_server" SCHEMA=&snow_schema &authdomain
+    &database;
+  %if %length(&open_passthrough)>0 %then %do;
+    proc sql;
+    connect using &libref as &open_passthrough;
+  %end;
+%end;
 %else %if &engine=TERADATA %then %do;
   %put NOTE: Obtaining &engine library details;
   data _null;
@@ -26426,7 +26512,8 @@ libname &libref1a JSON fileref=&fname1a;
 %let found=0;
 /* %put Getting object uri from &libref1a..items; */
 data _null_;
-  length contenttype name $1000;
+  length contenttype name uri $1000;
+  call missing(of _all_);
   set &libref1a..items;
   if contenttype='jobDefinition' and upcase(name)="%upcase(&name)" then do;
     call symputx('uri',cats("&base_uri",uri),'l');
@@ -28210,7 +28297,7 @@ libname &libref1 clear;
   @li mf_nobs.sas
   @li mp_abort.sas
 
-*/
+**/
 
 %macro mv_getViyaFileExtParms(
   ext,
@@ -28424,7 +28511,8 @@ libname &libref1 clear;
 
   %end;
 
-%mend mv_getViyaFileExtParms;/**
+%mend mv_getViyaFileExtParms;
+/**
   @file
   @brief Executes a SAS Viya Job
   @details Triggers a SAS Viya Job, with optional URL parameters, using
@@ -30908,6 +30996,117 @@ endsub;
 %end;
 
 %mend mcf_string2file;/**
+  @file mx_createjob.sas
+  @brief Create a job in SAS 9, Viya or SASjs
+  @details Creates a Stored Process in SAS 9, a Job Execution Service in SAS
+  Viya, or a Stored Program on SASjs Server - depending on the executing
+  environment.
+
+Usage:
+
+    %* compile macros ;
+    filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+    %inc mc;
+
+    %* write some code;
+    filename ft15f001 temp;
+    parmcards4;
+        data example1;
+          set sashelp.class;
+        run;
+    ;;;;
+
+    %* create the job;
+    %mx_createjob(path=/Public/app/jobs,name=myjob,replace=YES)
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mm_createstp.sas
+  @li ms_createfile.sas
+  @li mv_createjob.sas
+
+  @param [in,out] path= The full folder path where the job will be created
+  @param [in,out] name= Job name.  Avoid spaces.
+  @param [in] desc= The description of the job (optional)
+  @param [in] precode= Space separated list of filerefs, pointing to the code
+    that needs to be attached to the beginning of the job (optional)
+  @param [in] code= (ft15f001) Space seperated fileref(s) of the actual code to
+    be added
+  @param [in] replace= (YES) Select YES to replace any existing job in that
+    location
+  @param [in] mDebug= (0) set to 1 to show debug messages in the log
+
+  @author Allan Bowe
+
+  <h4> Related Macros </h4>
+  @li mx_createjob.test.sas
+  @li mx_createwebservice.sas
+
+**/
+
+%macro mx_createjob(path=HOME
+    ,name=initJob
+    ,precode=
+    ,code=ft15f001
+    ,desc=This job was created by the mx_createjob macro
+    ,replace=YES
+    ,mdebug=0
+)/*/STORE SOURCE*/;
+
+%if &syscc ge 4 %then %do;
+  %put syscc=&syscc - &sysmacroname will not execute in this state;
+  %return;
+%end;
+
+/* combine precode and code into a single file */
+%local tempref x fref freflist;
+%let tempref=%mf_getuniquefileref();
+%local work tmpfile;
+%let work=%sysfunc(pathname(work));
+%let tmpfile=&tempref..sas;
+filename &tempref "&work/&tmpfile";
+%let freflist=&precode &code ;
+%do x=1 %to %sysfunc(countw(&freflist));
+  %let fref=%scan(&freflist,&x);
+  %put &sysmacroname: adding &fref;
+  data _null_;
+    file &tempref lrecl=3000 termstr=crlf mod;
+    infile &fref lrecl=3000;
+    input;
+    put _infile_;
+  run;
+%end;
+
+%local platform; %let platform=%mf_getplatform();
+%if &platform=SASVIYA %then %do;
+  %if "&path"="HOME" %then %let path=/Users/&sysuserid/My Folder;
+  %mv_createjob(path=&path
+    ,name=&name
+    ,code=&tempref
+    ,desc=&desc
+    ,replace=&replace
+  )
+%end;
+%else %if &platform=SASJS %then %do;
+  %if "&path"="HOME" %then %let path=/Users/&_sasjs_username/My Folder;
+  %ms_createfile(&path/&name..sas
+    ,inref=&tempref
+    ,mdebug=&mdebug
+  )
+%end;
+%else %do;
+  %if "&path"="HOME" %then %let path=/User Folders/&_METAPERSON/My Folder;
+  %mm_createstp(stpname=&name
+    ,filename=&tmpfile
+    ,directory=&work
+    ,tree=&path
+    ,stpdesc=&desc
+    ,mDebug=&mdebug
+  )
+%end;
+filename &tempref clear;
+%mend mx_createjob;
+/**
   @file mx_createwebservice.sas
   @brief Create a web service in SAS 9, Viya or SASjs
   @details Creates a SASJS ready Stored Process in SAS 9, a Job Execution
@@ -30957,6 +31156,9 @@ Usage:
 
   @author Allan Bowe
 
+  <h4> Related Macros </h4>
+  @li mx_createjob.sas
+
 **/
 
 %macro mx_createwebservice(path=HOME

base/mf_getschema.sas

@@ -25,8 +25,9 @@
   %local dsid vnum rc schema;
   /* in case the parameter is a libref.tablename, pull off just the libref */
   %let libref = %upcase(%scan(&libref, 1, %str(.)));
+  /* sysname can be 'Schema/Owner' or just 'Schema' (eg snowflake) */
   %let dsid=%sysfunc(open(sashelp.vlibnam(where=(
-    libname="%upcase(&libref)" and sysname='Schema/Owner'
+    libname="%upcase(&libref)" and sysname=:'Schema'
   )),i));
   %if (&dsid ^= 0) %then %do;
     %let vnum=%sysfunc(varnum(&dsid,SYSVALUE));

base/mf_getvalue.sas

@@ -1,7 +1,8 @@
 /**
   @file
-  @brief Retrieves a value from a dataset.  If no filter supplied, then first
+  @brief Retrieves a value from a dataset. Returned value is fetched from the
-    record is used.
+  'fetchobs=' record (row 1 by default), after applying the optional filter.
+
   @details Be sure to <code>%quote()</code> your where clause.  Example usage:
 
       %put %mf_getvalue(sashelp.class,name,filter=%quote(age=15));
@@ -16,21 +17,39 @@
   @param [in] libds dataset to query
   @param [in] variable the variable which contains the value to return.
   @param [in] filter= (1) contents of where clause
+  @param [in] fetchobs= (1) observation to fetch. NB: Filter applies first.
 
   @version 9.2
   @author Allan Bowe
 **/
 
-%macro mf_getvalue(libds,variable,filter=1
+%macro mf_getvalue(libds,variable,filter=1,fetchobs=1
 )/*/STORE SOURCE*/;
-  %if %mf_getattrn(&libds,NLOBS)>0 %then %do;
+  %local dsid;
-    %local dsid rc &variable;
+
-    %let dsid=%sysfunc(open(&libds(where=(&filter))));
+  %let dsid=%sysfunc(open(&libds(where=(&filter))));
+  %if (&dsid) %then %do;
+    %local rc &variable;
     %syscall set(dsid);
-    %let rc = %sysfunc(fetch(&dsid));
+    %let rc = %sysfunc(fetchobs(&dsid,&fetchobs));
+    %if (&rc ne 0) %then %do;
+      %put NOTE: Problem reading obs &fetchobs from &libds..;
+      %put %sysfunc(sysmsg());
+      /* Coerce an rc value of -1 (read past end of data) to a 4
+      that, in SAS condition code terms, represents the sysmsg
+      w@rning it generates. */
+      %if &rc eq -1 %then %let rc = 4;
+      /* And update SYSCC if the &rc value is higher */
+      %let syscc = %sysfunc(max(&syscc,&rc));
+    %end;
     %let rc = %sysfunc(close(&dsid));
 
     %trim(&&&variable)
 
   %end;
-%mend mf_getvalue;
+  %else %do;
+    %put %sysfunc(sysmsg());
+    %let syscc = %sysfunc(max(&syscc,%sysfunc(sysrc())));
+  %end;
+
+%mend mf_getvalue;

base/mp_stripdiffs.sas

@@ -36,6 +36,7 @@
   @li mf_islibds.sas
   @li mf_wordsinstr1butnotstr2.sas
   @li mp_abort.sas
+  @li mp_ds2squeeze.sas
 
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas

base/mp_validatecol.sas

@@ -1,6 +1,6 @@
 /**
   @file
-  @brief Used to validate variables in a dataset
+  @brief Used to validate values in a data step
   @details Useful when sanitising inputs, to ensure that they arrive with a
   certain pattern.
   Usage:
@@ -27,6 +27,7 @@
   @param [in] incol The column to be validated
   @param [in] rule The rule to apply.  Current rules:
     @li ISINT - checks if the variable is an integer
+    @li ISLIB - checks if the value is a valid libref (NOT whether it exists)
     @li ISNUM - checks if the variable is numeric
     @li LIBDS - matches LIBREF.DATASET format
     @li FORMAT - checks if the provided format is syntactically valid
@@ -65,6 +66,19 @@
   else &outcol=1;
   drop &tempcol;
 %end;
+%else %if &rule=ISLIB %then %do;
+  if _n_=1 then do;
+    retain &tempcol;
+    &tempcol=prxparse('/^[_a-z]\w{0,7}$/i');
+    if missing(&tempcol) then do;
+      putlog 'ERR' +(-1) "OR: Invalid expression for ISLIB";
+      stop;
+    end;
+    drop &tempcol;
+  end;
+  if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
+  else &outcol=0;
+%end;
 %else %if &rule=LIBDS %then %do;
   /* match libref.dataset */
   if _n_=1 then do;

meta/mm_assigndirectlib.sas

@@ -213,7 +213,8 @@ run;
 %else %if &engine=ODBC %then %do;
   %&mD.put NOTE: Retrieving ODBC connection details;
   data _null_;
-    length connx_uri conprop_uri value datasource up_uri schema domprop_uri authdomain $256.;
+    length connx_uri conprop_uri value datasource up_uri schema domprop_uri
+      authdomain $256.;
     call missing (of _all_);
     /* get source connection ID */
     rc=metadata_getnasn("&liburi",'LibraryConnection',1,connx_uri);
@@ -413,6 +414,55 @@ run;
 
   libname &libref SQLSVR datasrc=&path schema=&schema user="&user" pass="&pass";
 %end;
+%else %if &engine=SASIOSNF or &engine=SNOW %then %do;
+  %&mD.put NOTE: Retrieving SNOW connection details;
+  data _null_;
+    length connx_uri conprop_uri value server up_uri schema domprop_uri
+      authdomain database $256.;
+    call missing (of _all_);
+    /* get source connection ID */
+    rc=metadata_getnasn("&liburi",'LibraryConnection',1,connx_uri);
+    /* get connection properties */
+    i=0;
+    do until (rc2<0);
+      i+1;
+      rc2=metadata_getnasn(connx_uri,'Properties',i,conprop_uri);
+      rc3=metadata_getattr(conprop_uri,'Name',value);
+      if value='Connection.DBMS.Property.SERVER.Name.xmlKey.txt' then do;
+        rc4=metadata_getattr(conprop_uri,'DefaultValue',server);
+        rc2=-1;
+      end;
+    end;
+
+    /* get auth domain */
+    autrc=metadata_getnasn(connx_uri,"Domain",1,domprop_uri);
+    arc=metadata_getattr(domprop_uri,"Name",authdomain);
+    if not missing(authdomain) then authdomain=cats('AUTHDOMAIN=',authdomain);
+    call symputx('authdomain',authdomain,'l');
+
+    /* get SCHEMA */
+    rc6=metadata_getnasn("&liburi",'UsingPackages',1,up_uri);
+    rc7=metadata_getattr(up_uri,'SchemaName',schema);
+    &mD.put rc= connx_uri= rc2= conprop_uri= rc3= value= rc4= server=
+      rc6= up_uri= rc7= schema=;
+
+    /* get database value */
+    prop='Connection.DBMS.Property.DB.Name.xmlKey.txt';
+    rc=metadata_getprop("&liburi",prop,database,"");
+    if database^='' then database='database='!!quote(trim(database));
+    call symputx('database',database,'l');
+
+    call symputx('snow_schema',schema,'l');
+    call symputx('snow_server',server,'l');
+  run;
+
+  libname &libref SNOW SERVER="&snow_server" SCHEMA=&snow_schema &authdomain
+    &database;
+  %if %length(&open_passthrough)>0 %then %do;
+    proc sql;
+    connect using &libref as &open_passthrough;
+  %end;
+%end;
 %else %if &engine=TERADATA %then %do;
   %put NOTE: Obtaining &engine library details;
   data _null;

sasjs/sasjsconfig.json

@@ -5,7 +5,10 @@
     "ddl",
     "fcmp",
     "lua",
+    "meta",
+    "metax",
     "server",
+    "viya",
     "xplatform",
     "tests/base",
     "tests/ddlonly",
@@ -42,7 +45,6 @@
         "deployScripts": []
       },
       "macroFolders": [
-        "viya",
         "tests/viyaonly"
       ],
       "contextName": "SAS Job Execution compute context"
@@ -56,8 +58,6 @@
       },
       "appLoc": "/Shared Data/temp/macrocore",
       "macroFolders": [
-        "meta",
-        "metax",
         "tests/sas9only"
       ],
       "programFolders": [],
@@ -82,7 +82,6 @@
         "deployScripts": []
       },
       "macroFolders": [
-        "server",
         "tests/serveronly"
       ]
     },

tests/base/mf_getvalue.test.sas

@@ -0,0 +1,91 @@
+/**
+  @file
+  @brief Testing mf_getvalue macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getvalue.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+data work.test_data;
+  do i = 1 to 10;
+    output;
+  end;
+  stop;
+run;
+
+/* - Test 1 -
+  Get value from default first observation.
+  No filter.
+*/
+%mp_assertscope(SNAPSHOT)
+%let test_value=%mf_getvalue(work.test_data,i);
+%mp_assertscope(COMPARE,ignorelist=test_value)
+
+%mp_assert(
+  iftrue=(&test_value=1 and &syscc eq 0),
+  desc=Basic test fetching value from default first obs,
+  outds=work.test_results
+)
+
+/* - Test 2 -
+  Get value from 10th observation.
+  No filter.
+*/
+%let test_value=%mf_getvalue(work.test_data,i,fetchobs=10);
+%mp_assert(
+  iftrue=(&test_value=10 and &syscc eq 0),
+  desc=Test fetching value from specifically the 10th row,
+  outds=work.test_results
+)
+
+/* - Test 3 -
+  Get value from default first observation.
+  With filter.
+*/
+%let test_value=%mf_getvalue(work.test_data,i,filter=(i>4));
+%mp_assert(
+  iftrue=(&test_value=5 and &syscc eq 0),
+  desc=Test fetching value from default row of filtered data,
+  outds=work.test_results
+)
+
+/* - Test 4 -
+  Get value from specified observation.
+  With filter.
+*/
+%let test_value=%mf_getvalue(work.test_data,i,filter=(i>4),fetchobs=5);
+%mp_assert(
+  iftrue=(&test_value=9 and &syscc eq 0),
+  desc=Test fetching value from 5th row of filtered data,
+  outds=work.test_results
+)
+
+/* - Test 5 -
+  Get value from default observation.
+  Filter removes all rows. This simulates providing an empty dataset
+  or specifying an observation number beyond the set returned by the filter.
+*/
+%let test_value=%mf_getvalue(work.test_data,i,filter=(i>10));
+%mp_assert(
+  iftrue=(&test_value=%str() and &syscc eq 4),
+  desc=Test fetching value from 1st row of empty (filtered) data,
+  outds=work.test_results
+)
+
+%let syscc = 0; /* Reset w@rning To ensure confidence in next test */
+
+/* - Test 6 -
+  Get value from default observation.
+  Dataset does not exist.
+*/
+%let test_value=%mf_getvalue(work.test_data_x,i);
+%mp_assert(
+  iftrue=(&test_value=%str() and &syscc gt 0),
+  desc=Test fetching value from 1st row of non-existent data,
+  outds=work.test_results
+)
+
+%let syscc = 0; /* To reset expected error and allow test job to exit clean. */

tests/base/mp_validatecol.test.sas

@@ -129,4 +129,30 @@ run;
   desc=Test4 - ISFORMAT,
   test=EQUALS 6,
   outds=work.test_results
+)
+
+/**
+  * Test 5 - ISLIB
+  */
+data test5;
+  infile datalines4 dsd;
+  input;
+  inf=_infile_;
+  %mp_validatecol(inf,ISLIB,islib)
+  if islib=1;
+datalines4;
+some
+!lib
+%abort
+definite
+2fail
+nineletrs
+.failalso
+_valid
+;;;;
+run;
+%mp_assertdsobs(work.test5,
+  desc=Testing ISLIB,
+  test=EQUALS 3,
+  outds=work.test_results
 )

tests/sas9only/mm_assigndirectlib.test.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief Testing mm_assigndirectlib macro
+  @details  A valid library must first be configured in metadata.
+  To test success, also define a table for which we can test the existence.
+  This is a unit test - not part of the full test run, as it would be a
+  lot of overhead to create an external DB and metadata setup each time.
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mm_assigndirectlib.sas
+
+**/
+
+%let runtest=0;
+%let libref=XXX;
+%let ds=XXXX;
+
+
+%mp_assertscope(SNAPSHOT)
+%mm_assigndirectlib(&libref)
+%mp_assertscope(COMPARE)
+
+
+%mp_assert(
+  iftrue=(&runtest=1 and %mf_existds(&libref..&ds)),
+  desc=Check if &libref..&ds exists
+)

tests/x-platform/mx_createjob.test.sas

@@ -0,0 +1,304 @@
+/**
+  @file
+  @brief Testing mx_createjob.sas macro
+
+  Be sure to run <code>%let mcTestAppLoc=/Public/temp/macrocore;</code> when
+  running in Studio
+
+  <h4> SAS Macros </h4>
+  @li mx_createjob.sas
+  @li mp_assert.sas
+  @li mf_getuniquefileref.sas
+  @li mp_assertscope.sas
+
+**/
+
+/**
+  * Test 1 - Basic job creation with default parameters
+  * Also checking for scope leakage
+  */
+filename ft15f001 temp;
+parmcards4;
+  data example1;
+    set sashelp.class;
+  run;
+  %put Job executed successfully;
+;;;;
+%mp_assertscope(SNAPSHOT)
+%mx_createjob(path=&mcTestAppLoc/jobs,name=testjob1,replace=YES)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 1: No errors after basic job creation,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - Job creation with custom description
+  */
+filename ft15f001 temp;
+parmcards4;
+  data example2;
+    set sashelp.cars;
+  run;
+;;;;
+%mx_createjob(
+  path=&mcTestAppLoc/jobs,
+  name=testjob2,
+  desc=Custom job description for testing,
+  replace=YES
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 2: Job created with custom description,
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - Job creation with precode
+  */
+filename precode1 temp;
+data _null_;
+  file precode1;
+  put '%let testvar=PreCodeValue;';
+  put '%put &=testvar;';
+run;
+
+filename ft15f001 temp;
+parmcards4;
+  data example3;
+    set sashelp.class;
+    precode_var="&testvar";
+  run;
+;;;;
+%mx_createjob(
+  path=&mcTestAppLoc/jobs,
+  name=testjob3,
+  precode=precode1,
+  replace=YES
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 3: Job created with precode parameter,
+  outds=work.test_results
+)
+
+filename precode1 clear;
+
+/**
+  * Test 4 - Job creation with multiple code filerefs
+  */
+%let code1=%mf_getuniquefileref();
+%let code2=%mf_getuniquefileref();
+
+filename &code1 temp;
+data _null_;
+  file &code1;
+  put 'data work.part1;';
+  put '  set sashelp.class(obs=5);';
+  put 'run;';
+run;
+
+filename &code2 temp;
+data _null_;
+  file &code2;
+  put 'data work.part2;';
+  put '  set sashelp.class(firstobs=6);';
+  put 'run;';
+run;
+
+%mx_createjob(
+  path=&mcTestAppLoc/jobs,
+  name=testjob4,
+  code=&code1 &code2,
+  replace=YES
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 4: Job created with multiple code filerefs,
+  outds=work.test_results
+)
+
+filename &code1 clear;
+filename &code2 clear;
+
+/**
+  * Test 5 - Job creation with both precode and multiple code files
+  */
+%let pre1=%mf_getuniquefileref();
+%let pre2=%mf_getuniquefileref();
+%let main1=%mf_getuniquefileref();
+
+filename &pre1 temp;
+data _null_;
+  file &pre1;
+  put '%let globalvar1=Value1;';
+run;
+
+filename &pre2 temp;
+data _null_;
+  file &pre2;
+  put '%let globalvar2=Value2;';
+run;
+
+filename &main1 temp;
+data _null_;
+  file &main1;
+  put 'data work.combined;';
+  put '  var1="&globalvar1";';
+  put '  var2="&globalvar2";';
+  put '  output;';
+  put 'run;';
+run;
+
+%mx_createjob(
+  path=&mcTestAppLoc/jobs,
+  name=testjob5,
+  precode=&pre1 &pre2,
+  code=&main1,
+  desc=Job with multiple precode and code files,
+  replace=YES
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 5: Job created with multiple precode and code files,
+  outds=work.test_results
+)
+
+filename &pre1 clear;
+filename &pre2 clear;
+filename &main1 clear;
+
+/**
+  * Test 6 - Job creation with special characters in code
+  */
+filename ft15f001 temp;
+parmcards4;
+  data example6;
+    length text $200;
+    text='Special chars: & % $ # @ !';
+    output;
+    text="Quotes: 'single' and ""double""";
+    output;
+  run;
+  %put Test with special characters;
+;;;;
+%mx_createjob(
+  path=&mcTestAppLoc/jobs,
+  name=testjob6,
+  desc=Job with special characters in code,
+  replace=YES
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 6: Job created with special characters in code,
+  outds=work.test_results
+)
+
+/**
+  * Test 7 - Job creation with macro code
+  */
+filename ft15f001 temp;
+parmcards4;
+  %macro testmacro();
+    data example7;
+      set sashelp.class;
+      where age > 12;
+    run;
+  %mend testmacro;
+
+  %testmacro()
+;;;;
+%mx_createjob(
+  path=&mcTestAppLoc/jobs,
+  name=testjob7,
+  desc=Job containing macro definitions,
+  replace=YES
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 7: Job created with macro code,
+  outds=work.test_results
+)
+
+/**
+  * Test 8 - Job creation with empty code (edge case)
+  */
+filename ft15f001 temp;
+data _null_;
+  file ft15f001;
+  put '/* Empty job for testing */';
+run;
+
+%mx_createjob(
+  path=&mcTestAppLoc/jobs,
+  name=testjob8,
+  desc=Job with minimal code,
+  replace=YES
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 8: Job created with minimal code,
+  outds=work.test_results
+)
+
+/**
+  * Test 9 - Job creation with long code block
+  */
+filename ft15f001 temp;
+data _null_;
+  file ft15f001;
+  put 'data work.longtest;';
+  do i=1 to 50;
+    put '  var' i +(-1) '=' i ';';
+  end;
+  put '  output;';
+  put 'run;';
+run;
+
+%mx_createjob(
+  path=&mcTestAppLoc/jobs,
+  name=testjob9,
+  desc=Job with many variables,
+  replace=YES
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 9: Job created with long code block,
+  outds=work.test_results
+)
+
+/**
+  * Test 10 - Replace existing job (replace=YES)
+  */
+filename ft15f001 temp;
+parmcards4;
+  data example10_v1;
+    set sashelp.class;
+  run;
+;;;;
+%mx_createjob(path=&mcTestAppLoc/jobs,name=testjob10,replace=YES)
+
+/* Now replace it */
+filename ft15f001 temp;
+parmcards4;
+  data example10_v2;
+    set sashelp.cars;
+  run;
+;;;;
+%mx_createjob(path=&mcTestAppLoc/jobs,name=testjob10,replace=YES)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 10: Job replaced successfully with replace=YES,
+  outds=work.test_results
+)

tests/x-platform/mx_testservice.test.sas

@@ -3,10 +3,10 @@
   @brief Testing mx_testservice.sas macro
 
   Be sure to run <code>%let mcTestAppLoc=/Public/temp/macrocore;</code> when
-  runnin in Studio
+  running in Studio
 
   <h4> SAS Macros </h4>
-  @li mp_createwebservice.sas
+  @li mx_createwebservice.sas
   @li mx_testservice.sas
   @li mp_assert.sas
 

viya/mv_deletejes.sas

@@ -117,7 +117,8 @@ libname &libref1a JSON fileref=&fname1a;
 %let found=0;
 /* %put Getting object uri from &libref1a..items; */
 data _null_;
-  length contenttype name $1000;
+  length contenttype name uri $1000;
+  call missing(of _all_);
   set &libref1a..items;
   if contenttype='jobDefinition' and upcase(name)="%upcase(&name)" then do;
     call symputx('uri',cats("&base_uri",uri),'l');

viya/mv_getviyafileextparms.sas

@@ -31,7 +31,7 @@
   @li mf_nobs.sas
   @li mp_abort.sas
 
-*/
+**/
 
 %macro mv_getViyaFileExtParms(
   ext,
@@ -245,4 +245,4 @@
 
   %end;
 
-%mend mv_getViyaFileExtParms;
+%mend mv_getViyaFileExtParms;

xplatform/mx_createjob.sas

@@ -0,0 +1,111 @@
+/**
+  @file mx_createjob.sas
+  @brief Create a job in SAS 9, Viya or SASjs
+  @details Creates a Stored Process in SAS 9, a Job Execution Service in SAS
+  Viya, or a Stored Program on SASjs Server - depending on the executing
+  environment.
+
+Usage:
+
+    %* compile macros ;
+    filename mc url "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+    %inc mc;
+
+    %* write some code;
+    filename ft15f001 temp;
+    parmcards4;
+        data example1;
+          set sashelp.class;
+        run;
+    ;;;;
+
+    %* create the job;
+    %mx_createjob(path=/Public/app/jobs,name=myjob,replace=YES)
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mm_createstp.sas
+  @li ms_createfile.sas
+  @li mv_createjob.sas
+
+  @param [in,out] path= The full folder path where the job will be created
+  @param [in,out] name= Job name.  Avoid spaces.
+  @param [in] desc= The description of the job (optional)
+  @param [in] precode= Space separated list of filerefs, pointing to the code
+    that needs to be attached to the beginning of the job (optional)
+  @param [in] code= (ft15f001) Space seperated fileref(s) of the actual code to
+    be added
+  @param [in] replace= (YES) Select YES to replace any existing job in that
+    location
+  @param [in] mDebug= (0) set to 1 to show debug messages in the log
+
+  @author Allan Bowe
+
+  <h4> Related Macros </h4>
+  @li mx_createjob.test.sas
+  @li mx_createwebservice.sas
+
+**/
+
+%macro mx_createjob(path=HOME
+    ,name=initJob
+    ,precode=
+    ,code=ft15f001
+    ,desc=This job was created by the mx_createjob macro
+    ,replace=YES
+    ,mdebug=0
+)/*/STORE SOURCE*/;
+
+%if &syscc ge 4 %then %do;
+  %put syscc=&syscc - &sysmacroname will not execute in this state;
+  %return;
+%end;
+
+/* combine precode and code into a single file */
+%local tempref x fref freflist;
+%let tempref=%mf_getuniquefileref();
+%local work tmpfile;
+%let work=%sysfunc(pathname(work));
+%let tmpfile=&tempref..sas;
+filename &tempref "&work/&tmpfile";
+%let freflist=&precode &code ;
+%do x=1 %to %sysfunc(countw(&freflist));
+  %let fref=%scan(&freflist,&x);
+  %put &sysmacroname: adding &fref;
+  data _null_;
+    file &tempref lrecl=3000 termstr=crlf mod;
+    infile &fref lrecl=3000;
+    input;
+    put _infile_;
+  run;
+%end;
+
+%local platform; %let platform=%mf_getplatform();
+%if &platform=SASVIYA %then %do;
+  %if "&path"="HOME" %then %let path=/Users/&sysuserid/My Folder;
+  %mv_createjob(path=&path
+    ,name=&name
+    ,code=&tempref
+    ,desc=&desc
+    ,replace=&replace
+  )
+%end;
+%else %if &platform=SASJS %then %do;
+  %if "&path"="HOME" %then %let path=/Users/&_sasjs_username/My Folder;
+  %ms_createfile(&path/&name..sas
+    ,inref=&tempref
+    ,mdebug=&mdebug
+  )
+%end;
+%else %do;
+  %if "&path"="HOME" %then %let path=/User Folders/&_METAPERSON/My Folder;
+  %mm_createstp(stpname=&name
+    ,filename=&tmpfile
+    ,directory=&work
+    ,tree=&path
+    ,stpdesc=&desc
+    ,mDebug=&mdebug
+  )
+%end;
+filename &tempref clear;
+%mend mx_createjob;

xplatform/mx_createwebservice.sas

@@ -48,6 +48,9 @@ Usage:
 
   @author Allan Bowe
 
+  <h4> Related Macros </h4>
+  @li mx_createjob.sas
+
 **/
 
 %macro mx_createwebservice(path=HOME