Merge pull request #432 from sasjs/mp_rowhash

feat: mp_rowhash macro and fixed failing tests

.github/workflows/main.yml

@@ -11,10 +11,24 @@ on:
 jobs:
   release:
     runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      issues: write
+      pull-requests: write
+      id-token: write # required for npm Trusted Publishing (OIDC)
     steps:
       - name: Checkout
         uses: actions/checkout@v6
 
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: 'lts/*'
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Ensure npm supports Trusted Publishing
+        run: npm install -g npm@latest
+
       - name: Install dependencies
         run: |
           npm ci
@@ -54,13 +68,40 @@ jobs:
           echo "REFRESH_TOKEN=${{secrets.SAS9_4GL_IO_REFRESH_TOKEN}}" >> .env.server
 
       - name: Semantic Release
+        id: makerelease
         uses: cycjimmy/semantic-release-action@v6
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
+          # NPM_TOKEN no longer required: package is published via npm
+          # Trusted Publishing (OIDC). The id-token: write permission above
+          # allows npm CLI to authenticate to the registry automatically.
 
       - name: SAS Packages Release
         run: |
           npx @sasjs/cli compile job -s sasjs/utils/create_sas_package.sas -o sasjsbuild -t server
           # need long duration token per https://github.com/sasjs/server/issues/307
           # npx @sasjs/cli run sasjsbuild/jobs/utils/create_sas_package.sas -t server
+
+      - name: Update Docs Site
+        if: steps.makerelease.outputs.new_release_published == 'true'
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y doxygen
+          npx @sasjs/cli doc -t docsonly 
+          git clone https://x-access-token:${{ secrets.CORESASJSIO_PAT }}@github.com/sasjs/core.github.io.git
+          cd core.github.io
+          rm -rf *.html
+          rm -rf *.js
+          rm -rf *.png
+          rm -rf *.dot
+          rm -rf *.css
+          rm -rf *.svg
+          rm -rf search
+          cp -R ../sasjsbuild/docs/* .
+          ls
+          git config user.name sasjs
+          echo 'core.sasjs.io' > CNAME
+          git add .
+          git commit -m "core pipeline build on $(date +%F:%H:%M:%S)"
+          git push
+          echo "check it out:  https://sasjs.github.io/core.github.io/files.html"

.github/workflows/notmain.yml

@@ -26,6 +26,7 @@ jobs:
           git config user.email github-actions@github.com
           python3 build.py
           git add all.sas
+          git add viya/mv_createwebservice.sas
           git commit -m "chore: updating all.sas" --allow-empty
           git push
 

.gitignore

@@ -14,3 +14,4 @@ mc_*
 
 ~
 
+.claude

AGENTS.md

@@ -0,0 +1,31 @@
+# Agent Instructions for @sasjs/core
+
+Follow these rules when editing or generating code for the @sasjs/core SAS macro library.
+
+## Project Context
+This repo is the SASjs Macro Core library — a collection of MIT-licensed, production-quality SAS macros for SAS application development.
+
+## Versioning
+- NEVER bump or modify the version in `package.json`.
+- Versioning is handled entirely by the CI/CD pipeline using semantic-release.
+
+## SAS Style & Standards
+- Read and follow the standards documented in `README.md` (Sections: Components, Standards, File Properties, Header Properties, Coding Standards).
+- Read and follow `.sasjslint`:
+  - No trailing spaces.
+  - Requires a Doxygen header on every macro (`@file`, `@brief`, etc.).
+  - Lowercase file names without spaces.
+  - Lowercase macro names.
+  - Macro definitions must use parentheses, e.g. `%macro x();` not `%macro x;`.
+  - Indentation = 2 spaces (or multiple thereof); no tabs.
+  - Max line length 300.
+  - No gremlins / invisible characters.
+- One macro per file; filename must match macro name.
+- Macro *calls* should NOT be terminated with a semicolon. Use `%my_macro()` not `%my_macro();`.
+- Macro variables must always be local, to prevent scope leakage.
+
+
+## Build / Generated Files
+- Do not run the build script locally; it is executed in the CI/CD pipeline.
+- Generated files, including the consolidated `all.sas`, the per-folder `mc_*.sas` files, and the LUA macro wrappers in the `lua` folder, can generally be ignored unless the pipeline requires an update. Do not edit generated files by hand.
+- run sasjs lint after each change

README.md

@@ -5,8 +5,6 @@
 ![npm](https://img.shields.io/npm/dt/@sasjs/core)
 ![GitHub top language](https://img.shields.io/github/languages/top/sasjs/core)
 [![GitHub closed issues](https://img.shields.io/github/issues-closed-raw/sasjs/core)](https://github.com/sasjs/core/issues?q=is%3Aissue+is%3Aclosed)
-[![GitHub issues](https://img.shields.io/github/issues-raw/sasjs/core)](https://github.com/sasjs/core/issues)
-![total lines](https://tokei.rs/b1/github/sasjs/core)
 
 
 Much quality. Many standards. The **Macro Core** library exists to save time and development effort! Herein ye shall find a veritable host of MIT-licenced, production quality SAS macros. These are a mix of tools, utilities, functions and code generators that are useful in the context of [Application Development](https://sasapps.io) on the SAS platform (eg https://datacontroller.io). [Contributions](https://github.com/sasjs/core/blob/main/.github/CONTRIBUTING.md) are welcome.
@@ -201,7 +199,7 @@ When contributing to this library, it is therefore important to ensure that all
 - All dataset references must be 2 level (eg `work.blah`, not `blah`). This is to avoid contention when options [DATASTMTCHK](https://support.sas.com/documentation/cdl/en/lrdict/64316/HTML/default/viewer.htm#a000279064.htm)=ALLKEYWORDS is in effect, or the [USER](https://documentation.sas.com/doc/en/pgmsascdc/9.4_3.5/lrcon/n18m1vkqmeo4esn1moikt23zhp8s.htm) library is active.
 - Avoid naming collisions! All macro variables should be local scope. Use system generated work tables where possible - eg `data ; set sashelp.class; run; data &output; set &syslast; run;`
 - Where global macro variables are absolutely necessary, they should make use of `&sasjs_prefix` - see mp_init.sas
-- The use of `quit;` for `proc sql` is optional unless you are looking to benefit from the timing statistics.
+- The use of `quit;` for `proc sql` is essential, to avoid `WARNING: You cannot disconnect or terminate session XXXX until the procedure completes.` when terminating CAS sessions in Viya.
 - Use [sasjs lint](https://github.com/sasjs/lint)!
 
 ## General Notes

base/mp_assert.sas

@@ -35,7 +35,7 @@
 )/*/STORE SOURCE*/;
 
   data ;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     test_comments="&sysmacroname: Test result of "!!symget('iftrue');
   %if %eval(%unquote(&iftrue)) %then %do;
@@ -52,5 +52,6 @@
   run;
   proc sql;
   drop table &ds;
+  quit;
 
 %mend mp_assert;

base/mp_assertcols.sas

@@ -108,7 +108,7 @@
   %end;
 
   data;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     if test_description='0'
     then test_description="Testing &inds for existence of &test of: &cols";

base/mp_assertcolvals.sas

@@ -143,7 +143,7 @@
   )
 
   data;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
     test_comments="&sysmacroname: &lib..&ds..&col has &result/&orig values "

base/mp_assertdsobs.sas

@@ -87,7 +87,7 @@
   %end;
 
   data &ds;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     test_result='FAIL';
     test_comments="&sysmacroname: Dataset &inds has &nobs observations.";

base/mp_assertscope.sas

@@ -92,6 +92,7 @@
     from dictionary.macros
     where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
     order by name,offset;
+  quit;
 %end;
 %else %if &action=COMPARE %then %do;
 
@@ -129,9 +130,8 @@
     %let test_comments=%str(Mod:(&mod) Add:(&add) Del:(&del));
   %end;
 
-
   data ;
-    length test_description $256 test_result $4 test_comments $256;
+    length test_result $4 test_description $256 test_comments $256;
     test_description=symget('desc');
     test_comments=symget('test_comments');
     test_result=symget('test_result');
@@ -142,6 +142,7 @@
   run;
   proc sql;
   drop table &ds;
+  quit;
 %end;
 
 %mend mp_assertscope;

base/mp_ds2csv.sas

@@ -130,10 +130,10 @@ data _null_;
         * there is not much point importing a short length numeric like this,
         * eg with best4., as the resulting variable will still be stored as
         * length 8.  We need a length or format statement to ensure variable
-        * is creatd with the smaller length...
+        * is created with the smaller length...
         **/
       else if vlen<8 then header=cats(varnm,':best',vlen,'.');
-      else header=cats(varnm,':best.');
+      else header=cats(varnm,':best32.');
     end;
   %end;
   %else %do;

base/mp_hashdataset.sas

@@ -17,7 +17,7 @@
   @li mf_getattrn.sas
   @li mf_getuniquename.sas
   @li mf_getvarlist.sas
-  @li mp_md5.sas
+  @li mp_rowhash.sas
 
   <h4> Related Files </h4>
   @li mp_hashdataset.test.sas
@@ -46,8 +46,7 @@
 
 %local keyvar /* roll up the md5 */
   prevkeyvar /* retain prev record md5 */
-  lastvar /* last var in input ds */
+  lastvar /* last var in input ds */;
-  cvars nvars;
 
 %if not(%eval(%unquote(&iftrue))) %then %return;
 
@@ -79,10 +78,11 @@
     retain &prevkeyvar;
     if _n_=1 then &prevkeyvar=put(md5("&salt"),$hex32.);
     set &libds end=&lastvar;
-    /* hash should include previous row */
+    /* hash should include previous row - listed first so it is hashed first */
-    &keyvar=%mp_md5(
+    %mp_rowhash(
-      cvars=%mf_getvarlist(&libds,typefilter=C) &prevkeyvar,
+      md5_col=&keyvar
-      nvars=%mf_getvarlist(&libds,typefilter=N)
+      ,cvars=&prevkeyvar %mf_getvarlist(&libds,typefilter=C)
+      ,nvars=%mf_getvarlist(&libds,typefilter=N)
     );
     &prevkeyvar=&keyvar;
     if &lastvar then output;

base/mp_init.sas

@@ -37,12 +37,14 @@
 
 %global
   SASJS_PREFIX       /* the ONLY hard-coded global macro variable in SASjs    */
+  &prefix.PROCESSMODE &prefix._STPSRV_HEADER_LOC /* SASjs server specific vars*/
   &prefix._FUNCTIONS /* used in mcf_init() to track core function compilation */
   &prefix._INIT_NUM  /* initialisation time as numeric                        */
   &prefix._INIT_DTTM /* initialisation time in E8601DT26.6 format             */
   &prefix.WORK       /* avoid typing %sysfunc(pathname(work)) every time      */
+  &prefix.PROCESSMODE
+  &prefix._STPSRV_HEADER_LOC
 ;
-
 %let sasjs_prefix=&prefix;
 
 data _null_;

base/mp_loadformat.sas

@@ -45,7 +45,7 @@
   @li mp_aligndecimal.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
-  @li mp_md5.sas
+  @li mp_rowhash.sas
   @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
@@ -174,14 +174,19 @@ select distinct
 %let nvars=FMTROW MIN MAX DEFAULT LENGTH FUZZ MULT NOEDIT;
 data &base_fmts/note2err;
   set &base_fmts;
-  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+  length fmthash $32;
+  %mp_rowhash(
+    md5_col=fmthash
+    ,cvars=&cvars
+    ,nvars=&nvars
+  )
 run;
 
 /**
   * Ensure input table and base_formats have consistent lengths and types
   */
 data &inlibds/nonote2err;
-  length &delete_col $3 FMTROW 8 start end label $32767;
+  length &delete_col $3 FMTROW 8 start end label $32767 fmthash $32;
   if 0 then set &base_fmts;
   set &libds;
   by type fmtname notsorted;
@@ -203,7 +208,11 @@ data &inlibds/nonote2err;
     %mp_aligndecimal(end,width=16)
   end;
 
-  fmthash=%mp_md5(cvars=&cvars, nvars=&nvars);
+  %mp_rowhash(
+    md5_col=fmthash
+    ,cvars=&cvars
+    ,nvars=&nvars
+  )
 run;
 
 /**

base/mp_md5.sas

@@ -28,11 +28,16 @@
   @li Global option:  `options dsoptions=nonote2err;`
   @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
 
+  For very wide tables (hundreds or thousands of columns) consider using
+  `mp_rowhash.sas`, which builds the same type of row hash iteratively and
+  avoids creating a single, very long concatenated SAS expression.
+
   @param [in] cvars= () Space seperated list of character variables
   @param [in] nvars= () Space seperated list of numeric variables
 
   <h4> Related Programs </h4>
   @li mp_init.sas
+  @li mp_rowhash.sas
 
   @version 9.2
   @author Allan Bowe

base/mp_retainedkey.sas

@@ -148,7 +148,8 @@
 %mp_abort(iftrue=(&iserr=1),mac=mp_retainedkey,msg=%superq(msg))
 
 proc sql noprint;
-select sum(max(&retained_key),0) into: maxkey from &base_libds;
+select sum(max(&retained_key),0) format=32. into: maxkey trimmed
+  from &base_libds;
 
 /**
   * get base table RK and bus field values for lookup
@@ -217,19 +218,19 @@ quit;
   insert into &maxkeytable
     set keytable="&base_libds"
       ,keycolumn="&retained_key"
-      ,max_key=%eval(&maxkey+&newkey_cnt)
+      ,max_key=%sysevalf(&maxkey+&newkey_cnt)
       ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt;
   %end;
   %else %do;
   update &maxkeytable
-    set max_key=%eval(&maxkey+&newkey_cnt)
+    set max_key=%sysevalf(&maxkey+&newkey_cnt)
       ,processed_dttm="%sysfunc(datetime(),%mf_fmtdttm())"dt
     where keytable="&base_libds";
   %end;
   %mp_lockanytable(UNLOCK
     ,lib=%scan(&maxkeytable,1,.)
     ,ds=%scan(&maxkeytable,2,.)
-    ,ref=Updating maxkeyvalues with maxkey=%eval(&maxkey+&newkey_cnt)
+    ,ref=Updating maxkeyvalues with maxkey=%sysevalf(&maxkey+&newkey_cnt)
     ,ctl_ds=&locktable
   )
 %end;
@@ -237,7 +238,7 @@ quit;
 /* fill in the missing retained key values */
 %let tempvar=%mf_getuniquename();
 data &outds(drop=&tempvar);
-  retain &tempvar %eval(&maxkey+1);
+  retain &tempvar %sysevalf(&maxkey+1);
   set &tempds2;
   if &retained_key =. then &retained_key=&tempvar;
   &tempvar=&tempvar+1;

base/mp_rowhash.sas

@@ -0,0 +1,90 @@
+/**
+  @file
+  @brief Iterative row-level MD5 hash generator
+  @details Generates DATA step statements that compute a deterministic,
+    row-level MD5 hash using a Merkle-style construction.  Each variable is
+    hashed independently and the resulting raw 16-byte digests are combined
+    iteratively.  This avoids concatenating the per-variable hex digests into a
+    single SAS character expression, which overflows when datasets contain many
+    variables.
+
+    Temporary variables use five leading underscores.
+
+    This macro is called from inside a DATA step.  The caller is responsible
+    for declaring the output hash column if it does not already exist on the
+    input dataset.
+
+    Variables are hashed in the order supplied.  If a particular variable
+    needs to influence the hash first (for example the previous row hash in
+    `mp_hashdataset`, or business dates in Data Controller's bitemporal
+    loader) simply list it first in `cvars` or `nvars`.
+
+  @param [in] md5_col= Name of the output hash column.
+  @param [in] cvars= Space separated list of character variables to hash.
+  @param [in] nvars= Space separated list of numeric variables to hash.
+
+  @version 9.3M5
+  @author Allan Bowe
+**/
+
+%macro mp_rowhash(
+    md5_col=
+    ,cvars=
+    ,nvars=
+  );
+
+  /* DATA step temp variables use five leading underscores.  The names
+      are generated at macro invocation to avoid clashing with data columns. */
+  %local state digest pair numtext normal i chars nums;
+  %let state=%mf_getuniquename(prefix=_____state_);
+  %let digest=%mf_getuniquename(prefix=_____digest_);
+  %let pair=%mf_getuniquename(prefix=_____pair_);
+  %let numtext=%mf_getuniquename(prefix=_____numtext_);
+  %let normal=%mf_getuniquename(prefix=_____normal_);
+  %let i=%mf_getuniquename(prefix=_____i_);
+  %let chars=%mf_getuniquename(prefix=_____chars_);
+  %let nums=%mf_getuniquename(prefix=_____nums_);
+
+  length &state $16
+        &digest $16
+        &pair $32
+        &numtext $64
+        &normal &i 8;
+  if _n_=1 then call missing(&state,&digest,&pair,&numtext,&normal,&i);
+  drop &state &digest &pair &numtext &normal &i;
+
+  /* Versioned seed prevents confusion with other hashing schemes. */
+  &state = md5('DC HASH v2');
+
+  %if %length(&cvars)>0 %then %do;
+    array &chars{*} $ &cvars;
+    do &i = 1 to dim(&chars);
+      /* Leading blanks are retained. */
+      &digest = md5(trimn(&chars[&i]));
+      substr(&pair,  1, 16) = &state;
+      substr(&pair, 17, 16) = &digest;
+      &state = md5(&pair);
+    end;
+  %end;
+
+  %if %length(&nvars)>0 %then %do;
+    array &nums{*} &nvars;
+    do &i = 1 to dim(&nums);
+      /*
+        multiply-by-one for consistent cross-system precision.
+        Ignore null to protect SAS special missing values.
+        Cannot use IFN() as it evaluates both sides
+      */
+      if missing(&nums[&i]) then &normal = &nums[&i];
+      else &normal = &nums[&i] * 1;
+      &numtext = put(&normal, binary64.);
+      &digest = md5(trim(&numtext));
+      substr(&pair,  1, 16) = &state;
+      substr(&pair, 17, 16) = &digest;
+      &state = md5(&pair);
+    end;
+  %end;
+
+  &md5_col = put(&state, $hex32.);
+
+%mend mp_rowhash;

base/mp_validatecol.sas

@@ -1,6 +1,6 @@
 /**
   @file
-  @brief Used to validate variables in a dataset
+  @brief Used to validate values in a data step
   @details Useful when sanitising inputs, to ensure that they arrive with a
   certain pattern.
   Usage:
@@ -27,6 +27,7 @@
   @param [in] incol The column to be validated
   @param [in] rule The rule to apply.  Current rules:
     @li ISINT - checks if the variable is an integer
+    @li ISLIB - checks if the value is a valid libref (NOT whether it exists)
     @li ISNUM - checks if the variable is numeric
     @li LIBDS - matches LIBREF.DATASET format
     @li FORMAT - checks if the provided format is syntactically valid
@@ -65,6 +66,19 @@
   else &outcol=1;
   drop &tempcol;
 %end;
+%else %if &rule=ISLIB %then %do;
+  if _n_=1 then do;
+    retain &tempcol;
+    &tempcol=prxparse('/^[_a-z]\w{0,7}$/i');
+    if missing(&tempcol) then do;
+      putlog 'ERR' +(-1) "OR: Invalid expression for ISLIB";
+      stop;
+    end;
+    drop &tempcol;
+  end;
+  if prxmatch(&tempcol, trim(&incol)) then &outcol=1;
+  else &outcol=0;
+%end;
 %else %if &rule=LIBDS %then %do;
   /* match libref.dataset */
   if _n_=1 then do;

package.json

@@ -31,5 +31,8 @@
     "test": "npx @sasjs/cli test -t server",
     "lint": "npx @sasjs/cli lint",
     "prepare": "git rev-parse --git-dir && git config core.hooksPath ./.git-hooks || true"
-  }
+  },
-}
+  "maintainers": [
+    "https://sasapps.io"
+  ]
+}

sasjs/doxy/new_header.html

@@ -21,6 +21,7 @@
   <link href="$relpath^tabs.css" rel="stylesheet" type="text/css" />
   <script type="text/javascript" src="$relpath^jquery.js"></script>
   <script type="text/javascript" src="$relpath^dynsections.js"></script>
+  <script type="text/javascript" src="$relpath^cookie.js"></script>
   $treeview $search $mathjax
   <link href="$relpath^$stylesheet" rel="stylesheet" type="text/css" />
   <link rel="shortcut icon" href="$relpath^favicon.ico" type="image/x-icon" />
@@ -39,6 +40,10 @@
       g.async = true; g.src = u + 'matomo.js'; s.parentNode.insertBefore(g, s);
     })();
   </script>
+  <noscript>
+    <p><img referrerpolicy="no-referrer-when-downgrade" src="https://analytics.4gl.io/matomo.php?idsite=6&amp;rec=1"
+        style="border:0;" alt="" /></p>
+  </noscript>
   <!-- End Matomo Code -->
 </head>
 
@@ -79,8 +84,4 @@
       </table>
     </div>
     <!--END TITLEAREA-->
-    <!-- end header part -->
+    <!-- end header part -->
-  </div>
-</body>
-
-</html>

sasjs/sasjsconfig.json

@@ -25,10 +25,7 @@
   },
   "testConfig": {
     "initProgram": "tests/testinit.sas",
-    "termProgram": "tests/testterm.sas",
+    "termProgram": "tests/testterm.sas"
-    "macroVars": {
-      "mcTestAppLoc": "/Public/temp/macrocore"
-    }
   },
   "defaultTarget": "server",
   "targets": [

tests/base/mf_getfilesize.test.sas

@@ -14,7 +14,7 @@ data test;
 run;
 
 %mp_assertscope(SNAPSHOT)
-%put %mf_getfilesize(libds=work.test)
+%put %mf_getfilesize(libds=work.test);
 %mp_assertscope(COMPARE)
 
 %mp_assert(

tests/base/mf_getfmtlist.test.sas

@@ -5,9 +5,14 @@
   <h4> SAS Macros </h4>
   @li mf_getfmtlist.sas
   @li mp_assert.sas
+  @li mp_assertscope.sas
 
 **/
 
+%mp_assertscope(SNAPSHOT)
+%put %mf_getfmtlist(sashelp.prdsale);
+%mp_assertscope(COMPARE)
+
 %mp_assert(
   iftrue=(
     "%mf_getfmtlist(sashelp.prdsale)"="DOLLAR $CHAR W MONNAME"
@@ -23,11 +28,3 @@
   desc=Checking basic char,
   outds=work.test_results
 )
-
-%mp_assert(
-  iftrue=(
-    "%mf_getfmtlist(sashelp.demographics)"="BEST Z $CHAR COMMA PERCENTN"
-  ),
-  desc=Checking longer numeric,
-  outds=work.test_results
-)

tests/base/mf_getvalue.test.sas

@@ -74,18 +74,5 @@ run;
   desc=Test fetching value from 1st row of empty (filtered) data,
   outds=work.test_results
 )
+%let syscc=0;
 
-%let syscc = 0; /* Reset w@rning To ensure confidence in next test */
-
-/* - Test 6 -
-  Get value from default observation.
-  Dataset does not exist.
-*/
-%let test_value=%mf_getvalue(work.test_data_x,i);
-%mp_assert(
-  iftrue=(&test_value=%str() and &syscc gt 0),
-  desc=Test fetching value from 1st row of non-existent data,
-  outds=work.test_results
-)
-
-%let syscc = 0; /* To reset expected error and allow test job to exit clean. */

tests/base/mf_getvarlist.test.sas

@@ -15,7 +15,7 @@
 %let test5=%mf_getvarlist(sashelp.class,typefilter=C);
 
 data work.test_results;
-  length test_description $256 test_result $4 test_comments base result  $256;
+  length test_result $4 test_description $256 test_comments base result  $256;
   test_description="Basic test";
   base=symget('test1');
   result='Name Sex Age Height Weight';

tests/base/mf_mimetype.test.sas

@@ -8,9 +8,12 @@
 
 **/
 
+%mp_assertscope(SNAPSHOT)
+%let test_value=%mf_mimetype(CSV);
+%mp_assertscope(COMPARE,ignorelist=test_value)
 
 %mp_assert(
-  iftrue=("%mf_mimetype(XLS)"="application/vnd.ms-excel",
+  iftrue=("%mf_mimetype(XLS)"="application/vnd.ms-excel"),
   desc=Checking correct value
 )
 

tests/base/mp_ds2csv.test.1.sas

@@ -85,7 +85,7 @@ data _null_;
 run;
 
 %mp_assert(
-  iftrue=("&test3a"="X:best. Y:$char7. Z:best."),
+  iftrue=("&test3a"="X:best32. Y:$char7. Z:best32."),
   desc=Checking header row Test 3,
   outds=work.test_results
 )
@@ -96,8 +96,87 @@ run;
 )
 
 /* test 4 - sasjs with compare */
-filename example temp;
+data work.baseball ;
-%mp_ds2csv(sashelp.air,outref=example,headerformat=SASJS)
+attrib
+Name                             length= $18 label="Player's Name"
+Team                             length= $14 label="Team at the End of 1986"
+nAtBat                           length= 8 label="Times at Bat in 1986"
+nHits                            length= 8 label="Hits in 1986"
+nHome                            length= 8 label="Home Runs in 1986"
+nRuns                            length= 8 label="Runs in 1986"
+nRBI                             length= 8 label="RBIs in 1986"
+nBB                              length= 8 label="Walks in 1986"
+YrMajor                          length= 8 label="Years in the Major Leagues"
+CrAtBat                          length= 8 label="Career Times at Bat"
+CrHits                           length= 8 label="Career Hits"
+CrHome                           length= 8 label="Career Home Runs"
+CrRuns                           length= 8 label="Career Runs"
+CrRbi                            length= 8 label="Career RBIs"
+CrBB                             length= 8 label="Career Walks"
+League                           length= $8 label="League at the End of 1986"
+Division                         length= $8 label="Division at the End of 1986"
+Position                         length= $8 label="Position(s) in 1986"
+nOuts                            length= 8 label="Put Outs in 1986"
+nAssts                           length= 8 label="Assists in 1986"
+nError                           length= 8 label="Errors in 1986"
+Salary                           length= 8 label="1987 Salary in $ Thousands"
+Div                              length= $16 label="League and Division"
+logSalary                        length= 8 label="Log Salary"
+;
+infile cards dsd;
+input
+  Name                             :$char.
+  Team                             :$char.
+  nAtBat
+  nHits
+  nHome
+  nRuns
+  nRBI
+  nBB
+  YrMajor
+  CrAtBat
+  CrHits
+  CrHome
+  CrRuns
+  CrRbi
+  CrBB
+  League                           :$char.
+  Division                         :$char.
+  Position                         :$char.
+  nOuts
+  nAssts
+  nError
+  Salary
+  Div                              :$char.
+  logSalary
+;
+missing a b c d e f g h i j k l m n o p q r s t u v w x y z _;
+/* fix precision issues */
+if not missing(logSalary) then logsalary=logSalary*1;
+datalines4;
+"Allanson, Andy",Cleveland,293,66,1,30,29,14,1,293,66,1,30,29,14,American,East,C,446,33,20,.,AE,.
+"Ashby, Alan",Houston,315,81,7,24,38,39,14,3449,835,69,321,414,375,National,West,C,632,43,10,475,NW,6.16331480403464
+"Davis, Alan",Seattle,479,130,18,66,72,76,3,1624,457,63,224,266,263,American,West,1B,880,82,14,480,AW,6.17378610390193
+"Dawson, Andre",Montreal,496,141,20,65,78,37,11,5628,1575,225,828,838,354,National,East,RF,200,11,3,500,NE,6.21460809842219
+"Galarraga, Andres",Montreal,321,87,10,39,42,30,2,396,101,12,48,46,33,National,East,1B,805,40,4,91.5,NE,4.51633897228147
+"Griffin, Alfredo",Oakland,594,169,4,74,51,35,11,4408,1133,19,501,336,194,American,West,SS,282,421,25,750,AW,6.62007320653035
+"Newman, Al",Montreal,185,37,1,23,8,21,2,214,42,1,30,9,24,National,East,2B,76,127,7,70,NE,4.24849524204936
+"Salazar, Argenis",Kansas City,298,73,0,24,24,7,3,509,108,0,41,37,12,American,West,SS,121,283,9,100,AW,4.60517018598809
+"Thomas, Andres",Atlanta,323,81,6,26,32,8,2,341,86,6,32,34,8,National,West,SS,143,290,19,75,NW,4.31748811353631
+"Thornton, Andre",Cleveland,401,92,17,49,66,65,13,5206,1332,253,784,890,866,American,East,DH,0,0,0,1100,AE,7.00306545878646
+"Trammell, Alan",Detroit,574,159,21,107,75,59,10,4631,1300,90,702,504,488,American,East,SS,238,445,22,517.143,AE,6.24831943200756
+"Trevino, Alex",Los Angeles,202,53,4,31,26,27,9,1876,467,15,192,186,161,National,West,C,304,45,11,512.5,NW,6.23930071101256
+"Van Slyke, Andy",St Louis,418,113,13,48,61,47,4,1512,392,41,205,204,203,National,East,RF,211,11,7,550,NE,6.30991827822651
+"Wiggins, Alan",Baltimore,239,60,0,30,11,22,6,1941,510,4,309,103,207,American,East,2B,121,151,6,700,AE,6.5510803350434
+"Almon, Bill",Pittsburgh,196,43,7,29,27,30,13,3231,825,36,376,290,238,National,East,UT,80,45,8,240,NE,5.48063892334199
+"Beane, Billy",Minneapolis,183,39,3,20,15,11,3,201,42,3,20,16,11,American,West,OF,118,0,0,.,AW,.
+"Bell, Buddy",Cincinnati,568,158,20,89,75,73,15,8068,2273,177,1045,993,732,National,West,3B,105,290,10,775,NW,6.65286302935334
+"Biancalana, Buddy",Kansas City,190,46,2,24,8,15,5,479,102,5,65,23,39,American,West,SS,102,177,16,175,AW,5.16478597392351
+"Bochte, Bruce",Oakland,407,104,6,57,43,65,12,5233,1478,100,643,658,653,American,West,1B,912,88,9,.,AW,.
+;;;;
+run;
+filename example temp lrecl=5000;
+%mp_ds2csv(work.baseball,outref=example,headerformat=SASJS)
 data _null_; infile example; input;put _infile_; if _n_>5 then stop;run;
 
 data _null_;
@@ -113,15 +192,16 @@ run;
 
 %mp_assert(
   iftrue=(&syscc =0),
-  desc=Checking syscc prior to compare of sashelp.air,
+  desc=Checking syscc prior to compare of work.baseball,
   outds=work.test_results
 )
 
-proc compare base=want compare=sashelp.air;
+proc compare base=want compare=work.baseball
+  method = absolute criterion = 0.0000000001   ;
 run;
 %mp_assert(
   iftrue=(&sysinfo le 41),
-  desc=Checking compare of sashelp.air,
+  desc=Checking compare of work.baseball,
   outds=work.test_results
 )
 

tests/base/mp_ds2csv.test.2.sas

@@ -32,7 +32,7 @@ data _null_;
 run;
 
 %mp_assert(
-  iftrue=("&test1a"="A:best3. B:best4. C:best."),
+  iftrue=("&test1a"="A:best3. B:best4. C:best32."),
   desc=Checking header row Test 1,
   outds=work.test_results
 )

tests/base/mp_gitreleaseinfo.test.sas

@@ -24,7 +24,7 @@ data _null_;
 run;
 
 %mp_assert(
-  iftrue=(&author=sasjsbot),
+  iftrue=("&author"="github-actions[bot]"),
   desc=release info extracted successfully,
   outds=work.test_results
 )

tests/base/mp_retainedkey.test.sas

@@ -5,6 +5,7 @@
   <h4> SAS Macros </h4>
   @li mp_assert.sas
   @li mp_assertcolvals.sas
+  @li mp_assertscope.sas
   @li mp_retainedkey.sas
 
 **/
@@ -114,3 +115,53 @@ run;
   desc=Checking append records created,
   outds=work.test_results
 )
+
+/** Test 3 - large numeric RK values (avoid exponential notation issue) **/
+data work.targetds3;
+  set sashelp.class;
+  rk_col=_n_+209100000000;
+run;
+
+data work.appendtable3;
+  set sashelp.class;
+  if mod(_n_,2)=0 then name=cats('New',_n_);
+  if _n_<7;
+run;
+
+%mp_assertscope(SNAPSHOT)
+
+%mp_retainedkey(
+  base_lib=X
+  ,base_dsn=targetds3
+  ,append_lib=X
+  ,append_dsn=APPENDTABLE3
+  ,retained_key=rk_col
+  ,business_key= name
+  ,check_uniqueness=NO
+  ,maxkeytable=0
+  ,locktable=0
+  ,outds=work.APPEND3
+  ,filter_str=
+)
+
+%mp_assertscope(COMPARE,
+  desc=Checking macro scope leakage in mp_retainedkey (test 3)
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Checking errors in test 3 (large numeric RK),
+  outds=work.test_results
+)
+
+data work.check3;
+  do val=209100000001,209100000003,209100000005,
+    209100000020,209100000021,209100000022;
+    output;
+  end;
+run;
+%mp_assertcolvals(work.append3.rk_col,
+  checkvals=work.check3.val,
+  desc=All large numeric values have a match,
+  test=ALLVALS
+)

tests/base/mp_rowhash.test.sas

@@ -0,0 +1,195 @@
+/**
+  @file
+  @brief Testing mp_rowhash.sas macro
+  @details The mp_rowhash macro generates DATA step statements that compute a
+    deterministic row-level MD5 hash.  These tests exercise the versioned seed,
+    character and numeric hashing rules, variable ordering, sensitivity to
+    whitespace, and reproducibility.
+
+  <h4> SAS Macros </h4>
+  @li mp_rowhash.sas
+  @li mp_assert.sas
+  @li mp_assertdsobs.sas
+  @li mp_assertscope.sas
+
+**/
+
+%mp_assertscope(SNAPSHOT)
+
+/* test 1 - versioned seed only (no input variables) */
+data _null_;
+  length actual expected $32;
+  actual='';
+  %mp_rowhash(md5_col=actual)
+  expected=put(md5('DC HASH v2'),$hex32.);
+  call symputx('t1_actual',actual);
+  call symputx('t1_expected',expected);
+run;
+%mp_assertscope(COMPARE,ignorelist=T1_ACTUAL T1_EXPECTED)
+%mp_assert(
+  iftrue=("&t1_actual"="&t1_expected"),
+  desc=Versioned seed is returned when no variables are supplied,
+  outds=work.test_results
+)
+
+/* test 2 - single character variable reference */
+data _null_;
+  length c1 $1 actual expected $32;
+  length state digest $16 pair $32;
+  c1='A';
+  actual='';
+  %mp_rowhash(md5_col=actual, cvars=c1)
+  state=md5('DC HASH v2');
+  digest=md5(trimn(c1));
+  pair=state||digest;
+  expected=put(md5(pair),$hex32.);
+  call symputx('t2_actual',actual);
+  call symputx('t2_expected',expected);
+run;
+%mp_assert(
+  iftrue=("&t2_actual"="&t2_expected"),
+  desc=Single character variable hash matches reference construction,
+  outds=work.test_results
+)
+
+/* test 3 - single numeric variable reference */
+data _null_;
+  length actual expected $32;
+  length state digest $16 numtext $64 pair $32;
+  n1=42;
+  actual='';
+  %mp_rowhash(md5_col=actual, nvars=n1)
+  state=md5('DC HASH v2');
+  /* multiply-by-one matches the internal normalisation */
+  normal=n1*1;
+  numtext=put(normal,binary64.);
+  digest=md5(trim(numtext));
+  pair=state||digest;
+  expected=put(md5(pair),$hex32.);
+  call symputx('t3_actual',actual);
+  call symputx('t3_expected',expected);
+run;
+%mp_assert(
+  iftrue=("&t3_actual"="&t3_expected"),
+  desc=Single numeric variable hash matches reference construction,
+  outds=work.test_results
+)
+
+/* test 4 - order of cvars affects the resulting hash */
+data _null_;
+  length c1 c2 $1 h1 h2 $32;
+  c1='A';
+  c2='B';
+  h1='';
+  %mp_rowhash(md5_col=h1, cvars=c1 c2)
+  h2='';
+  %mp_rowhash(md5_col=h2, cvars=c2 c1)
+  call symputx('t4_h1',h1);
+  call symputx('t4_h2',h2);
+run;
+%mp_assert(
+  iftrue=("&t4_h1" ne "&t4_h2"),
+  desc=Order of cvars changes the resulting hash,
+  outds=work.test_results
+)
+
+/* test 5 - order of nvars affects the resulting hash */
+data _null_;
+  length h1 h2 $32;
+  n1=1;
+  n2=2;
+  h1='';
+  %mp_rowhash(md5_col=h1, nvars=n1 n2)
+  h2='';
+  %mp_rowhash(md5_col=h2, nvars=n2 n1)
+  call symputx('t5_h1',h1);
+  call symputx('t5_h2',h2);
+run;
+%mp_assert(
+  iftrue=("&t5_h1" ne "&t5_h2"),
+  desc=Order of nvars changes the resulting hash,
+  outds=work.test_results
+)
+
+/* test 6 - identical rows produce identical hashes */
+data work.test6;
+  length c $5 n 8;
+  c='ABC';
+  n=42;
+  output;
+  output;
+run;
+
+data work.test6_hashed;
+  set work.test6;
+  length hash $32;
+  %mp_rowhash(md5_col=hash, cvars=c, nvars=n)
+run;
+
+proc sql noprint;
+  select count(distinct hash) into: t6_distinct trimmed
+    from work.test6_hashed;
+quit;
+
+%mp_assert(
+  iftrue=(&t6_distinct=1),
+  desc=Identical rows produce identical hashes,
+  outds=work.test_results
+)
+
+/* test 7 - different rows produce different hashes */
+data work.test7;
+  length c $5 n 8;
+  c='ABC';
+  n=42;
+  output;
+  c='ABD';
+  n=42;
+  output;
+run;
+
+data work.test7_hashed;
+  set work.test7;
+  length hash $32;
+  %mp_rowhash(md5_col=hash, cvars=c, nvars=n)
+run;
+
+proc sql noprint;
+  select count(distinct hash) into: t7_distinct trimmed
+    from work.test7_hashed;
+quit;
+
+%mp_assert(
+  iftrue=(&t7_distinct=2),
+  desc=Different rows produce different hashes,
+  outds=work.test_results
+)
+
+/* test 8 - leading blanks in character variables are retained */
+data work.test8;
+  length c $5;
+  c=' f';
+  output;
+  c='f';
+  output;
+run;
+
+data work.test8_hashed;
+  set work.test8;
+  length hash $32;
+  %mp_rowhash(md5_col=hash, cvars=c)
+run;
+
+proc sql noprint;
+  select count(distinct hash) into: t8_distinct trimmed
+    from work.test8_hashed;
+quit;
+
+%mp_assert(
+  iftrue=(&t8_distinct=2),
+  desc=Leading blanks are retained in character hashes,
+  outds=work.test_results
+)
+
+
+

tests/base/mp_validatecol.test.sas

@@ -129,4 +129,30 @@ run;
   desc=Test4 - ISFORMAT,
   test=EQUALS 6,
   outds=work.test_results
+)
+
+/**
+  * Test 5 - ISLIB
+  */
+data test5;
+  infile datalines4 dsd;
+  input;
+  inf=_infile_;
+  %mp_validatecol(inf,ISLIB,islib)
+  if islib=1;
+datalines4;
+some
+!lib
+%abort
+definite
+2fail
+nineletrs
+.failalso
+_valid
+;;;;
+run;
+%mp_assertdsobs(work.test5,
+  desc=Testing ISLIB,
+  test=EQUALS 3,
+  outds=work.test_results
 )

tests/serveronly/ms_adduser2group.test.sas

@@ -32,6 +32,7 @@ run;
 %ms_adduser2group(uid=1,gid=&groupid,mdebug=&sasjs_mdebug,outds=test1)
 %mp_assertscope(COMPARE
   ,ignorelist=MCLIB2_JADP1LEN MCLIB2_JADP2LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
+    MC2_JADP1LEN MC2_JADP2LEN MC2_JADPNUM MC2_JADVLEN
 )
 
 /* check the user is in the output list */

tests/serveronly/ms_creategroup.test.sas

@@ -16,7 +16,8 @@
 %mp_assertscope(SNAPSHOT)
 %ms_creategroup(&group, desc=The description,mdebug=&sasjs_mdebug,outds=test1)
 %mp_assertscope(COMPARE
-  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN
+  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN MC0_JADP1LEN
+    MC0_JADPNUM MC0_JADVLEN
 )
 
 %let id=0;

tests/serveronly/ms_createuser.test.sas

@@ -16,7 +16,8 @@
 %mp_assertscope(SNAPSHOT)
 %ms_createuser(&user,passwrd,outds=test1,mdebug=&sasjs_mdebug)
 %mp_assertscope(COMPARE
-  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN
+  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN MC0_JADP1LEN
+    MC0_JADPNUM MC0_JADVLEN
 )
 
 %let id=0;

tests/serveronly/ms_getgroups.test.sas

@@ -21,7 +21,8 @@
 %mp_assertscope(SNAPSHOT)
 %ms_getgroups(outds=work.test1,mdebug=&sasjs_mdebug)
 %mp_assertscope(COMPARE
-  ,ignorelist=MCLIB2_JADP1LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
+  ,ignorelist=MCLIB2_JADP1LEN MCLIB2_JADPNUM MCLIB2_JADVLEN MC2_JADP1LEN
+    MC2_JADPNUM MC2_JADVLEN
 )
 
 /* check the group was created */

tests/serveronly/ms_getusers.test.sas

@@ -15,7 +15,8 @@
 %mp_assertscope(SNAPSHOT)
 %ms_getusers(outds=work.test1,mdebug=&sasjs_mdebug)
 %mp_assertscope(COMPARE
-  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN
+  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN MC0_JADP1LEN
+    MC0_JADPNUM MC0_JADVLEN
 )
 
 %mp_assertdsobs(work.test1,test=ATLEAST 1)

tests/serveronly/ms_triggerstp.test.sas

@@ -51,12 +51,13 @@ options mprint;
   %ms_triggerstp(/sasjs/tests/&fname2
     ,outds=work.mySessions
   )
-%mp_assertscope(COMPARE
+%mp_assertscope(COMPARE,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM
-                ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADPNUM MCLIB0_JADVLEN)
+  MCLIB0_JADVLEN MC0_JADP1LEN MC0_JADPNUM MC0_JADVLEN
+)
 
 %mp_assert(iftrue=%str(%mf_existds(work.mySessions)=1)
-          ,desc=Testing output exists
+  ,desc=Testing output exists
-          ,outds=work.test_results
+  ,outds=work.test_results
 )
 
 %mp_assertdsobs(work.mySessions,

tests/testinit.sas

@@ -10,7 +10,9 @@
 **/
 
 /* location in metadata or SAS Drive for temporary files */
-%let mcTestAppLoc=/Public/testresults/sasjs_core/%mf_uid();
+%let mcTestAppLoc=/Users/&sysuserid/testresults/sasjs_core/%mf_uid();
+
+%let mcTestContext=SAS Job Execution compute context;
 
 /* set defaults */
 %mp_init()
@@ -23,6 +25,7 @@ options lrecl=80;
 
 %macro loglevel();
   %if "&_debug"="2477" or "&_debug"="fields,log,trace" or "&_debug"="131"
+  or "&_debug"="128"
   %then %do;
     %put debug mode activated;
     options mprint mprintnest;

tests/viyaonly/mfv_existsashdat.test.sas

@@ -0,0 +1,74 @@
+/**
+  @file
+  @brief Testing mfv_existsashdat macro function
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mfv_existsashdat.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+options mprint;
+
+/* ------------------------------------------------------------------------ */
+/* Setup: start a CAS session and stage a sashdat file in the Public caslib */
+/* ------------------------------------------------------------------------ */
+cas mysess;
+caslib _all_ assign;
+
+%let testcaslib = Public;   /* change this if Public isn't available */
+proc cas;
+  table.caslibInfo result=r / ;
+  found = 0;
+  do row over r.CASLibInfo;
+    if upcase(row.Name) = upcase("&testcaslib") then found = 1;
+  end;
+  if found = 0 then do;
+    print "ERROR: caslib &testcaslib not available";
+    exit;
+  end;
+quit;
+%put NOTE: Using testcaslib=&testcaslib;
+
+%let tab1=T%mf_uid();
+
+proc casutil;
+  load data=sashelp.baseball outcaslib="&testcaslib" casout="&tab1" replace;
+  save casdata="&tab1" incaslib="&testcaslib"
+    casout="&tab1..sashdat" outcaslib="&testcaslib" replace;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+quit;
+
+
+/* ------------------------------------------------------------------------ */
+%put TEST 1 - returns 1 when the sashdat file exists in the caslib;
+/* ------------------------------------------------------------------------ */
+%mp_assert(
+  iftrue=(%mfv_existsashdat(&testcaslib..&tab1)=1),
+  desc=Test 1 - Check returns 1 for a sashdat that exists
+)
+
+/* ------------------------------------------------------------------------ */
+%put TEST 2 - returns 0 when the file does not exist in the caslib;
+/* ------------------------------------------------------------------------ */
+%mp_assertscope(SNAPSHOT)
+%mp_assert(
+  iftrue=(%mfv_existsashdat(&testcaslib..DOESNOTEXIST_%mf_uid())=0),
+  desc=Check returns 0 for a sashdat that does not exist
+)
+%mp_assertscope(COMPARE,
+  desc=Check mfv_existsashdat does not leak macro variables into GLOBAL scope
+)
+
+/* ------------------------------------------------------------------------ */
+/* Teardown                                                                 */
+/* ------------------------------------------------------------------------ */
+proc casutil;
+  deletesource casdata="&tab1..sashdat" incaslib="&testcaslib" quiet;
+quit;
+
+cas mysess terminate;
+
+%let syscc=0;

tests/viyaonly/mfv_getcaslib.test.sas

@@ -0,0 +1,69 @@
+/**
+  @file
+  @brief Testing mfv_getcaslib macro function
+
+  <h4> SAS Macros </h4>
+  @li mfv_getcaslib.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+options mprint;
+
+/* ------------------------------------------------------------------------ */
+/* Setup: start a CAS session and assign caslibs                            */
+/* ------------------------------------------------------------------------ */
+cas mysess;
+caslib _all_ assign;
+
+%let testcaslib=Public;
+
+libname castest cas caslib=&testcaslib;
+
+/* ------------------------------------------------------------------------ */
+%put TEST 1 - returns the caslib name for a valid CAS libref;
+/* ------------------------------------------------------------------------ */
+%mp_assert(
+  iftrue=(%mfv_getcaslib(castest)=%upcase(&testcaslib)),
+  desc=Check correct caslib name returned for a valid CAS libref
+)
+
+
+/* ------------------------------------------------------------------------ */
+%put TEST 2 - returns empty for a non-CAS libref (WORK);
+/* ------------------------------------------------------------------------ */
+%mp_assert(
+  iftrue=(%mfv_getcaslib(WORK)=),
+  desc=Check empty string returned for a non-CAS libref
+)
+
+
+/* ------------------------------------------------------------------------ */
+%put TEST 3 - returns empty for a libref that does not exist;
+/* ------------------------------------------------------------------------ */
+%mp_assert(
+  iftrue=(%mfv_getcaslib(DOESNOTEXIST)=),
+  desc=Check empty string returned for a non-existent libref
+)
+
+
+/* ------------------------------------------------------------------------ */
+%put TEST 5 - no scope leakage into global macro variables;
+/* ------------------------------------------------------------------------ */
+%mp_assertscope(SNAPSHOT)
+
+%let _rc=%mfv_getcaslib(castest);
+
+%mp_assertscope(COMPARE,
+  desc=Check mfv_getcaslib does not leak macro variables into GLOBAL scope,
+  ignorelist=_RC
+)
+
+
+/* ------------------------------------------------------------------------ */
+/* Teardown                                                                  */
+/* ------------------------------------------------------------------------ */
+cas mysess terminate;
+
+

tests/viyaonly/mv_castabload.test.sas

@@ -0,0 +1,152 @@
+/**
+  @file
+  @brief Testing mv_castabload macro
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mv_castabload.sas
+
+**/
+
+options mprint;
+
+/* -------------------------------------------------------------------- */
+/* Setup: start a CAS session and stage a source file in the caslib     */
+/* -------------------------------------------------------------------- */
+cas mysess;
+caslib _all_ assign;
+
+%let testcaslib=Public;
+
+proc cas;
+  table.caslibInfo result=r / ;
+  found=0;
+  do row over r.CASLibInfo;
+    if upcase(row.Name)=upcase("&testcaslib") then found=1;
+  end;
+  if found=0 then do;
+    print "ERROR: caslib &testcaslib not available";
+    exit;
+  end;
+quit;
+%put NOTE: Using testcaslib=&testcaslib;
+
+%let tab1=T%mf_uid();
+
+/* Save a sashdat source file then drop the in-memory copy so the first
+    mv_castabload call has something to load                             */
+proc casutil;
+  load data=sashelp.baseball
+    outcaslib="&testcaslib" casout="&tab1" replace;
+  save casdata="&tab1" incaslib="&testcaslib"
+    casout="&tab1..sashdat" outcaslib="&testcaslib" replace;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+quit;
+
+libname mylib cas caslib="&testcaslib";
+
+
+/* -------------------------------------------------------------------- */
+%put TEST 1 - load a table that is not in memory;
+/* -------------------------------------------------------------------- */
+
+/* Confirm table is absent before the call */
+%let _tabexists=0;
+proc cas;
+  table.tableExists result=r /
+    caslib="&testcaslib" name="&tab1";
+  if r.exists > 0 then call symputx('_tabexists','1');
+quit;
+
+%mp_assert(
+  iftrue=(&_tabexists=0),
+  desc=Check table is not in memory before mv_castabload
+)
+
+%mv_castabload(lib=mylib, table=&tab1, mdebug=1)
+
+%let _tabexists=0;
+proc cas;
+  table.tableExists result=r /
+    caslib="&testcaslib" name="&tab1";
+  if r.exists > 0 then call symputx('_tabexists','1');
+quit;
+
+%mp_assert(
+  iftrue=(&_tabexists=1),
+  desc=Check table is in memory after mv_castabload
+)
+
+
+/* -------------------------------------------------------------------- */
+%put TEST 2 - reload fetches a fresh copy and discards in-memory changes;
+/* -------------------------------------------------------------------- */
+
+/* Append a sentinel row to the in-memory table */
+data work.extra;
+  set mylib.&tab1;
+  name='TESTROW';
+  output;
+  stop;
+run;
+proc casutil;
+  load data=work.extra casout="&tab1"
+    outcaslib="&testcaslib" append;
+quit;
+
+%let _modified=0;
+proc sql noprint;
+  select count(*) into :_modified
+  from mylib.&tab1
+  where name='TESTROW';
+quit;
+
+%mp_assert(
+  iftrue=(&_modified=1),
+  desc=Check sentinel row is present in memory before reload
+)
+
+/* Drop the table and reload - source file does not have the sentinel   */
+proc casutil;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+quit;
+
+%mp_assertscope(SNAPSHOT)
+
+%mv_castabload(lib=mylib, table=&tab1, mdebug=1)
+
+%mp_assertscope(COMPARE,
+  desc=Check mv_castabload does not leak macro variables into GLOBAL scope
+)
+
+%let _after=0;
+proc sql noprint;
+  select count(*) into :_after
+  from mylib.&tab1
+  where name='TESTROW';
+quit;
+
+%mp_assert(
+  iftrue=(&_after=0),
+  desc=Check sentinel row is absent after reload from source
+)
+
+
+/* -------------------------------------------------------------------- */
+/* Teardown                                                              */
+/* -------------------------------------------------------------------- */
+libname mylib clear;
+
+proc casutil;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  deletesource casdata="&tab1..sashdat"
+    incaslib="&testcaslib" quiet;
+quit;
+
+cas mysess terminate;
+
+%let syscc=0;

tests/viyaonly/mv_castabsave.test.sas

@@ -0,0 +1,158 @@
+/**
+  @file
+  @brief Testing mv_castabsave macro
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mv_castabsave.sas
+
+**/
+
+options mprint;
+
+/* -------------------------------------------------------------------- */
+/* Setup: start a CAS session and load a table that has a tracked       */
+/*        source file so mv_castabsave can discover it via the REST API */
+/* -------------------------------------------------------------------- */
+cas mysess;
+caslib _all_ assign;
+
+%let testcaslib=Public;
+
+proc cas;
+  table.caslibInfo result=r / ;
+  found=0;
+  do row over r.CASLibInfo;
+    if upcase(row.Name)=upcase("&testcaslib") then found=1;
+  end;
+  if found=0 then do;
+    print "ERROR: caslib &testcaslib not available";
+    exit;
+  end;
+quit;
+%put NOTE: Using testcaslib=&testcaslib;
+
+%let tab1=T%mf_uid();
+
+/* Load sashelp.class into CAS, save as sashdat, reload from that file
+    so the table has a tracked source path (needed for REST discovery) */
+proc casutil;
+  load data=sashelp.class
+    outcaslib="&testcaslib" casout="&tab1" replace;
+  save casdata="&tab1" incaslib="&testcaslib"
+    casout="&tab1..sashdat" outcaslib="&testcaslib" replace;
+  /* Drop any existing global-scope version before promoting */
+  /* runs twice (with quiet) as first would drop local scope if exists */
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+
+  load casdata="&tab1..sashdat" incaslib="&testcaslib"
+    casout="&tab1" outcaslib="&testcaslib" promote;
+quit;
+
+libname mylib cas caslib="&testcaslib";
+
+
+/* -------------------------------------------------------------------- */
+%put TEST 1 - save in-memory table back to disk + no scope leakage;
+/* -------------------------------------------------------------------- */
+
+/* Source file is removed so that the reload proves mv_castabsave
+    created the file from scratch, not that a prior version existed   */
+proc casutil;
+  deletesource casdata="&tab1..sashdat"
+    incaslib="&testcaslib" quiet;
+quit;
+
+/* Insert a sentinel row - it must survive the full save/drop/reload   */
+data work.appendme;
+  set mylib.&tab1;
+  name='TESTROW';
+  output;
+  stop;
+proc casutil;
+  load data=work.appendme casout="&tab1" outcaslib="&testcaslib" append;
+quit;
+
+%mp_assertscope(SNAPSHOT)
+
+%mv_castabsave(lib=mylib, table=&tab1, mdebug=1)
+
+%mp_assertscope(COMPARE,
+  desc=Check mv_castabsave does not leak macro variables into GLOBAL scope,
+  ignorelist=MC0_JADP1LEN MC0_JADP2LEN MC0_JADP3LEN MC0_JADPNUM MC0_JADVLEN
+)
+
+proc casutil;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  load casdata="&tab1..sashdat" incaslib="&testcaslib"
+    casout="&tab1" outcaslib="&testcaslib" promote;
+quit;
+
+%let _rowcount=0;
+proc sql noprint;
+  select count(*) into :_rowcount
+  from mylib.&tab1
+  where name='TESTROW';
+quit;
+
+%mp_assert(
+  iftrue=(&_rowcount=1),
+  desc=Check inserted row survives mv_castabsave round-trip to disk
+)
+
+
+/* -------------------------------------------------------------------- */
+%put TEST 2 - save overwrites an existing source file;
+/* -------------------------------------------------------------------- */
+
+/* Source file already exists from the TEST 1 save - append a new row  */
+data work.appendme;
+  set mylib.&tab1;
+  name='TESTROW2';
+  output;
+  stop;
+proc casutil;
+  load data=work.appendme casout="&tab1"
+    outcaslib="&testcaslib" append;
+quit;
+
+%mv_castabsave(lib=mylib, table=&tab1, mdebug=1)
+
+proc casutil;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  load casdata="&tab1..sashdat" incaslib="&testcaslib"
+    casout="&tab1" outcaslib="&testcaslib" promote;
+quit;
+
+%let _rowcount=0;
+proc sql noprint;
+  select count(*) into :_rowcount
+  from mylib.&tab1
+  where name='TESTROW2';
+quit;
+
+%mp_assert(
+  iftrue=(&_rowcount=1),
+  desc=Check inserted row survives save over an existing source file
+)
+
+
+/* -------------------------------------------------------------------- */
+/* Teardown                                                              */
+/* -------------------------------------------------------------------- */
+libname mylib clear;
+
+proc casutil;
+  droptable casdata="&tab1" incaslib="&testcaslib" quiet;
+  deletesource casdata="&tab1..sashdat"
+    incaslib="&testcaslib" quiet;
+quit;
+
+cas mysess terminate;
+
+%let syscc=0;

tests/viyaonly/mv_createfolder.test.sas

@@ -5,6 +5,7 @@
   <h4> SAS Macros </h4>
   @li mf_uid.sas
   @li mp_assert.sas
+  @li mp_assertscope.sas
   @li mv_createfolder.sas
   @li mv_deleteviyafolder.sas
   @li mv_getfoldermembers.sas
@@ -15,7 +16,11 @@
 %let folder=%mf_uid();
 
 /* create a folder */
+%mp_assertscope(SNAPSHOT)
 %mv_createfolder(path=&mcTestAppLoc/temp/&folder/&folder)
+%mp_assertscope(COMPARE, ignorelist=MC0_JADP1LEN MC0_JADP2LEN MC0_JADPNUM
+  MC0_JADVLEN MC2_JADP1LEN MC2_JADP2LEN MC2_JADPNUM MC2_JADVLEN
+)
 
 %mv_getfoldermembers(root=&mcTestAppLoc/temp/&folder, outds=work.folders)
 
@@ -32,17 +37,39 @@ run;
 )
 
 /* create a folder without output dataset as part of the original macro */
-%mv_createfolder(path=&mcTestAppLoc/temp/&folder/folder2,outds=folders2)
+%mv_createfolder(path=&mcTestAppLoc/temp/&folder/f2
+  ,outds=folders2,mdebug=&sasjs_mdebug
+)
 
 %let test=0;
 data _null_;
   set work.folders2;
   putlog (_all_)(=);
-  if not missing(self_uri) and not missing(parent_uri)
+  if not missing(self_uri) then call symputx('test2',1);
-  then call symputx('test2',1);
 run;
 
 %mp_assert(
   iftrue=(&test2=1),
   desc=Check if outds param works
+)
+
+/* create a folder with full stops */
+%let newfolder=%mf_uid().2.1;
+%mv_createfolder(path=&mcTestAppLoc/temp/&newfolder
+  ,outds=work.folders3
+  ,mdebug=&sasjs_mdebug
+)
+
+%mv_getfoldermembers(root=&mcTestAppLoc/temp, outds=work.folders3)
+
+%let test3=0;
+data _null_;
+  set work.folders3;
+  putlog (_all_)(=);
+  if name="&newfolder" then call symputx('test3',1);
+run;
+
+%mp_assert(
+  iftrue=(&test3=1),
+  desc=Check if folder with full stops can be successfully created
 )

tests/viyaonly/mv_createwebservice.test.sas

@@ -32,7 +32,7 @@ run;
 )
 %put TEST1: checking web service code;
 data work.test_results;
-  length test_description $256 test_result $4 test_comments $256;
+  length test_result $4 test_description $256 test_comments $256;
   if _n_=1 then call missing (of _all_);
   infile compare end=eof;
   input;

tests/viyaonly/mv_getviyafileextparms.test.sas

@@ -80,7 +80,7 @@ options mprint;
   typeDefNameVar=invalidTypeDefName,
   propertiesVar=invalidProperties,
   mediaTypeVar=invalidMediaType
-  )
+)
 %mp_assertscope(COMPARE
   ,ignorelist=
     &mvarIgnoreList invalidTypeDefName invalidProperties invalidMediaType

tests/viyaonly/mv_jobflow.test.1.sas

@@ -33,19 +33,19 @@ data _null_;
   ;
 run;
 
-%mv_createjob(path=/Public/temp,name=demo1,code=testprog)
+%mv_createjob(path=&mcTestAppLoc,name=demo1,code=testprog)
-%mv_createjob(path=/Public/temp,name=demo2,code=testprog)
+%mv_createjob(path=&mcTestAppLoc,name=demo2,code=testprog)
 
 data work.inputjobs;
-  _contextName='SAS Job Execution compute context';
+  _contextName="&mcTestContext";
   do flow_id=1 to 2;
     do i=1 to 4;
-      _program='/Public/temp/demo1';
+      _program="&mcTestAppLoc/demo1";
       macrovar1=10*i;
       macrovar2=4*i;
       output;
       i+1;
-      _program='/Public/temp/demo2';
+      _program="&mcTestAppLoc/demo2";
       macrovar1=40*i;
       macrovar2=44*i;
       output;

tests/viyaonly/mv_jobflow.test.2.sas

@@ -29,19 +29,19 @@ data _null_;
   ;
 run;
 
-%mv_createjob(path=/Public/temp,name=demo1,code=testprog)
+%mv_createjob(path=&mcTestAppLoc,name=demo1,code=testprog)
-%mv_createjob(path=/Public/temp,name=demo2,code=testprog)
+%mv_createjob(path=&mcTestAppLoc,name=demo2,code=testprog)
 
 data work.inputjobs;
-  _contextName='SAS Job Execution compute context';
+  _contextName="&mcTestContext";
   do flow_id=1 to 2;
     do i=1 to 4;
-      _program='/Public/temp/demo1';
+      _program="&mcTestAppLoc/demo1";
       macrovar1=10*i;
       macrovar2=4*i;
       output;
       i+1;
-      _program='/Public/temp/demo2';
+      _program="&mcTestAppLoc/demo2";
       macrovar1=40*i;
       macrovar2=44*i;
       output;

tests/x-platform/mx_append2pgm.test.sas

@@ -0,0 +1,192 @@
+/**
+  @file
+  @brief Testing mx_append2pgm.sas macro
+
+  Be sure to run <code>%let mcTestAppLoc=/Public/temp/macrocore;</code> when
+  running in Studio
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mf_uid.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li ms_createfile.sas
+  @li mv_createfile.sas
+  @li mm_createstp.sas
+  @li mx_append2pgm.sas
+  @li mx_getcode.sas
+
+**/
+
+/**
+  * Test 1 - Append content to an existing program and verify combined output
+  * Also checking for scope leakage
+  */
+
+/* create a unique name for the program */
+%let item=test_%mf_uid();
+
+/* create the initial program with some code */
+filename initpgm temp;
+data _null_;
+  file initpgm;
+  put '%put ORIGINAL LINE;';
+run;
+
+%macro setup_pgm();
+  %let platform=%mf_getplatform();
+  %if &platform=SASJS %then %do;
+    %ms_createfile(&mcTestAppLoc/temp/&item..sas, inref=initpgm)
+  %end;
+  %else %if &platform=SASVIYA %then %do;
+    %mv_createfile(path=&mcTestAppLoc/temp, name=&item..sas, inref=initpgm)
+  %end;
+  %else %do;
+    %let work=%sysfunc(pathname(work));
+    data _null_;
+      file "&work/&item..sas";
+      infile initpgm;
+      input;
+      put _infile_;
+    run;
+    %mm_createstp(stpname=&item
+      ,filename=&item..sas
+      ,directory=&work
+      ,tree=&mcTestAppLoc/temp
+      ,stptype=2
+      ,minify=NO
+    )
+  %end;
+%mend setup_pgm;
+%setup_pgm()
+
+/* create the content to append */
+filename toappnd temp;
+data _null_;
+  file toappnd;
+  put '%put APPENDED LINE;';
+run;
+
+/* run the macro under test with scope checks */
+%mp_assertscope(SNAPSHOT)
+%mx_append2pgm(&mcTestAppLoc/temp/&item, inref=toappnd)
+%mp_assertscope(COMPARE,
+  desc=Test 1: mx_append2pgm does not leak scope,
+  outds=work.test_results,
+  ignorelist=MC2_JADP1LEN MC2_JADP2LEN MC2_JADPNUM MC2_JADVLEN MC2_JADP3LEN
+)
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Test 1: No errors after appending content to program,
+  outds=work.test_results
+)
+
+/**
+  * Test 2 - Verify the appended content is present
+  * Fetch the modified program and check both original and appended lines exist
+  */
+
+%let test2_orig=0;
+%let test2_appd=0;
+
+%macro verify_test2();
+  %let platform=%mf_getplatform();
+  %if &platform=SASVIYA %then %do;
+    filename verifrf filesrvc folderpath="&mcTestAppLoc/temp";
+    data _null_;
+      infile verifrf("&item..sas") lrecl=32000;
+      input;
+      if index(_infile_,'ORIGINAL LINE') then call symputx('test2_orig','1');
+      if index(_infile_,'APPENDED LINE') then call symputx('test2_appd','1');
+    run;
+    filename verifrf clear;
+  %end;
+  %else %do;
+    %mx_getcode(&mcTestAppLoc/temp/&item, outref=verifrf)
+    data _null_;
+      infile verifrf lrecl=32000;
+      input;
+      if index(_infile_,'ORIGINAL LINE') then call symputx('test2_orig','1');
+      if index(_infile_,'APPENDED LINE') then call symputx('test2_appd','1');
+    run;
+  %end;
+%mend verify_test2;
+%verify_test2()
+
+%mp_assert(
+  iftrue=(&test2_orig=1),
+  desc=Test 2a: Original content is preserved after append,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&test2_appd=1),
+  desc=Test 2b: Appended content is present in modified program,
+  outds=work.test_results
+)
+
+/**
+  * Test 3 - Append multiple times to ensure repeated appends work
+  */
+filename toappd2 temp;
+data _null_;
+  file toappd2;
+  put '%put SECOND APPEND;';
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mx_append2pgm(&mcTestAppLoc/temp/&item, inref=toappd2)
+%mp_assertscope(COMPARE,
+  desc=Test 3: mx_append2pgm does not leak scope on second call,
+  outds=work.test_results
+)
+
+/* verify all three pieces of content exist */
+%let test3_orig=0;
+%let test3_appd=0;
+%let test3_app2=0;
+
+%macro verify_test3();
+  %let platform=%mf_getplatform();
+  %if &platform=SASVIYA %then %do;
+    filename verifr2 filesrvc folderpath="&mcTestAppLoc/temp";
+    data _null_;
+      infile verifr2("&item..sas") lrecl=32000;
+      input;
+      if index(_infile_,'ORIGINAL LINE') then call symputx('test3_orig','1');
+      if index(_infile_,'APPENDED LINE') then call symputx('test3_appd','1');
+      if index(_infile_,'SECOND APPEND') then call symputx('test3_app2','1');
+    run;
+    filename verifr2 clear;
+  %end;
+  %else %do;
+    %mx_getcode(&mcTestAppLoc/temp/&item, outref=verifr2)
+    data _null_;
+      infile verifr2 lrecl=32000;
+      input;
+      if index(_infile_,'ORIGINAL LINE') then call symputx('test3_orig','1');
+      if index(_infile_,'APPENDED LINE') then call symputx('test3_appd','1');
+      if index(_infile_,'SECOND APPEND') then call symputx('test3_app2','1');
+    run;
+  %end;
+%mend verify_test3;
+%verify_test3()
+
+%mp_assert(
+  iftrue=(&test3_orig=1),
+  desc=Test 3a: Original content still present after second append,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&test3_appd=1),
+  desc=Test 3b: First appended content still present after second append,
+  outds=work.test_results
+)
+
+%mp_assert(
+  iftrue=(&test3_app2=1),
+  desc=Test 3c: Second appended content is present,
+  outds=work.test_results
+)

tests/x-platform/mx_createjob.test.sas

@@ -26,7 +26,10 @@ parmcards4;
 ;;;;
 %mp_assertscope(SNAPSHOT)
 %mx_createjob(path=&mcTestAppLoc/jobs,name=testjob1,replace=YES)
-%mp_assertscope(COMPARE)
+%mp_assertscope(COMPARE,ignorelist=MC0_JADP1LEN MC0_JADP2LEN MC0_JADPNUM
+  MC0_JADVLEN MC2_JADP1LEN MC2_JADP2LEN MC2_JADPNUM MC2_JADVLEN MC4_JADP1LEN
+  MC4_JADP2LEN MC4_JADPNUM MC4_JADVLEN
+)
 
 %mp_assert(
   iftrue=(&syscc=0),

viya/mfv_existsashdat.sas

@@ -6,12 +6,12 @@
       %if %mfv_existsashdat(libds=casuser.sometable) %then %put  yes it does!;
 
   The function uses `dosubl()` to run the `table.fileinfo` action, for the
-  specified library, filtering for `*.sashdat` tables.  The results are stored
+  specified library, filtering for `*.sashdat` tables.
-  in a WORK table (&outprefix._&lib). If that table already exists, it is
-  queried instead, to avoid the dosubl() performance hit.
 
-  To force a rescan, just use a new `&outprefix` value, or delete the table(s)
+  Results are cached in a WORK table (&outprefix._&lib).  If that table
-  before running the function.
+  already exists it is queried directly to avoid the dosubl() overhead.
+  To force a rescan, use a new `&outprefix` value or delete the cache
+  table before calling.
 
   @param [in] libds library.dataset
   @param [out] outprefix= (work.mfv_existsashdat)
@@ -24,13 +24,12 @@
   @author Mathieu Blauw
 **/
 
-%macro mfv_existsashdat(libds,outprefix=work.mfv_existsashdat
+%macro mfv_existsashdat(libds,outprefix=work.mfv_existsashdat);
-);
 %local rc dsid name lib ds;
 %let lib=%upcase(%scan(&libds,1,'.'));
 %let ds=%upcase(%scan(&libds,-1,'.'));
 
-/* if table does not exist, create it */
+/* if cache table does not exist, build it */
 %if %sysfunc(exist(&outprefix._&lib)) ne 1 %then %do;
   %let rc=%sysfunc(dosubl(%nrstr(
     /* Read in table list (once per &lib per session) */
@@ -41,7 +40,7 @@
     quit;
     /* Only keep name, without file extension */
     data &outprefix._&lib;
-      set &outprefix._&lib(where=(Name like '%.sashdat') keep=Name);
+      set &outprefix._&lib(where=(upcase(Name) like '%.SASHDAT') keep=Name);
       Name=upcase(scan(Name,1,'.'));
     run;
   )));

viya/mfv_getcaslib.sas

@@ -0,0 +1,37 @@
+/**
+  @file mfv_getcaslib.sas
+  @brief Returns the CAS caslib name for a given SAS libref
+  @details Pure macro function.  Reads sashelp.vlibnam and returns
+    the sysvalue where sysname='Caslib' for the given libref.  This
+    is useful when the caslib name and libref name may differ.
+
+    Usage:
+
+        %put %mfv_getcaslib(lib=PUBLIC);
+
+  @param [in] lib SAS libref for which to return the CAS caslib name
+
+  @return Returns the CAS caslib name, or empty string if not found
+
+**/
+
+%macro mfv_getcaslib(lib);
+
+%local dsid rc result;
+
+%let dsid=%sysfunc(open(sashelp.vlibnam(
+  where=(libname="%upcase(&lib)" and sysname="Caslib")
+)));
+
+%if &dsid %then %do;
+  %let rc=%sysfunc(fetch(&dsid));
+  %if &rc=0 %then
+    %let result=%sysfunc(
+      getvarc(&dsid,%sysfunc(varnum(&dsid,SYSVALUE)))
+    );
+  %let rc=%sysfunc(close(&dsid));
+%end;
+
+&result
+
+%mend mfv_getcaslib;

viya/mv_castabload.sas

@@ -0,0 +1,207 @@
+/**
+  @file mv_castabload.sas
+  @brief Checks if a CAS table is loaded; if not, loads and promotes it
+  @details Runs in SPRE against an active CAS session.  Accepts a
+    SAS libref, derives the CAS caslib and session UUID from
+    sashelp.vlibnam, then checks whether the table is already
+    in-memory.  If not, locates the owning CAS server via the
+    casManagement REST API, queries the table endpoint to discover
+    the source file and caslib, then loads and promotes the table.
+
+    A CAS session must already be established by the caller, eg:
+
+        cas mysess;
+        libname mylib cas caslib=Public;
+        %mv_castabload(lib=mylib, table=BASEBALL)
+
+  @param [in] lib=    SAS libref for the CAS caslib
+  @param [in] table=  Name of the CAS table to load
+  @param [in] mdebug= (0) Set to 1 to enable verbose logging:
+                        - echoes resolved parameters
+                        - prints tableExists result
+                        - enables mprint/notes during PROC calls
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mp_abort.sas
+
+**/
+
+%macro mv_castabload(
+    lib=
+    ,table=
+    ,mdebug=0
+);
+
+%local _sysopts base_uri caslib uuid server
+      srcfile srccaslib fname1 libref1 ftmp i _svcount _exists;
+%let _sysopts=%sysfunc(getoption(mprint)) %sysfunc(getoption(notes));
+
+/* ---- input validation -------------------------------------------------- */
+%mp_abort(
+  iftrue=("&lib"="" or "&table"=""),
+  msg=%str(lib= and table= are required)
+)
+
+%if &mdebug=1 %then %do;
+  %put &=lib;
+  %put &=table;
+  options mprint notes;
+%end;
+
+/* ---- derive caslib and session UUID from sashelp.vlibnam --------------- */
+data _null_;
+  set sashelp.vlibnam(
+    where=(libname="%upcase(&lib)"
+      and sysname in ("Caslib","Session UUID"))
+  );
+  if sysname="Caslib" then call symputx('caslib',sysvalue,'L');
+  else call symputx('uuid',sysvalue,'L');
+  %if &mdebug=1 %then %do;
+    putlog sysname sysvalue;
+  %end;
+run;
+
+%mp_abort(
+  iftrue=("&caslib"=""),
+  msg=%str(&lib is not an assigned CAS libref)
+)
+
+%mp_abort(
+  iftrue=("&uuid"=""),
+  msg=%str(No session UUID found for libref &lib)
+)
+
+/* ---- existence check --------------------------------------------------- */
+proc cas;
+  table.tableExists result=r /
+      caslib="&caslib"
+      name="&table";
+  %if &mdebug=1 %then %do;
+    print r;
+  %end;
+  if r.exists > 0 then call symputx('_exists', '1', 'L');
+  else call symputx('_exists', '0', 'L');
+quit;
+
+/* ---- already loaded: skip ---------------------------------------------- */
+%if &_exists=1 %then %do;
+  %put NOTE: Table &caslib..&table already loaded - skipping;
+  %return;
+%end;
+
+/* ---- get list of CAS servers ----------------------------------------- */
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+%let fname1=%mf_getuniquefileref();
+%let libref1=%mf_getuniquelibref();
+
+proc http method='GET' out=&fname1 oauth_bearer=sas_services
+    url="&base_uri/casManagement/servers";
+run;
+
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200),
+  msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+libname &libref1 JSON fileref=&fname1;
+
+data _null_;
+  set &libref1..items;
+  call symputx(cats('_sv_', _n_), name, 'L');
+  call symputx('_svcount', _n_, 'L');
+run;
+
+libname &libref1 clear;
+filename &fname1 clear;
+
+/* ---- find which server owns this session ------------------------------ */
+%do i=1 %to &_svcount;
+  %if "&server"="" %then %do;
+    %if &mdebug=1 %then %put checking server: &&_sv_&i;
+    %let ftmp=%mf_getuniquefileref();
+    proc http method='GET' out=&ftmp oauth_bearer=sas_services
+      url="&base_uri/casManagement/servers/&&_sv_&i/sessions/&uuid";
+    run;
+    %if &SYS_PROCHTTP_STATUS_CODE=200
+      %then %let server=&&_sv_&i;
+    filename &ftmp clear;
+  %end;
+%end;
+
+%mp_abort(
+  iftrue=("&server"=""),
+  msg=%str(Could not find owning server for CAS session &uuid)
+)
+
+%if &mdebug=1 %then %put &=server;
+
+/* ---- discover source file from REST endpoint -------------------------- */
+%let fname1=%mf_getuniquefileref();
+%let libref1=%mf_getuniquelibref();
+
+proc http method='GET' out=&fname1 oauth_bearer=sas_services
+  url="&base_uri/casManagement/servers/&server/caslibs/&caslib/tables/&table";
+run;
+
+%if &mdebug=1 %then %do;
+  %put &=SYS_PROCHTTP_STATUS_CODE &=SYS_PROCHTTP_STATUS_PHRASE;
+  data _null_;
+    infile &fname1;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE=404),
+  msg=%str(&caslib..&table not found - is a source file registered?)
+)
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200),
+  msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+libname &libref1 JSON fileref=&fname1;
+
+data _null_;
+  set &libref1..tablereference;
+  call symputx('srcfile', sourceTableName, 'L');
+  call symputx('srccaslib', sourceCaslibName, 'L');
+  stop;
+run;
+
+libname &libref1 clear;
+filename &fname1 clear;
+
+%mp_abort(
+  iftrue=("&srcfile"="" or "&srccaslib"=""),
+  msg=%str(No sourceTableName/sourceCaslibName for &caslib..&table)
+)
+
+%if &mdebug=1 %then %put &=srcfile &=srccaslib;
+
+/* ---- load from discovered source -------------------------------------- */
+proc casutil;
+  load casdata="&srcfile"
+      incaslib="&srccaslib"
+      casout="&table"
+      outcaslib="&caslib"
+      promote;
+quit;
+
+%mp_abort(
+  iftrue=(&syscc ne 0),
+  msg=%str(Load failed for &caslib..&table)
+)
+
+%put NOTE: Table &caslib..&table loaded and promoted from &srcfile;
+
+/* ---- restore options --------------------------------------------------- */
+%if &mdebug=1 %then %do;
+  options &_sysopts;
+%end;
+
+%mend mv_castabload;

viya/mv_castabsave.sas

@@ -0,0 +1,192 @@
+/**
+  @file mv_castabsave.sas
+  @brief Saves an in-memory CAS table back to persistent storage
+  @details Runs in SPRE against an active CAS session.  Accepts a
+    SAS libref, derives the CAS caslib and session UUID from
+    sashelp.vlibnam, locates the owning CAS server via the
+    casManagement REST API, then queries the table endpoint to
+    discover the original source file and saves back to that path.
+    CASUTIL infers the file type from the output file extension.
+
+    A CAS session must already be established by the caller, eg:
+
+        cas mysess;
+        libname mylib cas caslib=Public;
+        %mv_castabsave(lib=mylib, table=BASEBALL)
+
+  @param [in] lib=    SAS libref for the CAS caslib
+  @param [in] table=  Name of the in-memory CAS table to save
+  @param [in] mdebug= (0) Set to 1 to enable verbose logging:
+                        - echoes resolved parameters
+                        - prints HTTP response body
+                        - enables mprint/notes during PROC calls
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
+  @li mp_abort.sas
+
+**/
+
+%macro mv_castabsave(
+    lib=
+    ,table=
+    ,mdebug=0
+);
+
+%local _sysopts base_uri caslib uuid server
+      srcfile srccaslib fname1 libref1 ftmp i _svcount;
+%let _sysopts=%sysfunc(getoption(mprint)) %sysfunc(getoption(notes));
+
+/* ---- input validation -------------------------------------------------- */
+%mp_abort(
+  iftrue=("&lib"="" or "&table"=""),
+  msg=%str(lib= and table= are required)
+)
+
+%if &mdebug=1 %then %do;
+  %put &=lib;
+  %put &=table;
+  options mprint notes;
+%end;
+
+/* ---- derive caslib and session UUID from sashelp.vlibnam --------------- */
+data _null_;
+  set sashelp.vlibnam(
+    where=(libname="%upcase(&lib)"
+      and sysname in ("Caslib","Session UUID"))
+  );
+  if sysname="Caslib" then call symputx('caslib',sysvalue,'L');
+  else call symputx('uuid',sysvalue,'L');
+run;
+
+%mp_abort(
+  iftrue=("&caslib"=""),
+  msg=%str(&lib is not an assigned CAS libref)
+)
+
+%mp_abort(
+  iftrue=("&uuid"=""),
+  msg=%str(No session UUID found for libref &lib)
+)
+
+%if &mdebug=1 %then %do;
+  %put &=caslib;
+  %put &=uuid;
+%end;
+
+%let base_uri=%mf_getplatform(VIYARESTAPI);
+
+/* ---- get list of CAS servers ------------------------------------------- */
+%let fname1=%mf_getuniquefileref();
+%let libref1=%mf_getuniquelibref();
+
+proc http method='GET' out=&fname1 oauth_bearer=sas_services
+    url="&base_uri/casManagement/servers";
+run;
+
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200),
+  msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+libname &libref1 JSON fileref=&fname1;
+
+data _null_;
+  set &libref1..items;
+  call symputx(cats('_sv_', _n_), name, 'L');
+  call symputx('_svcount', _n_, 'L');
+run;
+
+libname &libref1 clear;
+filename &fname1 clear;
+
+/* ---- find which server owns this session ------------------------------- */
+%do i=1 %to &_svcount;
+  %if "&server"="" %then %do;
+    %if &mdebug=1 %then %put checking server: &&_sv_&i;
+    %let ftmp=%mf_getuniquefileref();
+    proc http method='GET' out=&ftmp oauth_bearer=sas_services
+        url="&base_uri/casManagement/servers/&&_sv_&i/sessions/&uuid";
+    run;
+    %if &SYS_PROCHTTP_STATUS_CODE=200
+      %then %let server=&&_sv_&i;
+    filename &ftmp clear;
+  %end;
+%end;
+
+%mp_abort(
+  iftrue=("&server"=""),
+  msg=%str(Could not find owning server for CAS session &uuid)
+)
+
+%if &mdebug=1 %then %put &=server;
+
+/* ---- discover srcfile from REST endpoint ------------------------------- */
+%let fname1=%mf_getuniquefileref();
+%let libref1=%mf_getuniquelibref();
+
+proc http method='GET' out=&fname1 oauth_bearer=sas_services
+    url="&base_uri/casManagement/servers/&server/caslibs/&caslib/tables/&table";
+run;
+
+%if &mdebug=1 %then %do;
+  %put &=SYS_PROCHTTP_STATUS_CODE &=SYS_PROCHTTP_STATUS_PHRASE;
+  data _null_;
+    infile &fname1;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE=404),
+  msg=%str(&caslib..&table not found - is it loaded in memory?)
+)
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200),
+  msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+libname &libref1 JSON fileref=&fname1;
+
+data _null_;
+  set &libref1..tablereference;
+  call symputx('srcfile', sourceTableName, 'L');
+  call symputx('srccaslib', sourceCaslibName, 'L');
+  stop;
+run;
+
+libname &libref1 clear;
+filename &fname1 clear;
+
+%mp_abort(
+  iftrue=("&srcfile"="" or "&srccaslib"=""),
+  msg=%str(No sourceTableName/sourceCaslibName for &caslib..&table)
+)
+
+%if &mdebug=1 %then %put &=srcfile;
+
+/* ---- save to disk ------------------------------------------------------- */
+proc casutil;
+  save casdata="&table"
+      incaslib="&caslib"
+      casout="&srcfile"
+      outcaslib="&srccaslib"
+      replace;
+quit;
+
+%mp_abort(
+  iftrue=(&syscc ne 0),
+  msg=%str(Save failed for &caslib..&table)
+)
+
+%put NOTE: Table &caslib..&table saved to &srcfile;
+
+/* ---- restore options --------------------------------------------------- */
+%if &mdebug=1 %then %do;
+  options &_sysopts;
+%end;
+
+%mend mv_castabsave;

viya/mv_createfile.sas

@@ -204,9 +204,10 @@ run;
 
 /* Get Viya file-extension details into some macro variables */
 %mv_getViyaFileExtParms(&ext
-                        ,propertiesVar=viyaProperties
+  ,propertiesVar=viyaProperties
-                        ,typeDefNameVar=viyaTypeDefName
+  ,typeDefNameVar=viyaTypeDefName
-                        ,mdebug=&mdebug);
+  ,mdebug=&mdebug
+)
 
 /* fetch job info */
 %local fname1;
@@ -274,9 +275,9 @@ run;
 
 /* If properties were found then patch the file to include them */
 %if not %mf_isBlank(%superq(viyaProperties)) %then %do;
-  /* Wrap the properties object in a root object also containing the file name */
+  /* Wrap the properties object in a root object also containing the filename */
   %local viyapatch;
-  %let viyapatch = %sysfunc(pathname(work))/%mf_getuniquename(prefix=patch_json_);
+  %let viyapatch=%sysfunc(pathname(work))/%mf_getuniquename(prefix=patch_json_);
   data _null_;
     length line $32767;
     file "&viyapatch" lrecl=32767;
@@ -296,7 +297,7 @@ run;
     run;
   %end;
 
-  /* And apply the properties to the newly created file, using the PATCH method */
+  /* Apply the properties to the newly created file, using the PATCH method */
   %let fref=%mf_getuniquefileref();
   filename &fref "&viyapatch";
   %let url=&base_uri&fileuri;

viya/mv_createfolder.sas

@@ -145,7 +145,6 @@ options noquotelenmax;
     run;
   %end;
   %if &SYS_PROCHTTP_STATUS_CODE=200 %then %do;
-    %*put &sysmacroname &newpath exists so grab the follow on link ;
     data _null_;
       set &libref1..links;
       if rel='createChild' then

viya/mv_createwebservice.sas

@@ -594,14 +594,14 @@ data _null_;
   put '  ,showmeta=N,maxobs=MAX,workobs=0 ';
   put '); ';
   put '%global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name ';
-  put '  sasjs_tables SYS_JES_JOB_URI; ';
+  put '  sasjs_tables SYS_JES_JOB_URI _EXECUTIONTASKS; ';
-  put '%if %index("&_debug",log) %then %let _debug=131; ';
+  put '%if %index("&_debug",log) %then %let _debug=128; ';
   put ' ';
   put '%local i tempds table; ';
   put '%let action=%upcase(&action); ';
   put ' ';
   put '%if &action=FETCH %then %do; ';
-  put '  %if %upcase(&_omittextlog)=FALSE or %str(&_debug) ge 131 %then %do; ';
+  put '  %if %upcase(&_omittextlog)=FALSE or %str(&_debug) ge 128 %then %do; ';
   put '    options mprint notes mprintnest; ';
   put '  %end; ';
   put ' ';
@@ -609,6 +609,11 @@ data _null_;
   put '    %let _webin_file_count=%eval(&_webin_file_count+0); ';
   put '    %let _webin_fileuri1=&_webin_fileuri; ';
   put '    %let _webin_name1=&_webin_name; ';
+  put '    %if &_EXECUTIONTASKS=true %then %do; ';
+  put '      /* TODO - remove this once SAS Track CS0409737 is resolved */ ';
+  put '      /* links:  https://github.com/sasjs/adapter/issues/884 */ ';
+  put '      %if %upcase(&_webin_name)=_SASJS_NOOP %then %let _webin_file_count=0; ';
+  put '    %end; ';
   put '  %end; ';
   put ' ';
   put '  /* if the sasjs_tables param is passed, we expect param based upload */ ';
@@ -649,13 +654,17 @@ data _null_;
   put '  %end; ';
   put '  %else %do i=1 %to &_webin_file_count; ';
   put '    /* read in any files that are sent */ ';
-  put '    /* this part needs refactoring for wide files */ ';
+  put '    %if &_EXECUTIONTASKS=true %then %do; ';
-  put '    filename indata filesrvc "&&_webin_fileuri&i" lrecl=999999; ';
+  put '      filename indata "%sysfunc(pathname(&&_webin_fileref&i))" lrecl=999999; ';
+  put '    %end; ';
+  put '    %else %do; ';
+  put '      filename indata filesrvc "&&_webin_fileuri&i" lrecl=999999; ';
+  put '    %end; ';
   put '    data _null_; ';
   put '      infile indata termstr=crlf lrecl=32767; ';
   put '      input; ';
   put '      if _n_=1 then call symputx(''input_statement'',_infile_); ';
-  put '      %if %str(&_debug) ge 131 %then %do; ';
+  put '      %if %str(&_debug) ge 128 %then %do; ';
   put '        if _n_<20 then putlog _infile_; ';
   put '        else stop; ';
   put '      %end; ';
@@ -688,6 +697,9 @@ data _null_;
   put ' ';
   put '  /* setup json */ ';
   put '  data _null_;file &fref; ';
+  put '  %if %str(&_debug) ge 128 and &_EXECUTIONTASKS=true %then %do; ';
+  put '    put ''>>weboutBEGIN<<''; ';
+  put '  %end; ';
   put '    put ''{"SYSDATE" : "'' "&SYSDATE" ''"''; ';
   put '    put '',"SYSTIME" : "'' "&SYSTIME" ''"''; ';
   put '  run; ';
@@ -800,7 +812,9 @@ data _null_;
   put '    memsize=quote(cats(memsize)); ';
   put '    put '',"MEMSIZE" : '' memsize; ';
   put '    put "}"; ';
-  put ' ';
+  put '  %if %str(&_debug) ge 128 and &_EXECUTIONTASKS=true %then %do; ';
+  put '    put ''>>weboutEND<<''; ';
+  put '  %end; ';
   put '  %if %upcase(&fref) ne _WEBOUT and &stream=Y %then %do; ';
   put '    data _null_; rc=fcopy("&fref","_webout");run; ';
   put '  %end; ';

viya/mv_getviyafileextparms.sas

@@ -23,6 +23,7 @@
   @li mf_existds.sas
   @li mf_getplatform.sas
   @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
   @li mf_getuniquename.sas
   @li mf_getvalue.sas
   @li mf_getvarlist.sas
@@ -48,7 +49,7 @@
     iftrue=(%mf_isBlank(&ext))
     ,msg=%str(No file extension provided.)
     ,mac=MV_GETVIYAFILEEXTPARMS
-  );
+  )
 
   %mp_abort(
     iftrue=(%mf_isBlank(&typeDefNameVar) and
@@ -56,13 +57,13 @@
             %mf_isBlank(&mediaTypeVar))
     ,msg=%str(MV_GETVIYAFILEEXTPARMS - No parameter was requested.)
     ,mac=MV_GETVIYAFILEEXTPARMS
-  );
+  )
 
   %mp_abort(
     iftrue=(%mf_isBlank(&viyaFileExtRespLibDs))
     ,msg=%str(No <libname.>dataset name provided to cache inital response.)
     ,mac=MV_GETVIYAFILEEXTPARMS
-  );
+  )
 
   /* Declare requested parameters as global macro vars and initialize blank */
   %if not %mf_isBlank(&typeDefNameVar) %then %do;
@@ -79,9 +80,7 @@
   %end;
 
   %let base_uri=%mf_getplatform(VIYARESTAPI);
-  %if &mdebug=1 %then %do;
+  %if &mdebug=1 %then %put DEBUG: &=base_uri;
-    %put DEBUG: &=base_uri;
-  %end;
 
   %let ext=%lowcase(&ext);
 
@@ -105,7 +104,7 @@
 
     %if (&SYS_PROCHTTP_STATUS_CODE ne 200) %then %do;
       /* To avoid a breaking change, exit early if the request failed.
-        The calling process will proceed with empty requested macro variables. */
+          The calling process will proceed with empty macro variables. */
       %put INFO: &sysmacroname File extension details were not retrieved.;
       filename &viyatypedefs clear;
       %return;
@@ -126,11 +125,12 @@
     /* Convert the content of that JSON into SAS datasets */
       /* First prepare a new WORK-based folder to receive the datasets */
     %local jsonworkfolder jsonlib opt_dlcreatedir;
-    %let jsonworkfolder=%sysfunc(pathname(work))/%mf_getuniquename(prefix=json_);
+    %let jsonworkfolder=%sysfunc(pathname(work))/%mf_getuniquename(prefix=jsn_);
     %let jsonlib=%mf_getuniquelibref(prefix=json);
-      /* And point a libname at it */
+    /* And point a libname at it */
     %let opt_dlcreatedir = %sysfunc(getoption(dlcreatedir));
-    options dlcreatedir; libname &jsonlib "&jsonworkfolder"; options &opt_dlcreatedir;
+    options dlcreatedir; libname &jsonlib "&jsonworkfolder";
+    options &opt_dlcreatedir;
 
     /* Read the json output once and copy datasets to its work folder */
     %local libref1;
@@ -159,20 +159,22 @@
 
   %end; /* If initial filetype query response didn't exist */
 
-  /* Find the row-group for the current file extension */
+  %if &mdebug %then %put DEBUG: Find the row-group for extension &ext;
   %local itemRowGroup;
-  %let itemRowGroup =
+  data _null_;
-    %mf_getValue(
+    set &viyaFileExtRespLibDs;
-      &viyaFileExtRespLibDs
+    where p1='items' and p2='extensions' and value="&ext";
-      ,_viyaItemIdx
+    call symputx('itemRowGroup',_viyaItemIdx,'l');
-      ,filter=%quote(p1='items' and p2='extensions' and value="&ext")
+  %if &mdebug %then %do;
-    );
+    putlog (_all_)(=);
+  %end;
+  run;
 
   %if &mdebug %then %put DEBUG: &=itemRowGroup;
 
   %if %mf_isBlank(&itemRowGroup) %then %do;
     /* extension was not found */
-    %if(&mdebug=1) %then %put DEBUG: No type details found for extension "&ext".;
+    %if &mdebug %then %put DEBUG: No type details found for extension "&ext";
     %return;
   %end;
 
@@ -186,14 +188,17 @@
 
   /* Populate typeDefName, if requested */
   %if (not %mf_isBlank(&typeDefNameVar)) %then %do;
-    %let &typeDefNameVar = %mf_getvalue(&dsItems,value,filter=%quote(p1="items" and p2="name"));
+    %let &typeDefNameVar = %mf_getvalue(
-    %if &mdebug=1 %then %put DEBUG: &=typeDefNameVar &typeDefNameVar=&&&typeDefNameVar;
+        &dsItems,value,filter=%quote(p1="items" and p2="name"));
+    %if &mdebug %then
+      %put DEBUG: &=typeDefNameVar &typeDefNameVar=&&&typeDefNameVar;
   %end;
 
   /* Populate mediaType, if requested */
   %if (not %mf_isBlank(&mediaTypeVar)) %then %do;
-    %let &mediaTypeVar = %mf_getvalue(&dsItems,value,filter=%quote(p1="items" and p2="mediaType"));
+    %let &mediaTypeVar = %mf_getvalue(
-    %if &mdebug=1 %then %put DEBUG: &=mediaTypeVar &mediaTypeVar=&&&mediaTypeVar;
+      &dsItems,value,filter=%quote(p1="items" and p2="mediaType"));
+    %if &mdebug %then %put DEBUG: &=mediaTypeVar &mediaTypeVar=&&&mediaTypeVar;
   %end;
 
   /* Populate properties macro variable, if requested */
@@ -210,7 +215,8 @@
     /* Check for 1+ properties */
     %if ( %mf_nobs(&dsProperties) = 0 ) %then %do;
       %let &propertiesVar = %str();
-      %if &mdebug=1 %then %put DEBUG: &SYSMACRONAME - No Viya properties found for file suffix %str(%')&ext%str(%');
+      %if &mdebug %then %put DEBUG: &SYSMACRONAME - No Viya properties %trim(
+        )found for file suffix %str(%')&ext%str(%');
     %end;
     %else %do;
       /* Properties potentially span multiple rows in the input table */
@@ -240,7 +246,8 @@
         end;
       run;
 
-      %if &mdebug=1 %then %put DEBUG: &=propertiesVar &propertiesVar=&&&propertiesVar;
+      %if &mdebug %then
+        %put DEBUG: &=propertiesVar &propertiesVar=&&&propertiesVar;
     %end;
 
   %end;

viya/mv_jobflow.sas

@@ -121,7 +121,7 @@
   @li mf_nobs.sas
   @li mp_abort.sas
   @li mf_getplatform.sas
-  @li mf_getuniquefileref.sas
+  @li mf_getvarlist.sas
   @li mf_existvarlist.sas
   @li mv_jobwaitfor.sas
   @li mv_jobexecute.sas
@@ -234,14 +234,12 @@ data _null_;
   if last then call symputx('flowcnt',cnt,'l');
 run;
 
-/* prepare temporary datasets and frefs */
+/* prepare temporary datasets */
-%local fid jid jds jjson jdsapp jdsrunning jdswaitfor jfref;
+%local fid jid jds jdsapp jdsrunning jdswaitfor;
 data;run;%let jds=&syslast;
-data;run;%let jjson=&syslast;
 data;run;%let jdsapp=&syslast;
 data;run;%let jdsrunning=&syslast;
 data;run;%let jdswaitfor=&syslast;
-%let jfref=%mf_getuniquefileref();
 
 /* start loop */
 %do fid=1 %to &flowcnt;
@@ -259,23 +257,39 @@ data;run;%let jdswaitfor=&syslast;
       &dbg. if _n_= 1 then putlog "Loop &fid";
       &dbg. putlog (_all_)(=);
     run;
+    /* build list of char and num vars in json format */
+    /* Viya 2026 expects all values to be strings */
+    %local nvars cvars ii _vnm;
+    %let cvars=%mf_getvarlist(&jds,typefilter=C);
+    %let nvars=%mf_getvarlist(&jds,typefilter=N);
     %put exporting job variables in json format;
     %do jid=1 %to &jcnt;
-      data &jjson;
-        set &jds;
-        if _n_=&jid then do;
-          output;
-          stop;
-        end;
-      run;
-      proc json out=&jfref;
-        export &jjson / nosastags fmtnumeric;
-      run;
       data _null_;
-        infile &jfref lrecl=32767;
+        set &jds;
-        input;
+        if _n_=&jid;
-        jparams=cats('jparams',symget('jid'));
+        length _param $32767;
-        call symputx(jparams,substr(_infile_,3,length(_infile_)-4));
+        _param='';
+      %if %length(&cvars)>0 %then %do ii=1 %to %sysfunc(countw(&cvars,%str( )));
+        %let _vnm=%scan(&cvars,&ii,%str( ));
+        if _param ne '' then _param=cats(_param,',');
+        _param=cats(_param,'"'
+          ,"%lowcase(&_vnm)"
+          ,'":'
+          ,quote(trim(&_vnm))
+        );
+      %end;
+      %if %length(&nvars)>0 %then %do ii=1 %to %sysfunc(countw(&nvars,%str( )));
+        %let _vnm=%scan(&nvars,&ii,%str( ));
+        if _param ne '' then _param=cats(_param,',');
+        _param=cats(_param,'"'
+          ,"%lowcase(&_vnm)"
+          ,'":"'
+          ,strip(put(&_vnm,best32.))
+          ,'"'
+        );
+      %end;
+        call symputx(cats('jparams',&jid),_param,'l');
+        stop;
       run;
       %local jobuid&jid;
       %let jobuid&jid=0; /* used in next loop */
@@ -320,6 +334,7 @@ data;run;%let jdswaitfor=&syslast;
               ,paramstring=%superq(jparams&jid)
               ,outds=&jdsapp
               ,contextname=&&context&jid
+              ,mdebug=&mdebug
             )
             data &jdsapp;
               format jobparams $32767.;

viya/mv_webout.sas

@@ -55,14 +55,14 @@
   ,showmeta=N,maxobs=MAX,workobs=0
 );
 %global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name
-  sasjs_tables SYS_JES_JOB_URI;
+  sasjs_tables SYS_JES_JOB_URI _EXECUTIONTASKS;
-%if %index("&_debug",log) %then %let _debug=131;
+%if %index("&_debug",log) %then %let _debug=128;
 
 %local i tempds table;
 %let action=%upcase(&action);
 
 %if &action=FETCH %then %do;
-  %if %upcase(&_omittextlog)=FALSE or %str(&_debug) ge 131 %then %do;
+  %if %upcase(&_omittextlog)=FALSE or %str(&_debug) ge 128 %then %do;
     options mprint notes mprintnest;
   %end;
 
@@ -70,6 +70,11 @@
     %let _webin_file_count=%eval(&_webin_file_count+0);
     %let _webin_fileuri1=&_webin_fileuri;
     %let _webin_name1=&_webin_name;
+    %if &_EXECUTIONTASKS=true %then %do;
+      /* TODO - remove this once SAS Track CS0409737 is resolved */
+      /* links:  https://github.com/sasjs/adapter/issues/884 */
+      %if %upcase(&_webin_name)=_SASJS_NOOP %then %let _webin_file_count=0;
+    %end;
   %end;
 
   /* if the sasjs_tables param is passed, we expect param based upload */
@@ -110,13 +115,17 @@
   %end;
   %else %do i=1 %to &_webin_file_count;
     /* read in any files that are sent */
-    /* this part needs refactoring for wide files */
+    %if &_EXECUTIONTASKS=true %then %do;
-    filename indata filesrvc "&&_webin_fileuri&i" lrecl=999999;
+      filename indata "%sysfunc(pathname(&&_webin_fileref&i))" lrecl=999999;
+    %end;
+    %else %do;
+      filename indata filesrvc "&&_webin_fileuri&i" lrecl=999999;
+    %end;
     data _null_;
       infile indata termstr=crlf lrecl=32767;
       input;
       if _n_=1 then call symputx('input_statement',_infile_);
-      %if %str(&_debug) ge 131 %then %do;
+      %if %str(&_debug) ge 128 %then %do;
         if _n_<20 then putlog _infile_;
         else stop;
       %end;
@@ -149,6 +158,9 @@
 
   /* setup json */
   data _null_;file &fref;
+  %if %str(&_debug) ge 128 and &_EXECUTIONTASKS=true %then %do;
+    put '>>weboutBEGIN<<';
+  %end;
     put '{"SYSDATE" : "' "&SYSDATE" '"';
     put ',"SYSTIME" : "' "&SYSTIME" '"';
   run;
@@ -261,7 +273,9 @@
     memsize=quote(cats(memsize));
     put ',"MEMSIZE" : ' memsize;
     put "}";
-
+  %if %str(&_debug) ge 128 and &_EXECUTIONTASKS=true %then %do;
+    put '>>weboutEND<<';
+  %end;
   %if %upcase(&fref) ne _WEBOUT and &stream=Y %then %do;
     data _null_; rc=fcopy("&fref","_webout");run;
   %end;

xplatform/mx_append2pgm.sas

@@ -0,0 +1,149 @@
+/**
+  @file
+  @brief Appends a text file to a SASjs Stored Program, Viya SAS program, or
+    SAS 9 Stored Process
+  @details Extracts the source code from a SASjs Stored Program, Viya SAS
+  program (file in SAS Drive), or SAS 9 Stored Process, appends the contents
+  of a provided text file, then deletes and recreates the target item with the
+  combined content.
+
+  This is useful for dynamically modifying deployed programs, for example to
+  add test-specific configuration or runtime settings.
+
+  Usage:
+
+      %* compile macros ;
+      filename mc url
+        "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
+      %inc mc;
+
+      %* write some content to append;
+      filename append temp;
+      data _null_;
+        file append;
+        put "libname mylib '/some/path';";
+      run;
+
+      %* append to existing program;
+      %mx_append2pgm(/Public/app/common/settings, inref=append)
+
+  @param [in] loc The full path to the Viya SAS program, SAS 9 Stored Process,
+    or SASjs Stored Program in Drive or Metadata, WITHOUT the .sas extension
+    (SASjs only)
+  @param [in] inref= (0) Fileref pointing to the content to be appended to the
+    target program.
+  @param [in] mdebug= (0) Set to 1 to show debug messages in the log
+
+  <h4> SAS Macros </h4>
+  @li mf_getplatform.sas
+  @li mf_getuniquefileref.sas
+  @li mm_createstp.sas
+  @li mm_deletestp.sas
+  @li mm_getstpcode.sas
+  @li ms_createfile.sas
+  @li ms_deletefile.sas
+  @li mv_createfile.sas
+  @li mv_deletefoldermember.sas
+  @li mx_getcode.sas
+
+  <h4> Related Macros </h4>
+  @li mx_append2pgm.test.sas
+  @li mx_getcode.sas
+  @li mx_createjob.sas
+
+  @author Allan Bowe
+
+**/
+
+%macro mx_append2pgm(loc
+    ,inref=0
+    ,mdebug=0
+)/*/STORE SOURCE*/;
+
+%local platform name shortloc coderef combref work tmpfile viyaref;
+%let platform=%mf_getplatform();
+
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%if &syscc ne 0 %then %do;
+  %put syscc=&syscc - &sysmacroname will not execute in this state;
+  %return;
+%end;
+
+/* extract name & path from loc */
+data _null_;
+  length name shortloc $500;
+  loc=symget('loc');
+  name=scan(loc,-1,'/');
+  shortloc=substr(loc,1,length(loc)-length(name)-1);
+  call symputx('name',name,'l');
+  call symputx('shortloc',shortloc,'l');
+run;
+
+/* create a combined fileref with original + appended content */
+%let combref=%mf_getuniquefileref();
+%let work=%sysfunc(pathname(work));
+%let tmpfile=&combref..sas;
+filename &combref "&work/&tmpfile" lrecl=32000;
+
+%if &platform=SASVIYA %then %do;
+  /* On Viya, read the SAS program file from SAS Drive using filesrvc */
+  %let viyaref=%mf_getuniquefileref();
+  filename &viyaref filesrvc folderpath="&shortloc";
+  data _null_;
+    file &combref lrecl=32000 termstr=crlf;
+    infile &viyaref("&name..sas") lrecl=32000 end=last;
+    input;
+    put _infile_;
+  run;
+  filename &viyaref clear;
+  %symdel _FILESRVC_&viyaref._URI;
+%end;
+%else %do;
+  /* For SAS9 and SASJS, use mx_getcode */
+  %let coderef=%mf_getuniquefileref();
+  %mx_getcode(&loc, outref=&coderef)
+  data _null_;
+    file &combref lrecl=32000 termstr=crlf;
+    infile &coderef lrecl=32000 end=last;
+    input;
+    put _infile_;
+  run;
+  filename &coderef clear;
+%end;
+
+/* append the new content */
+data _null_;
+  file &combref lrecl=32000 termstr=crlf mod;
+  infile &inref lrecl=32000;
+  input;
+  put _infile_;
+run;
+
+/* delete and recreate the target item */
+%if &platform=SASJS %then %do;
+  %ms_deletefile(&loc..sas)
+  %ms_createfile(&loc..sas, inref=&combref, mdebug=&mdebug)
+%end;
+%else %if &platform=SASVIYA %then %do;
+  %mv_deletefoldermember(path=&shortloc, name=&name..sas, contenttype=file)
+  %mv_createfile(path=&shortloc, name=&name..sas, inref=&combref)
+%end;
+%else %do;
+  /* SAS 9 */
+  %mm_deletestp(target=&loc)
+  %mm_createstp(stpname=&name
+    ,filename=&tmpfile
+    ,directory=&work
+    ,tree=&shortloc
+    ,stptype=2
+    ,mDebug=&mdebug
+    ,minify=NO
+  )
+%end;
+
+filename &combref clear;
+
+%mend mx_append2pgm;