diff --git a/.agents/docs/tests.md b/.agents/docs/tests.md new file mode 100644 index 0000000..a448a13 --- /dev/null +++ b/.agents/docs/tests.md @@ -0,0 +1,59 @@ +# Testing in @sasjs/core + +## Overview + +Tests are executed on a real SAS server using the SASjs CLI (`sasjs test`), not locally. Each test is a self-contained `.sas` file that is submitted to the server, and results are collected in the `sasjsresults` folder. + +## Running Tests + +```bash +npm test # runs: npx @sasjs/cli test -t server +``` + +The `-t server` flag selects the target (server type) from `sasjs/sasjsconfig.json`. + +## Test Structure + +Test files live under `tests/` in subfolders by platform applicability: + +- `tests/base` — run on all platforms (SAS 9 and Viya) +- `tests/sas9only` — SAS 9 only (metadata server macros) +- `tests/viyaonly` — Viya only +- `tests/serveronly` — SASjs Server only +- `tests/x-platform` — cross-platform (both SAS 9 and Viya) +- `tests/ddlonly` — DDL-related tests + +Naming convention: `.test.sas`, with numbered variants (`.test.1.sas`, `.test.2.sas`, ...) for multiple tests of the same macro. File names are lowercase, matching the lint rules. + +## Test Flow + +1. **Init**: `tests/testinit.sas` runs before every test (configured in `sasjsconfig.json` under `testConfig.initProgram`). It sets up a unique app location (`mcTestAppLoc`), the compute context, calls `%mp_init()`, and enables debug options when `_debug` is set. +2. **Test body**: the test file itself runs. It should use `%mp_assert()` to record results into `work.test_results`: + +```sas +%mp_assert( + iftrue=(&syscc=0), + desc=Checking for error condition, + outds=work.test_results +) +``` + +3. **Term**: `tests/testterm.sas` runs after every test (`testConfig.termProgram`). It adds a final assertion that `&syscc=0`, then writes the results as JSON via `%webout(OPEN) / %webout(OBJ,TEST_RESULTS) / %webout(CLOSE)`. + +## Results + +After a test run, check the `sasjsresults` folder: + +- `testResults.json` / `testResults.xml` / `testResults.csv` — per-test PASS/FAIL with descriptions and comments +- `logs/.log` — the full SAS log for each test; check here first when a test fails +- `coverage.lcov` — coverage data + +## Writing Tests — Things to Know + +- Tests follow the same Doxygen header and lint standards as regular macros (`@file`, `@brief`, `

SAS Macros

` listing macros used). +- Macro *calls* are not terminated with semicolons: use `%mp_assert(...)` not `%mp_assert(...);`. +- Use `%mp_assert(iftrue=(...), desc=..., outds=work.test_results)` for every check — always append to `work.test_results`. +- When comparing datasets after a round trip (eg through JSON), do not assert `proc compare` SYSINFO=0 directly — SYSINFO is a bitmask that includes attribute differences (length, format, label) which round trips legitimately change. Mask it to data-related bits only (64=missing obs in compare, 128=extra obs in compare, 4096=unequal values, 32768=obs count differs). Note `SYSINFO` is a read-only automatic macro variable, so store the masked value in a new variable. +- Be careful with character data round trips: `cats()` and the plain `$` informat strip leading blanks; use `trim()` and `$char` where leading blanks must be preserved. +- After any change, run `npx sasjs lint`. +- Do not edit generated copies under `sasjsbuild/` — they are refreshed by the CI build. diff --git a/.npmignore b/.npmignore index c9a7666..545ecea 100644 --- a/.npmignore +++ b/.npmignore @@ -8,3 +8,4 @@ sasjs/ make_singlefile.sh *.md .all-contributorsrc +.agents \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 487194c..2477ab0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,6 +25,12 @@ This repo is the SASjs Macro Core library — a collection of MIT-licensed, prod - Macro variables must always be local, to prevent scope leakage. +## Testing +- Read `.agents/docs/tests.md` for details on how the testing process works (how to run tests, structure, assertions, and where to find logs/results). + +## Markdown Files +- Markdown files must not use word-wrap: never insert carriage returns mid-sentence. Each sentence/paragraph stays on one line. + ## Build / Generated Files - Do not run the build script locally; it is executed in the CI/CD pipeline. - Generated files, including the consolidated `all.sas`, the per-folder `mc_*.sas` files, and the LUA macro wrappers in the `lua` folder, can generally be ignored unless the pipeline requires an update. Do not edit generated files by hand. diff --git a/all.sas b/all.sas index a1e784e..2361bf8 100644 --- a/all.sas +++ b/all.sas @@ -10464,7 +10464,8 @@ options prxchange('s/\\/\\\\/',-1,&&name&i) )))))))))))))!!'"'; end; - else &&name&i=quote(cats(&&name&i)); + /* trim (not cats) so leading blanks are retained */ + else &&name&i='"'!!trim(&&name&i)!!'"'; %end; %end; run; @@ -17547,7 +17548,8 @@ data _null_; put ' prxchange(''s/\\/\\\\/'',-1,&&name&i) '; put ' )))))))))))))!!''"''; '; put ' end; '; - put ' else &&name&i=quote(cats(&&name&i)); '; + put ' /* trim (not cats) so leading blanks are retained */ '; + put ' else &&name&i=''"''!!trim(&&name&i)!!''"''; '; put ' %end; '; put ' %end; '; put ' run; '; @@ -17666,7 +17668,10 @@ data _null_; put ' data _null_; '; put ' infile &&_webin_fileref&i termstr=crlf; '; put ' input; '; - put ' call symputx(''input_statement'',_infile_); '; + put ' /* a plain $ informat strips leading blanks - use $char instead */ '; + put ' call symputx(''input_statement'' '; + put ' ,prxchange(''s/:\$(?=[0-9 ])/:\$char/i'',-1,_infile_) '; + put ' ); '; put ' putlog "&&_webin_name&i input statement: " _infile_; '; put ' stop; '; put ' data &&_webin_name&i; '; @@ -21365,7 +21370,10 @@ run; data _null_; infile &&_webin_fileref&i termstr=crlf; input; - call symputx('input_statement',_infile_); + /* a plain $ informat strips leading blanks - use $char instead */ + call symputx('input_statement' + ,prxchange('s/:\$(?=[0-9 ])/:\$char/i',-1,_infile_) + ); putlog "&&_webin_name&i input statement: " _infile_; stop; data &&_webin_name&i; @@ -22659,7 +22667,8 @@ data _null_; put ' prxchange(''s/\\/\\\\/'',-1,&&name&i) '; put ' )))))))))))))!!''"''; '; put ' end; '; - put ' else &&name&i=quote(cats(&&name&i)); '; + put ' /* trim (not cats) so leading blanks are retained */ '; + put ' else &&name&i=''"''!!trim(&&name&i)!!''"''; '; put ' %end; '; put ' %end; '; put ' run; '; @@ -22776,7 +22785,10 @@ data _null_; put ' data _null_; '; put ' infile &&_webin_fileref&i termstr=crlf lrecl=32767; '; put ' input; '; - put ' call symputx(''input_statement'',_infile_); '; + put ' /* a plain $ informat strips leading blanks - use $char instead */ '; + put ' call symputx(''input_statement'' '; + put ' ,prxchange(''s/:\$(?=[0-9 ])/:\$char/i'',-1,_infile_) '; + put ' ); '; put ' putlog "&&_webin_name&i input statement: " _infile_; '; put ' stop; '; put ' data &&_webin_name&i; '; @@ -24045,7 +24057,10 @@ run; data _null_; infile &&_webin_fileref&i termstr=crlf lrecl=32767; input; - call symputx('input_statement',_infile_); + /* a plain $ informat strips leading blanks - use $char instead */ + call symputx('input_statement' + ,prxchange('s/:\$(?=[0-9 ])/:\$char/i',-1,_infile_) + ); putlog "&&_webin_name&i input statement: " _infile_; stop; data &&_webin_name&i; @@ -26310,7 +26325,8 @@ data _null_; put ' prxchange(''s/\\/\\\\/'',-1,&&name&i) '; put ' )))))))))))))!!''"''; '; put ' end; '; - put ' else &&name&i=quote(cats(&&name&i)); '; + put ' /* trim (not cats) so leading blanks are retained */ '; + put ' else &&name&i=''"''!!trim(&&name&i)!!''"''; '; put ' %end; '; put ' %end; '; put ' run; '; @@ -26456,7 +26472,10 @@ data _null_; put ' data _null_; '; put ' infile "%sysfunc(pathname(work))/&table..csv" termstr=crlf ; '; put ' input; '; - put ' if _n_=1 then call symputx(''input_statement'',_infile_); '; + put ' /* a plain $ informat strips leading blanks - use $char instead */ '; + put ' if _n_=1 then call symputx(''input_statement'' '; + put ' ,prxchange(''s/:\$(?=[0-9 ])/:\$char/i'',-1,_infile_) '; + put ' ); '; put ' list; '; put ' data work.&table; '; put ' infile "%sysfunc(pathname(work))/&table..csv" firstobs=2 dsd '; @@ -26476,7 +26495,10 @@ data _null_; put ' data _null_; '; put ' infile indata termstr=crlf lrecl=32767; '; put ' input; '; - put ' if _n_=1 then call symputx(''input_statement'',_infile_); '; + put ' /* a plain $ informat strips leading blanks - use $char instead */ '; + put ' if _n_=1 then call symputx(''input_statement'' '; + put ' ,prxchange(''s/:\$(?=[0-9 ])/:\$char/i'',-1,_infile_) '; + put ' ); '; put ' %if %str(&_debug) ge 128 %then %do; '; put ' if _n_<20 then putlog _infile_; '; put ' else stop; '; @@ -30602,7 +30624,10 @@ filename &fref1 clear; data _null_; infile "%sysfunc(pathname(work))/&table..csv" termstr=crlf ; input; - if _n_=1 then call symputx('input_statement',_infile_); + /* a plain $ informat strips leading blanks - use $char instead */ + if _n_=1 then call symputx('input_statement' + ,prxchange('s/:\$(?=[0-9 ])/:\$char/i',-1,_infile_) + ); list; data work.&table; infile "%sysfunc(pathname(work))/&table..csv" firstobs=2 dsd @@ -30622,7 +30647,10 @@ filename &fref1 clear; data _null_; infile indata termstr=crlf lrecl=32767; input; - if _n_=1 then call symputx('input_statement',_infile_); + /* a plain $ informat strips leading blanks - use $char instead */ + if _n_=1 then call symputx('input_statement' + ,prxchange('s/:\$(?=[0-9 ])/:\$char/i',-1,_infile_) + ); %if %str(&_debug) ge 128 %then %do; if _n_<20 then putlog _infile_; else stop; diff --git a/base/mp_jsonout.sas b/base/mp_jsonout.sas index f4bb74b..6a341cd 100644 --- a/base/mp_jsonout.sas +++ b/base/mp_jsonout.sas @@ -331,7 +331,8 @@ prxchange('s/\\/\\\\/',-1,&&name&i) )))))))))))))!!'"'; end; - else &&name&i=quote(cats(&&name&i)); + /* trim (not cats) so leading blanks are retained */ + else &&name&i='"'!!trim(&&name&i)!!'"'; %end; %end; run; diff --git a/meta/mm_createwebservice.sas b/meta/mm_createwebservice.sas index 5c30c69..e7c1592 100644 --- a/meta/mm_createwebservice.sas +++ b/meta/mm_createwebservice.sas @@ -355,7 +355,8 @@ data _null_; put ' prxchange(''s/\\/\\\\/'',-1,&&name&i) '; put ' )))))))))))))!!''"''; '; put ' end; '; - put ' else &&name&i=quote(cats(&&name&i)); '; + put ' /* trim (not cats) so leading blanks are retained */ '; + put ' else &&name&i=''"''!!trim(&&name&i)!!''"''; '; put ' %end; '; put ' %end; '; put ' run; '; @@ -474,7 +475,10 @@ data _null_; put ' data _null_; '; put ' infile &&_webin_fileref&i termstr=crlf; '; put ' input; '; - put ' call symputx(''input_statement'',_infile_); '; + put ' /* a plain $ informat strips leading blanks - use $char instead */ '; + put ' call symputx(''input_statement'' '; + put ' ,prxchange(''s/:\$(?=[0-9 ])/:\$char/i'',-1,_infile_) '; + put ' ); '; put ' putlog "&&_webin_name&i input statement: " _infile_; '; put ' stop; '; put ' data &&_webin_name&i; '; diff --git a/meta/mm_webout.sas b/meta/mm_webout.sas index f41af4f..653388e 100644 --- a/meta/mm_webout.sas +++ b/meta/mm_webout.sas @@ -77,7 +77,10 @@ data _null_; infile &&_webin_fileref&i termstr=crlf; input; - call symputx('input_statement',_infile_); + /* a plain $ informat strips leading blanks - use $char instead */ + call symputx('input_statement' + ,prxchange('s/:\$(?=[0-9 ])/:\$char/i',-1,_infile_) + ); putlog "&&_webin_name&i input statement: " _infile_; stop; data &&_webin_name&i; diff --git a/server/ms_createwebservice.sas b/server/ms_createwebservice.sas index 351f0d6..2950f06 100644 --- a/server/ms_createwebservice.sas +++ b/server/ms_createwebservice.sas @@ -355,7 +355,8 @@ data _null_; put ' prxchange(''s/\\/\\\\/'',-1,&&name&i) '; put ' )))))))))))))!!''"''; '; put ' end; '; - put ' else &&name&i=quote(cats(&&name&i)); '; + put ' /* trim (not cats) so leading blanks are retained */ '; + put ' else &&name&i=''"''!!trim(&&name&i)!!''"''; '; put ' %end; '; put ' %end; '; put ' run; '; @@ -472,7 +473,10 @@ data _null_; put ' data _null_; '; put ' infile &&_webin_fileref&i termstr=crlf lrecl=32767; '; put ' input; '; - put ' call symputx(''input_statement'',_infile_); '; + put ' /* a plain $ informat strips leading blanks - use $char instead */ '; + put ' call symputx(''input_statement'' '; + put ' ,prxchange(''s/:\$(?=[0-9 ])/:\$char/i'',-1,_infile_) '; + put ' ); '; put ' putlog "&&_webin_name&i input statement: " _infile_; '; put ' stop; '; put ' data &&_webin_name&i; '; diff --git a/server/ms_webout.sas b/server/ms_webout.sas index 0a6e253..216a18f 100644 --- a/server/ms_webout.sas +++ b/server/ms_webout.sas @@ -74,7 +74,10 @@ data _null_; infile &&_webin_fileref&i termstr=crlf lrecl=32767; input; - call symputx('input_statement',_infile_); + /* a plain $ informat strips leading blanks - use $char instead */ + call symputx('input_statement' + ,prxchange('s/:\$(?=[0-9 ])/:\$char/i',-1,_infile_) + ); putlog "&&_webin_name&i input statement: " _infile_; stop; data &&_webin_name&i; diff --git a/tests/base/mp_jsonout.test.3.sas b/tests/base/mp_jsonout.test.3.sas index ea98eb6..56bbf03 100644 --- a/tests/base/mp_jsonout.test.3.sas +++ b/tests/base/mp_jsonout.test.3.sas @@ -11,9 +11,20 @@ filename webref temp; data demo; + length x $100; do x='"','0A'x,'0D'x,'09'x,'00'x,'0E'x,'0F'x,'01'x,'02'x,'10'x,'11'x,'\'; output; end; + /* embedded quote variants */ + x='say "hi" there'; output; + x='"fully quoted"'; output; + x='back\slash'; output; + x='quote and back\"slash'; output; + /* leading / trailing blank variants */ + x=' leading blanks'; output; + x=' "leading blanks and quotes"'; output; + x='trailing blanks '; output; + x=' both '; output; run; %mp_jsonout(OPEN,jref=webref) %mp_jsonout(OBJ,demo,jref=webref) @@ -46,8 +57,19 @@ describe table web.demo; proc compare base=work.demo compare=web.demo(keep=x); quit; +/* sysinfo is a bitmask - keep only data-related bits, ie: + 64 Base data set has observation not in comparison + 128 Comparison data set has observation not in base + 4096 A value comparison was unequal + 32768 Number of observations differ + Attribute diffs (eg 16 - variable length) are ignored, as a JSON + round trip will not preserve lengths/formats/labels. +*/ +/* SYSINFO is read only, so store the masked value in a new variable */ +%let sysinfo_masked=%sysfunc(band(&sysinfo, 64+128+4096+32768)); + %mp_assert( - iftrue=(&sysinfo=0), + iftrue=(&sysinfo_masked=0), desc=Returned json is identical to input table for all special chars, outds=work.test_results ) diff --git a/tests/serveronly/ms_webout.test.sas b/tests/serveronly/ms_webout.test.sas index bd68244..dff7f2f 100644 --- a/tests/serveronly/ms_webout.test.sas +++ b/tests/serveronly/ms_webout.test.sas @@ -32,4 +32,55 @@ run; %mp_assert( iftrue=(%str(&checkval)=%str(&sysvlong)), desc=Check if the sysvlong value was created +) + +/* + Test that ms_webout(FETCH) retains leading blanks in character values + (simulates the CSV format generated by the sasjs adapter, ie an input + statement in the first row followed by unquoted data) +*/ +%let fref2=%mf_getuniquefileref(); +data _null_; + file &fref2 lrecl=32767 termstr=crlf; + put 'col1:$char10. col2:best.'; + put ' padded,1'; +run; +%global _webin_fileref _webin_name; +%let _webin_fileref=&fref2; +%let _webin_name=leadblank; +%let _webin_file_count=1; +%ms_webout(FETCH) + +data _null_; + set leadblank; + if col1=' padded' then call symputx('checkblank','PASS'); + else call symputx('checkblank',cats('FAIL:',col1)); +run; + +%mp_assert( + iftrue=(&checkblank=PASS), + desc=ms_webout FETCH retains leading blanks with $char informat +) + +/* same test, but with a plain $ informat (as sent by some clients) */ +%let fref3=%mf_getuniquefileref(); +data _null_; + file &fref3 lrecl=32767 termstr=crlf; + put 'col1:$10. col2:best.'; + put ' padded,1'; +run; +%let _webin_fileref=&fref3; +%let _webin_name=leadblank2; +%let _webin_file_count=1; +%ms_webout(FETCH) + +data _null_; + set leadblank2; + if col1=' padded' then call symputx('checkblank2','PASS'); + else call symputx('checkblank2',cats('FAIL:',col1)); +run; + +%mp_assert( + iftrue=(&checkblank2=PASS), + desc=ms_webout FETCH retains leading blanks with $ informat ) \ No newline at end of file diff --git a/tests/x-platform/mx_testservice.test.sas b/tests/x-platform/mx_testservice.test.sas index 28e7e89..ee82b56 100644 --- a/tests/x-platform/mx_testservice.test.sas +++ b/tests/x-platform/mx_testservice.test.sas @@ -44,6 +44,7 @@ data work.somedata1 work.somedata2; x=1; y=' t"w"o'; z=.z; + y2=' two'; label x='x factor'; output; run; @@ -58,14 +59,14 @@ run; %let test1=FAIL; data _null_; set testlib1.somedata1; - if x=1 and y=' t"w"o' and z="Z" then call symputx('test1','PASS'); + if x=1 and y=' t"w"o' and z="Z" and y2=' two' then call symputx('test1','PASS'); putlog (_all_)(=); run; %let test2=FAIL; data _null_; set testlib1.somedata2; - if x=1 and y=' t"w"o' and z="Z" then call symputx('test2','PASS'); + if x=1 and y=' t"w"o' and z="Z" and y2=' two' then call symputx('test2','PASS'); putlog (_all_)(=); run; diff --git a/viya/mv_createwebservice.sas b/viya/mv_createwebservice.sas index 0732f35..1baa5e3 100644 --- a/viya/mv_createwebservice.sas +++ b/viya/mv_createwebservice.sas @@ -497,7 +497,8 @@ data _null_; put ' prxchange(''s/\\/\\\\/'',-1,&&name&i) '; put ' )))))))))))))!!''"''; '; put ' end; '; - put ' else &&name&i=quote(cats(&&name&i)); '; + put ' /* trim (not cats) so leading blanks are retained */ '; + put ' else &&name&i=''"''!!trim(&&name&i)!!''"''; '; put ' %end; '; put ' %end; '; put ' run; '; @@ -643,7 +644,10 @@ data _null_; put ' data _null_; '; put ' infile "%sysfunc(pathname(work))/&table..csv" termstr=crlf ; '; put ' input; '; - put ' if _n_=1 then call symputx(''input_statement'',_infile_); '; + put ' /* a plain $ informat strips leading blanks - use $char instead */ '; + put ' if _n_=1 then call symputx(''input_statement'' '; + put ' ,prxchange(''s/:\$(?=[0-9 ])/:\$char/i'',-1,_infile_) '; + put ' ); '; put ' list; '; put ' data work.&table; '; put ' infile "%sysfunc(pathname(work))/&table..csv" firstobs=2 dsd '; @@ -663,7 +667,10 @@ data _null_; put ' data _null_; '; put ' infile indata termstr=crlf lrecl=32767; '; put ' input; '; - put ' if _n_=1 then call symputx(''input_statement'',_infile_); '; + put ' /* a plain $ informat strips leading blanks - use $char instead */ '; + put ' if _n_=1 then call symputx(''input_statement'' '; + put ' ,prxchange(''s/:\$(?=[0-9 ])/:\$char/i'',-1,_infile_) '; + put ' ); '; put ' %if %str(&_debug) ge 128 %then %do; '; put ' if _n_<20 then putlog _infile_; '; put ' else stop; '; diff --git a/viya/mv_webout.sas b/viya/mv_webout.sas index 96901c7..daa1bd2 100644 --- a/viya/mv_webout.sas +++ b/viya/mv_webout.sas @@ -104,7 +104,10 @@ data _null_; infile "%sysfunc(pathname(work))/&table..csv" termstr=crlf ; input; - if _n_=1 then call symputx('input_statement',_infile_); + /* a plain $ informat strips leading blanks - use $char instead */ + if _n_=1 then call symputx('input_statement' + ,prxchange('s/:\$(?=[0-9 ])/:\$char/i',-1,_infile_) + ); list; data work.&table; infile "%sysfunc(pathname(work))/&table..csv" firstobs=2 dsd @@ -124,7 +127,10 @@ data _null_; infile indata termstr=crlf lrecl=32767; input; - if _n_=1 then call symputx('input_statement',_infile_); + /* a plain $ informat strips leading blanks - use $char instead */ + if _n_=1 then call symputx('input_statement' + ,prxchange('s/:\$(?=[0-9 ])/:\$char/i',-1,_infile_) + ); %if %str(&_debug) ge 128 %then %do; if _n_<20 then putlog _infile_; else stop;