Merge pull request #36 from sasjs/filter_json_fixes

Filter json fixes

SECURITY.md

@@ -0,0 +1,18 @@
+# Security Policy
+
+Security is an extremely high priority when it comes to the SASjs product suite.  We take a number of steps across all repositories to minimise risk, such as:
+
+* Regular dependabot updates
+* Snyk reports
+* Minimising dependencies, especially production dependencies (sasjs/core has NONE)
+* Testing & Code review process
+
+## Supported Versions
+
+We support only the latest version
+
+## Reporting a Vulnerability
+
+We welcome disclosures of all kinds in relation to all the SASjs libraries.  You can submit them here: [https://sasapps.io/contact-us](https://sasapps.io/contact-us)
+
+

base/mp_abort.sas

@@ -99,7 +99,7 @@
 
     /* send response in SASjs JSON format */
     data _null_;
-      file _webout mod lrecl=32000;
+      file _webout mod lrecl=32000 encoding='utf-8';
       length msg $32767 debug $8;
       sasdatetime=datetime();
       msg=cats(symget('msg'),'\n\nLog Extract:\n',symget('logmsg'));
@@ -133,6 +133,8 @@
       put ",""SYSCC"" : ""&syscc"" ";
       put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
       put ",""SYSJOBID"" : ""&sysjobid"" ";
+      sysvlong=quote(trim(symget('sysvlong')));
+      put ',"SYSVLONG" : ' sysvlong;
       put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
       put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
       put "}" @;

base/mp_filtervalidate.sas

@@ -78,7 +78,8 @@ run;
 
 data &outds;
   if &sqlrc or &syscc or &syserr then do;
-    REASON_CD=coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
+    REASON_CD='VALIDATION_ERROR: '!!
+      coalescec(symget('SYSERRORTEXT'),symget('SYSWARNINGTEXT'));
     output;
   end;
   else stop;
@@ -102,4 +103,4 @@ filename &fref1 clear;
   %let syscc=1008;
 %end;
 
-%mend;
+%mend mp_filtervalidate;

base/mp_jsonout.sas

@@ -147,7 +147,7 @@
       run;
       %let ds=&fmtds;
     %end; /* &fmt=Y */
-    data _null_;file &jref mod ;
+    data _null_;file &jref mod encoding='utf-8';
       put "["; call symputx('cols',0,'l');
     proc sort
       data=sashelp.vcolumn(where=(libname='WORK' & memname="%upcase(&ds)"))

meta/mm_webout.sas

@@ -26,7 +26,8 @@
   @param action Either FETCH, OPEN, ARR, OBJ or CLOSE
   @param ds The dataset to send back to the frontend
   @param dslabel= value to use instead of the real name for sending to JSON
-  @param fmt= set to N to send back unformatted values
+  @param fmt=(Y) Set to N to send back unformatted values
+  @param fref=(_webout) The fileref to which to write the JSON
 
   @version 9.3
   @author Allan Bowe
@@ -91,7 +92,7 @@
 %end;
 
 %else %if &action=ARR or &action=OBJ %then %do;
-  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt
+  %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt,jref=&fref
     ,engine=DATASTEP,dbg=%str(&_debug)
   )
 %end;
@@ -150,6 +151,8 @@
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSJOBID"" : ""&sysjobid"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
+    sysvlong=quote(trim(symget('sysvlong')));
+    put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
     put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
     put "}" @;
@@ -159,4 +162,4 @@
   run;
 %end;
 
-%mend;
+%mend mm_webout;

package.json

@@ -35,10 +35,5 @@
   "devDependencies": {
     "@sasjs/cli": "^2.22.2"
   },
-  "config": {
-    "ghooks": {
-      "pre-commit": "sasjs lint"
-    }
-  },
   "dependencies": {}
 }

sasjs/sasjsconfig.json

@@ -5,7 +5,8 @@
     "meta",
     "metax",
     "viya",
-    "lua"
+    "lua",
+    "tests/base"
   ],
   "docConfig": {
     "displayMacroCore": false,
@@ -16,6 +17,13 @@
       "readMe": "../../README.md"
     }
   },
+  "testConfig": {
+    "initProgram": "tests/testinit.sas",
+    "termProgram": "tests/testterm.sas",
+    "macroVars": {
+      "mcTestAppLoc": "/Public/temp/macrocore"
+    }
+  },
   "defaultTarget": "viya",
   "targets": [
     {
@@ -27,22 +35,18 @@
         "deployServicePack": true
       },
       "macroFolders": [
-        "base",
-        "meta",
-        "metax",
-        "viya",
-        "lua",
-        "tests/base",
         "tests/viya"
       ],
-      "contextName": "SAS Job Execution compute context",
+      "contextName": "SAS Job Execution compute context"
-      "testConfig": {
+    },
-        "initProgram": "tests/testinit.sas",
+    {
-        "termProgram": "tests/testterm.sas",
+      "name": "sas9",
-        "macroVars": {
+      "serverUrl": "https://sas.analytium.co.uk:8343",
-          "mcTestAppLoc": "/Public/temp/macrocore"
+      "serverType": "SAS9",
-        }
+      "appLoc": "/Shared Data/temp/macrocore",
-      }
+      "macroFolders": [
+        "tests/meta"
+      ]
     }
   ]
 }

tests/base/mp_filtervalidate.test.sas

@@ -6,6 +6,7 @@
   @li mp_filtergenerate.sas
   @li mp_filtervalidate.sas
   @li mp_assertdsobs.sas
+  @li mp_assert.sas
 
 **/
 
@@ -41,14 +42,14 @@ run;
 %mp_filtergenerate(work.inds,outref=myfilter)
 %mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
 %mp_assertdsobs(work.results,
-  desc=Valid filter,
+  desc=Empty filter,
   test=EMPTY,
   outds=work.test_results
 )
 
 
 
-/* invalid filter*/
+/* invalid filter - char var, num val */
 data work.inds;
   infile datalines4 dsd;
   input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
@@ -60,9 +61,40 @@ run;
 %mp_filtergenerate(work.inds,outref=myfilter)
 %mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
 %let syscc=0;
-%mp_assertdsobs(work.results,
+%let test3=0;
-  desc=Valid filter,
+data _null_;
-  test=EQUALS 1,
+  set work.results;
+  if REASON_CD=:'VALIDATION_ERROR' then call symputx('test3',1);
+  putlog (_all_)(=);
+  stop;
+run;
+%mp_assert(
+  iftrue=(&test3=1),
+  desc=Checking char var could not receive num val,
   outds=work.test_results
 )
 
+/* invalid filter - num var, char val */
+data work.inds;
+  infile datalines4 dsd;
+  input GROUP_LOGIC:$3. SUBGROUP_LOGIC:$3. SUBGROUP_ID:8. VARIABLE_NM:$32.
+    OPERATOR_NM:$10. RAW_VALUE:$4000.;
+datalines4;
+AND,AND,1,age,NE,"'M'"
+;;;;
+run;
+%mp_filtergenerate(work.inds,outref=myfilter)
+%mp_filtervalidate(myfilter,sashelp.class,outds=work.results,abort=NO)
+%let syscc=0;
+%let test4=0;
+data _null_;
+  set work.results;
+  if REASON_CD=:'VALIDATION_ERROR' then call symputx('test4',1);
+  putlog (_all_)(=);
+  stop;
+run;
+%mp_assert(
+  iftrue=(&test4=1),
+  desc=Checking num var could not receive char val,
+  outds=work.test_results
+)

tests/meta/mm_webout.test.sas

@@ -0,0 +1,35 @@
+/**
+  @file
+  @brief Testing mm_webout macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mm_webout.sas
+  @li mp_assert.sas
+
+**/
+
+
+%let fref=%mf_getuniquefileref();
+%global _metaperson;
+data some datasets;
+  x=1;
+run;
+%mm_webout(OPEN,fref=&fref)
+%mm_webout(ARR,some,fref=&fref)
+%mm_webout(OBJ,datasets,fref=&fref)
+%mm_webout(CLOSE,fref=&fref)
+
+libname test JSON (&fref);
+data root;
+  set test.root;
+  call symputx('checkval',sysvlong);
+run;
+data alldata;
+  set test.alldata;
+run;
+
+%mp_assert(
+  iftrue=(%str(&checkval)=%str(&sysvlong)),
+  desc=Check if the sysvlong value was created
+)

tests/viya/mv_webout.test.sas

@@ -0,0 +1,41 @@
+/**
+  @file
+  @brief Testing mm_webout macro
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mv_webout.sas
+  @li mp_assert.sas
+
+**/
+
+
+%let fref=%mf_getuniquefileref();
+%global _metaperson;
+data some datasets;
+  x=1;
+run;
+%mv_webout(OPEN,fref=&fref,stream=N)
+%mv_webout(ARR,some,fref=&fref,stream=N)
+%mv_webout(OBJ,datasets,fref=&fref,stream=N)
+%mv_webout(CLOSE,fref=&fref,stream=N)
+
+data _null_;
+  infile &fref;
+  input;
+  putlog _infile_;
+run;
+
+libname test JSON (&fref);
+data root;
+  set test.root;
+  call symputx('checkval',sysvlong);
+run;
+data alldata;
+  set test.alldata;
+run;
+
+%mp_assert(
+  iftrue=(%str(&checkval)=%str(&sysvlong)),
+  desc=Check if the sysvlong value was created
+)

viya/mv_webout.sas

@@ -23,9 +23,10 @@
   @param action Either OPEN, ARR, OBJ or CLOSE
   @param ds The dataset to send back to the frontend
   @param _webout= fileref for returning the json
-  @param fref= temp fref
+  @param fref=(_mvwtemp) Temp fileref to which to write the output
   @param dslabel= value to use instead of the real name for sending to JSON
-  @param fmt= change to N to strip formats from output
+  @param fmt=(Y) change to N to strip formats from output
+  @param stream=(Y) Change to N if not streaming to _webout
 
   <h4> SAS Macros </h4>
   @li mp_jsonout.sas
@@ -35,7 +36,7 @@
   @author Allan Bowe, source: https://github.com/sasjs/core
 
 **/
-%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y);
+%macro mv_webout(action,ds,fref=_mvwtemp,dslabel=,fmt=Y,stream=Y);
 %global _webin_file_count _webin_fileuri _debug _omittextlog _webin_name
   sasjs_tables SYS_JES_JOB_URI;
 %if %index("&_debug",log) %then %let _debug=131;
@@ -214,14 +215,16 @@
     put ",""SYSERRORTEXT"" : ""&syserrortext"" ";
     put ",""SYSHOSTNAME"" : ""&syshostname"" ";
     put ",""SYSSITE"" : ""&syssite"" ";
+    sysvlong=quote(trim(symget('sysvlong')));
+    put ',"SYSVLONG" : ' sysvlong;
     put ",""SYSWARNINGTEXT"" : ""&syswarningtext"" ";
     put ',"END_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '" ';
     put "}";
 
-  %if %upcase(&fref) ne _WEBOUT %then %do;
+  %if %upcase(&fref) ne _WEBOUT and &stream=Y %then %do;
     data _null_; rc=fcopy("&fref","_webout");run;
   %end;
 
 %end;
 
-%mend;
+%mend mv_webout;