sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2025-12-11 06:24:35 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #25 from sasjs/dcfixes

Browse Source 
fix: more logging in mp_abort, fixing job test, better return values …
This commit is contained in:
Allan Bowe
2021-05-11 23:37:25 +03:00
committed by GitHub
parent 1a595c64c6 2fa9e48286
commit b7bafb49f4
5 changed files with 45 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
base/mp_abort.sas
View File

@@ -74,7 +74,7 @@
input;
i=1;
stoploop=0;
if _n_ ge &logline-5 and stoploop=0 then do until (i>12);
if _n_ ge &logline-15 and stoploop=0 then do until (i>22);
call symputx('logmsg',catx('\n',symget('logmsg'),_infile_));
input;
i+1;

51
base/mp_filtercheck.sas
View File

@@ -33,12 +33,13 @@
@returns The &outds table containing any bad rows, plus a REASON_CD column.
@param [in] inds The table to be checked, with the format above
@param [in] targetds= The target dataset against which to verify VARIABLE_NM
@param [in] targetds= The target dataset against which to verify VARIABLE_NM.
This must be available (ie, the library must be assigned).
@param [out] abort= (YES) If YES will call mp_abort.sas on any exceptions
@param [out] outds= The output table, which is a copy of the &inds. table
plus a REASON_CD column, containing only bad records. If bad records found,
the SYSCC value will be set to 1008 (general data problem). Downstream
processes should check this table (and return code) before continuing.
plus a REASON_CD column, containing only bad records. If bad records found,
the SYSCC value will be set to 1008 (general data problem). Downstream
processes should check this table (and return code) before continuing.
<h4> SAS Macros </h4>
@li mp_abort.sas
@@ -84,41 +85,52 @@
* quotes, commas, periods and spaces.
* Only numeric values should remain
*/
%local reason_cd;
%local reason_cd nobs;
%let nobs=0;
data &outds;
/*length GROUP_LOGIC SUBGROUP_LOGIC $3 SUBGROUP_ID 8 VARIABLE_NM $32
OPERATOR_NM $10 RAW_VALUE $4000;*/
set &inds;
length reason_cd $32;
length reason_cd $4032;
/* closed list checks */
if GROUP_LOGIC not in ('AND','OR') then do;
REASON_CD='GROUP_LOGIC should be either AND or OR';
REASON_CD='GROUP_LOGIC should be AND/OR, not:'!!cats(GROUP_LOGIC);
putlog REASON_CD= GROUP_LOGIC=;
call symputx('reason_cd',reason_cd,'l');
call symputx('nobs',_n_,'l');
output;
end;
if SUBGROUP_LOGIC not in ('AND','OR') then do;
REASON_CD='SUBGROUP_LOGIC should be either AND or OR';
REASON_CD='SUBGROUP_LOGIC should be AND/OR, not:'!!cats(SUBGROUP_LOGIC);
putlog REASON_CD= SUBGROUP_LOGIC=;
call symputx('reason_cd',reason_cd,'l');
call symputx('nobs',_n_,'l');
output;
end;
if mod(SUBGROUP_ID,1) ne 0 then do;
REASON_CD='SUBGROUP_ID should be integer';
REASON_CD='SUBGROUP_ID should be integer, not '!!left(subgroup_id);
putlog REASON_CD= SUBGROUP_ID=;
call symputx('reason_cd',reason_cd,'l');
call symputx('nobs',_n_,'l');
output;
end;
if upcase(VARIABLE_NM) not in
(%upcase(%mf_getvarlist(&targetds,dlm=%str(,),quote=SINGLE)))
then do;
REASON_CD="VARIABLE_NM not in &targetds";
REASON_CD="Variable "!!cats(variable_nm)!!" not in &targetds";
putlog REASON_CD= VARIABLE_NM=;
call symputx('reason_cd',reason_cd,'l');
call symputx('nobs',_n_,'l');
output;
end;
if OPERATOR_NM not in
('=','>','<','<=','>=','BETWEEN','IN','NOT IN','NE','CONTAINS')
then do;
REASON_CD='Invalid OPERATOR_NM';
REASON_CD='Invalid OPERATOR_NM: '!!left(OPERATOR_NM);
putlog REASON_CD= OPERATOR_NM=;
call symputx('reason_cd',reason_cd,'l');
call symputx('nobs',_n_,'l');
output;
end;
@@ -128,8 +140,10 @@ data &outds;
if substr(raw_value,1,1) ne '('
or substr(cats(reverse(raw_value)),1,1) ne ')'
then do;
REASON_CD='Missing brackets in RAW_VALUE';
REASON_CD='Missing start/end bracket in RAW_VALUE';
putlog REASON_CD= OPERATOR_NM= raw_value= raw_value1= ;
call symputx('reason_cd',reason_cd,'l');
call symputx('nobs',_n_,'l');
output;
end;
else raw_value1=substr(raw_value,2,max(length(raw_value)-2,0));
@@ -150,27 +164,24 @@ data &outds;
/* output records that contain values other than digits and spaces */
if notdigit(compress(raw_value3,' '))>0 then do;
putlog raw_value3= $hex32.;
REASON_CD='Invalid RAW_VALUE';
REASON_CD=cats('Invalid RAW_VALUE:',raw_value);
putlog REASON_CD= raw_value= raw_value1= raw_value2= raw_value3=;
call symputx('reason_cd',reason_cd,'l');
call symputx('nobs',_n_,'l');
output;
end;
run;
%local nobs;
%let nobs=0;
data _null_;
set &outds end=last;
putlog (_all_)(=);
if last then do;
call symputx('REASON_CD',reason_cd,'l');
call symputx('nobs',_n_,'l');
end;
run;
%mp_abort(iftrue=(&abort=YES and &nobs>0),
mac=&sysmacroname,
msg=%str(&nobs filter issues in &inds, reason: &reason_cd, details in &outds)
msg=%str(Data issue: %superq(reason_cd))
)
%if &nobs>0 %then %do;

3
base/mp_filtervalidate.sas
View File

@@ -95,7 +95,8 @@ filename &fref1 clear;
run;
%mp_abort(
mac=&sysmacroname,
msg=%str(Filter issues in &inref: %quote(&reason_cd))
msg=%str(Filter validation issues. ERR=%superq(SYSERRORTEXT)
, WARN=%superq(SYSWARNINGTEXT) )
)
%end;
%let syscc=1008;

2
base/mp_jsonout.sas
View File

@@ -61,7 +61,7 @@
)/*/STORE SOURCE*/;
%put output location=&jref;
%if &action=OPEN %then %do;
OPTIONS NOBOMFILE;
options nobomfile;
data _null_;file &jref encoding='utf-8';
put '{"START_DTTM" : "' "%sysfunc(datetime(),datetime20.3)" '"';
run;

14
tests/viya/mv_getjoblog.test.sas
View File

@@ -19,6 +19,7 @@
filename testref temp;
data _null_;
file testref;
put 'data;run;';
put 'endsas;';
run;
%mv_createjob(
@@ -31,7 +32,7 @@ run;
%mv_jobexecute(
path=&mcTestAppLoc/jobs/temp,
name=testjob,
outds=work.info,
outds=work.info
)
%* Wait for it to finish;
@@ -52,10 +53,15 @@ run;
data _null_;
infile mylog;
infile mylog end=eof;
input;
if index(_infile_,'endsas;') then call symputx('found',1);
else call symputx('found',0);
putlog _infile_;
retain found 0;
if index(_infile_,'endsas;') then do;
found=1;
call symputx('found',found);
end;
else if eof and found ne 1 then call symputx('found',0);
run;
%mp_assert(