sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2026-01-03 23:50:06 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #205 from sasjs/issue204

Browse Source 
fix: updating mp_hashdataset to cope with STRICT mode.  Adding test a…
This commit is contained in:
Allan Bowe
2022-03-29 15:47:17 +03:00
committed by GitHub
parent 40d694eec8 72d6b446c3
commit 3ce771d587
5 changed files with 155 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
all.sas
View File

@@ -7954,8 +7954,8 @@ run;
%mend mp_guesspk;/** %mend mp_guesspk;/**
@file @file
@brief Returns a unique hash for a dataset @brief Returns a unique hash for a dataset
@details Ignores metadata attributes, used only to hash values. Compared @details Ignores metadata attributes, used only to hash values. If used to
datasets must be in the same order. compare datasets, they must have their columns and rows in the same order.
%mp_hashdataset(sashelp.class,outds=myhash) %mp_hashdataset(sashelp.class,outds=myhash)
@@ -7970,7 +7970,10 @@ run;
@li mf_getattrn.sas @li mf_getattrn.sas
@li mf_getuniquename.sas @li mf_getuniquename.sas
@li mf_getvarlist.sas @li mf_getvarlist.sas
@li mf_getvartype.sas @li mp_md5.sas
<h4> Related Files </h4>
@li mp_hashdataset.test.sas
@param [in] libds dataset to hash @param [in] libds dataset to hash
@param [in] salt= Provide a salt (could be, for instance, the dataset name) @param [in] salt= Provide a salt (could be, for instance, the dataset name)
@@ -7988,51 +7991,52 @@ run;
%macro mp_hashdataset( %macro mp_hashdataset(
libds, libds,
outds=, outds=work._data_,
salt=, salt=,
iftrue=%str(1=1) iftrue=%str(1=1)
)/*/STORE SOURCE*/; )/*/STORE SOURCE*/;
%if not(%eval(%unquote(&iftrue))) %then %return; %local keyvar /* roll up the md5 */
prevkeyvar /* retain prev record md5 */
lastvar /* last var in input ds */
cvars nvars;
%if %mf_getattrn(&libds,NLOBS)=0 %then %do; %if not(%eval(%unquote(&iftrue))) %then %return;
%put %str(WARN)ING: Dataset &libds is empty, or is not a dataset;
%end; /* avoid naming conflict for hash key vars */
%else %if %mf_getattrn(&libds,NLOBS)<0 %then %do; %let keyvar=%mf_getuniquename();
%put %str(ERR)OR: Dataset &libds is not a dataset; %let prevkeyvar=%mf_getuniquename();
%end; %let lastvar=%mf_getuniquename();
%else %do;
%local keyvar /* roll up the md5 */ %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
prevkeyvar /* retain prev record md5 */ data &outds;
lastvar /* last var in input ds */ length hashkey $32;
varlist var i; retain hashkey "%sysfunc(md5(%str(&salt)),$hex32.)";
/* avoid naming conflict for hash key vars */ output;
%let keyvar=%mf_getuniquename(); stop;
%let prevkeyvar=%mf_getuniquename(); run;
%let lastvar=%mf_getuniquename(); %put &sysmacroname: Dataset &libds is empty, or is not a dataset;
%let varlist=%mf_getvarlist(&libds); %put &sysmacroname: hashkey of &outds is based on salt (&salt) only;
data &outds(rename=(&keyvar=hashkey) keep=&keyvar); %end;
length &prevkeyvar &keyvar $32; %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)"; %put %str(ERR)OR: Dataset &libds is not a dataset;
set &libds end=&lastvar; %end;
/* hash should include previous row */ %else %do;
&keyvar=put(md5(&prevkeyvar data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
/* loop every column, hashing every individual value */ length &prevkeyvar &keyvar $32;
%do i=1 %to %sysfunc(countw(&varlist)); retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
%let var=%scan(&varlist,&i,%str( )); set &libds end=&lastvar;
%if %mf_getvartype(&libds,&var)=C %then %do; /* hash should include previous row */
!!put(md5(trim(&var)),$hex32.) &keyvar=%mp_md5(
%end; cvars=%mf_getvarlist(&libds,typefilter=C) &prevkeyvar,
%else %do; nvars=%mf_getvarlist(&libds,typefilter=N)
!!put(md5(trim(put(&var*1,binary64.))),$hex32.) );
%end; &prevkeyvar=&keyvar;
%end; if &lastvar then output;
),$hex32.); run;
&prevkeyvar=&keyvar; %end;
if &lastvar then output; %mend mp_hashdataset;
run; /**
%end;
%mend mp_hashdataset;/**
@file @file
@brief Performs a wrapped \%include @brief Performs a wrapped \%include
@details This macro wrapper is necessary if you need your included code to @details This macro wrapper is necessary if you need your included code to
@@ -19224,6 +19228,7 @@ run;
data _null_; data _null_;
set &tempds; set &tempds;
if not (upcase(name) =:"DATA"); /* ignore temp datasets */ if not (upcase(name) =:"DATA"); /* ignore temp datasets */
if not (upcase(name)=:"_DATA_");
i+1; i+1;
call symputx(cats('wt',i),name,'l'); call symputx(cats('wt',i),name,'l');
call symputx('wtcnt',i,'l'); call symputx('wtcnt',i,'l');

89
base/mp_hashdataset.sas
View File

@@ -1,8 +1,8 @@
/** /**
@file @file
@brief Returns a unique hash for a dataset @brief Returns a unique hash for a dataset
@details Ignores metadata attributes, used only to hash values. Compared @details Ignores metadata attributes, used only to hash values. If used to
datasets must be in the same order. compare datasets, they must have their columns and rows in the same order.
%mp_hashdataset(sashelp.class,outds=myhash) %mp_hashdataset(sashelp.class,outds=myhash)
@@ -17,7 +17,10 @@
@li mf_getattrn.sas @li mf_getattrn.sas
@li mf_getuniquename.sas @li mf_getuniquename.sas
@li mf_getvarlist.sas @li mf_getvarlist.sas
@li mf_getvartype.sas @li mp_md5.sas
<h4> Related Files </h4>
@li mp_hashdataset.test.sas
@param [in] libds dataset to hash @param [in] libds dataset to hash
@param [in] salt= Provide a salt (could be, for instance, the dataset name) @param [in] salt= Provide a salt (could be, for instance, the dataset name)
@@ -35,48 +38,48 @@
%macro mp_hashdataset( %macro mp_hashdataset(
libds, libds,
outds=, outds=work._data_,
salt=, salt=,
iftrue=%str(1=1) iftrue=%str(1=1)
)/*/STORE SOURCE*/; )/*/STORE SOURCE*/;
%if not(%eval(%unquote(&iftrue))) %then %return; %local keyvar /* roll up the md5 */
prevkeyvar /* retain prev record md5 */
lastvar /* last var in input ds */
cvars nvars;
%if %mf_getattrn(&libds,NLOBS)=0 %then %do; %if not(%eval(%unquote(&iftrue))) %then %return;
%put %str(WARN)ING: Dataset &libds is empty, or is not a dataset;
%end; /* avoid naming conflict for hash key vars */
%else %if %mf_getattrn(&libds,NLOBS)<0 %then %do; %let keyvar=%mf_getuniquename();
%put %str(ERR)OR: Dataset &libds is not a dataset; %let prevkeyvar=%mf_getuniquename();
%end; %let lastvar=%mf_getuniquename();
%else %do;
%local keyvar /* roll up the md5 */ %if %mf_getattrn(&libds,NLOBS)=0 %then %do;
prevkeyvar /* retain prev record md5 */ data &outds;
lastvar /* last var in input ds */ length hashkey $32;
varlist var i; retain hashkey "%sysfunc(md5(%str(&salt)),$hex32.)";
/* avoid naming conflict for hash key vars */ output;
%let keyvar=%mf_getuniquename(); stop;
%let prevkeyvar=%mf_getuniquename(); run;
%let lastvar=%mf_getuniquename(); %put &sysmacroname: Dataset &libds is empty, or is not a dataset;
%let varlist=%mf_getvarlist(&libds); %put &sysmacroname: hashkey of &outds is based on salt (&salt) only;
data &outds(rename=(&keyvar=hashkey) keep=&keyvar); %end;
length &prevkeyvar &keyvar $32; %else %if %mf_getattrn(&libds,NLOBS)<0 %then %do;
retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)"; %put %str(ERR)OR: Dataset &libds is not a dataset;
set &libds end=&lastvar; %end;
/* hash should include previous row */ %else %do;
&keyvar=put(md5(&prevkeyvar data &outds(rename=(&keyvar=hashkey) keep=&keyvar);
/* loop every column, hashing every individual value */ length &prevkeyvar &keyvar $32;
%do i=1 %to %sysfunc(countw(&varlist)); retain &prevkeyvar "%sysfunc(md5(%str(&salt)),$hex32.)";
%let var=%scan(&varlist,&i,%str( )); set &libds end=&lastvar;
%if %mf_getvartype(&libds,&var)=C %then %do; /* hash should include previous row */
!!put(md5(trim(&var)),$hex32.) &keyvar=%mp_md5(
%end; cvars=%mf_getvarlist(&libds,typefilter=C) &prevkeyvar,
%else %do; nvars=%mf_getvarlist(&libds,typefilter=N)
!!put(md5(trim(put(&var*1,binary64.))),$hex32.) );
%end; &prevkeyvar=&keyvar;
%end; if &lastvar then output;
),$hex32.); run;
&prevkeyvar=&keyvar; %end;
if &lastvar then output; %mend mp_hashdataset;
run;
%end;
%mend mp_hashdataset;

2
sasjs/sasjsconfig.json
View File

@@ -71,7 +71,7 @@
}, },
{ {
"name": "server", "name": "server",
"serverUrl": "https://sas.analytium.co.uk:5000", "serverUrl": " ",
"serverType": "SASJS", "serverType": "SASJS",
"httpsAgentOptions": { "httpsAgentOptions": {
"allowInsecureRequests": false "allowInsecureRequests": false

1
server/ms_webout.sas
View File

@@ -114,6 +114,7 @@
data _null_; data _null_;
set &tempds; set &tempds;
if not (upcase(name) =:"DATA"); /* ignore temp datasets */ if not (upcase(name) =:"DATA"); /* ignore temp datasets */
if not (upcase(name)=:"_DATA_");
i+1; i+1;
call symputx(cats('wt',i),name,'l'); call symputx(cats('wt',i),name,'l');
call symputx('wtcnt',i,'l'); call symputx('wtcnt',i,'l');

59
tests/crossplatform/mp_hashdataset.test.sas Normal file
View File

@@ -0,0 +1,59 @@
/**
@file
@brief Testing mp_hashdataset.sas macro
<h4> SAS Macros </h4>
@li mp_hashdataset.sas
@li mp_assert.sas
**/
/* test 1 - regular DS */
data work.test;
set sashelp.vextfl;
missval=.;
misscval='';
run;
%mp_assertscope(SNAPSHOT)
%mp_hashdataset(test)
%mp_assertscope(COMPARE)
%mp_assert(
iftrue=(&syscc=0),
desc=Regular test works,
outds=work.test_results
)
%mp_hashdataset(test,outds=work.test2)
%mp_assert(
iftrue=(&syscc=0),
desc=hash with output runs without errors,
outds=work.test_results
)
%mp_assert(
iftrue=(%mf_nobs(work.test2)=1),
desc=output has 1 row,
outds=work.test_results
)
data work.test3a;
set work.test;
stop;
run;
%mp_hashdataset(test3a,outds=work.test3b)
%mp_assert(
iftrue=(&syscc=0),
desc=hash with zero-row input runs without errors,
outds=work.test_results
)
%mp_assert(
iftrue=(%mf_nobs(work.test3b)=1),
desc=test 3 output has 1 row,
outds=work.test_results
)