sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2026-01-18 14:00:05 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #265 from sasjs/binaryfix

Browse Source 
enable file copy of files with an encoding that does not match session encoding
This commit is contained in:
Allan Bowe
2022-06-26 17:14:25 +02:00
committed by GitHub
parent b72e404d52 7a69698178
commit 6d0e34ba1d
9 changed files with 163 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
all.sas
View File

@@ -3470,8 +3470,8 @@ run;
@details Reads in a file byte by byte and writes it back out. Is an @details Reads in a file byte by byte and writes it back out. Is an
os-independent method to copy files. In case of naming collision, the os-independent method to copy files. In case of naming collision, the
default filerefs can be modified. default filerefs can be modified.
Based on: Note that if you have a new enough version of SAS, and you don't need features
https://stackoverflow.com/questions/13046116/using-sas-to-copy-a-text-file such as APPEND, you may be better of using the fcopy() function instead.
%mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout) %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
@@ -3511,14 +3511,9 @@ run;
,outref=____out /* override default to use own filerefs */ ,outref=____out /* override default to use own filerefs */
,mode=CREATE ,mode=CREATE
)/*/STORE SOURCE*/; )/*/STORE SOURCE*/;
%local mod outmode; %local mod;
%if &mode=APPEND %then %do; %if &mode=APPEND %then %let mod=mod;
%let mod=mod;
%let outmode='a';
%end;
%else %do;
%let outmode='o';
%end;
/* these IN and OUT filerefs can point to anything */ /* these IN and OUT filerefs can point to anything */
%if &inref = ____in %then %do; %if &inref = ____in %then %do;
filename &inref &inloc lrecl=1048576 ; filename &inref &inloc lrecl=1048576 ;
@@ -3529,25 +3524,21 @@ run;
/* copy the file byte-for-byte */ /* copy the file byte-for-byte */
data _null_; data _null_;
length filein 8 fileid 8; infile &inref lrecl=1 recfm=n;
filein = fopen("&inref",'I',1,'B'); file &outref &mod recfm=n;
fileid = fopen("&outref",&outmode,1,'B'); input sourcechar $char1. @@;
rec = '20'x; format sourcechar hex2.;
do while(fread(filein)=0); put sourcechar char1. @@;
rc = fget(filein,rec,1);
rc = fput(fileid, rec);
rc =fwrite(fileid);
end;
rc = fclose(filein);
rc = fclose(fileid);
run; run;
%if &inref = ____in %then %do; %if &inref = ____in %then %do;
filename &inref clear; filename &inref clear;
%end; %end;
%if &outref=____out %then %do; %if &outref=____out %then %do;
filename &outref clear; filename &outref clear;
%end; %end;
%mend mp_binarycopy;/** %mend mp_binarycopy;
/**
@file @file
@brief Splits a file of ANY SIZE by reference to a search string. @brief Splits a file of ANY SIZE by reference to a search string.
@details Provide a fileref and a search string to chop off part of a file. @details Provide a fileref and a search string to chop off part of a file.
@@ -8817,22 +8808,18 @@ options
"&&name&i"n /* name literal for reserved variable names */ "&&name&i"n /* name literal for reserved variable names */
%end; %end;
%if &action=ARR %then "]" ; %else "}" ; ; %if &action=ARR %then "]" ; %else "}" ; ;
/* now write the long strings to _webout 1 byte at a time */ /* now write the long strings to _webout 1 char at a time */
data _null_; data _null_;
length filein 8 fileid 8; infile _sjs lrecl=1 recfm=n;
filein=fopen("_sjs",'I',1,'B'); file &jref mod lrecl=1 recfm=n;
fileid=fopen("&jref",'A',1,'B'); input sourcechar $char1. @@;
rec='20'x; format sourcechar hex2.;
do while(fread(filein)=0); put sourcechar char1. @@;
rc=fget(filein,rec,1); run;
rc=fput(fileid, rec);
rc=fwrite(fileid);
end;
/* close out the table */ /* close out the table */
rc=fput(fileid, "]"); data _null_;
rc=fwrite(fileid); file &jref mod;
rc=fclose(filein); put ']';
rc=fclose(fileid);
run; run;
filename _sjs clear; filename _sjs clear;
%end; %end;
@@ -14066,7 +14053,7 @@ run;
%mm_createfolder(path=/some/meta/folder) %mm_createfolder(path=/some/meta/folder)
@param [in] path= Name of the folder to create. @param [in] path= Name of the folder to create.
@param [in] mdebug= set DBG to 1 to disable DEBUG messages @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
@version 9.4 @version 9.4
@@ -14932,7 +14919,8 @@ run;
@file mm_createwebservice.sas @file mm_createwebservice.sas
@brief Create a Web Ready Stored Process @brief Create a Web Ready Stored Process
@details This macro creates a Type 2 Stored Process with the mm_webout macro @details This macro creates a Type 2 Stored Process with the mm_webout macro
included as pre-code. (and dependencies) included as pre-code.
Usage: Usage:
%* compile macros ; %* compile macros ;
@@ -15197,22 +15185,18 @@ data _null_;
put ' "&&name&i"n /* name literal for reserved variable names */ '; put ' "&&name&i"n /* name literal for reserved variable names */ ';
put ' %end; '; put ' %end; ';
put ' %if &action=ARR %then "]" ; %else "}" ; ; '; put ' %if &action=ARR %then "]" ; %else "}" ; ; ';
put ' /* now write the long strings to _webout 1 byte at a time */ '; put ' /* now write the long strings to _webout 1 char at a time */ ';
put ' data _null_; '; put ' data _null_; ';
put ' length filein 8 fileid 8; '; put ' infile _sjs lrecl=1 recfm=n; ';
put ' filein=fopen("_sjs",''I'',1,''B''); '; put ' file &jref mod lrecl=1 recfm=n; ';
put ' fileid=fopen("&jref",''A'',1,''B''); '; put ' input sourcechar $char1. @@; ';
put ' rec=''20''x; '; put ' format sourcechar hex2.; ';
put ' do while(fread(filein)=0); '; put ' put sourcechar char1. @@; ';
put ' rc=fget(filein,rec,1); '; put ' run; ';
put ' rc=fput(fileid, rec); ';
put ' rc=fwrite(fileid); ';
put ' end; ';
put ' /* close out the table */ '; put ' /* close out the table */ ';
put ' rc=fput(fileid, "]"); '; put ' data _null_; ';
put ' rc=fwrite(fileid); '; put ' file &jref mod; ';
put ' rc=fclose(filein); '; put ' put '']''; ';
put ' rc=fclose(fileid); ';
put ' run; '; put ' run; ';
put ' filename _sjs clear; '; put ' filename _sjs clear; ';
put ' %end; '; put ' %end; ';
@@ -15424,7 +15408,7 @@ run;
%if &x>1 %then %let mod=mod; %if &x>1 %then %let mod=mod;
%let fref=%scan(&freflist,&x); %let fref=%scan(&freflist,&x);
%put &sysmacroname: adding &fref; %&mD.put &sysmacroname: adding &fref;
data _null_; data _null_;
file "&work/&tmpfile" lrecl=3000 &mod; file "&work/&tmpfile" lrecl=3000 &mod;
infile &fref; infile &fref;
@@ -15460,13 +15444,11 @@ data _null_;
if rc=0 then call symputx('url',url,'l'); if rc=0 then call symputx('url',url,'l');
run; run;
%put ;%put ;%put ;%put ;%put ;%put ;
%put &sysmacroname: STP &name successfully created in &path; %put &sysmacroname: STP &name successfully created in &path;
%put ;%put ;%put ;
%put Check it out here:; %put Check it out here:;
%put ;%put ;%put ; %put ;%put ;%put ;
%put &url?_PROGRAM=&path/&name; %put &url?_PROGRAM=&path/&name;
%put ;%put ;%put ;%put ;%put ;%put ; %put ;%put ;%put ;
%mend mm_createwebservice; %mend mm_createwebservice;
/** /**
@@ -19936,22 +19918,18 @@ data _null_;
put ' "&&name&i"n /* name literal for reserved variable names */ '; put ' "&&name&i"n /* name literal for reserved variable names */ ';
put ' %end; '; put ' %end; ';
put ' %if &action=ARR %then "]" ; %else "}" ; ; '; put ' %if &action=ARR %then "]" ; %else "}" ; ; ';
put ' /* now write the long strings to _webout 1 byte at a time */ '; put ' /* now write the long strings to _webout 1 char at a time */ ';
put ' data _null_; '; put ' data _null_; ';
put ' length filein 8 fileid 8; '; put ' infile _sjs lrecl=1 recfm=n; ';
put ' filein=fopen("_sjs",''I'',1,''B''); '; put ' file &jref mod lrecl=1 recfm=n; ';
put ' fileid=fopen("&jref",''A'',1,''B''); '; put ' input sourcechar $char1. @@; ';
put ' rec=''20''x; '; put ' format sourcechar hex2.; ';
put ' do while(fread(filein)=0); '; put ' put sourcechar char1. @@; ';
put ' rc=fget(filein,rec,1); '; put ' run; ';
put ' rc=fput(fileid, rec); ';
put ' rc=fwrite(fileid); ';
put ' end; ';
put ' /* close out the table */ '; put ' /* close out the table */ ';
put ' rc=fput(fileid, "]"); '; put ' data _null_; ';
put ' rc=fwrite(fileid); '; put ' file &jref mod; ';
put ' rc=fclose(filein); '; put ' put '']''; ';
put ' rc=fclose(fileid); ';
put ' run; '; put ' run; ';
put ' filename _sjs clear; '; put ' filename _sjs clear; ';
put ' %end; '; put ' %end; ';
@@ -22281,22 +22259,18 @@ data _null_;
put ' "&&name&i"n /* name literal for reserved variable names */ '; put ' "&&name&i"n /* name literal for reserved variable names */ ';
put ' %end; '; put ' %end; ';
put ' %if &action=ARR %then "]" ; %else "}" ; ; '; put ' %if &action=ARR %then "]" ; %else "}" ; ; ';
put ' /* now write the long strings to _webout 1 byte at a time */ '; put ' /* now write the long strings to _webout 1 char at a time */ ';
put ' data _null_; '; put ' data _null_; ';
put ' length filein 8 fileid 8; '; put ' infile _sjs lrecl=1 recfm=n; ';
put ' filein=fopen("_sjs",''I'',1,''B''); '; put ' file &jref mod lrecl=1 recfm=n; ';
put ' fileid=fopen("&jref",''A'',1,''B''); '; put ' input sourcechar $char1. @@; ';
put ' rec=''20''x; '; put ' format sourcechar hex2.; ';
put ' do while(fread(filein)=0); '; put ' put sourcechar char1. @@; ';
put ' rc=fget(filein,rec,1); '; put ' run; ';
put ' rc=fput(fileid, rec); ';
put ' rc=fwrite(fileid); ';
put ' end; ';
put ' /* close out the table */ '; put ' /* close out the table */ ';
put ' rc=fput(fileid, "]"); '; put ' data _null_; ';
put ' rc=fwrite(fileid); '; put ' file &jref mod; ';
put ' rc=fclose(filein); '; put ' put '']''; ';
put ' rc=fclose(fileid); ';
put ' run; '; put ' run; ';
put ' filename _sjs clear; '; put ' filename _sjs clear; ';
put ' %end; '; put ' %end; ';

32
base/mp_binarycopy.sas
View File

@@ -4,8 +4,8 @@
@details Reads in a file byte by byte and writes it back out. Is an @details Reads in a file byte by byte and writes it back out. Is an
os-independent method to copy files. In case of naming collision, the os-independent method to copy files. In case of naming collision, the
default filerefs can be modified. default filerefs can be modified.
Based on: Note that if you have a new enough version of SAS, and you don't need features
https://stackoverflow.com/questions/13046116/using-sas-to-copy-a-text-file such as APPEND, you may be better of using the fcopy() function instead.
%mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout) %mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
@@ -45,14 +45,9 @@
,outref=____out /* override default to use own filerefs */ ,outref=____out /* override default to use own filerefs */
,mode=CREATE ,mode=CREATE
)/*/STORE SOURCE*/; )/*/STORE SOURCE*/;
%local mod outmode; %local mod;
%if &mode=APPEND %then %do; %if &mode=APPEND %then %let mod=mod;
%let mod=mod;
%let outmode='a';
%end;
%else %do;
%let outmode='o';
%end;
/* these IN and OUT filerefs can point to anything */ /* these IN and OUT filerefs can point to anything */
%if &inref = ____in %then %do; %if &inref = ____in %then %do;
filename &inref &inloc lrecl=1048576 ; filename &inref &inloc lrecl=1048576 ;
@@ -63,18 +58,13 @@
/* copy the file byte-for-byte */ /* copy the file byte-for-byte */
data _null_; data _null_;
length filein 8 fileid 8; infile &inref lrecl=1 recfm=n;
filein = fopen("&inref",'I',1,'B'); file &outref &mod recfm=n;
fileid = fopen("&outref",&outmode,1,'B'); input sourcechar $char1. @@;
rec = '20'x; format sourcechar hex2.;
do while(fread(filein)=0); put sourcechar char1. @@;
rc = fget(filein,rec,1);
rc = fput(fileid, rec);
rc =fwrite(fileid);
end;
rc = fclose(filein);
rc = fclose(fileid);
run; run;
%if &inref = ____in %then %do; %if &inref = ____in %then %do;
filename &inref clear; filename &inref clear;
%end; %end;

24
base/mp_jsonout.sas
View File

@@ -234,22 +234,18 @@
"&&name&i"n /* name literal for reserved variable names */ "&&name&i"n /* name literal for reserved variable names */
%end; %end;
%if &action=ARR %then "]" ; %else "}" ; ; %if &action=ARR %then "]" ; %else "}" ; ;
/* now write the long strings to _webout 1 byte at a time */ /* now write the long strings to _webout 1 char at a time */
data _null_; data _null_;
length filein 8 fileid 8; infile _sjs lrecl=1 recfm=n;
filein=fopen("_sjs",'I',1,'B'); file &jref mod lrecl=1 recfm=n;
fileid=fopen("&jref",'A',1,'B'); input sourcechar $char1. @@;
rec='20'x; format sourcechar hex2.;
do while(fread(filein)=0); put sourcechar char1. @@;
rc=fget(filein,rec,1); run;
rc=fput(fileid, rec);
rc=fwrite(fileid);
end;
/* close out the table */ /* close out the table */
rc=fput(fileid, "]"); data _null_;
rc=fwrite(fileid); file &jref mod;
rc=fclose(filein); put ']';
rc=fclose(fileid);
run; run;
filename _sjs clear; filename _sjs clear;
%end; %end;

2
meta/mm_createfolder.sas
View File

@@ -17,7 +17,7 @@
%mm_createfolder(path=/some/meta/folder) %mm_createfolder(path=/some/meta/folder)
@param [in] path= Name of the folder to create. @param [in] path= Name of the folder to create.
@param [in] mdebug= set DBG to 1 to disable DEBUG messages @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
@version 9.4 @version 9.4

33
meta/mm_createwebservice.sas
View File

@@ -2,7 +2,8 @@
@file mm_createwebservice.sas @file mm_createwebservice.sas
@brief Create a Web Ready Stored Process @brief Create a Web Ready Stored Process
@details This macro creates a Type 2 Stored Process with the mm_webout macro @details This macro creates a Type 2 Stored Process with the mm_webout macro
included as pre-code. (and dependencies) included as pre-code.
Usage: Usage:
%* compile macros ; %* compile macros ;
@@ -267,22 +268,18 @@ data _null_;
put ' "&&name&i"n /* name literal for reserved variable names */ '; put ' "&&name&i"n /* name literal for reserved variable names */ ';
put ' %end; '; put ' %end; ';
put ' %if &action=ARR %then "]" ; %else "}" ; ; '; put ' %if &action=ARR %then "]" ; %else "}" ; ; ';
put ' /* now write the long strings to _webout 1 byte at a time */ '; put ' /* now write the long strings to _webout 1 char at a time */ ';
put ' data _null_; '; put ' data _null_; ';
put ' length filein 8 fileid 8; '; put ' infile _sjs lrecl=1 recfm=n; ';
put ' filein=fopen("_sjs",''I'',1,''B''); '; put ' file &jref mod lrecl=1 recfm=n; ';
put ' fileid=fopen("&jref",''A'',1,''B''); '; put ' input sourcechar $char1. @@; ';
put ' rec=''20''x; '; put ' format sourcechar hex2.; ';
put ' do while(fread(filein)=0); '; put ' put sourcechar char1. @@; ';
put ' rc=fget(filein,rec,1); '; put ' run; ';
put ' rc=fput(fileid, rec); ';
put ' rc=fwrite(fileid); ';
put ' end; ';
put ' /* close out the table */ '; put ' /* close out the table */ ';
put ' rc=fput(fileid, "]"); '; put ' data _null_; ';
put ' rc=fwrite(fileid); '; put ' file &jref mod; ';
put ' rc=fclose(filein); '; put ' put '']''; ';
put ' rc=fclose(fileid); ';
put ' run; '; put ' run; ';
put ' filename _sjs clear; '; put ' filename _sjs clear; ';
put ' %end; '; put ' %end; ';
@@ -494,7 +491,7 @@ run;
%if &x>1 %then %let mod=mod; %if &x>1 %then %let mod=mod;
%let fref=%scan(&freflist,&x); %let fref=%scan(&freflist,&x);
%put &sysmacroname: adding &fref; %&mD.put &sysmacroname: adding &fref;
data _null_; data _null_;
file "&work/&tmpfile" lrecl=3000 &mod; file "&work/&tmpfile" lrecl=3000 &mod;
infile &fref; infile &fref;
@@ -530,12 +527,10 @@ data _null_;
if rc=0 then call symputx('url',url,'l'); if rc=0 then call symputx('url',url,'l');
run; run;
%put ;%put ;%put ;%put ;%put ;%put ;
%put &sysmacroname: STP &name successfully created in &path; %put &sysmacroname: STP &name successfully created in &path;
%put ;%put ;%put ;
%put Check it out here:; %put Check it out here:;
%put ;%put ;%put ; %put ;%put ;%put ;
%put &url?_PROGRAM=&path/&name; %put &url?_PROGRAM=&path/&name;
%put ;%put ;%put ;%put ;%put ;%put ; %put ;%put ;%put ;
%mend mm_createwebservice; %mend mm_createwebservice;

1
sasjs/sasjsconfig.json
View File

@@ -67,6 +67,7 @@
}, },
{ {
"name": "server", "name": "server",
"serverUrl": "",
"serverType": "SASJS", "serverType": "SASJS",
"httpsAgentOptions": { "httpsAgentOptions": {
"allowInsecureRequests": false "allowInsecureRequests": false

24
server/ms_createwebservice.sas
View File

@@ -269,22 +269,18 @@ data _null_;
put ' "&&name&i"n /* name literal for reserved variable names */ '; put ' "&&name&i"n /* name literal for reserved variable names */ ';
put ' %end; '; put ' %end; ';
put ' %if &action=ARR %then "]" ; %else "}" ; ; '; put ' %if &action=ARR %then "]" ; %else "}" ; ; ';
put ' /* now write the long strings to _webout 1 byte at a time */ '; put ' /* now write the long strings to _webout 1 char at a time */ ';
put ' data _null_; '; put ' data _null_; ';
put ' length filein 8 fileid 8; '; put ' infile _sjs lrecl=1 recfm=n; ';
put ' filein=fopen("_sjs",''I'',1,''B''); '; put ' file &jref mod lrecl=1 recfm=n; ';
put ' fileid=fopen("&jref",''A'',1,''B''); '; put ' input sourcechar $char1. @@; ';
put ' rec=''20''x; '; put ' format sourcechar hex2.; ';
put ' do while(fread(filein)=0); '; put ' put sourcechar char1. @@; ';
put ' rc=fget(filein,rec,1); '; put ' run; ';
put ' rc=fput(fileid, rec); ';
put ' rc=fwrite(fileid); ';
put ' end; ';
put ' /* close out the table */ '; put ' /* close out the table */ ';
put ' rc=fput(fileid, "]"); '; put ' data _null_; ';
put ' rc=fwrite(fileid); '; put ' file &jref mod; ';
put ' rc=fclose(filein); '; put ' put '']''; ';
put ' rc=fclose(fileid); ';
put ' run; '; put ' run; ';
put ' filename _sjs clear; '; put ' filename _sjs clear; ';
put ' %end; '; put ' %end; ';

37
tests/base/mp_binarycopy.test.sas
View File

@@ -5,6 +5,7 @@
<h4> SAS Macros </h4> <h4> SAS Macros </h4>
@li mp_binarycopy.sas @li mp_binarycopy.sas
@li mp_assert.sas @li mp_assert.sas
@li mp_hashdataset.sas
**/ **/
@@ -97,3 +98,39 @@ run;
desc=Append Check (ref to file), desc=Append Check (ref to file),
outds=work.test_results outds=work.test_results
) )
/* test 5 - ensure copy works for binary characters */
/* do this backwards to avoid null chars in JSON preview */
data work.test5;
do i=255 to 1 by -1;
str=byte(i);
output;
end;
run;
/* get an md5 hash of the ds */
%mp_hashdataset(work.test5,outds=myhash)
/* copy it */
%mp_binarycopy(inloc="%sysfunc(pathname(work))/test5.sas7bdat",
outloc="%sysfunc(pathname(work))/test5copy.sas7bdat"
)
/* get an md5 hash of the copied ds */
%mp_hashdataset(work.test5copy,outds=myhash2)
/* compare hashes */
%let test5a=0;
%let test5b=1;
data _null_;
set myhash;
call symputx('test5a',hashkey);
run;
data _null_;
set myhash2;
call symputx('test5b',hashkey);
run;
%mp_assert(
iftrue=("&test5a"="&test5b"),
desc=Ensuring binary copy works on binary characters,
outds=work.test_results
)

24
viya/mv_createwebservice.sas
View File

@@ -411,22 +411,18 @@ data _null_;
put ' "&&name&i"n /* name literal for reserved variable names */ '; put ' "&&name&i"n /* name literal for reserved variable names */ ';
put ' %end; '; put ' %end; ';
put ' %if &action=ARR %then "]" ; %else "}" ; ; '; put ' %if &action=ARR %then "]" ; %else "}" ; ; ';
put ' /* now write the long strings to _webout 1 byte at a time */ '; put ' /* now write the long strings to _webout 1 char at a time */ ';
put ' data _null_; '; put ' data _null_; ';
put ' length filein 8 fileid 8; '; put ' infile _sjs lrecl=1 recfm=n; ';
put ' filein=fopen("_sjs",''I'',1,''B''); '; put ' file &jref mod lrecl=1 recfm=n; ';
put ' fileid=fopen("&jref",''A'',1,''B''); '; put ' input sourcechar $char1. @@; ';
put ' rec=''20''x; '; put ' format sourcechar hex2.; ';
put ' do while(fread(filein)=0); '; put ' put sourcechar char1. @@; ';
put ' rc=fget(filein,rec,1); '; put ' run; ';
put ' rc=fput(fileid, rec); ';
put ' rc=fwrite(fileid); ';
put ' end; ';
put ' /* close out the table */ '; put ' /* close out the table */ ';
put ' rc=fput(fileid, "]"); '; put ' data _null_; ';
put ' rc=fwrite(fileid); '; put ' file &jref mod; ';
put ' rc=fclose(filein); '; put ' put '']''; ';
put ' rc=fclose(fileid); ';
put ' run; '; put ' run; ';
put ' filename _sjs clear; '; put ' filename _sjs clear; ';
put ' %end; '; put ' %end; ';