sasjs/core
1
0
Fork 0
mirror of https://github.com/sasjs/core.git synced 2025-12-11 06:24:35 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #265 from sasjs/binaryfix

Browse Source 
enable file copy of files with an encoding that does not match session encoding
This commit is contained in:
Allan Bowe
2022-06-26 17:14:25 +02:00
committed by GitHub
parent b72e404d52 7a69698178
commit 6d0e34ba1d
9 changed files with 163 additions and 178 deletions
Download Patch File Download Diff File Expand all files Collapse all files

150
all.sas
View File

@@ -3470,8 +3470,8 @@ run;
@details Reads in a file byte by byte and writes it back out. Is an
os-independent method to copy files. In case of naming collision, the
default filerefs can be modified.
Based on:
https://stackoverflow.com/questions/13046116/using-sas-to-copy-a-text-file
Note that if you have a new enough version of SAS, and you don't need features
such as APPEND, you may be better of using the fcopy() function instead.
%mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
@@ -3511,14 +3511,9 @@ run;
,outref=____out /* override default to use own filerefs */
,mode=CREATE
)/*/STORE SOURCE*/;
%local mod outmode;
%if &mode=APPEND %then %do;
%let mod=mod;
%let outmode='a';
%end;
%else %do;
%let outmode='o';
%end;
%local mod;
%if &mode=APPEND %then %let mod=mod;
/* these IN and OUT filerefs can point to anything */
%if &inref = ____in %then %do;
filename &inref &inloc lrecl=1048576 ;
@@ -3529,25 +3524,21 @@ run;
/* copy the file byte-for-byte */
data _null_;
length filein 8 fileid 8;
filein = fopen("&inref",'I',1,'B');
fileid = fopen("&outref",&outmode,1,'B');
rec = '20'x;
do while(fread(filein)=0);
rc = fget(filein,rec,1);
rc = fput(fileid, rec);
rc =fwrite(fileid);
end;
rc = fclose(filein);
rc = fclose(fileid);
infile &inref lrecl=1 recfm=n;
file &outref &mod recfm=n;
input sourcechar $char1. @@;
format sourcechar hex2.;
put sourcechar char1. @@;
run;
%if &inref = ____in %then %do;
filename &inref clear;
%end;
%if &outref=____out %then %do;
filename &outref clear;
%end;
%mend mp_binarycopy;/**
%mend mp_binarycopy;
/**
@file
@brief Splits a file of ANY SIZE by reference to a search string.
@details Provide a fileref and a search string to chop off part of a file.
@@ -8817,22 +8808,18 @@ options
"&&name&i"n /* name literal for reserved variable names */
%end;
%if &action=ARR %then "]" ; %else "}" ; ;
/* now write the long strings to _webout 1 byte at a time */
/* now write the long strings to _webout 1 char at a time */
data _null_;
length filein 8 fileid 8;
filein=fopen("_sjs",'I',1,'B');
fileid=fopen("&jref",'A',1,'B');
rec='20'x;
do while(fread(filein)=0);
rc=fget(filein,rec,1);
rc=fput(fileid, rec);
rc=fwrite(fileid);
end;
/* close out the table */
rc=fput(fileid, "]");
rc=fwrite(fileid);
rc=fclose(filein);
rc=fclose(fileid);
infile _sjs lrecl=1 recfm=n;
file &jref mod lrecl=1 recfm=n;
input sourcechar $char1. @@;
format sourcechar hex2.;
put sourcechar char1. @@;
run;
/* close out the table */
data _null_;
file &jref mod;
put ']';
run;
filename _sjs clear;
%end;
@@ -14066,7 +14053,7 @@ run;
%mm_createfolder(path=/some/meta/folder)
@param [in] path= Name of the folder to create.
@param [in] mdebug= set DBG to 1 to disable DEBUG messages
@param [in] mdebug= (0) Set to 1 to enable DEBUG messages
@version 9.4
@@ -14932,7 +14919,8 @@ run;
@file mm_createwebservice.sas
@brief Create a Web Ready Stored Process
@details This macro creates a Type 2 Stored Process with the mm_webout macro
included as pre-code.
(and dependencies) included as pre-code.
Usage:
%* compile macros ;
@@ -15197,22 +15185,18 @@ data _null_;
put ' "&&name&i"n /* name literal for reserved variable names */ ';
put ' %end; ';
put ' %if &action=ARR %then "]" ; %else "}" ; ; ';
put ' /* now write the long strings to _webout 1 byte at a time */ ';
put ' /* now write the long strings to _webout 1 char at a time */ ';
put ' data _null_; ';
put ' length filein 8 fileid 8; ';
put ' filein=fopen("_sjs",''I'',1,''B''); ';
put ' fileid=fopen("&jref",''A'',1,''B''); ';
put ' rec=''20''x; ';
put ' do while(fread(filein)=0); ';
put ' rc=fget(filein,rec,1); ';
put ' rc=fput(fileid, rec); ';
put ' rc=fwrite(fileid); ';
put ' end; ';
put ' /* close out the table */ ';
put ' rc=fput(fileid, "]"); ';
put ' rc=fwrite(fileid); ';
put ' rc=fclose(filein); ';
put ' rc=fclose(fileid); ';
put ' infile _sjs lrecl=1 recfm=n; ';
put ' file &jref mod lrecl=1 recfm=n; ';
put ' input sourcechar $char1. @@; ';
put ' format sourcechar hex2.; ';
put ' put sourcechar char1. @@; ';
put ' run; ';
put ' /* close out the table */ ';
put ' data _null_; ';
put ' file &jref mod; ';
put ' put '']''; ';
put ' run; ';
put ' filename _sjs clear; ';
put ' %end; ';
@@ -15424,7 +15408,7 @@ run;
%if &x>1 %then %let mod=mod;
%let fref=%scan(&freflist,&x);
%put &sysmacroname: adding &fref;
%&mD.put &sysmacroname: adding &fref;
data _null_;
file "&work/&tmpfile" lrecl=3000 &mod;
infile &fref;
@@ -15460,13 +15444,11 @@ data _null_;
if rc=0 then call symputx('url',url,'l');
run;
%put ;%put ;%put ;%put ;%put ;%put ;
%put &sysmacroname: STP &name successfully created in &path;
%put ;%put ;%put ;
%put Check it out here:;
%put ;%put ;%put ;
%put &url?_PROGRAM=&path/&name;
%put ;%put ;%put ;%put ;%put ;%put ;
%put ;%put ;%put ;
%mend mm_createwebservice;
/**
@@ -19936,22 +19918,18 @@ data _null_;
put ' "&&name&i"n /* name literal for reserved variable names */ ';
put ' %end; ';
put ' %if &action=ARR %then "]" ; %else "}" ; ; ';
put ' /* now write the long strings to _webout 1 byte at a time */ ';
put ' /* now write the long strings to _webout 1 char at a time */ ';
put ' data _null_; ';
put ' length filein 8 fileid 8; ';
put ' filein=fopen("_sjs",''I'',1,''B''); ';
put ' fileid=fopen("&jref",''A'',1,''B''); ';
put ' rec=''20''x; ';
put ' do while(fread(filein)=0); ';
put ' rc=fget(filein,rec,1); ';
put ' rc=fput(fileid, rec); ';
put ' rc=fwrite(fileid); ';
put ' end; ';
put ' /* close out the table */ ';
put ' rc=fput(fileid, "]"); ';
put ' rc=fwrite(fileid); ';
put ' rc=fclose(filein); ';
put ' rc=fclose(fileid); ';
put ' infile _sjs lrecl=1 recfm=n; ';
put ' file &jref mod lrecl=1 recfm=n; ';
put ' input sourcechar $char1. @@; ';
put ' format sourcechar hex2.; ';
put ' put sourcechar char1. @@; ';
put ' run; ';
put ' /* close out the table */ ';
put ' data _null_; ';
put ' file &jref mod; ';
put ' put '']''; ';
put ' run; ';
put ' filename _sjs clear; ';
put ' %end; ';
@@ -22281,22 +22259,18 @@ data _null_;
put ' "&&name&i"n /* name literal for reserved variable names */ ';
put ' %end; ';
put ' %if &action=ARR %then "]" ; %else "}" ; ; ';
put ' /* now write the long strings to _webout 1 byte at a time */ ';
put ' /* now write the long strings to _webout 1 char at a time */ ';
put ' data _null_; ';
put ' length filein 8 fileid 8; ';
put ' filein=fopen("_sjs",''I'',1,''B''); ';
put ' fileid=fopen("&jref",''A'',1,''B''); ';
put ' rec=''20''x; ';
put ' do while(fread(filein)=0); ';
put ' rc=fget(filein,rec,1); ';
put ' rc=fput(fileid, rec); ';
put ' rc=fwrite(fileid); ';
put ' end; ';
put ' /* close out the table */ ';
put ' rc=fput(fileid, "]"); ';
put ' rc=fwrite(fileid); ';
put ' rc=fclose(filein); ';
put ' rc=fclose(fileid); ';
put ' infile _sjs lrecl=1 recfm=n; ';
put ' file &jref mod lrecl=1 recfm=n; ';
put ' input sourcechar $char1. @@; ';
put ' format sourcechar hex2.; ';
put ' put sourcechar char1. @@; ';
put ' run; ';
put ' /* close out the table */ ';
put ' data _null_; ';
put ' file &jref mod; ';
put ' put '']''; ';
put ' run; ';
put ' filename _sjs clear; ';
put ' %end; ';

34
base/mp_binarycopy.sas
View File

@@ -4,8 +4,8 @@
@details Reads in a file byte by byte and writes it back out. Is an
os-independent method to copy files. In case of naming collision, the
default filerefs can be modified.
Based on:
https://stackoverflow.com/questions/13046116/using-sas-to-copy-a-text-file
Note that if you have a new enough version of SAS, and you don't need features
such as APPEND, you may be better of using the fcopy() function instead.
%mp_binarycopy(inloc="/home/me/blah.txt", outref=_webout)
@@ -45,14 +45,9 @@
,outref=____out /* override default to use own filerefs */
,mode=CREATE
)/*/STORE SOURCE*/;
%local mod outmode;
%if &mode=APPEND %then %do;
%let mod=mod;
%let outmode='a';
%end;
%else %do;
%let outmode='o';
%end;
%local mod;
%if &mode=APPEND %then %let mod=mod;
/* these IN and OUT filerefs can point to anything */
%if &inref = ____in %then %do;
filename &inref &inloc lrecl=1048576 ;
@@ -63,22 +58,17 @@
/* copy the file byte-for-byte */
data _null_;
length filein 8 fileid 8;
filein = fopen("&inref",'I',1,'B');
fileid = fopen("&outref",&outmode,1,'B');
rec = '20'x;
do while(fread(filein)=0);
rc = fget(filein,rec,1);
rc = fput(fileid, rec);
rc =fwrite(fileid);
end;
rc = fclose(filein);
rc = fclose(fileid);
infile &inref lrecl=1 recfm=n;
file &outref &mod recfm=n;
input sourcechar $char1. @@;
format sourcechar hex2.;
put sourcechar char1. @@;
run;
%if &inref = ____in %then %do;
filename &inref clear;
%end;
%if &outref=____out %then %do;
filename &outref clear;
%end;
%mend mp_binarycopy;
%mend mp_binarycopy;

26
base/mp_jsonout.sas
View File

@@ -234,22 +234,18 @@
"&&name&i"n /* name literal for reserved variable names */
%end;
%if &action=ARR %then "]" ; %else "}" ; ;
/* now write the long strings to _webout 1 byte at a time */
/* now write the long strings to _webout 1 char at a time */
data _null_;
length filein 8 fileid 8;
filein=fopen("_sjs",'I',1,'B');
fileid=fopen("&jref",'A',1,'B');
rec='20'x;
do while(fread(filein)=0);
rc=fget(filein,rec,1);
rc=fput(fileid, rec);
rc=fwrite(fileid);
end;
/* close out the table */
rc=fput(fileid, "]");
rc=fwrite(fileid);
rc=fclose(filein);
rc=fclose(fileid);
infile _sjs lrecl=1 recfm=n;
file &jref mod lrecl=1 recfm=n;
input sourcechar $char1. @@;
format sourcechar hex2.;
put sourcechar char1. @@;
run;
/* close out the table */
data _null_;
file &jref mod;
put ']';
run;
filename _sjs clear;
%end;

2
meta/mm_createfolder.sas
View File

@@ -17,7 +17,7 @@
%mm_createfolder(path=/some/meta/folder)
@param [in] path= Name of the folder to create.
@param [in] mdebug= set DBG to 1 to disable DEBUG messages
@param [in] mdebug= (0) Set to 1 to enable DEBUG messages
@version 9.4

35
meta/mm_createwebservice.sas
View File

@@ -2,7 +2,8 @@
@file mm_createwebservice.sas
@brief Create a Web Ready Stored Process
@details This macro creates a Type 2 Stored Process with the mm_webout macro
included as pre-code.
(and dependencies) included as pre-code.
Usage:
%* compile macros ;
@@ -267,22 +268,18 @@ data _null_;
put ' "&&name&i"n /* name literal for reserved variable names */ ';
put ' %end; ';
put ' %if &action=ARR %then "]" ; %else "}" ; ; ';
put ' /* now write the long strings to _webout 1 byte at a time */ ';
put ' /* now write the long strings to _webout 1 char at a time */ ';
put ' data _null_; ';
put ' length filein 8 fileid 8; ';
put ' filein=fopen("_sjs",''I'',1,''B''); ';
put ' fileid=fopen("&jref",''A'',1,''B''); ';
put ' rec=''20''x; ';
put ' do while(fread(filein)=0); ';
put ' rc=fget(filein,rec,1); ';
put ' rc=fput(fileid, rec); ';
put ' rc=fwrite(fileid); ';
put ' end; ';
put ' /* close out the table */ ';
put ' rc=fput(fileid, "]"); ';
put ' rc=fwrite(fileid); ';
put ' rc=fclose(filein); ';
put ' rc=fclose(fileid); ';
put ' infile _sjs lrecl=1 recfm=n; ';
put ' file &jref mod lrecl=1 recfm=n; ';
put ' input sourcechar $char1. @@; ';
put ' format sourcechar hex2.; ';
put ' put sourcechar char1. @@; ';
put ' run; ';
put ' /* close out the table */ ';
put ' data _null_; ';
put ' file &jref mod; ';
put ' put '']''; ';
put ' run; ';
put ' filename _sjs clear; ';
put ' %end; ';
@@ -494,7 +491,7 @@ run;
%if &x>1 %then %let mod=mod;
%let fref=%scan(&freflist,&x);
%put &sysmacroname: adding &fref;
%&mD.put &sysmacroname: adding &fref;
data _null_;
file "&work/&tmpfile" lrecl=3000 &mod;
infile &fref;
@@ -530,12 +527,10 @@ data _null_;
if rc=0 then call symputx('url',url,'l');
run;
%put ;%put ;%put ;%put ;%put ;%put ;
%put &sysmacroname: STP &name successfully created in &path;
%put ;%put ;%put ;
%put Check it out here:;
%put ;%put ;%put ;
%put &url?_PROGRAM=&path/&name;
%put ;%put ;%put ;%put ;%put ;%put ;
%put ;%put ;%put ;
%mend mm_createwebservice;

3
sasjs/sasjsconfig.json
View File

@@ -67,6 +67,7 @@
},
{
"name": "server",
"serverUrl": "",
"serverType": "SASJS",
"httpsAgentOptions": {
"allowInsecureRequests": false
@@ -106,4 +107,4 @@
"contextName": "SAS Job Execution compute context"
}
]
}
}

26
server/ms_createwebservice.sas
View File

@@ -269,22 +269,18 @@ data _null_;
put ' "&&name&i"n /* name literal for reserved variable names */ ';
put ' %end; ';
put ' %if &action=ARR %then "]" ; %else "}" ; ; ';
put ' /* now write the long strings to _webout 1 byte at a time */ ';
put ' /* now write the long strings to _webout 1 char at a time */ ';
put ' data _null_; ';
put ' length filein 8 fileid 8; ';
put ' filein=fopen("_sjs",''I'',1,''B''); ';
put ' fileid=fopen("&jref",''A'',1,''B''); ';
put ' rec=''20''x; ';
put ' do while(fread(filein)=0); ';
put ' rc=fget(filein,rec,1); ';
put ' rc=fput(fileid, rec); ';
put ' rc=fwrite(fileid); ';
put ' end; ';
put ' /* close out the table */ ';
put ' rc=fput(fileid, "]"); ';
put ' rc=fwrite(fileid); ';
put ' rc=fclose(filein); ';
put ' rc=fclose(fileid); ';
put ' infile _sjs lrecl=1 recfm=n; ';
put ' file &jref mod lrecl=1 recfm=n; ';
put ' input sourcechar $char1. @@; ';
put ' format sourcechar hex2.; ';
put ' put sourcechar char1. @@; ';
put ' run; ';
put ' /* close out the table */ ';
put ' data _null_; ';
put ' file &jref mod; ';
put ' put '']''; ';
put ' run; ';
put ' filename _sjs clear; ';
put ' %end; ';

39
tests/base/mp_binarycopy.test.sas
View File

@@ -5,6 +5,7 @@
<h4> SAS Macros </h4>
@li mp_binarycopy.sas
@li mp_assert.sas
@li mp_hashdataset.sas
**/
@@ -96,4 +97,40 @@ run;
iftrue=("&string4"="&string4_check"),
desc=Append Check (ref to file),
outds=work.test_results
)
)
/* test 5 - ensure copy works for binary characters */
/* do this backwards to avoid null chars in JSON preview */
data work.test5;
do i=255 to 1 by -1;
str=byte(i);
output;
end;
run;
/* get an md5 hash of the ds */
%mp_hashdataset(work.test5,outds=myhash)
/* copy it */
%mp_binarycopy(inloc="%sysfunc(pathname(work))/test5.sas7bdat",
outloc="%sysfunc(pathname(work))/test5copy.sas7bdat"
)
/* get an md5 hash of the copied ds */
%mp_hashdataset(work.test5copy,outds=myhash2)
/* compare hashes */
%let test5a=0;
%let test5b=1;
data _null_;
set myhash;
call symputx('test5a',hashkey);
run;
data _null_;
set myhash2;
call symputx('test5b',hashkey);
run;
%mp_assert(
iftrue=("&test5a"="&test5b"),
desc=Ensuring binary copy works on binary characters,
outds=work.test_results
)

26
viya/mv_createwebservice.sas
View File

@@ -411,22 +411,18 @@ data _null_;
put ' "&&name&i"n /* name literal for reserved variable names */ ';
put ' %end; ';
put ' %if &action=ARR %then "]" ; %else "}" ; ; ';
put ' /* now write the long strings to _webout 1 byte at a time */ ';
put ' /* now write the long strings to _webout 1 char at a time */ ';
put ' data _null_; ';
put ' length filein 8 fileid 8; ';
put ' filein=fopen("_sjs",''I'',1,''B''); ';
put ' fileid=fopen("&jref",''A'',1,''B''); ';
put ' rec=''20''x; ';
put ' do while(fread(filein)=0); ';
put ' rc=fget(filein,rec,1); ';
put ' rc=fput(fileid, rec); ';
put ' rc=fwrite(fileid); ';
put ' end; ';
put ' /* close out the table */ ';
put ' rc=fput(fileid, "]"); ';
put ' rc=fwrite(fileid); ';
put ' rc=fclose(filein); ';
put ' rc=fclose(fileid); ';
put ' infile _sjs lrecl=1 recfm=n; ';
put ' file &jref mod lrecl=1 recfm=n; ';
put ' input sourcechar $char1. @@; ';
put ' format sourcechar hex2.; ';
put ' put sourcechar char1. @@; ';
put ' run; ';
put ' /* close out the table */ ';
put ' data _null_; ';
put ' file &jref mod; ';
put ' put '']''; ';
put ' run; ';
put ' filename _sjs clear; ';
put ' %end; ';