sasjs/adapter
1
0
Fork 0
mirror of https://github.com/sasjs/adapter.git synced 2026-01-06 20:10:05 +00:00
Code Issues Projects Releases Wiki Activity

refactor(convert-to-csv): change func arguments

Browse Source
This commit is contained in:
Yury Shkoda
2022-03-03 15:07:01 +03:00
parent ff5dc5f196
commit 5927910a52
7 changed files with 165 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/api/viya/uploadTables.ts
View File

@@ -18,7 +18,7 @@ export async function uploadTables(
const uploadedFiles = [] const uploadedFiles = []
for (const tableName in data) { for (const tableName in data) {
const csv = convertToCSV(data[tableName]) const csv = convertToCSV(data, tableName)
if (csv === 'ERROR: LARGE STRING LENGTH') { if (csv === 'ERROR: LARGE STRING LENGTH') {
throw new Error( throw new Error(
'The max length of a string value in SASjs is 32765 characters.' 'The max length of a string value in SASjs is 32765 characters.'

6
src/file/generateFileUploadForm.ts
View File

@@ -18,11 +18,7 @@ export const generateFileUploadForm = (
if (!Array.isArray(data[tableName])) continue if (!Array.isArray(data[tableName])) continue
const name = tableName const name = tableName
const formatsObj = const csv = convertToCSV(data, tableName)
data[`$${tableName}`] && data[`$${tableName}`].formats
? data[`$${tableName}`]
: undefined
const csv = convertToCSV(data[tableName], formatsObj)
if (csv === 'ERROR: LARGE STRING LENGTH') { if (csv === 'ERROR: LARGE STRING LENGTH') {
throw new Error( throw new Error(

9
src/file/generateTableUploadForm.ts
View File

@@ -9,18 +9,24 @@ export const generateTableUploadForm = (
const sasjsTables = [] const sasjsTables = []
const requestParams: any = {} const requestParams: any = {}
let tableCounter = 0 let tableCounter = 0
for (const tableName in data) { for (const tableName in data) {
tableCounter++ tableCounter++
sasjsTables.push(tableName) sasjsTables.push(tableName)
const csv = convertToCSV(data[tableName])
const csv = convertToCSV(data, tableName)
if (csv === 'ERROR: LARGE STRING LENGTH') { if (csv === 'ERROR: LARGE STRING LENGTH') {
throw new Error( throw new Error(
'The max length of a string value in SASjs is 32765 characters.' 'The max length of a string value in SASjs is 32765 characters.'
) )
} }
// if csv has length more then 16k, send in chunks // if csv has length more then 16k, send in chunks
if (csv.length > 16000) { if (csv.length > 16000) {
const csvChunks = splitChunks(csv) const csvChunks = splitChunks(csv)
// append chunks to form data with same key // append chunks to form data with same key
csvChunks.map((chunk) => { csvChunks.map((chunk) => {
formData.append(`sasjs${tableCounter}data`, chunk) formData.append(`sasjs${tableCounter}data`, chunk)
@@ -29,6 +35,7 @@ export const generateTableUploadForm = (
requestParams[`sasjs${tableCounter}data`] = csv requestParams[`sasjs${tableCounter}data`] = csv
} }
} }
requestParams['sasjs_tables'] = sasjsTables.join(' ') requestParams['sasjs_tables'] = sasjsTables.join(' ')
return { formData, requestParams } return { formData, requestParams }

3
src/job-execution/Sas9JobExecutor.ts
View File

@@ -125,7 +125,8 @@ const generateFileUploadForm = (
): NodeFormData => { ): NodeFormData => {
for (const tableName in data) { for (const tableName in data) {
const name = tableName const name = tableName
const csv = convertToCSV(data[tableName]) const csv = convertToCSV(data, tableName)
if (csv === 'ERROR: LARGE STRING LENGTH') { if (csv === 'ERROR: LARGE STRING LENGTH') {
throw new Error( throw new Error(
'The max length of a string value in SASjs is 32765 characters.' 'The max length of a string value in SASjs is 32765 characters.'

229
src/utils/convertToCsv.spec.ts
View File

@@ -1,183 +1,222 @@
import { convertToCSV } from './convertToCsv' import { convertToCSV } from './convertToCsv'
describe('convertToCsv', () => { describe('convertToCsv', () => {
const tableName = 'testTable'
it('should convert single quoted values', () => { it('should convert single quoted values', () => {
const data = [ const data = {
{ foo: `'bar'`, bar: 'abc' }, [tableName]: [
{ foo: 'sadf', bar: 'def' }, { foo: `'bar'`, bar: 'abc' },
{ foo: 'asd', bar: `'qwert'` } { foo: 'sadf', bar: 'def' },
] { foo: 'asd', bar: `'qwert'` }
]
}
const expectedOutput = `foo:$char5. bar:$char7.\r\n"'bar'",abc\r\nsadf,def\r\nasd,"'qwert'"` const expectedOutput = `foo:$char5. bar:$char7.\r\n"'bar'",abc\r\nsadf,def\r\nasd,"'qwert'"`
expect(convertToCSV(data)).toEqual(expectedOutput) expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
}) })
it('should convert double quoted values', () => { it('should convert double quoted values', () => {
const data = [ const data = {
{ foo: `"bar"`, bar: 'abc' }, [tableName]: [
{ foo: 'sadf', bar: 'def' }, { foo: `"bar"`, bar: 'abc' },
{ foo: 'asd', bar: `"qwert"` } { foo: 'sadf', bar: 'def' },
] { foo: 'asd', bar: `"qwert"` }
]
}
const expectedOutput = `foo:$char5. bar:$char7.\r\n"""bar""",abc\r\nsadf,def\r\nasd,"""qwert"""` const expectedOutput = `foo:$char5. bar:$char7.\r\n"""bar""",abc\r\nsadf,def\r\nasd,"""qwert"""`
expect(convertToCSV(data)).toEqual(expectedOutput) expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
}) })
it('should convert values with mixed quotes', () => { it('should convert values with mixed quotes', () => {
const data = [{ foo: `'blah'`, bar: `"blah"` }] const data = { [tableName]: [{ foo: `'blah'`, bar: `"blah"` }] }
const expectedOutput = `foo:$char6. bar:$char6.\r\n"'blah'","""blah"""` const expectedOutput = `foo:$char6. bar:$char6.\r\n"'blah'","""blah"""`
expect(convertToCSV(data)).toEqual(expectedOutput) expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
}) })
it('should convert values with mixed quotes', () => { it('should convert values with mixed quotes', () => {
const data = [{ foo: `'blah,"'`, bar: `"blah,blah" "` }] const data = { [tableName]: [{ foo: `'blah,"'`, bar: `"blah,blah" "` }] }
const expectedOutput = `foo:$char8. bar:$char13.\r\n"'blah,""'","""blah,blah"" """` const expectedOutput = `foo:$char8. bar:$char13.\r\n"'blah,""'","""blah,blah"" """`
expect(convertToCSV(data)).toEqual(expectedOutput) expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
}) })
it('should convert values with mixed quotes', () => { it('should convert values with mixed quotes', () => {
const data = [{ foo: `',''`, bar: `","` }] const data = { [tableName]: [{ foo: `',''`, bar: `","` }] }
const expectedOutput = `foo:$char4. bar:$char3.\r\n"',''",""","""` const expectedOutput = `foo:$char4. bar:$char3.\r\n"',''",""","""`
expect(convertToCSV(data)).toEqual(expectedOutput) expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
}) })
it('should convert values with mixed quotes', () => { it('should convert values with mixed quotes', () => {
const data = [{ foo: `','`, bar: `,"` }] const data = { [tableName]: [{ foo: `','`, bar: `,"` }] }
const expectedOutput = `foo:$char3. bar:$char2.\r\n"','",","""` const expectedOutput = `foo:$char3. bar:$char2.\r\n"','",","""`
expect(convertToCSV(data)).toEqual(expectedOutput) expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
}) })
it('should convert values with mixed quotes', () => { it('should convert values with mixed quotes', () => {
const data = [{ foo: `"`, bar: `'` }] const data = { [tableName]: [{ foo: `"`, bar: `'` }] }
const expectedOutput = `foo:$char1. bar:$char1.\r\n"""","'"` const expectedOutput = `foo:$char1. bar:$char1.\r\n"""","'"`
expect(convertToCSV(data)).toEqual(expectedOutput) expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
}) })
it('should convert values with mixed quotes', () => { it('should convert values with mixed quotes', () => {
const data = [{ foo: `,`, bar: `',` }] const data = { [tableName]: [{ foo: `,`, bar: `',` }] }
const expectedOutput = `foo:$char1. bar:$char2.\r\n",","',"` const expectedOutput = `foo:$char1. bar:$char2.\r\n",","',"`
expect(convertToCSV(data)).toEqual(expectedOutput) expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
}) })
it('should convert values with number cases 1', () => { it('should convert values with number cases 1', () => {
const data = [ const data = {
{ col1: 42, col2: null, col3: 'x', col4: null }, [tableName]: [
{ col1: 42, col2: null, col3: 'x', col4: null }, { col1: 42, col2: null, col3: 'x', col4: null },
{ col1: 42, col2: null, col3: 'x', col4: null }, { col1: 42, col2: null, col3: 'x', col4: null },
{ col1: 42, col2: null, col3: 'x', col4: '' }, { col1: 42, col2: null, col3: 'x', col4: null },
{ col1: 42, col2: null, col3: 'x', col4: '' } { col1: 42, col2: null, col3: 'x', col4: '' },
] { col1: 42, col2: null, col3: 'x', col4: '' }
]
}
const expectedOutput = `col1:best. col2:best. col3:$char1. col4:$char1.\r\n42,.,x,\r\n42,.,x,\r\n42,.,x,\r\n42,.,x,\r\n42,.,x,` const expectedOutput = `col1:best. col2:best. col3:$char1. col4:$char1.\r\n42,.,x,\r\n42,.,x,\r\n42,.,x,\r\n42,.,x,\r\n42,.,x,`
expect(convertToCSV(data)).toEqual(expectedOutput) expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
}) })
it('should convert values with number cases 2', () => { it('should convert values with number cases 2', () => {
const data = [ const data = {
{ col1: 42, col2: null, col3: 'x', col4: '' }, [tableName]: [
{ col1: 42, col2: null, col3: 'x', col4: '' }, { col1: 42, col2: null, col3: 'x', col4: '' },
{ col1: 42, col2: null, col3: 'x', col4: '' }, { col1: 42, col2: null, col3: 'x', col4: '' },
{ col1: 42, col2: 1.62, col3: 'x', col4: 'x' }, { col1: 42, col2: null, col3: 'x', col4: '' },
{ col1: 42, col2: 1.62, col3: 'x', col4: 'x' } { col1: 42, col2: 1.62, col3: 'x', col4: 'x' },
] { col1: 42, col2: 1.62, col3: 'x', col4: 'x' }
]
}
const expectedOutput = `col1:best. col2:best. col3:$char1. col4:$char1.\r\n42,.,x,\r\n42,.,x,\r\n42,.,x,\r\n42,1.62,x,x\r\n42,1.62,x,x` const expectedOutput = `col1:best. col2:best. col3:$char1. col4:$char1.\r\n42,.,x,\r\n42,.,x,\r\n42,.,x,\r\n42,1.62,x,x\r\n42,1.62,x,x`
expect(convertToCSV(data)).toEqual(expectedOutput) expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
}) })
it('should convert values with common special characters', () => { it('should convert values with common special characters', () => {
expect(convertToCSV([{ tab: '\t' }])).toEqual(`tab:$char1.\r\n\"\t\"`) expect(convertToCSV({ [tableName]: [{ tab: '\t' }] }, tableName)).toEqual(
expect(convertToCSV([{ lf: '\n' }])).toEqual(`lf:$char1.\r\n\"\n\"`) `tab:$char1.\r\n\"\t\"`
expect(convertToCSV([{ semicolon: ';semi' }])).toEqual(
`semicolon:$char5.\r\n;semi`
) )
expect(convertToCSV([{ percent: '%' }])).toEqual(`percent:$char1.\r\n%`) expect(convertToCSV({ [tableName]: [{ lf: '\n' }] }, tableName)).toEqual(
expect(convertToCSV([{ singleQuote: "'" }])).toEqual( `lf:$char1.\r\n\"\n\"`
`singleQuote:$char1.\r\n\"'\"`
)
expect(convertToCSV([{ doubleQuote: '"' }])).toEqual(
`doubleQuote:$char1.\r\n""""`
)
expect(convertToCSV([{ crlf: '\r\n' }])).toEqual(`crlf:$char2.\r\n\"\n\"`)
expect(convertToCSV([{ euro: '€euro' }])).toEqual(`euro:$char7.\r\n€euro`)
expect(convertToCSV([{ banghash: '!#banghash' }])).toEqual(
`banghash:$char10.\r\n!#banghash`
) )
expect(
convertToCSV({ [tableName]: [{ semicolon: ';semi' }] }, tableName)
).toEqual(`semicolon:$char5.\r\n;semi`)
expect(
convertToCSV({ [tableName]: [{ percent: '%' }] }, tableName)
).toEqual(`percent:$char1.\r\n%`)
expect(
convertToCSV({ [tableName]: [{ singleQuote: "'" }] }, tableName)
).toEqual(`singleQuote:$char1.\r\n\"'\"`)
expect(
convertToCSV({ [tableName]: [{ doubleQuote: '"' }] }, tableName)
).toEqual(`doubleQuote:$char1.\r\n""""`)
expect(
convertToCSV({ [tableName]: [{ crlf: '\r\n' }] }, tableName)
).toEqual(`crlf:$char2.\r\n\"\n\"`)
expect(
convertToCSV({ [tableName]: [{ euro: '€euro' }] }, tableName)
).toEqual(`euro:$char7.\r\n€euro`)
expect(
convertToCSV({ [tableName]: [{ banghash: '!#banghash' }] }, tableName)
).toEqual(`banghash:$char10.\r\n!#banghash`)
}) })
it('should convert values with other special characters', () => { it('should convert values with other special characters', () => {
const data = [ const data = {
{ [tableName]: [
speech0: '"speech', {
pct: '%percent', speech0: '"speech',
speech: '"speech', pct: '%percent',
slash: '\\slash', speech: '"speech',
slashWithSpecial: '\\\tslash', slash: '\\slash',
macvar: '&sysuserid', slashWithSpecial: '\\\tslash',
chinese: '传/傳chinese', macvar: '&sysuserid',
sigma: 'Σsigma', chinese: '传/傳chinese',
at: '@at', sigma: 'Σsigma',
serbian: 'Српски', at: '@at',
dollar: '$' serbian: 'Српски',
} dollar: '$'
] }
]
}
const expectedOutput = `speech0:$char7. pct:$char8. speech:$char7. slash:$char6. slashWithSpecial:$char7. macvar:$char10. chinese:$char14. sigma:$char7. at:$char3. serbian:$char12. dollar:$char1.\r\n"""speech",%percent,"""speech",\\slash,\"\\\tslash\",&sysuserid,传/傳chinese,Σsigma,@at,Српски,$` const expectedOutput = `speech0:$char7. pct:$char8. speech:$char7. slash:$char6. slashWithSpecial:$char7. macvar:$char10. chinese:$char14. sigma:$char7. at:$char3. serbian:$char12. dollar:$char1.\r\n"""speech",%percent,"""speech",\\slash,\"\\\tslash\",&sysuserid,传/傳chinese,Σsigma,@at,Српски,$`
expect(convertToCSV(data)).toEqual(expectedOutput) expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
expect(convertToCSV([{ speech: 'menext' }])).toEqual(
`speech:$char6.\r\nmenext`
)
expect(convertToCSV([{ speech: 'me\nnext' }])).toEqual(
`speech:$char7.\r\n\"me\nnext\"`
)
expect(convertToCSV([{ speech: `me'next` }])).toEqual(
`speech:$char7.\r\n\"me'next\"`
)
expect(convertToCSV([{ speech: `me"next` }])).toEqual(
`speech:$char7.\r\n\"me""next\"`
)
expect(convertToCSV([{ speech: `me""next` }])).toEqual(
`speech:$char8.\r\n\"me""""next\"`
)
expect(convertToCSV([{ slashWithSpecial: '\\\tslash' }])).toEqual(
`slashWithSpecial:$char7.\r\n\"\\\tslash\"`
)
expect(convertToCSV([{ slashWithSpecial: '\\ \tslash' }])).toEqual(
`slashWithSpecial:$char8.\r\n\"\\ \tslash\"`
)
expect( expect(
convertToCSV([{ slashWithSpecialExtra: '\\\ts\tl\ta\ts\t\th\t' }]) convertToCSV({ [tableName]: [{ speech: 'menext' }] }, tableName)
).toEqual(`speech:$char6.\r\nmenext`)
expect(
convertToCSV({ [tableName]: [{ speech: 'me\nnext' }] }, tableName)
).toEqual(`speech:$char7.\r\n\"me\nnext\"`)
expect(
convertToCSV({ [tableName]: [{ speech: `me'next` }] }, tableName)
).toEqual(`speech:$char7.\r\n\"me'next\"`)
expect(
convertToCSV({ [tableName]: [{ speech: `me"next` }] }, tableName)
).toEqual(`speech:$char7.\r\n\"me""next\"`)
expect(
convertToCSV({ [tableName]: [{ speech: `me""next` }] }, tableName)
).toEqual(`speech:$char8.\r\n\"me""""next\"`)
expect(
convertToCSV(
{ [tableName]: [{ slashWithSpecial: '\\\tslash' }] },
tableName
)
).toEqual(`slashWithSpecial:$char7.\r\n\"\\\tslash\"`)
expect(
convertToCSV(
{ [tableName]: [{ slashWithSpecial: '\\ \tslash' }] },
tableName
)
).toEqual(`slashWithSpecial:$char8.\r\n\"\\ \tslash\"`)
expect(
convertToCSV(
{ [tableName]: [{ slashWithSpecialExtra: '\\\ts\tl\ta\ts\t\th\t' }] },
tableName
)
).toEqual(`slashWithSpecialExtra:$char13.\r\n\"\\\ts\tl\ta\ts\t\th\t\"`) ).toEqual(`slashWithSpecialExtra:$char13.\r\n\"\\\ts\tl\ta\ts\t\th\t\"`)
}) })
it('should console log error if data has mixed types', () => { it('should console log error if data has mixed types', () => {
const colName = 'var1' const colName = 'var1'
const data = [{ [colName]: 'string' }, { [colName]: 232 }] const data = { [tableName]: [{ [colName]: 'string' }, { [colName]: 232 }] }
jest.spyOn(console, 'error').mockImplementation(() => {}) jest.spyOn(console, 'error').mockImplementation(() => {})
convertToCSV(data) convertToCSV(data, tableName)
expect(console.error).toHaveBeenCalledWith( expect(console.error).toHaveBeenCalledWith(
`Row (2), Column (${colName}) has mixed types: ERROR` `Row (2), Column (${colName}) has mixed types: ERROR`
) )
}) })
it('should throw an error if table was not found in data object', () => {
const data = { [tableName]: [{ var1: 'string' }] }
expect(() => convertToCSV(data, 'wrongTableName')).toThrow(
new Error('No table provided to be converted to CSV')
)
})
}) })

21
src/utils/convertToCsv.ts
View File

@@ -3,10 +3,15 @@
* @param data - the array of JSON objects to convert. * @param data - the array of JSON objects to convert.
*/ */
export const convertToCSV = ( export const convertToCSV = (
data: any[], data: { [key: string]: any },
sasFormats?: { formats: { [key: string]: string } } tableName: string
) => { ) => {
let formats = sasFormats?.formats if (!data[tableName]) {
throw new Error('No table provided to be converted to CSV')
}
const table = data[tableName]
let formats = data[`$${tableName}`]?.formats
let headers: string[] = [] let headers: string[] = []
let csvTest let csvTest
let invalidString = false let invalidString = false
@@ -16,14 +21,14 @@ export const convertToCSV = (
headers = Object.keys(formats).map((key) => `${key}:${formats![key]}`) headers = Object.keys(formats).map((key) => `${key}:${formats![key]}`)
} }
const headerFields = Object.keys(data[0]) const headerFields = Object.keys(table[0])
headerFields.forEach((field) => { headerFields.forEach((field) => {
if (!formats || !Object.keys(formats).includes(field)) { if (!formats || !Object.keys(formats).includes(field)) {
let hasNullOrNumber = false let hasNullOrNumber = false
let hasSpecialMissingString = false let hasSpecialMissingString = false
data.forEach((row: { [key: string]: any }) => { table.forEach((row: { [key: string]: any }) => {
if (row[field] === null || typeof row[field] === 'number') { if (row[field] === null || typeof row[field] === 'number') {
hasNullOrNumber = true hasNullOrNumber = true
} else if ( } else if (
@@ -45,7 +50,7 @@ export const convertToCSV = (
let hasMixedTypes: boolean = false let hasMixedTypes: boolean = false
let rowNumError: number = -1 let rowNumError: number = -1
const longestValueForField = data const longestValueForField = table
.map((row: any, index: number) => { .map((row: any, index: number) => {
if (row[field] || row[field] === '') { if (row[field] || row[field] === '') {
if (firstFoundType) { if (firstFoundType) {
@@ -101,7 +106,7 @@ export const convertToCSV = (
} }
}) })
if (sasFormats) { if (formats) {
headers = headers.sort( headers = headers.sort(
(a, b) => (a, b) =>
headerFields.indexOf(a.replace(/:.*/, '')) - headerFields.indexOf(a.replace(/:.*/, '')) -
@@ -111,7 +116,7 @@ export const convertToCSV = (
if (invalidString) return 'ERROR: LARGE STRING LENGTH' if (invalidString) return 'ERROR: LARGE STRING LENGTH'
csvTest = data.map((row: any) => { csvTest = table.map((row: any) => {
const fields = Object.keys(row).map((fieldName, index) => { const fields = Object.keys(row).map((fieldName, index) => {
let value let value
const currentCell = row[fieldName] const currentCell = row[fieldName]

7
src/utils/formatDataForRequest.ts
View File

@@ -15,19 +15,24 @@ export const formatDataForRequest = (data: any) => {
} }
tableCounter++ tableCounter++
sasjsTables.push(tableName) sasjsTables.push(tableName)
const csv = convertToCSV(data[tableName], data[`$${tableName}`])
const csv = convertToCSV(data, tableName)
if (csv === 'ERROR: LARGE STRING LENGTH') { if (csv === 'ERROR: LARGE STRING LENGTH') {
throw new Error( throw new Error(
'The max length of a string value in SASjs is 32765 characters.' 'The max length of a string value in SASjs is 32765 characters.'
) )
} }
// if csv has length more then 16k, send in chunks // if csv has length more then 16k, send in chunks
if (csv.length > 16000) { if (csv.length > 16000) {
const csvChunks = splitChunks(csv) const csvChunks = splitChunks(csv)
// append chunks to form data with same key // append chunks to form data with same key
result[`sasjs${tableCounter}data0`] = csvChunks.length result[`sasjs${tableCounter}data0`] = csvChunks.length
csvChunks.forEach((chunk, index) => { csvChunks.forEach((chunk, index) => {
result[`sasjs${tableCounter}data${index + 1}`] = chunk result[`sasjs${tableCounter}data${index + 1}`] = chunk
}) })