diff --git a/src/api/viya/uploadTables.ts b/src/api/viya/uploadTables.ts index b9e4402..47f3208 100644 --- a/src/api/viya/uploadTables.ts +++ b/src/api/viya/uploadTables.ts @@ -18,7 +18,7 @@ export async function uploadTables( const uploadedFiles = [] for (const tableName in data) { - const csv = convertToCSV(data[tableName]) + const csv = convertToCSV(data, tableName) if (csv === 'ERROR: LARGE STRING LENGTH') { throw new Error( 'The max length of a string value in SASjs is 32765 characters.' diff --git a/src/file/generateFileUploadForm.ts b/src/file/generateFileUploadForm.ts index 7a3f7da..04e9c30 100644 --- a/src/file/generateFileUploadForm.ts +++ b/src/file/generateFileUploadForm.ts @@ -18,11 +18,7 @@ export const generateFileUploadForm = ( if (!Array.isArray(data[tableName])) continue const name = tableName - const formatsObj = - data[`$${tableName}`] && data[`$${tableName}`].formats - ? data[`$${tableName}`] - : undefined - const csv = convertToCSV(data[tableName], formatsObj) + const csv = convertToCSV(data, tableName) if (csv === 'ERROR: LARGE STRING LENGTH') { throw new Error( diff --git a/src/file/generateTableUploadForm.ts b/src/file/generateTableUploadForm.ts index 450333b..f2e8b08 100644 --- a/src/file/generateTableUploadForm.ts +++ b/src/file/generateTableUploadForm.ts @@ -9,18 +9,24 @@ export const generateTableUploadForm = ( const sasjsTables = [] const requestParams: any = {} let tableCounter = 0 + for (const tableName in data) { tableCounter++ + sasjsTables.push(tableName) - const csv = convertToCSV(data[tableName]) + + const csv = convertToCSV(data, tableName) + if (csv === 'ERROR: LARGE STRING LENGTH') { throw new Error( 'The max length of a string value in SASjs is 32765 characters.' ) } + // if csv has length more then 16k, send in chunks if (csv.length > 16000) { const csvChunks = splitChunks(csv) + // append chunks to form data with same key csvChunks.map((chunk) => { formData.append(`sasjs${tableCounter}data`, chunk) @@ -29,6 +35,7 @@ export const generateTableUploadForm = ( requestParams[`sasjs${tableCounter}data`] = csv } } + requestParams['sasjs_tables'] = sasjsTables.join(' ') return { formData, requestParams } diff --git a/src/job-execution/Sas9JobExecutor.ts b/src/job-execution/Sas9JobExecutor.ts index 82336b4..b1c1fd0 100644 --- a/src/job-execution/Sas9JobExecutor.ts +++ b/src/job-execution/Sas9JobExecutor.ts @@ -125,7 +125,8 @@ const generateFileUploadForm = ( ): NodeFormData => { for (const tableName in data) { const name = tableName - const csv = convertToCSV(data[tableName]) + const csv = convertToCSV(data, tableName) + if (csv === 'ERROR: LARGE STRING LENGTH') { throw new Error( 'The max length of a string value in SASjs is 32765 characters.' diff --git a/src/utils/convertToCsv.spec.ts b/src/utils/convertToCsv.spec.ts index 9ce9f9d..5d2af1c 100644 --- a/src/utils/convertToCsv.spec.ts +++ b/src/utils/convertToCsv.spec.ts @@ -1,183 +1,222 @@ import { convertToCSV } from './convertToCsv' describe('convertToCsv', () => { + const tableName = 'testTable' + it('should convert single quoted values', () => { - const data = [ - { foo: `'bar'`, bar: 'abc' }, - { foo: 'sadf', bar: 'def' }, - { foo: 'asd', bar: `'qwert'` } - ] + const data = { + [tableName]: [ + { foo: `'bar'`, bar: 'abc' }, + { foo: 'sadf', bar: 'def' }, + { foo: 'asd', bar: `'qwert'` } + ] + } const expectedOutput = `foo:$char5. bar:$char7.\r\n"'bar'",abc\r\nsadf,def\r\nasd,"'qwert'"` - expect(convertToCSV(data)).toEqual(expectedOutput) + expect(convertToCSV(data, tableName)).toEqual(expectedOutput) }) it('should convert double quoted values', () => { - const data = [ - { foo: `"bar"`, bar: 'abc' }, - { foo: 'sadf', bar: 'def' }, - { foo: 'asd', bar: `"qwert"` } - ] + const data = { + [tableName]: [ + { foo: `"bar"`, bar: 'abc' }, + { foo: 'sadf', bar: 'def' }, + { foo: 'asd', bar: `"qwert"` } + ] + } const expectedOutput = `foo:$char5. bar:$char7.\r\n"""bar""",abc\r\nsadf,def\r\nasd,"""qwert"""` - expect(convertToCSV(data)).toEqual(expectedOutput) + expect(convertToCSV(data, tableName)).toEqual(expectedOutput) }) it('should convert values with mixed quotes', () => { - const data = [{ foo: `'blah'`, bar: `"blah"` }] + const data = { [tableName]: [{ foo: `'blah'`, bar: `"blah"` }] } const expectedOutput = `foo:$char6. bar:$char6.\r\n"'blah'","""blah"""` - expect(convertToCSV(data)).toEqual(expectedOutput) + expect(convertToCSV(data, tableName)).toEqual(expectedOutput) }) it('should convert values with mixed quotes', () => { - const data = [{ foo: `'blah,"'`, bar: `"blah,blah" "` }] + const data = { [tableName]: [{ foo: `'blah,"'`, bar: `"blah,blah" "` }] } const expectedOutput = `foo:$char8. bar:$char13.\r\n"'blah,""'","""blah,blah"" """` - expect(convertToCSV(data)).toEqual(expectedOutput) + expect(convertToCSV(data, tableName)).toEqual(expectedOutput) }) it('should convert values with mixed quotes', () => { - const data = [{ foo: `',''`, bar: `","` }] + const data = { [tableName]: [{ foo: `',''`, bar: `","` }] } const expectedOutput = `foo:$char4. bar:$char3.\r\n"',''",""","""` - expect(convertToCSV(data)).toEqual(expectedOutput) + expect(convertToCSV(data, tableName)).toEqual(expectedOutput) }) it('should convert values with mixed quotes', () => { - const data = [{ foo: `','`, bar: `,"` }] + const data = { [tableName]: [{ foo: `','`, bar: `,"` }] } const expectedOutput = `foo:$char3. bar:$char2.\r\n"','",","""` - expect(convertToCSV(data)).toEqual(expectedOutput) + expect(convertToCSV(data, tableName)).toEqual(expectedOutput) }) it('should convert values with mixed quotes', () => { - const data = [{ foo: `"`, bar: `'` }] + const data = { [tableName]: [{ foo: `"`, bar: `'` }] } const expectedOutput = `foo:$char1. bar:$char1.\r\n"""","'"` - expect(convertToCSV(data)).toEqual(expectedOutput) + expect(convertToCSV(data, tableName)).toEqual(expectedOutput) }) it('should convert values with mixed quotes', () => { - const data = [{ foo: `,`, bar: `',` }] + const data = { [tableName]: [{ foo: `,`, bar: `',` }] } const expectedOutput = `foo:$char1. bar:$char2.\r\n",","',"` - expect(convertToCSV(data)).toEqual(expectedOutput) + expect(convertToCSV(data, tableName)).toEqual(expectedOutput) }) it('should convert values with number cases 1', () => { - const data = [ - { col1: 42, col2: null, col3: 'x', col4: null }, - { col1: 42, col2: null, col3: 'x', col4: null }, - { col1: 42, col2: null, col3: 'x', col4: null }, - { col1: 42, col2: null, col3: 'x', col4: '' }, - { col1: 42, col2: null, col3: 'x', col4: '' } - ] + const data = { + [tableName]: [ + { col1: 42, col2: null, col3: 'x', col4: null }, + { col1: 42, col2: null, col3: 'x', col4: null }, + { col1: 42, col2: null, col3: 'x', col4: null }, + { col1: 42, col2: null, col3: 'x', col4: '' }, + { col1: 42, col2: null, col3: 'x', col4: '' } + ] + } const expectedOutput = `col1:best. col2:best. col3:$char1. col4:$char1.\r\n42,.,x,\r\n42,.,x,\r\n42,.,x,\r\n42,.,x,\r\n42,.,x,` - expect(convertToCSV(data)).toEqual(expectedOutput) + expect(convertToCSV(data, tableName)).toEqual(expectedOutput) }) it('should convert values with number cases 2', () => { - const data = [ - { col1: 42, col2: null, col3: 'x', col4: '' }, - { col1: 42, col2: null, col3: 'x', col4: '' }, - { col1: 42, col2: null, col3: 'x', col4: '' }, - { col1: 42, col2: 1.62, col3: 'x', col4: 'x' }, - { col1: 42, col2: 1.62, col3: 'x', col4: 'x' } - ] + const data = { + [tableName]: [ + { col1: 42, col2: null, col3: 'x', col4: '' }, + { col1: 42, col2: null, col3: 'x', col4: '' }, + { col1: 42, col2: null, col3: 'x', col4: '' }, + { col1: 42, col2: 1.62, col3: 'x', col4: 'x' }, + { col1: 42, col2: 1.62, col3: 'x', col4: 'x' } + ] + } const expectedOutput = `col1:best. col2:best. col3:$char1. col4:$char1.\r\n42,.,x,\r\n42,.,x,\r\n42,.,x,\r\n42,1.62,x,x\r\n42,1.62,x,x` - expect(convertToCSV(data)).toEqual(expectedOutput) + expect(convertToCSV(data, tableName)).toEqual(expectedOutput) }) it('should convert values with common special characters', () => { - expect(convertToCSV([{ tab: '\t' }])).toEqual(`tab:$char1.\r\n\"\t\"`) - expect(convertToCSV([{ lf: '\n' }])).toEqual(`lf:$char1.\r\n\"\n\"`) - expect(convertToCSV([{ semicolon: ';semi' }])).toEqual( - `semicolon:$char5.\r\n;semi` + expect(convertToCSV({ [tableName]: [{ tab: '\t' }] }, tableName)).toEqual( + `tab:$char1.\r\n\"\t\"` ) - expect(convertToCSV([{ percent: '%' }])).toEqual(`percent:$char1.\r\n%`) - expect(convertToCSV([{ singleQuote: "'" }])).toEqual( - `singleQuote:$char1.\r\n\"'\"` - ) - expect(convertToCSV([{ doubleQuote: '"' }])).toEqual( - `doubleQuote:$char1.\r\n""""` - ) - expect(convertToCSV([{ crlf: '\r\n' }])).toEqual(`crlf:$char2.\r\n\"\n\"`) - expect(convertToCSV([{ euro: '€euro' }])).toEqual(`euro:$char7.\r\n€euro`) - expect(convertToCSV([{ banghash: '!#banghash' }])).toEqual( - `banghash:$char10.\r\n!#banghash` + expect(convertToCSV({ [tableName]: [{ lf: '\n' }] }, tableName)).toEqual( + `lf:$char1.\r\n\"\n\"` ) + expect( + convertToCSV({ [tableName]: [{ semicolon: ';semi' }] }, tableName) + ).toEqual(`semicolon:$char5.\r\n;semi`) + expect( + convertToCSV({ [tableName]: [{ percent: '%' }] }, tableName) + ).toEqual(`percent:$char1.\r\n%`) + expect( + convertToCSV({ [tableName]: [{ singleQuote: "'" }] }, tableName) + ).toEqual(`singleQuote:$char1.\r\n\"'\"`) + expect( + convertToCSV({ [tableName]: [{ doubleQuote: '"' }] }, tableName) + ).toEqual(`doubleQuote:$char1.\r\n""""`) + expect( + convertToCSV({ [tableName]: [{ crlf: '\r\n' }] }, tableName) + ).toEqual(`crlf:$char2.\r\n\"\n\"`) + expect( + convertToCSV({ [tableName]: [{ euro: '€euro' }] }, tableName) + ).toEqual(`euro:$char7.\r\n€euro`) + expect( + convertToCSV({ [tableName]: [{ banghash: '!#banghash' }] }, tableName) + ).toEqual(`banghash:$char10.\r\n!#banghash`) }) it('should convert values with other special characters', () => { - const data = [ - { - speech0: '"speech', - pct: '%percent', - speech: '"speech', - slash: '\\slash', - slashWithSpecial: '\\\tslash', - macvar: '&sysuserid', - chinese: '传/傳chinese', - sigma: 'Σsigma', - at: '@at', - serbian: 'Српски', - dollar: '$' - } - ] + const data = { + [tableName]: [ + { + speech0: '"speech', + pct: '%percent', + speech: '"speech', + slash: '\\slash', + slashWithSpecial: '\\\tslash', + macvar: '&sysuserid', + chinese: '传/傳chinese', + sigma: 'Σsigma', + at: '@at', + serbian: 'Српски', + dollar: '$' + } + ] + } const expectedOutput = `speech0:$char7. pct:$char8. speech:$char7. slash:$char6. slashWithSpecial:$char7. macvar:$char10. chinese:$char14. sigma:$char7. at:$char3. serbian:$char12. dollar:$char1.\r\n"""speech",%percent,"""speech",\\slash,\"\\\tslash\",&sysuserid,传/傳chinese,Σsigma,@at,Српски,$` - expect(convertToCSV(data)).toEqual(expectedOutput) + expect(convertToCSV(data, tableName)).toEqual(expectedOutput) - expect(convertToCSV([{ speech: 'menext' }])).toEqual( - `speech:$char6.\r\nmenext` - ) - expect(convertToCSV([{ speech: 'me\nnext' }])).toEqual( - `speech:$char7.\r\n\"me\nnext\"` - ) - expect(convertToCSV([{ speech: `me'next` }])).toEqual( - `speech:$char7.\r\n\"me'next\"` - ) - expect(convertToCSV([{ speech: `me"next` }])).toEqual( - `speech:$char7.\r\n\"me""next\"` - ) - expect(convertToCSV([{ speech: `me""next` }])).toEqual( - `speech:$char8.\r\n\"me""""next\"` - ) - expect(convertToCSV([{ slashWithSpecial: '\\\tslash' }])).toEqual( - `slashWithSpecial:$char7.\r\n\"\\\tslash\"` - ) - expect(convertToCSV([{ slashWithSpecial: '\\ \tslash' }])).toEqual( - `slashWithSpecial:$char8.\r\n\"\\ \tslash\"` - ) expect( - convertToCSV([{ slashWithSpecialExtra: '\\\ts\tl\ta\ts\t\th\t' }]) + convertToCSV({ [tableName]: [{ speech: 'menext' }] }, tableName) + ).toEqual(`speech:$char6.\r\nmenext`) + expect( + convertToCSV({ [tableName]: [{ speech: 'me\nnext' }] }, tableName) + ).toEqual(`speech:$char7.\r\n\"me\nnext\"`) + expect( + convertToCSV({ [tableName]: [{ speech: `me'next` }] }, tableName) + ).toEqual(`speech:$char7.\r\n\"me'next\"`) + expect( + convertToCSV({ [tableName]: [{ speech: `me"next` }] }, tableName) + ).toEqual(`speech:$char7.\r\n\"me""next\"`) + expect( + convertToCSV({ [tableName]: [{ speech: `me""next` }] }, tableName) + ).toEqual(`speech:$char8.\r\n\"me""""next\"`) + expect( + convertToCSV( + { [tableName]: [{ slashWithSpecial: '\\\tslash' }] }, + tableName + ) + ).toEqual(`slashWithSpecial:$char7.\r\n\"\\\tslash\"`) + expect( + convertToCSV( + { [tableName]: [{ slashWithSpecial: '\\ \tslash' }] }, + tableName + ) + ).toEqual(`slashWithSpecial:$char8.\r\n\"\\ \tslash\"`) + expect( + convertToCSV( + { [tableName]: [{ slashWithSpecialExtra: '\\\ts\tl\ta\ts\t\th\t' }] }, + tableName + ) ).toEqual(`slashWithSpecialExtra:$char13.\r\n\"\\\ts\tl\ta\ts\t\th\t\"`) }) it('should console log error if data has mixed types', () => { const colName = 'var1' - const data = [{ [colName]: 'string' }, { [colName]: 232 }] + const data = { [tableName]: [{ [colName]: 'string' }, { [colName]: 232 }] } jest.spyOn(console, 'error').mockImplementation(() => {}) - convertToCSV(data) + convertToCSV(data, tableName) expect(console.error).toHaveBeenCalledWith( `Row (2), Column (${colName}) has mixed types: ERROR` ) }) + + it('should throw an error if table was not found in data object', () => { + const data = { [tableName]: [{ var1: 'string' }] } + + expect(() => convertToCSV(data, 'wrongTableName')).toThrow( + new Error('No table provided to be converted to CSV') + ) + }) }) diff --git a/src/utils/convertToCsv.ts b/src/utils/convertToCsv.ts index 73af3ba..9d439a7 100644 --- a/src/utils/convertToCsv.ts +++ b/src/utils/convertToCsv.ts @@ -3,10 +3,15 @@ * @param data - the array of JSON objects to convert. */ export const convertToCSV = ( - data: any[], - sasFormats?: { formats: { [key: string]: string } } + data: { [key: string]: any }, + tableName: string ) => { - let formats = sasFormats?.formats + if (!data[tableName]) { + throw new Error('No table provided to be converted to CSV') + } + + const table = data[tableName] + let formats = data[`$${tableName}`]?.formats let headers: string[] = [] let csvTest let invalidString = false @@ -16,14 +21,14 @@ export const convertToCSV = ( headers = Object.keys(formats).map((key) => `${key}:${formats![key]}`) } - const headerFields = Object.keys(data[0]) + const headerFields = Object.keys(table[0]) headerFields.forEach((field) => { if (!formats || !Object.keys(formats).includes(field)) { let hasNullOrNumber = false let hasSpecialMissingString = false - data.forEach((row: { [key: string]: any }) => { + table.forEach((row: { [key: string]: any }) => { if (row[field] === null || typeof row[field] === 'number') { hasNullOrNumber = true } else if ( @@ -45,7 +50,7 @@ export const convertToCSV = ( let hasMixedTypes: boolean = false let rowNumError: number = -1 - const longestValueForField = data + const longestValueForField = table .map((row: any, index: number) => { if (row[field] || row[field] === '') { if (firstFoundType) { @@ -101,7 +106,7 @@ export const convertToCSV = ( } }) - if (sasFormats) { + if (formats) { headers = headers.sort( (a, b) => headerFields.indexOf(a.replace(/:.*/, '')) - @@ -111,7 +116,7 @@ export const convertToCSV = ( if (invalidString) return 'ERROR: LARGE STRING LENGTH' - csvTest = data.map((row: any) => { + csvTest = table.map((row: any) => { const fields = Object.keys(row).map((fieldName, index) => { let value const currentCell = row[fieldName] diff --git a/src/utils/formatDataForRequest.ts b/src/utils/formatDataForRequest.ts index d223432..bf73457 100644 --- a/src/utils/formatDataForRequest.ts +++ b/src/utils/formatDataForRequest.ts @@ -15,19 +15,24 @@ export const formatDataForRequest = (data: any) => { } tableCounter++ + sasjsTables.push(tableName) - const csv = convertToCSV(data[tableName], data[`$${tableName}`]) + + const csv = convertToCSV(data, tableName) if (csv === 'ERROR: LARGE STRING LENGTH') { throw new Error( 'The max length of a string value in SASjs is 32765 characters.' ) } + // if csv has length more then 16k, send in chunks if (csv.length > 16000) { const csvChunks = splitChunks(csv) + // append chunks to form data with same key result[`sasjs${tableCounter}data0`] = csvChunks.length + csvChunks.forEach((chunk, index) => { result[`sasjs${tableCounter}data${index + 1}`] = chunk })