From fbce35b27272a211add6b0ee15ab13146d3ef01d Mon Sep 17 00:00:00 2001 From: Yury Shkoda Date: Wed, 5 Jan 2022 16:54:16 +0300 Subject: [PATCH] feat(nullVars): add SAS null vars support --- sasjs-tests/src/testSuites/SpecialCases.ts | 33 ++++++ src/SASjs.ts | 11 +- src/utils/convertToCsv.spec.ts | 38 +++++++ src/utils/convertToCsv.ts | 122 +++++++++++++-------- src/utils/formatDataForRequest.ts | 11 +- 5 files changed, 163 insertions(+), 52 deletions(-) diff --git a/sasjs-tests/src/testSuites/SpecialCases.ts b/sasjs-tests/src/testSuites/SpecialCases.ts index 1ec227a..f4e0345 100644 --- a/sasjs-tests/src/testSuites/SpecialCases.ts +++ b/sasjs-tests/src/testSuites/SpecialCases.ts @@ -79,6 +79,19 @@ const errorAndCsrfData: any = { _csrf: [{ col1: 'q', col2: 'w', col3: 'e', col4: 'r' }] } +const testTable = 'sometable' +const testTableWithNullVars = { + [testTable]: [ + { var1: 'string', var2: 232, nullvar: 'A' }, + { var1: 'string', var2: 232, nullvar: 'B' }, + { var1: 'string', var2: 232, nullvar: '_' }, + { var1: 'string', var2: 232, nullvar: 0 }, + { var1: 'string', var2: 232, nullvar: 'z' }, + { var1: 'string', var2: 232, nullvar: null } + ], + [`$${testTable}`]: { formats: { var1: '$char12.', nullvar: 'best.' } } +} + export const specialCaseTests = (adapter: SASjs): TestSuite => ({ name: 'Special Cases', tests: [ @@ -247,6 +260,26 @@ export const specialCaseTests = (adapter: SASjs): TestSuite => ({ res._csrf[0].COL4 === errorAndCsrfData._csrf[0].col4 ) } + }, + { + title: 'Special missing values', + description: 'Should support special missing values', + test: () => { + return adapter.request('common/sendObj', testTableWithNullVars) + }, + assertion: (res: any) => { + let assertionRes = true + + testTableWithNullVars[testTable].forEach((row: {}, i: number) => + Object.keys(row).forEach((col: string) => { + if (col !== res[testTable][i][col.toUpperCase()]) { + assertionRes = false + } + }) + ) + + return assertionRes + } } ] }) diff --git a/src/SASjs.ts b/src/SASjs.ts index 3103457..2728f16 100644 --- a/src/SASjs.ts +++ b/src/SASjs.ts @@ -736,15 +736,19 @@ export default class SASjs { msg: string } { if (data === null) return { status: true, msg: '' } + + const isSasFormatsTable = (key: string) => + key.match(/^\$.*/) && Object.keys(data).includes(key.replace(/^\$/, '')) + for (const key in data) { - if (!key.match(/^[a-zA-Z_]/)) { + if (!key.match(/^[a-zA-Z_]/) && !isSasFormatsTable(key)) { return { status: false, msg: 'First letter of table should be alphabet or underscore.' } } - if (!key.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) { + if (!key.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/) && !isSasFormatsTable(key)) { return { status: false, msg: 'Table name should be alphanumeric.' } } @@ -755,7 +759,7 @@ export default class SASjs { } } - if (this.getType(data[key]) !== 'Array') { + if (this.getType(data[key]) !== 'Array' && !isSasFormatsTable(key)) { return { status: false, msg: 'Parameter data contains invalid table structure.' @@ -771,6 +775,7 @@ export default class SASjs { } } } + return { status: true, msg: '' } } diff --git a/src/utils/convertToCsv.spec.ts b/src/utils/convertToCsv.spec.ts index 8a56335..0d5dd25 100644 --- a/src/utils/convertToCsv.spec.ts +++ b/src/utils/convertToCsv.spec.ts @@ -167,4 +167,42 @@ describe('convertToCsv', () => { convertToCSV([{ slashWithSpecialExtra: '\\\ts\tl\ta\ts\t\th\t' }]) ).toEqual(`slashWithSpecialExtra:$char13.\r\n\"\\\ts\tl\ta\ts\t\th\t\"`) }) + + it('should convert not null values', () => { + const data = [ + { var1: 'string', nullvar: 'A', var2: 232 }, + { var1: 'string', nullvar: 'B', var2: 232 }, + { var1: 'string', nullvar: '_', var2: 232 }, + { var1: 'string', nullvar: 0, var2: 232 }, + { var1: 'string', nullvar: 'z', var2: 232 }, + { var1: 'string', nullvar: null, var2: 232 } + ] + + const expectedOutput = `var1:$char6. nullvar:best. var2:best.\r\nstring,.a,232\r\nstring,.b,232\r\nstring,._,232\r\nstring,0,232\r\nstring,.z,232\r\nstring,.,232` + + expect( + convertToCSV(data, { + formats: { var1: '$char6.', nullvar: 'best.' } + }) + ).toEqual(expectedOutput) + }) + + it('should return error if string is more than maxFieldValue', () => { + const data = [{ var1: 'z'.repeat(32765 + 1) }] + + expect(convertToCSV(data)).toEqual('ERROR: LARGE STRING LENGTH') + }) + + it('should console log error if data has mixed types', () => { + const colName = 'var1' + const data = [{ [colName]: 'string' }, { [colName]: 232 }] + + jest.spyOn(console, 'error').mockImplementation(() => {}) + + convertToCSV(data) + + expect(console.error).toHaveBeenCalledWith( + `Row (2), Column (${colName}) has mixed types: ERROR` + ) + }) }) diff --git a/src/utils/convertToCsv.ts b/src/utils/convertToCsv.ts index 9496514..e7a424e 100644 --- a/src/utils/convertToCsv.ts +++ b/src/utils/convertToCsv.ts @@ -2,70 +2,92 @@ * Converts the given JSON object array to a CSV string. * @param data - the array of JSON objects to convert. */ -export const convertToCSV = (data: any) => { - const replacer = (key: any, value: any) => (value === null ? '' : value) - const headerFields = Object.keys(data[0]) +export const convertToCSV = ( + data: any, + sasFormats?: { formats: { [key: string]: string } } +) => { + const formats = sasFormats?.formats + let headers: string[] = [] let csvTest let invalidString = false - const headers = headerFields.map((field) => { - let firstFoundType: string | null = null - let hasMixedTypes: boolean = false - let rowNumError: number = -1 - const longestValueForField = data - .map((row: any, index: number) => { - if (row[field] || row[field] === '') { - if (firstFoundType) { - let currentFieldType = - row[field] === '' || typeof row[field] === 'string' - ? 'chars' - : 'number' + if (formats) { + headers = Object.keys(formats).map((key) => `${key}:${formats[key]}`) + } - if (!hasMixedTypes) { - hasMixedTypes = currentFieldType !== firstFoundType - rowNumError = hasMixedTypes ? index + 1 : -1 - } - } else { - if (row[field] === '') { - firstFoundType = 'chars' + const headerFields = Object.keys(data[0]) + + headerFields.forEach((field) => { + if (!formats || !Object.keys(formats).includes(field)) { + let firstFoundType: string | null = null + let hasMixedTypes: boolean = false + let rowNumError: number = -1 + + const longestValueForField = data + .map((row: any, index: number) => { + if (row[field] || row[field] === '') { + if (firstFoundType) { + let currentFieldType = + row[field] === '' || typeof row[field] === 'string' + ? 'chars' + : 'number' + + if (!hasMixedTypes) { + hasMixedTypes = currentFieldType !== firstFoundType + rowNumError = hasMixedTypes ? index + 1 : -1 + } } else { - firstFoundType = - typeof row[field] === 'string' ? 'chars' : 'number' + if (row[field] === '') { + firstFoundType = 'chars' + } else { + firstFoundType = + typeof row[field] === 'string' ? 'chars' : 'number' + } } + + let byteSize + + if (typeof row[field] === 'string') { + byteSize = getByteSize(row[field]) + } + + return byteSize } + }) + .sort((a: number, b: number) => b - a)[0] - let byteSize + if (longestValueForField && longestValueForField > 32765) { + invalidString = true + } - if (typeof row[field] === 'string') { - byteSize = getByteSize(row[field]) - } + if (hasMixedTypes) { + console.error( + `Row (${rowNumError}), Column (${field}) has mixed types: ERROR` + ) + } - return byteSize - } - }) - .sort((a: number, b: number) => b - a)[0] - if (longestValueForField && longestValueForField > 32765) { - invalidString = true - } - if (hasMixedTypes) { - console.error( - `Row (${rowNumError}), Column (${field}) has mixed types: ERROR` + headers.push( + `${field}:${firstFoundType === 'chars' ? '$char' : ''}${ + longestValueForField + ? longestValueForField + : firstFoundType === 'chars' + ? '1' + : 'best' + }.` ) } - - return `${field}:${firstFoundType === 'chars' ? '$char' : ''}${ - longestValueForField - ? longestValueForField - : firstFoundType === 'chars' - ? '1' - : 'best' - }.` }) - if (invalidString) { - return 'ERROR: LARGE STRING LENGTH' + if (sasFormats) { + headers = headers.sort( + (a, b) => + headerFields.indexOf(a.replace(/:.*/, '')) - + headerFields.indexOf(b.replace(/:.*/, '')) + ) } + if (invalidString) return 'ERROR: LARGE STRING LENGTH' + csvTest = data.map((row: any) => { const fields = Object.keys(row).map((fieldName, index) => { let value @@ -76,6 +98,10 @@ export const convertToCSV = (data: any) => { // stringify with replacer converts null values to empty strings value = currentCell === null ? '' : currentCell + if (formats && formats[fieldName] === 'best.') { + return `.${value.toLowerCase()}` + } + // if there any present, it should have preceding (") for escaping value = value.replace(/"/g, `""`) diff --git a/src/utils/formatDataForRequest.ts b/src/utils/formatDataForRequest.ts index caf157d..d223432 100644 --- a/src/utils/formatDataForRequest.ts +++ b/src/utils/formatDataForRequest.ts @@ -7,9 +7,17 @@ export const formatDataForRequest = (data: any) => { const result: any = {} for (const tableName in data) { + if ( + tableName.match(/^\$.*/) && + Object.keys(data).includes(tableName.replace(/^\$/, '')) + ) { + continue + } + tableCounter++ sasjsTables.push(tableName) - const csv = convertToCSV(data[tableName]) + const csv = convertToCSV(data[tableName], data[`$${tableName}`]) + if (csv === 'ERROR: LARGE STRING LENGTH') { throw new Error( 'The max length of a string value in SASjs is 32765 characters.' @@ -27,6 +35,7 @@ export const formatDataForRequest = (data: any) => { result[`sasjs${tableCounter}data`] = csv } } + result['sasjs_tables'] = sasjsTables.join(' ') return result