From dbb95b7763a4ed434f5dfe017d682b04bd87e793 Mon Sep 17 00:00:00 2001 From: Yury Shkoda Date: Thu, 6 Jan 2022 16:49:58 +0300 Subject: [PATCH] feat: improve convertToCsv function --- sasjs-tests/src/testSuites/SpecialCases.ts | 27 +++-- src/test/utils/formatDataForRequest.spec.ts | 45 ++++++- src/utils/convertToCsv.ts | 126 ++++++++++++-------- 3 files changed, 140 insertions(+), 58 deletions(-) diff --git a/sasjs-tests/src/testSuites/SpecialCases.ts b/sasjs-tests/src/testSuites/SpecialCases.ts index f4e0345..636c9c3 100644 --- a/sasjs-tests/src/testSuites/SpecialCases.ts +++ b/sasjs-tests/src/testSuites/SpecialCases.ts @@ -80,7 +80,7 @@ const errorAndCsrfData: any = { } const testTable = 'sometable' -const testTableWithNullVars = { +const testTableWithNullVars: { [key: string]: any } = { [testTable]: [ { var1: 'string', var2: 232, nullvar: 'A' }, { var1: 'string', var2: 232, nullvar: 'B' }, @@ -270,12 +270,25 @@ export const specialCaseTests = (adapter: SASjs): TestSuite => ({ assertion: (res: any) => { let assertionRes = true - testTableWithNullVars[testTable].forEach((row: {}, i: number) => - Object.keys(row).forEach((col: string) => { - if (col !== res[testTable][i][col.toUpperCase()]) { - assertionRes = false - } - }) + testTableWithNullVars[testTable].forEach( + (row: { [key: string]: any }, i: number) => + Object.keys(row).forEach((col: string) => { + const resValue = res[testTable][i][col.toUpperCase()] + + if ( + typeof row[col] === 'string' && + testTableWithNullVars[`$${testTable}`].formats[col] === + 'best.' && + row[col].toUpperCase() !== resValue + ) { + assertionRes = false + } else if ( + typeof row[col] !== 'string' && + row[col] !== resValue + ) { + assertionRes = false + } + }) ) return assertionRes diff --git a/src/test/utils/formatDataForRequest.spec.ts b/src/test/utils/formatDataForRequest.spec.ts index b560144..9a74998 100644 --- a/src/test/utils/formatDataForRequest.spec.ts +++ b/src/test/utils/formatDataForRequest.spec.ts @@ -3,8 +3,8 @@ import { formatDataForRequest } from '../../utils/formatDataForRequest' describe('formatDataForRequest', () => { const testTable = 'sometable' - it('should', () => { - const testTableWithNullVars = { + it('should format table with special missing values', () => { + const tableWithMissingValues = { [testTable]: [ { var1: 'string', var2: 232, nullvar: 'A' }, { var1: 'string', var2: 232, nullvar: 'B' }, @@ -21,7 +21,7 @@ describe('formatDataForRequest', () => { sasjs_tables: testTable } - expect(formatDataForRequest(testTableWithNullVars)).toEqual(expectedOutput) + expect(formatDataForRequest(tableWithMissingValues)).toEqual(expectedOutput) }) it('should return error if string is more than 32765 characters', () => { @@ -53,4 +53,43 @@ describe('formatDataForRequest', () => { expect(formatDataForRequest(data)).toEqual(expectedOutput) }) + + it('should throw an error if special missing values is not valid', () => { + let tableWithMissingValues = { + [testTable]: [{ var: 'AA' }, { var: 0 }], + [`$${testTable}`]: { formats: { var: 'best.' } } + } + + expect(() => formatDataForRequest(tableWithMissingValues)).toThrow( + new Error( + 'Special missing value can only be a single character from A to Z or _' + ) + ) + }) + + it('should auto-detect special missing values type as best.', () => { + const tableWithMissingValues = { + [testTable]: [{ var: 'a' }, { var: 'A' }, { var: '_' }, { var: 0 }] + } + + const expectedOutput = { + sasjs1data: `var:best.\r\n.a\r\n.a\r\n._\r\n0`, + sasjs_tables: testTable + } + + expect(formatDataForRequest(tableWithMissingValues)).toEqual(expectedOutput) + }) + + it('should auto-detect values type as $char1.', () => { + const tableWithMissingValues = { + [testTable]: [{ var: 'a' }, { var: 'A' }, { var: '_' }] + } + + const expectedOutput = { + sasjs1data: `var:$char1.\r\na\r\nA\r\n_`, + sasjs_tables: testTable + } + + expect(formatDataForRequest(tableWithMissingValues)).toEqual(expectedOutput) + }) }) diff --git a/src/utils/convertToCsv.ts b/src/utils/convertToCsv.ts index e7a424e..9cc5565 100644 --- a/src/utils/convertToCsv.ts +++ b/src/utils/convertToCsv.ts @@ -6,75 +6,98 @@ export const convertToCSV = ( data: any, sasFormats?: { formats: { [key: string]: string } } ) => { - const formats = sasFormats?.formats + let formats = sasFormats?.formats let headers: string[] = [] let csvTest let invalidString = false + const specialMissingValueRegExp = /^[a-z_]{1}$/i if (formats) { - headers = Object.keys(formats).map((key) => `${key}:${formats[key]}`) + headers = Object.keys(formats).map((key) => `${key}:${formats![key]}`) } const headerFields = Object.keys(data[0]) headerFields.forEach((field) => { if (!formats || !Object.keys(formats).includes(field)) { - let firstFoundType: string | null = null - let hasMixedTypes: boolean = false - let rowNumError: number = -1 + let hasNullOrNumber = false + let hasSpecialMissingString = false - const longestValueForField = data - .map((row: any, index: number) => { - if (row[field] || row[field] === '') { - if (firstFoundType) { - let currentFieldType = - row[field] === '' || typeof row[field] === 'string' - ? 'chars' - : 'number' + data.forEach((row: { [key: string]: any }) => { + if (row[field] === null || typeof row[field] === 'number') { + hasNullOrNumber = true + } else if ( + typeof row[field] === 'string' && + specialMissingValueRegExp.test(row[field]) + ) { + hasSpecialMissingString = true + } + }) - if (!hasMixedTypes) { - hasMixedTypes = currentFieldType !== firstFoundType - rowNumError = hasMixedTypes ? index + 1 : -1 - } - } else { - if (row[field] === '') { - firstFoundType = 'chars' + if (hasNullOrNumber && hasSpecialMissingString) { + headers.push(`${field}:best.`) + + if (!formats) formats = {} + + formats[field] = 'best.' + } else { + let firstFoundType: string | null = null + let hasMixedTypes: boolean = false + let rowNumError: number = -1 + + const longestValueForField = data + .map((row: any, index: number) => { + if (row[field] || row[field] === '') { + if (firstFoundType) { + let currentFieldType = + row[field] === '' || typeof row[field] === 'string' + ? 'chars' + : 'number' + + if (!hasMixedTypes) { + hasMixedTypes = currentFieldType !== firstFoundType + rowNumError = hasMixedTypes ? index + 1 : -1 + } } else { - firstFoundType = - typeof row[field] === 'string' ? 'chars' : 'number' + if (row[field] === '') { + firstFoundType = 'chars' + } else { + firstFoundType = + typeof row[field] === 'string' ? 'chars' : 'number' + } } + + let byteSize + + if (typeof row[field] === 'string') { + byteSize = getByteSize(row[field]) + } + + return byteSize } + }) + .sort((a: number, b: number) => b - a)[0] - let byteSize + if (longestValueForField && longestValueForField > 32765) { + invalidString = true + } - if (typeof row[field] === 'string') { - byteSize = getByteSize(row[field]) - } + if (hasMixedTypes) { + console.error( + `Row (${rowNumError}), Column (${field}) has mixed types: ERROR` + ) + } - return byteSize - } - }) - .sort((a: number, b: number) => b - a)[0] - - if (longestValueForField && longestValueForField > 32765) { - invalidString = true - } - - if (hasMixedTypes) { - console.error( - `Row (${rowNumError}), Column (${field}) has mixed types: ERROR` + headers.push( + `${field}:${firstFoundType === 'chars' ? '$char' : ''}${ + longestValueForField + ? longestValueForField + : firstFoundType === 'chars' + ? '1' + : 'best' + }.` ) } - - headers.push( - `${field}:${firstFoundType === 'chars' ? '$char' : ''}${ - longestValueForField - ? longestValueForField - : firstFoundType === 'chars' - ? '1' - : 'best' - }.` - ) } }) @@ -99,6 +122,13 @@ export const convertToCSV = ( value = currentCell === null ? '' : currentCell if (formats && formats[fieldName] === 'best.') { + if (value && !specialMissingValueRegExp.test(value)) { + console.log(`🤖[value]🤖`, value) + throw new Error( + 'Special missing value can only be a single character from A to Z or _' + ) + } + return `.${value.toLowerCase()}` }