refactor(convert-to-csv): change func arguments

2026-01-04 11:10:05 +00:00 · 2022-03-03 15:07:01 +03:00
parent ff5dc5f196
commit 5927910a52
7 changed files with 165 additions and 112 deletions
--- a/src/api/viya/uploadTables.ts
+++ b/src/api/viya/uploadTables.ts
@@ -18,7 +18,7 @@ export async function uploadTables(
  const uploadedFiles = []

  for (const tableName in data) {
-    const csv = convertToCSV(data[tableName])
+    const csv = convertToCSV(data, tableName)
    if (csv === 'ERROR: LARGE STRING LENGTH') {
      throw new Error(
        'The max length of a string value in SASjs is 32765 characters.'
--- a/src/file/generateFileUploadForm.ts
+++ b/src/file/generateFileUploadForm.ts
@@ -18,11 +18,7 @@ export const generateFileUploadForm = (
    if (!Array.isArray(data[tableName])) continue

    const name = tableName
-    const formatsObj =
-      data[`$${tableName}`] && data[`$${tableName}`].formats
-        ? data[`$${tableName}`]
-        : undefined
-    const csv = convertToCSV(data[tableName], formatsObj)
+    const csv = convertToCSV(data, tableName)

    if (csv === 'ERROR: LARGE STRING LENGTH') {
      throw new Error(
--- a/src/file/generateTableUploadForm.ts
+++ b/src/file/generateTableUploadForm.ts
@@ -9,18 +9,24 @@ export const generateTableUploadForm = (
  const sasjsTables = []
  const requestParams: any = {}
  let tableCounter = 0
+
  for (const tableName in data) {
    tableCounter++
+
    sasjsTables.push(tableName)
-    const csv = convertToCSV(data[tableName])
+
+    const csv = convertToCSV(data, tableName)
+
    if (csv === 'ERROR: LARGE STRING LENGTH') {
      throw new Error(
        'The max length of a string value in SASjs is 32765 characters.'
      )
    }
+
    // if csv has length more then 16k, send in chunks
    if (csv.length > 16000) {
      const csvChunks = splitChunks(csv)
+
      // append chunks to form data with same key
      csvChunks.map((chunk) => {
        formData.append(`sasjs${tableCounter}data`, chunk)
@@ -29,6 +35,7 @@ export const generateTableUploadForm = (
      requestParams[`sasjs${tableCounter}data`] = csv
    }
  }
+
  requestParams['sasjs_tables'] = sasjsTables.join(' ')

  return { formData, requestParams }
--- a/src/job-execution/Sas9JobExecutor.ts
+++ b/src/job-execution/Sas9JobExecutor.ts
@@ -125,7 +125,8 @@ const generateFileUploadForm = (
 ): NodeFormData => {
  for (const tableName in data) {
    const name = tableName
-    const csv = convertToCSV(data[tableName])
+    const csv = convertToCSV(data, tableName)
+
    if (csv === 'ERROR: LARGE STRING LENGTH') {
      throw new Error(
        'The max length of a string value in SASjs is 32765 characters.'
--- a/src/utils/convertToCsv.spec.ts
+++ b/src/utils/convertToCsv.spec.ts
@@ -1,183 +1,222 @@
 import { convertToCSV } from './convertToCsv'

 describe('convertToCsv', () => {
+  const tableName = 'testTable'
+
  it('should convert single quoted values', () => {
-    const data = [
-      { foo: `'bar'`, bar: 'abc' },
-      { foo: 'sadf', bar: 'def' },
-      { foo: 'asd', bar: `'qwert'` }
-    ]
+    const data = {
+      [tableName]: [
+        { foo: `'bar'`, bar: 'abc' },
+        { foo: 'sadf', bar: 'def' },
+        { foo: 'asd', bar: `'qwert'` }
+      ]
+    }

    const expectedOutput = `foo:$char5. bar:$char7.\r\n"'bar'",abc\r\nsadf,def\r\nasd,"'qwert'"`

-    expect(convertToCSV(data)).toEqual(expectedOutput)
+    expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
  })

  it('should convert double quoted values', () => {
-    const data = [
-      { foo: `"bar"`, bar: 'abc' },
-      { foo: 'sadf', bar: 'def' },
-      { foo: 'asd', bar: `"qwert"` }
-    ]
+    const data = {
+      [tableName]: [
+        { foo: `"bar"`, bar: 'abc' },
+        { foo: 'sadf', bar: 'def' },
+        { foo: 'asd', bar: `"qwert"` }
+      ]
+    }

    const expectedOutput = `foo:$char5. bar:$char7.\r\n"""bar""",abc\r\nsadf,def\r\nasd,"""qwert"""`

-    expect(convertToCSV(data)).toEqual(expectedOutput)
+    expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
  })

  it('should convert values with mixed quotes', () => {
-    const data = [{ foo: `'blah'`, bar: `"blah"` }]
+    const data = { [tableName]: [{ foo: `'blah'`, bar: `"blah"` }] }

    const expectedOutput = `foo:$char6. bar:$char6.\r\n"'blah'","""blah"""`

-    expect(convertToCSV(data)).toEqual(expectedOutput)
+    expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
  })

  it('should convert values with mixed quotes', () => {
-    const data = [{ foo: `'blah,"'`, bar: `"blah,blah" "` }]
+    const data = { [tableName]: [{ foo: `'blah,"'`, bar: `"blah,blah" "` }] }

    const expectedOutput = `foo:$char8. bar:$char13.\r\n"'blah,""'","""blah,blah"" """`

-    expect(convertToCSV(data)).toEqual(expectedOutput)
+    expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
  })

  it('should convert values with mixed quotes', () => {
-    const data = [{ foo: `',''`, bar: `","` }]
+    const data = { [tableName]: [{ foo: `',''`, bar: `","` }] }

    const expectedOutput = `foo:$char4. bar:$char3.\r\n"',''",""","""`

-    expect(convertToCSV(data)).toEqual(expectedOutput)
+    expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
  })

  it('should convert values with mixed quotes', () => {
-    const data = [{ foo: `','`, bar: `,"` }]
+    const data = { [tableName]: [{ foo: `','`, bar: `,"` }] }

    const expectedOutput = `foo:$char3. bar:$char2.\r\n"','",","""`

-    expect(convertToCSV(data)).toEqual(expectedOutput)
+    expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
  })

  it('should convert values with mixed quotes', () => {
-    const data = [{ foo: `"`, bar: `'` }]
+    const data = { [tableName]: [{ foo: `"`, bar: `'` }] }

    const expectedOutput = `foo:$char1. bar:$char1.\r\n"""","'"`

-    expect(convertToCSV(data)).toEqual(expectedOutput)
+    expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
  })

  it('should convert values with mixed quotes', () => {
-    const data = [{ foo: `,`, bar: `',` }]
+    const data = { [tableName]: [{ foo: `,`, bar: `',` }] }

    const expectedOutput = `foo:$char1. bar:$char2.\r\n",","',"`

-    expect(convertToCSV(data)).toEqual(expectedOutput)
+    expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
  })

  it('should convert values with number cases 1', () => {
-    const data = [
-      { col1: 42, col2: null, col3: 'x', col4: null },
-      { col1: 42, col2: null, col3: 'x', col4: null },
-      { col1: 42, col2: null, col3: 'x', col4: null },
-      { col1: 42, col2: null, col3: 'x', col4: '' },
-      { col1: 42, col2: null, col3: 'x', col4: '' }
-    ]
+    const data = {
+      [tableName]: [
+        { col1: 42, col2: null, col3: 'x', col4: null },
+        { col1: 42, col2: null, col3: 'x', col4: null },
+        { col1: 42, col2: null, col3: 'x', col4: null },
+        { col1: 42, col2: null, col3: 'x', col4: '' },
+        { col1: 42, col2: null, col3: 'x', col4: '' }
+      ]
+    }

    const expectedOutput = `col1:best. col2:best. col3:$char1. col4:$char1.\r\n42,.,x,\r\n42,.,x,\r\n42,.,x,\r\n42,.,x,\r\n42,.,x,`

-    expect(convertToCSV(data)).toEqual(expectedOutput)
+    expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
  })

  it('should convert values with number cases 2', () => {
-    const data = [
-      { col1: 42, col2: null, col3: 'x', col4: '' },
-      { col1: 42, col2: null, col3: 'x', col4: '' },
-      { col1: 42, col2: null, col3: 'x', col4: '' },
-      { col1: 42, col2: 1.62, col3: 'x', col4: 'x' },
-      { col1: 42, col2: 1.62, col3: 'x', col4: 'x' }
-    ]
+    const data = {
+      [tableName]: [
+        { col1: 42, col2: null, col3: 'x', col4: '' },
+        { col1: 42, col2: null, col3: 'x', col4: '' },
+        { col1: 42, col2: null, col3: 'x', col4: '' },
+        { col1: 42, col2: 1.62, col3: 'x', col4: 'x' },
+        { col1: 42, col2: 1.62, col3: 'x', col4: 'x' }
+      ]
+    }

    const expectedOutput = `col1:best. col2:best. col3:$char1. col4:$char1.\r\n42,.,x,\r\n42,.,x,\r\n42,.,x,\r\n42,1.62,x,x\r\n42,1.62,x,x`

-    expect(convertToCSV(data)).toEqual(expectedOutput)
+    expect(convertToCSV(data, tableName)).toEqual(expectedOutput)
  })

  it('should convert values with common special characters', () => {
-    expect(convertToCSV([{ tab: '\t' }])).toEqual(`tab:$char1.\r\n\"\t\"`)
-    expect(convertToCSV([{ lf: '\n' }])).toEqual(`lf:$char1.\r\n\"\n\"`)
-    expect(convertToCSV([{ semicolon: ';semi' }])).toEqual(
-      `semicolon:$char5.\r\n;semi`
+    expect(convertToCSV({ [tableName]: [{ tab: '\t' }] }, tableName)).toEqual(
+      `tab:$char1.\r\n\"\t\"`
    )
-    expect(convertToCSV([{ percent: '%' }])).toEqual(`percent:$char1.\r\n%`)
-    expect(convertToCSV([{ singleQuote: "'" }])).toEqual(
-      `singleQuote:$char1.\r\n\"'\"`
-    )
-    expect(convertToCSV([{ doubleQuote: '"' }])).toEqual(
-      `doubleQuote:$char1.\r\n""""`
-    )
-    expect(convertToCSV([{ crlf: '\r\n' }])).toEqual(`crlf:$char2.\r\n\"\n\"`)
-    expect(convertToCSV([{ euro: '€euro' }])).toEqual(`euro:$char7.\r\n€euro`)
-    expect(convertToCSV([{ banghash: '!#banghash' }])).toEqual(
-      `banghash:$char10.\r\n!#banghash`
+    expect(convertToCSV({ [tableName]: [{ lf: '\n' }] }, tableName)).toEqual(
+      `lf:$char1.\r\n\"\n\"`
    )
+    expect(
+      convertToCSV({ [tableName]: [{ semicolon: ';semi' }] }, tableName)
+    ).toEqual(`semicolon:$char5.\r\n;semi`)
+    expect(
+      convertToCSV({ [tableName]: [{ percent: '%' }] }, tableName)
+    ).toEqual(`percent:$char1.\r\n%`)
+    expect(
+      convertToCSV({ [tableName]: [{ singleQuote: "'" }] }, tableName)
+    ).toEqual(`singleQuote:$char1.\r\n\"'\"`)
+    expect(
+      convertToCSV({ [tableName]: [{ doubleQuote: '"' }] }, tableName)
+    ).toEqual(`doubleQuote:$char1.\r\n""""`)
+    expect(
+      convertToCSV({ [tableName]: [{ crlf: '\r\n' }] }, tableName)
+    ).toEqual(`crlf:$char2.\r\n\"\n\"`)
+    expect(
+      convertToCSV({ [tableName]: [{ euro: '€euro' }] }, tableName)
+    ).toEqual(`euro:$char7.\r\n€euro`)
+    expect(
+      convertToCSV({ [tableName]: [{ banghash: '!#banghash' }] }, tableName)
+    ).toEqual(`banghash:$char10.\r\n!#banghash`)
  })

  it('should convert values with other special characters', () => {
-    const data = [
-      {
-        speech0: '"speech',
-        pct: '%percent',
-        speech: '"speech',
-        slash: '\\slash',
-        slashWithSpecial: '\\\tslash',
-        macvar: '&sysuserid',
-        chinese: '传/傳chinese',
-        sigma: 'Σsigma',
-        at: '@at',
-        serbian: 'Српски',
-        dollar: '$'
-      }
-    ]
+    const data = {
+      [tableName]: [
+        {
+          speech0: '"speech',
+          pct: '%percent',
+          speech: '"speech',
+          slash: '\\slash',
+          slashWithSpecial: '\\\tslash',
+          macvar: '&sysuserid',
+          chinese: '传/傳chinese',
+          sigma: 'Σsigma',
+          at: '@at',
+          serbian: 'Српски',
+          dollar: '$'
+        }
+      ]
+    }

    const expectedOutput = `speech0:$char7. pct:$char8. speech:$char7. slash:$char6. slashWithSpecial:$char7. macvar:$char10. chinese:$char14. sigma:$char7. at:$char3. serbian:$char12. dollar:$char1.\r\n"""speech",%percent,"""speech",\\slash,\"\\\tslash\",&sysuserid,传/傳chinese,Σsigma,@at,Српски,$`

-    expect(convertToCSV(data)).toEqual(expectedOutput)
+    expect(convertToCSV(data, tableName)).toEqual(expectedOutput)

-    expect(convertToCSV([{ speech: 'menext' }])).toEqual(
-      `speech:$char6.\r\nmenext`
-    )
-    expect(convertToCSV([{ speech: 'me\nnext' }])).toEqual(
-      `speech:$char7.\r\n\"me\nnext\"`
-    )
-    expect(convertToCSV([{ speech: `me'next` }])).toEqual(
-      `speech:$char7.\r\n\"me'next\"`
-    )
-    expect(convertToCSV([{ speech: `me"next` }])).toEqual(
-      `speech:$char7.\r\n\"me""next\"`
-    )
-    expect(convertToCSV([{ speech: `me""next` }])).toEqual(
-      `speech:$char8.\r\n\"me""""next\"`
-    )
-    expect(convertToCSV([{ slashWithSpecial: '\\\tslash' }])).toEqual(
-      `slashWithSpecial:$char7.\r\n\"\\\tslash\"`
-    )
-    expect(convertToCSV([{ slashWithSpecial: '\\ \tslash' }])).toEqual(
-      `slashWithSpecial:$char8.\r\n\"\\ \tslash\"`
-    )
    expect(
-      convertToCSV([{ slashWithSpecialExtra: '\\\ts\tl\ta\ts\t\th\t' }])
+      convertToCSV({ [tableName]: [{ speech: 'menext' }] }, tableName)
+    ).toEqual(`speech:$char6.\r\nmenext`)
+    expect(
+      convertToCSV({ [tableName]: [{ speech: 'me\nnext' }] }, tableName)
+    ).toEqual(`speech:$char7.\r\n\"me\nnext\"`)
+    expect(
+      convertToCSV({ [tableName]: [{ speech: `me'next` }] }, tableName)
+    ).toEqual(`speech:$char7.\r\n\"me'next\"`)
+    expect(
+      convertToCSV({ [tableName]: [{ speech: `me"next` }] }, tableName)
+    ).toEqual(`speech:$char7.\r\n\"me""next\"`)
+    expect(
+      convertToCSV({ [tableName]: [{ speech: `me""next` }] }, tableName)
+    ).toEqual(`speech:$char8.\r\n\"me""""next\"`)
+    expect(
+      convertToCSV(
+        { [tableName]: [{ slashWithSpecial: '\\\tslash' }] },
+        tableName
+      )
+    ).toEqual(`slashWithSpecial:$char7.\r\n\"\\\tslash\"`)
+    expect(
+      convertToCSV(
+        { [tableName]: [{ slashWithSpecial: '\\ \tslash' }] },
+        tableName
+      )
+    ).toEqual(`slashWithSpecial:$char8.\r\n\"\\ \tslash\"`)
+    expect(
+      convertToCSV(
+        { [tableName]: [{ slashWithSpecialExtra: '\\\ts\tl\ta\ts\t\th\t' }] },
+        tableName
+      )
    ).toEqual(`slashWithSpecialExtra:$char13.\r\n\"\\\ts\tl\ta\ts\t\th\t\"`)
  })

  it('should console log error if data has mixed types', () => {
    const colName = 'var1'
-    const data = [{ [colName]: 'string' }, { [colName]: 232 }]
+    const data = { [tableName]: [{ [colName]: 'string' }, { [colName]: 232 }] }

    jest.spyOn(console, 'error').mockImplementation(() => {})

-    convertToCSV(data)
+    convertToCSV(data, tableName)

    expect(console.error).toHaveBeenCalledWith(
      `Row (2), Column (${colName}) has mixed types: ERROR`
    )
  })
+
+  it('should throw an error if table was not found in data object', () => {
+    const data = { [tableName]: [{ var1: 'string' }] }
+
+    expect(() => convertToCSV(data, 'wrongTableName')).toThrow(
+      new Error('No table provided to be converted to CSV')
+    )
+  })
 })
--- a/src/utils/convertToCsv.ts
+++ b/src/utils/convertToCsv.ts
@@ -3,10 +3,15 @@
 * @param data - the array of JSON objects to convert.
 */
 export const convertToCSV = (
-  data: any[],
-  sasFormats?: { formats: { [key: string]: string } }
+  data: { [key: string]: any },
+  tableName: string
 ) => {
-  let formats = sasFormats?.formats
+  if (!data[tableName]) {
+    throw new Error('No table provided to be converted to CSV')
+  }
+
+  const table = data[tableName]
+  let formats = data[`$${tableName}`]?.formats
  let headers: string[] = []
  let csvTest
  let invalidString = false
@@ -16,14 +21,14 @@ export const convertToCSV = (
    headers = Object.keys(formats).map((key) => `${key}:${formats![key]}`)
  }

-  const headerFields = Object.keys(data[0])
+  const headerFields = Object.keys(table[0])

  headerFields.forEach((field) => {
    if (!formats || !Object.keys(formats).includes(field)) {
      let hasNullOrNumber = false
      let hasSpecialMissingString = false

-      data.forEach((row: { [key: string]: any }) => {
+      table.forEach((row: { [key: string]: any }) => {
        if (row[field] === null || typeof row[field] === 'number') {
          hasNullOrNumber = true
        } else if (
@@ -45,7 +50,7 @@ export const convertToCSV = (
        let hasMixedTypes: boolean = false
        let rowNumError: number = -1

-        const longestValueForField = data
+        const longestValueForField = table
          .map((row: any, index: number) => {
            if (row[field] || row[field] === '') {
              if (firstFoundType) {
@@ -101,7 +106,7 @@ export const convertToCSV = (
    }
  })

-  if (sasFormats) {
+  if (formats) {
    headers = headers.sort(
      (a, b) =>
        headerFields.indexOf(a.replace(/:.*/, '')) -
@@ -111,7 +116,7 @@ export const convertToCSV = (

  if (invalidString) return 'ERROR: LARGE STRING LENGTH'

-  csvTest = data.map((row: any) => {
+  csvTest = table.map((row: any) => {
    const fields = Object.keys(row).map((fieldName, index) => {
      let value
      const currentCell = row[fieldName]
--- a/src/utils/formatDataForRequest.ts
+++ b/src/utils/formatDataForRequest.ts
@@ -15,19 +15,24 @@ export const formatDataForRequest = (data: any) => {
    }

    tableCounter++
+
    sasjsTables.push(tableName)
-    const csv = convertToCSV(data[tableName], data[`$${tableName}`])
+
+    const csv = convertToCSV(data, tableName)

    if (csv === 'ERROR: LARGE STRING LENGTH') {
      throw new Error(
        'The max length of a string value in SASjs is 32765 characters.'
      )
    }
+
    // if csv has length more then 16k, send in chunks
    if (csv.length > 16000) {
      const csvChunks = splitChunks(csv)
+
      // append chunks to form data with same key
      result[`sasjs${tableCounter}data0`] = csvChunks.length
+
      csvChunks.forEach((chunk, index) => {
        result[`sasjs${tableCounter}data${index + 1}`] = chunk
      })