feat: improve convertToCsv function

sasjs-tests/src/testSuites/SpecialCases.ts

@@ -80,7 +80,7 @@ const errorAndCsrfData: any = {
 }
 
 const testTable = 'sometable'
-const testTableWithNullVars = {
+const testTableWithNullVars: { [key: string]: any } = {
   [testTable]: [
     { var1: 'string', var2: 232, nullvar: 'A' },
     { var1: 'string', var2: 232, nullvar: 'B' },
@@ -270,9 +270,22 @@ export const specialCaseTests = (adapter: SASjs): TestSuite => ({
       assertion: (res: any) => {
         let assertionRes = true
 
-        testTableWithNullVars[testTable].forEach((row: {}, i: number) =>
+        testTableWithNullVars[testTable].forEach(
+          (row: { [key: string]: any }, i: number) =>
             Object.keys(row).forEach((col: string) => {
-            if (col !== res[testTable][i][col.toUpperCase()]) {
+              const resValue = res[testTable][i][col.toUpperCase()]
+
+              if (
+                typeof row[col] === 'string' &&
+                testTableWithNullVars[`$${testTable}`].formats[col] ===
+                  'best.' &&
+                row[col].toUpperCase() !== resValue
+              ) {
+                assertionRes = false
+              } else if (
+                typeof row[col] !== 'string' &&
+                row[col] !== resValue
+              ) {
                 assertionRes = false
               }
             })

src/test/utils/formatDataForRequest.spec.ts

@@ -3,8 +3,8 @@ import { formatDataForRequest } from '../../utils/formatDataForRequest'
 describe('formatDataForRequest', () => {
   const testTable = 'sometable'
 
-  it('should', () => {
+  it('should format table with special missing values', () => {
-    const testTableWithNullVars = {
+    const tableWithMissingValues = {
       [testTable]: [
         { var1: 'string', var2: 232, nullvar: 'A' },
         { var1: 'string', var2: 232, nullvar: 'B' },
@@ -21,7 +21,7 @@ describe('formatDataForRequest', () => {
       sasjs_tables: testTable
     }
 
-    expect(formatDataForRequest(testTableWithNullVars)).toEqual(expectedOutput)
+    expect(formatDataForRequest(tableWithMissingValues)).toEqual(expectedOutput)
   })
 
   it('should return error if string is more than 32765 characters', () => {
@@ -53,4 +53,43 @@ describe('formatDataForRequest', () => {
 
     expect(formatDataForRequest(data)).toEqual(expectedOutput)
   })
+
+  it('should throw an error if special missing values is not valid', () => {
+    let tableWithMissingValues = {
+      [testTable]: [{ var: 'AA' }, { var: 0 }],
+      [`$${testTable}`]: { formats: { var: 'best.' } }
+    }
+
+    expect(() => formatDataForRequest(tableWithMissingValues)).toThrow(
+      new Error(
+        'Special missing value can only be a single character from A to Z or _'
+      )
+    )
+  })
+
+  it('should auto-detect special missing values type as best.', () => {
+    const tableWithMissingValues = {
+      [testTable]: [{ var: 'a' }, { var: 'A' }, { var: '_' }, { var: 0 }]
+    }
+
+    const expectedOutput = {
+      sasjs1data: `var:best.\r\n.a\r\n.a\r\n._\r\n0`,
+      sasjs_tables: testTable
+    }
+
+    expect(formatDataForRequest(tableWithMissingValues)).toEqual(expectedOutput)
+  })
+
+  it('should auto-detect values type as $char1.', () => {
+    const tableWithMissingValues = {
+      [testTable]: [{ var: 'a' }, { var: 'A' }, { var: '_' }]
+    }
+
+    const expectedOutput = {
+      sasjs1data: `var:$char1.\r\na\r\nA\r\n_`,
+      sasjs_tables: testTable
+    }
+
+    expect(formatDataForRequest(tableWithMissingValues)).toEqual(expectedOutput)
+  })
 })

src/utils/convertToCsv.ts

@@ -6,19 +6,41 @@ export const convertToCSV = (
   data: any,
   sasFormats?: { formats: { [key: string]: string } }
 ) => {
-  const formats = sasFormats?.formats
+  let formats = sasFormats?.formats
   let headers: string[] = []
   let csvTest
   let invalidString = false
+  const specialMissingValueRegExp = /^[a-z_]{1}$/i
 
   if (formats) {
-    headers = Object.keys(formats).map((key) => `${key}:${formats[key]}`)
+    headers = Object.keys(formats).map((key) => `${key}:${formats![key]}`)
   }
 
   const headerFields = Object.keys(data[0])
 
   headerFields.forEach((field) => {
     if (!formats || !Object.keys(formats).includes(field)) {
+      let hasNullOrNumber = false
+      let hasSpecialMissingString = false
+
+      data.forEach((row: { [key: string]: any }) => {
+        if (row[field] === null || typeof row[field] === 'number') {
+          hasNullOrNumber = true
+        } else if (
+          typeof row[field] === 'string' &&
+          specialMissingValueRegExp.test(row[field])
+        ) {
+          hasSpecialMissingString = true
+        }
+      })
+
+      if (hasNullOrNumber && hasSpecialMissingString) {
+        headers.push(`${field}:best.`)
+
+        if (!formats) formats = {}
+
+        formats[field] = 'best.'
+      } else {
         let firstFoundType: string | null = null
         let hasMixedTypes: boolean = false
         let rowNumError: number = -1
@@ -76,6 +98,7 @@ export const convertToCSV = (
           }.`
         )
       }
+    }
   })
 
   if (sasFormats) {
@@ -99,6 +122,13 @@ export const convertToCSV = (
       value = currentCell === null ? '' : currentCell
 
       if (formats && formats[fieldName] === 'best.') {
+        if (value && !specialMissingValueRegExp.test(value)) {
+          console.log(`🤖[value]🤖`, value)
+          throw new Error(
+            'Special missing value can only be a single character from A to Z or _'
+          )
+        }
+
         return `.${value.toLowerCase()}`
       }