feat: improve convertToCsv function

sasjs-tests/src/testSuites/SpecialCases.ts

@@ -80,7 +80,7 @@ const errorAndCsrfData: any = {
 }
 
 const testTable = 'sometable'
-const testTableWithNullVars = {
+const testTableWithNullVars: { [key: string]: any } = {
   [testTable]: [
     { var1: 'string', var2: 232, nullvar: 'A' },
     { var1: 'string', var2: 232, nullvar: 'B' },
@@ -270,12 +270,25 @@ export const specialCaseTests = (adapter: SASjs): TestSuite => ({
       assertion: (res: any) => {
         let assertionRes = true
 
-        testTableWithNullVars[testTable].forEach((row: {}, i: number) =>
-          Object.keys(row).forEach((col: string) => {
-            if (col !== res[testTable][i][col.toUpperCase()]) {
-              assertionRes = false
-            }
-          })
+        testTableWithNullVars[testTable].forEach(
+          (row: { [key: string]: any }, i: number) =>
+            Object.keys(row).forEach((col: string) => {
+              const resValue = res[testTable][i][col.toUpperCase()]
+
+              if (
+                typeof row[col] === 'string' &&
+                testTableWithNullVars[`$${testTable}`].formats[col] ===
+                  'best.' &&
+                row[col].toUpperCase() !== resValue
+              ) {
+                assertionRes = false
+              } else if (
+                typeof row[col] !== 'string' &&
+                row[col] !== resValue
+              ) {
+                assertionRes = false
+              }
+            })
         )
 
         return assertionRes

src/test/utils/formatDataForRequest.spec.ts

@@ -3,8 +3,8 @@ import { formatDataForRequest } from '../../utils/formatDataForRequest'
 describe('formatDataForRequest', () => {
   const testTable = 'sometable'
 
-  it('should', () => {
-    const testTableWithNullVars = {
+  it('should format table with special missing values', () => {
+    const tableWithMissingValues = {
       [testTable]: [
         { var1: 'string', var2: 232, nullvar: 'A' },
         { var1: 'string', var2: 232, nullvar: 'B' },
@@ -21,7 +21,7 @@ describe('formatDataForRequest', () => {
       sasjs_tables: testTable
     }
 
-    expect(formatDataForRequest(testTableWithNullVars)).toEqual(expectedOutput)
+    expect(formatDataForRequest(tableWithMissingValues)).toEqual(expectedOutput)
   })
 
   it('should return error if string is more than 32765 characters', () => {
@@ -53,4 +53,43 @@ describe('formatDataForRequest', () => {
 
     expect(formatDataForRequest(data)).toEqual(expectedOutput)
   })
+
+  it('should throw an error if special missing values is not valid', () => {
+    let tableWithMissingValues = {
+      [testTable]: [{ var: 'AA' }, { var: 0 }],
+      [`$${testTable}`]: { formats: { var: 'best.' } }
+    }
+
+    expect(() => formatDataForRequest(tableWithMissingValues)).toThrow(
+      new Error(
+        'Special missing value can only be a single character from A to Z or _'
+      )
+    )
+  })
+
+  it('should auto-detect special missing values type as best.', () => {
+    const tableWithMissingValues = {
+      [testTable]: [{ var: 'a' }, { var: 'A' }, { var: '_' }, { var: 0 }]
+    }
+
+    const expectedOutput = {
+      sasjs1data: `var:best.\r\n.a\r\n.a\r\n._\r\n0`,
+      sasjs_tables: testTable
+    }
+
+    expect(formatDataForRequest(tableWithMissingValues)).toEqual(expectedOutput)
+  })
+
+  it('should auto-detect values type as $char1.', () => {
+    const tableWithMissingValues = {
+      [testTable]: [{ var: 'a' }, { var: 'A' }, { var: '_' }]
+    }
+
+    const expectedOutput = {
+      sasjs1data: `var:$char1.\r\na\r\nA\r\n_`,
+      sasjs_tables: testTable
+    }
+
+    expect(formatDataForRequest(tableWithMissingValues)).toEqual(expectedOutput)
+  })
 })

src/utils/convertToCsv.ts

@@ -6,75 +6,98 @@ export const convertToCSV = (
   data: any,
   sasFormats?: { formats: { [key: string]: string } }
 ) => {
-  const formats = sasFormats?.formats
+  let formats = sasFormats?.formats
   let headers: string[] = []
   let csvTest
   let invalidString = false
+  const specialMissingValueRegExp = /^[a-z_]{1}$/i
 
   if (formats) {
-    headers = Object.keys(formats).map((key) => `${key}:${formats[key]}`)
+    headers = Object.keys(formats).map((key) => `${key}:${formats![key]}`)
   }
 
   const headerFields = Object.keys(data[0])
 
   headerFields.forEach((field) => {
     if (!formats || !Object.keys(formats).includes(field)) {
-      let firstFoundType: string | null = null
-      let hasMixedTypes: boolean = false
-      let rowNumError: number = -1
+      let hasNullOrNumber = false
+      let hasSpecialMissingString = false
 
-      const longestValueForField = data
-        .map((row: any, index: number) => {
-          if (row[field] || row[field] === '') {
-            if (firstFoundType) {
-              let currentFieldType =
-                row[field] === '' || typeof row[field] === 'string'
-                  ? 'chars'
-                  : 'number'
+      data.forEach((row: { [key: string]: any }) => {
+        if (row[field] === null || typeof row[field] === 'number') {
+          hasNullOrNumber = true
+        } else if (
+          typeof row[field] === 'string' &&
+          specialMissingValueRegExp.test(row[field])
+        ) {
+          hasSpecialMissingString = true
+        }
+      })
 
-              if (!hasMixedTypes) {
-                hasMixedTypes = currentFieldType !== firstFoundType
-                rowNumError = hasMixedTypes ? index + 1 : -1
-              }
-            } else {
-              if (row[field] === '') {
-                firstFoundType = 'chars'
+      if (hasNullOrNumber && hasSpecialMissingString) {
+        headers.push(`${field}:best.`)
+
+        if (!formats) formats = {}
+
+        formats[field] = 'best.'
+      } else {
+        let firstFoundType: string | null = null
+        let hasMixedTypes: boolean = false
+        let rowNumError: number = -1
+
+        const longestValueForField = data
+          .map((row: any, index: number) => {
+            if (row[field] || row[field] === '') {
+              if (firstFoundType) {
+                let currentFieldType =
+                  row[field] === '' || typeof row[field] === 'string'
+                    ? 'chars'
+                    : 'number'
+
+                if (!hasMixedTypes) {
+                  hasMixedTypes = currentFieldType !== firstFoundType
+                  rowNumError = hasMixedTypes ? index + 1 : -1
+                }
               } else {
-                firstFoundType =
-                  typeof row[field] === 'string' ? 'chars' : 'number'
+                if (row[field] === '') {
+                  firstFoundType = 'chars'
+                } else {
+                  firstFoundType =
+                    typeof row[field] === 'string' ? 'chars' : 'number'
+                }
               }
+
+              let byteSize
+
+              if (typeof row[field] === 'string') {
+                byteSize = getByteSize(row[field])
+              }
+
+              return byteSize
             }
+          })
+          .sort((a: number, b: number) => b - a)[0]
 
-            let byteSize
+        if (longestValueForField && longestValueForField > 32765) {
+          invalidString = true
+        }
 
-            if (typeof row[field] === 'string') {
-              byteSize = getByteSize(row[field])
-            }
+        if (hasMixedTypes) {
+          console.error(
+            `Row (${rowNumError}), Column (${field}) has mixed types: ERROR`
+          )
+        }
 
-            return byteSize
-          }
-        })
-        .sort((a: number, b: number) => b - a)[0]
-
-      if (longestValueForField && longestValueForField > 32765) {
-        invalidString = true
-      }
-
-      if (hasMixedTypes) {
-        console.error(
-          `Row (${rowNumError}), Column (${field}) has mixed types: ERROR`
+        headers.push(
+          `${field}:${firstFoundType === 'chars' ? '$char' : ''}${
+            longestValueForField
+              ? longestValueForField
+              : firstFoundType === 'chars'
+              ? '1'
+              : 'best'
+          }.`
         )
       }
-
-      headers.push(
-        `${field}:${firstFoundType === 'chars' ? '$char' : ''}${
-          longestValueForField
-            ? longestValueForField
-            : firstFoundType === 'chars'
-            ? '1'
-            : 'best'
-        }.`
-      )
     }
   })
 
@@ -99,6 +122,13 @@ export const convertToCSV = (
       value = currentCell === null ? '' : currentCell
 
       if (formats && formats[fieldName] === 'best.') {
+        if (value && !specialMissingValueRegExp.test(value)) {
+          console.log(`🤖[value]🤖`, value)
+          throw new Error(
+            'Special missing value can only be a single character from A to Z or _'
+          )
+        }
+
         return `.${value.toLowerCase()}`
       }