feat(nullVars): add SAS null vars support

sasjs-tests/src/testSuites/SpecialCases.ts

@@ -79,6 +79,19 @@ const errorAndCsrfData: any = {
   _csrf: [{ col1: 'q', col2: 'w', col3: 'e', col4: 'r' }]
 }
 
+const testTable = 'sometable'
+const testTableWithNullVars = {
+  [testTable]: [
+    { var1: 'string', var2: 232, nullvar: 'A' },
+    { var1: 'string', var2: 232, nullvar: 'B' },
+    { var1: 'string', var2: 232, nullvar: '_' },
+    { var1: 'string', var2: 232, nullvar: 0 },
+    { var1: 'string', var2: 232, nullvar: 'z' },
+    { var1: 'string', var2: 232, nullvar: null }
+  ],
+  [`$${testTable}`]: { formats: { var1: '$char12.', nullvar: 'best.' } }
+}
+
 export const specialCaseTests = (adapter: SASjs): TestSuite => ({
   name: 'Special Cases',
   tests: [
@@ -247,6 +260,26 @@ export const specialCaseTests = (adapter: SASjs): TestSuite => ({
           res._csrf[0].COL4 === errorAndCsrfData._csrf[0].col4
         )
       }
+    },
+    {
+      title: 'Special missing values',
+      description: 'Should support special missing values',
+      test: () => {
+        return adapter.request('common/sendObj', testTableWithNullVars)
+      },
+      assertion: (res: any) => {
+        let assertionRes = true
+
+        testTableWithNullVars[testTable].forEach((row: {}, i: number) =>
+          Object.keys(row).forEach((col: string) => {
+            if (col !== res[testTable][i][col.toUpperCase()]) {
+              assertionRes = false
+            }
+          })
+        )
+
+        return assertionRes
+      }
     }
   ]
 })

src/SASjs.ts

@@ -736,15 +736,19 @@ export default class SASjs {
     msg: string
   } {
     if (data === null) return { status: true, msg: '' }
+
+    const isSasFormatsTable = (key: string) =>
+      key.match(/^\$.*/) && Object.keys(data).includes(key.replace(/^\$/, ''))
+
     for (const key in data) {
-      if (!key.match(/^[a-zA-Z_]/)) {
+      if (!key.match(/^[a-zA-Z_]/) && !isSasFormatsTable(key)) {
         return {
           status: false,
           msg: 'First letter of table should be alphabet or underscore.'
         }
       }
 
-      if (!key.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) {
+      if (!key.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/) && !isSasFormatsTable(key)) {
         return { status: false, msg: 'Table name should be alphanumeric.' }
       }
 
@@ -755,7 +759,7 @@ export default class SASjs {
         }
       }
 
-      if (this.getType(data[key]) !== 'Array') {
+      if (this.getType(data[key]) !== 'Array' && !isSasFormatsTable(key)) {
         return {
           status: false,
           msg: 'Parameter data contains invalid table structure.'
@@ -771,6 +775,7 @@ export default class SASjs {
         }
       }
     }
+
     return { status: true, msg: '' }
   }
 

src/utils/convertToCsv.spec.ts

@@ -167,4 +167,42 @@ describe('convertToCsv', () => {
       convertToCSV([{ slashWithSpecialExtra: '\\\ts\tl\ta\ts\t\th\t' }])
     ).toEqual(`slashWithSpecialExtra:$char13.\r\n\"\\\ts\tl\ta\ts\t\th\t\"`)
   })
+
+  it('should convert not null values', () => {
+    const data = [
+      { var1: 'string', nullvar: 'A', var2: 232 },
+      { var1: 'string', nullvar: 'B', var2: 232 },
+      { var1: 'string', nullvar: '_', var2: 232 },
+      { var1: 'string', nullvar: 0, var2: 232 },
+      { var1: 'string', nullvar: 'z', var2: 232 },
+      { var1: 'string', nullvar: null, var2: 232 }
+    ]
+
+    const expectedOutput = `var1:$char6. nullvar:best. var2:best.\r\nstring,.a,232\r\nstring,.b,232\r\nstring,._,232\r\nstring,0,232\r\nstring,.z,232\r\nstring,.,232`
+
+    expect(
+      convertToCSV(data, {
+        formats: { var1: '$char6.', nullvar: 'best.' }
+      })
+    ).toEqual(expectedOutput)
+  })
+
+  it('should return error if string is more than maxFieldValue', () => {
+    const data = [{ var1: 'z'.repeat(32765 + 1) }]
+
+    expect(convertToCSV(data)).toEqual('ERROR: LARGE STRING LENGTH')
+  })
+
+  it('should console log error if data has mixed types', () => {
+    const colName = 'var1'
+    const data = [{ [colName]: 'string' }, { [colName]: 232 }]
+
+    jest.spyOn(console, 'error').mockImplementation(() => {})
+
+    convertToCSV(data)
+
+    expect(console.error).toHaveBeenCalledWith(
+      `Row (2), Column (${colName}) has mixed types: ERROR`
+    )
+  })
 })

src/utils/convertToCsv.ts

@@ -2,70 +2,92 @@
  * Converts the given JSON object array to a CSV string.
  * @param data - the array of JSON objects to convert.
  */
-export const convertToCSV = (data: any) => {
+export const convertToCSV = (
-  const replacer = (key: any, value: any) => (value === null ? '' : value)
+  data: any,
-  const headerFields = Object.keys(data[0])
+  sasFormats?: { formats: { [key: string]: string } }
+) => {
+  const formats = sasFormats?.formats
+  let headers: string[] = []
   let csvTest
   let invalidString = false
-  const headers = headerFields.map((field) => {
-    let firstFoundType: string | null = null
-    let hasMixedTypes: boolean = false
-    let rowNumError: number = -1
 
-    const longestValueForField = data
+  if (formats) {
-      .map((row: any, index: number) => {
+    headers = Object.keys(formats).map((key) => `${key}:${formats[key]}`)
-        if (row[field] || row[field] === '') {
+  }
-          if (firstFoundType) {
-            let currentFieldType =
-              row[field] === '' || typeof row[field] === 'string'
-                ? 'chars'
-                : 'number'
 
-            if (!hasMixedTypes) {
+  const headerFields = Object.keys(data[0])
-              hasMixedTypes = currentFieldType !== firstFoundType
+
-              rowNumError = hasMixedTypes ? index + 1 : -1
+  headerFields.forEach((field) => {
-            }
+    if (!formats || !Object.keys(formats).includes(field)) {
-          } else {
+      let firstFoundType: string | null = null
-            if (row[field] === '') {
+      let hasMixedTypes: boolean = false
-              firstFoundType = 'chars'
+      let rowNumError: number = -1
+
+      const longestValueForField = data
+        .map((row: any, index: number) => {
+          if (row[field] || row[field] === '') {
+            if (firstFoundType) {
+              let currentFieldType =
+                row[field] === '' || typeof row[field] === 'string'
+                  ? 'chars'
+                  : 'number'
+
+              if (!hasMixedTypes) {
+                hasMixedTypes = currentFieldType !== firstFoundType
+                rowNumError = hasMixedTypes ? index + 1 : -1
+              }
             } else {
-              firstFoundType =
+              if (row[field] === '') {
-                typeof row[field] === 'string' ? 'chars' : 'number'
+                firstFoundType = 'chars'
+              } else {
+                firstFoundType =
+                  typeof row[field] === 'string' ? 'chars' : 'number'
+              }
             }
+
+            let byteSize
+
+            if (typeof row[field] === 'string') {
+              byteSize = getByteSize(row[field])
+            }
+
+            return byteSize
           }
+        })
+        .sort((a: number, b: number) => b - a)[0]
 
-          let byteSize
+      if (longestValueForField && longestValueForField > 32765) {
+        invalidString = true
+      }
 
-          if (typeof row[field] === 'string') {
+      if (hasMixedTypes) {
-            byteSize = getByteSize(row[field])
+        console.error(
-          }
+          `Row (${rowNumError}), Column (${field}) has mixed types: ERROR`
+        )
+      }
 
-          return byteSize
+      headers.push(
-        }
+        `${field}:${firstFoundType === 'chars' ? '$char' : ''}${
-      })
+          longestValueForField
-      .sort((a: number, b: number) => b - a)[0]
+            ? longestValueForField
-    if (longestValueForField && longestValueForField > 32765) {
+            : firstFoundType === 'chars'
-      invalidString = true
+            ? '1'
-    }
+            : 'best'
-    if (hasMixedTypes) {
+        }.`
-      console.error(
-        `Row (${rowNumError}), Column (${field}) has mixed types: ERROR`
       )
     }
-
-    return `${field}:${firstFoundType === 'chars' ? '$char' : ''}${
-      longestValueForField
-        ? longestValueForField
-        : firstFoundType === 'chars'
-        ? '1'
-        : 'best'
-    }.`
   })
 
-  if (invalidString) {
+  if (sasFormats) {
-    return 'ERROR: LARGE STRING LENGTH'
+    headers = headers.sort(
+      (a, b) =>
+        headerFields.indexOf(a.replace(/:.*/, '')) -
+        headerFields.indexOf(b.replace(/:.*/, ''))
+    )
   }
 
+  if (invalidString) return 'ERROR: LARGE STRING LENGTH'
+
   csvTest = data.map((row: any) => {
     const fields = Object.keys(row).map((fieldName, index) => {
       let value
@@ -76,6 +98,10 @@ export const convertToCSV = (data: any) => {
       // stringify with replacer converts null values to empty strings
       value = currentCell === null ? '' : currentCell
 
+      if (formats && formats[fieldName] === 'best.') {
+        return `.${value.toLowerCase()}`
+      }
+
       // if there any present, it should have preceding (") for escaping
       value = value.replace(/"/g, `""`)
 

src/utils/formatDataForRequest.ts

@@ -7,9 +7,17 @@ export const formatDataForRequest = (data: any) => {
   const result: any = {}
 
   for (const tableName in data) {
+    if (
+      tableName.match(/^\$.*/) &&
+      Object.keys(data).includes(tableName.replace(/^\$/, ''))
+    ) {
+      continue
+    }
+
     tableCounter++
     sasjsTables.push(tableName)
-    const csv = convertToCSV(data[tableName])
+    const csv = convertToCSV(data[tableName], data[`$${tableName}`])
+
     if (csv === 'ERROR: LARGE STRING LENGTH') {
       throw new Error(
         'The max length of a string value in SASjs is 32765 characters.'
@@ -27,6 +35,7 @@ export const formatDataForRequest = (data: any) => {
       result[`sasjs${tableCounter}data`] = csv
     }
   }
+
   result['sasjs_tables'] = sasjsTables.join(' ')
 
   return result