Merge pull request #615 from sasjs/issue-607

Support special missing values

.gitpod.yml

@@ -0,0 +1,2 @@
+tasks:
+  - init: npm install && npm run build

README.md

@@ -142,6 +142,71 @@ The response object will contain returned tables and columns.  Table names are a
 
 The adapter will also cache the logs (if debug enabled) and even the work tables.  For performance, it is best to keep debug mode off.
 
+### Variable Types
+
+The SAS type (char/numeric) of the values is determined according to a set of rules:
+
+* If the values are numeric, the SAS type is numeric
+* If the values are all string, the SAS type is character
+* If the values contain a single character (a-Z + underscore) AND a numeric, then the SAS type is numeric (with special missing values).  
+* `null` is set to either '.' or '' depending on the assigned or derived type per the above rules.  If entire column is `null` then the type will be numeric.
+
+The following table illustrates the formats applied to columns under various scenarios:
+
+|JS Values |SAS Format|
+|---|---|
+|'a', 'a' |$char1.|
+|0, '_' |best.|
+|'Z', 0 |best.|
+|'a', 'aaa' |$char3.|
+|null, 'a', 'aaa' | $char3.|
+|null, 'a', 0 | best.|
+|null, null | best.|
+|null, '' | $char1.|
+|null, 'a' | $char1.|
+|'a' | $char1.|
+|'a', null | $char1.|
+|'a', null, 0 | best.|
+
+Validation is also performed on the values.  The following combinations will throw errors:
+
+|JS Values |SAS Format|
+|---|---|
+|null, 'aaaa', 0 | Error: mixed types. 'aaaa' is not a special missing value.|
+|0, 'a', '!' | Error: mixed types. '!' is not a special missing value|
+|1.1, '.', 0| Error: mixed types.  For regular nulls, use `null`|
+
+### Variable Format Override
+The auto-detect functionality above is thwarted in the following scenarios:
+
+* A character column containing only `null` values (is considered numeric)
+* A numeric column containing only special missing values (is considered character)
+
+To cater for these scenarios, an optional array of formats can be passed along with the data to ensure that SAS will read them in correctly.
+
+To understand these formats, it should be noted that the JSON data is NOT passed directly (as JSON) to SAS. It is first converted into CSV, and the header row is actually an `infile` statement in disguise.  It looks a bit like this:
+
+```csv
+CHARVAR1:$char4. CHARVAR2:$char1. NUMVAR:best.
+LOAD,,0
+ABCD,X,.
+```
+
+To provide overrides to this header row, the tables object can be constructed as follows (with a leading '$' in the table name):
+
+```javascript
+let specialData={
+  "tablewith2cols2rows": [
+    {"col1": "val1","specialMissingsCol": "A"},
+    {"col1": "val2","specialMissingsCol": "_"}
+  ],
+  "$tablewith2cols2rows":{"formats":{"specialMissingsCol":"best."}
+  }
+};
+```
+
+It is not necessary to provide formats for ALL the columns, only the ones that need to be overridden.
+
 ## SAS Inputs / Outputs
 
 The SAS side is handled by a number of macros in the [macro core](https://github.com/sasjs/core) library.
@@ -153,16 +218,29 @@ The following snippet shows the process of SAS tables arriving / leaving:
 %webout(FETCH)
 
 /* some sas code */
-data some sas tables;
+data a b c;
   set from js;
 run;
 
-%webout(OPEN)  /* open the JSON to be returned */
-%webout(OBJ,some) /* `some` table is sent in object format */
-%webout(ARR,sas) /* `sas` table is sent in array format, smaller filesize */
-%webout(OBJ,tables,fmt=N) /* unformatted (raw) data */
-%webout(OBJ,tables,label=newtable) /* rename tables on export */
-%webout(CLOSE) /* close the JSON and send some extra useful variables too */
+%webout(OPEN)  /* Open the JSON to be returned */
+%webout(OBJ,a) /* Rows in table `a` are objects (easy to use) */
+%webout(ARR,b) /* Rows in table `b` are arrays (compact) */
+%webout(OBJ,c,fmt=N) /* Table `c` is sent unformatted (raw) */
+%webout(OBJ,c,label=d) /* Rename as `d` on JS side */
+%webout(CLOSE) /* Close the JSON and add default variables */
+```
+
+By default, special SAS numeric missings (_a-Z) are converted to `null` in the JSON.  If you'd like to preserve these, use the `missing=STRING` option as follows:
+
+```sas
+%webout(OBJ,a,missing=STRING)
+```
+In this case, special missings (such as `.a`, `.b`) are converted to javascript string values (`'A', 'B'`).
+
+Where an entire column is made up of special missing numerics, there would be no way to distinguish it from a single-character column by looking at the values.  To cater for this scenario, it is possible to export the variable types (and other attributes such as label and format) by adding a `showmeta` param to the `webout()` macro as follows:
+
+```sas
+%webout(OBJ,a,missing=STRING,showmeta=YES)
 ```
 
 ## Configuration
@@ -170,7 +248,7 @@ run;
 Configuration on the client side involves passing an object on startup, which can also be passed with each request.  Technical documentation on the SASjsConfig class is available [here](https://adapter.sasjs.io/classes/types.sasjsconfig.html).  The main config items are:
 
 * `appLoc` - this is the folder under which the SAS services will be created.
-* `serverType` - either `SAS9` or `SASVIYA`.
+* `serverType` - either `SAS9`, `SASVIYA` or `SASJS`.  The `SASJS` server type is for use with [sasjs/server](https://github.com/sasjs/server).
 * `serverUrl` - the location (including http protocol and port) of the SAS Server. Can be omitted, eg if serving directly from the SAS Web Server, or in streaming mode.
 * `debug` - if `true` then SAS Logs and extra debug information is returned.
 * `LoginMechanism` - either `Default` or `Redirected`.  If `Redirected` then authentication occurs through the injection of an additional screen, which contains the SASLogon prompt.  This allows for more complex authentication flows (such as 2FA) and avoids the need to handle passwords in the application itself.  The styling of the redirect flow can also be modified.  If left at "Default" then the developer must capture the username and password and use these with the `.login()` method.

sasjs-tests/package.json

@@ -5,7 +5,7 @@
   "private": true,
   "dependencies": {
     "@sasjs/adapter": "file:../build/sasjs-adapter-5.0.0.tgz",
-    "@sasjs/test-framework": "^1.4.2",
+    "@sasjs/test-framework": "^1.4.3",
     "@types/jest": "^26.0.20",
     "@types/node": "^14.14.41",
     "@types/react": "^17.0.1",
@@ -22,7 +22,7 @@
     "build": "react-scripts build",
     "test": "react-scripts test",
     "eject": "react-scripts eject",
-    "update:adapter": "cd .. && npm run package:lib && cd sasjs-tests && npm i ../build/sasjs-adapter-5.0.0.tgz",
+    "update:adapter": "cd .. && npm run package:lib && cd sasjs-tests && npm i ../build/sasjs-adapter-5.0.0.tgz --legacy-peer-deps",
     "deploy:tests": "rsync -avhe ssh ./build/* --delete $SSH_ACCOUNT:$DEPLOY_PATH || npm run deploy:tests-win",
     "deploy:tests-win": "scp %DEPLOY_PATH% ./build/*",
     "deploy": "npm run update:adapter && npm run build && npm run deploy:tests"
@@ -43,6 +43,6 @@
     ]
   },
   "devDependencies": {
-    "node-sass": "^5.0.0"
+    "node-sass": "^6.0.1"
   }
 }

sasjs-tests/src/testSuites/SpecialCases.ts

@@ -79,6 +79,19 @@ const errorAndCsrfData: any = {
   _csrf: [{ col1: 'q', col2: 'w', col3: 'e', col4: 'r' }]
 }
 
+const testTable = 'sometable'
+const testTableWithNullVars: { [key: string]: any } = {
+  [testTable]: [
+    { var1: 'string', var2: 232, nullvar: 'A' },
+    { var1: 'string', var2: 232, nullvar: 'B' },
+    { var1: 'string', var2: 232, nullvar: '_' },
+    { var1: 'string', var2: 232, nullvar: 0 },
+    { var1: 'string', var2: 232, nullvar: 'z' },
+    { var1: 'string', var2: 232, nullvar: null }
+  ],
+  [`$${testTable}`]: { formats: { var1: '$char12.', nullvar: 'best.' } }
+}
+
 export const specialCaseTests = (adapter: SASjs): TestSuite => ({
   name: 'Special Cases',
   tests: [
@@ -247,6 +260,39 @@ export const specialCaseTests = (adapter: SASjs): TestSuite => ({
           res._csrf[0].COL4 === errorAndCsrfData._csrf[0].col4
         )
       }
+    },
+    {
+      title: 'Special missing values',
+      description: 'Should support special missing values',
+      test: () => {
+        return adapter.request('common/sendObj', testTableWithNullVars)
+      },
+      assertion: (res: any) => {
+        let assertionRes = true
+
+        testTableWithNullVars[testTable].forEach(
+          (row: { [key: string]: any }, i: number) =>
+            Object.keys(row).forEach((col: string) => {
+              const resValue = res[testTable][i][col.toUpperCase()]
+
+              if (
+                typeof row[col] === 'string' &&
+                testTableWithNullVars[`$${testTable}`].formats[col] ===
+                  'best.' &&
+                row[col].toUpperCase() !== resValue
+              ) {
+                assertionRes = false
+              } else if (
+                typeof row[col] !== 'string' &&
+                row[col] !== resValue
+              ) {
+                assertionRes = false
+              }
+            })
+        )
+
+        return assertionRes
+      }
     }
   ]
 })

src/SASjs.ts

@@ -736,15 +736,19 @@ export default class SASjs {
     msg: string
   } {
     if (data === null) return { status: true, msg: '' }
+
+    const isSasFormatsTable = (key: string) =>
+      key.match(/^\$.*/) && Object.keys(data).includes(key.replace(/^\$/, ''))
+
     for (const key in data) {
-      if (!key.match(/^[a-zA-Z_]/)) {
+      if (!key.match(/^[a-zA-Z_]/) && !isSasFormatsTable(key)) {
         return {
           status: false,
           msg: 'First letter of table should be alphabet or underscore.'
         }
       }
 
-      if (!key.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) {
+      if (!key.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/) && !isSasFormatsTable(key)) {
         return { status: false, msg: 'Table name should be alphanumeric.' }
       }
 
@@ -755,7 +759,7 @@ export default class SASjs {
         }
       }
 
-      if (this.getType(data[key]) !== 'Array') {
+      if (this.getType(data[key]) !== 'Array' && !isSasFormatsTable(key)) {
         return {
           status: false,
           msg: 'Parameter data contains invalid table structure.'
@@ -771,6 +775,7 @@ export default class SASjs {
         }
       }
     }
+
     return { status: true, msg: '' }
   }
 

src/test/utils/formatDataForRequest.spec.ts

@@ -0,0 +1,95 @@
+import { formatDataForRequest } from '../../utils/formatDataForRequest'
+
+describe('formatDataForRequest', () => {
+  const testTable = 'sometable'
+
+  it('should format table with special missing values', () => {
+    const tableWithMissingValues = {
+      [testTable]: [
+        { var1: 'string', var2: 232, nullvar: 'A' },
+        { var1: 'string', var2: 232, nullvar: 'B' },
+        { var1: 'string', var2: 232, nullvar: '_' },
+        { var1: 'string', var2: 232, nullvar: 0 },
+        { var1: 'string', var2: 232, nullvar: 'z' },
+        { var1: 'string', var2: 232, nullvar: null }
+      ],
+      [`$${testTable}`]: { formats: { var1: '$char12.', nullvar: 'best.' } }
+    }
+
+    const expectedOutput = {
+      sasjs1data: `var1:$char12. var2:best. nullvar:best.\r\nstring,232,.a\r\nstring,232,.b\r\nstring,232,._\r\nstring,232,0\r\nstring,232,.z\r\nstring,232,.`,
+      sasjs_tables: testTable
+    }
+
+    expect(formatDataForRequest(tableWithMissingValues)).toEqual(expectedOutput)
+  })
+
+  it('should return error if string is more than 32765 characters', () => {
+    const data = { testTable: [{ var1: 'z'.repeat(32765 + 1) }] }
+
+    expect(() => formatDataForRequest(data)).toThrow(
+      new Error(
+        'The max length of a string value in SASjs is 32765 characters.'
+      )
+    )
+  })
+
+  it('should return error if string is more than 32765 characters', () => {
+    const charsCount = 16 * 1000 + 1
+    const allChars = 'z'.repeat(charsCount)
+    const data = { [testTable]: [{ var1: allChars }] }
+    const firstChunk = `var1:$char${charsCount}.\r\n`
+    const firstChunkChars = 'z'.repeat(16000 - firstChunk.length)
+    const secondChunkChars = 'z'.repeat(
+      charsCount - (16000 - firstChunk.length)
+    )
+
+    const expectedOutput = {
+      sasjs1data0: 2,
+      sasjs1data1: `${firstChunk}${firstChunkChars}`,
+      sasjs1data2: secondChunkChars,
+      sasjs_tables: testTable
+    }
+
+    expect(formatDataForRequest(data)).toEqual(expectedOutput)
+  })
+
+  it('should throw an error if special missing values is not valid', () => {
+    let tableWithMissingValues = {
+      [testTable]: [{ var: 'AA' }, { var: 0 }],
+      [`$${testTable}`]: { formats: { var: 'best.' } }
+    }
+
+    expect(() => formatDataForRequest(tableWithMissingValues)).toThrow(
+      new Error(
+        'Special missing value can only be a single character from A to Z or _'
+      )
+    )
+  })
+
+  it('should auto-detect special missing values type as best.', () => {
+    const tableWithMissingValues = {
+      [testTable]: [{ var: 'a' }, { var: 'A' }, { var: '_' }, { var: 0 }]
+    }
+
+    const expectedOutput = {
+      sasjs1data: `var:best.\r\n.a\r\n.a\r\n._\r\n0`,
+      sasjs_tables: testTable
+    }
+
+    expect(formatDataForRequest(tableWithMissingValues)).toEqual(expectedOutput)
+  })
+
+  it('should auto-detect values type as $char1.', () => {
+    const tableWithMissingValues = {
+      [testTable]: [{ var: 'a' }, { var: 'A' }, { var: '_' }]
+    }
+
+    const expectedOutput = {
+      sasjs1data: `var:$char1.\r\na\r\nA\r\n_`,
+      sasjs_tables: testTable
+    }
+
+    expect(formatDataForRequest(tableWithMissingValues)).toEqual(expectedOutput)
+  })
+})

src/utils/convertToCsv.spec.ts

@@ -167,4 +167,17 @@ describe('convertToCsv', () => {
       convertToCSV([{ slashWithSpecialExtra: '\\\ts\tl\ta\ts\t\th\t' }])
     ).toEqual(`slashWithSpecialExtra:$char13.\r\n\"\\\ts\tl\ta\ts\t\th\t\"`)
   })
+
+  it('should console log error if data has mixed types', () => {
+    const colName = 'var1'
+    const data = [{ [colName]: 'string' }, { [colName]: 232 }]
+
+    jest.spyOn(console, 'error').mockImplementation(() => {})
+
+    convertToCSV(data)
+
+    expect(console.error).toHaveBeenCalledWith(
+      `Row (2), Column (${colName}) has mixed types: ERROR`
+    )
+  })
 })

src/utils/convertToCsv.ts

@@ -2,70 +2,115 @@
  * Converts the given JSON object array to a CSV string.
  * @param data - the array of JSON objects to convert.
  */
-export const convertToCSV = (data: any) => {
-  const replacer = (key: any, value: any) => (value === null ? '' : value)
-  const headerFields = Object.keys(data[0])
+export const convertToCSV = (
+  data: any,
+  sasFormats?: { formats: { [key: string]: string } }
+) => {
+  let formats = sasFormats?.formats
+  let headers: string[] = []
   let csvTest
   let invalidString = false
-  const headers = headerFields.map((field) => {
-    let firstFoundType: string | null = null
-    let hasMixedTypes: boolean = false
-    let rowNumError: number = -1
+  const specialMissingValueRegExp = /^[a-z_]{1}$/i
 
-    const longestValueForField = data
-      .map((row: any, index: number) => {
-        if (row[field] || row[field] === '') {
-          if (firstFoundType) {
-            let currentFieldType =
-              row[field] === '' || typeof row[field] === 'string'
-                ? 'chars'
-                : 'number'
+  if (formats) {
+    headers = Object.keys(formats).map((key) => `${key}:${formats![key]}`)
+  }
 
-            if (!hasMixedTypes) {
-              hasMixedTypes = currentFieldType !== firstFoundType
-              rowNumError = hasMixedTypes ? index + 1 : -1
-            }
-          } else {
-            if (row[field] === '') {
-              firstFoundType = 'chars'
-            } else {
-              firstFoundType =
-                typeof row[field] === 'string' ? 'chars' : 'number'
-            }
-          }
+  const headerFields = Object.keys(data[0])
 
-          let byteSize
+  headerFields.forEach((field) => {
+    if (!formats || !Object.keys(formats).includes(field)) {
+      let hasNullOrNumber = false
+      let hasSpecialMissingString = false
 
-          if (typeof row[field] === 'string') {
-            byteSize = getByteSize(row[field])
-          }
-
-          return byteSize
+      data.forEach((row: { [key: string]: any }) => {
+        if (row[field] === null || typeof row[field] === 'number') {
+          hasNullOrNumber = true
+        } else if (
+          typeof row[field] === 'string' &&
+          specialMissingValueRegExp.test(row[field])
+        ) {
+          hasSpecialMissingString = true
         }
       })
-      .sort((a: number, b: number) => b - a)[0]
-    if (longestValueForField && longestValueForField > 32765) {
-      invalidString = true
-    }
-    if (hasMixedTypes) {
-      console.error(
-        `Row (${rowNumError}), Column (${field}) has mixed types: ERROR`
-      )
-    }
 
-    return `${field}:${firstFoundType === 'chars' ? '$char' : ''}${
-      longestValueForField
-        ? longestValueForField
-        : firstFoundType === 'chars'
-        ? '1'
-        : 'best'
-    }.`
+      if (hasNullOrNumber && hasSpecialMissingString) {
+        headers.push(`${field}:best.`)
+
+        if (!formats) formats = {}
+
+        formats[field] = 'best.'
+      } else {
+        let firstFoundType: string | null = null
+        let hasMixedTypes: boolean = false
+        let rowNumError: number = -1
+
+        const longestValueForField = data
+          .map((row: any, index: number) => {
+            if (row[field] || row[field] === '') {
+              if (firstFoundType) {
+                let currentFieldType =
+                  row[field] === '' || typeof row[field] === 'string'
+                    ? 'chars'
+                    : 'number'
+
+                if (!hasMixedTypes) {
+                  hasMixedTypes = currentFieldType !== firstFoundType
+                  rowNumError = hasMixedTypes ? index + 1 : -1
+                }
+              } else {
+                if (row[field] === '') {
+                  firstFoundType = 'chars'
+                } else {
+                  firstFoundType =
+                    typeof row[field] === 'string' ? 'chars' : 'number'
+                }
+              }
+
+              let byteSize
+
+              if (typeof row[field] === 'string') {
+                byteSize = getByteSize(row[field])
+              }
+
+              return byteSize
+            }
+          })
+          .sort((a: number, b: number) => b - a)[0]
+
+        if (longestValueForField && longestValueForField > 32765) {
+          invalidString = true
+        }
+
+        if (hasMixedTypes) {
+          console.error(
+            `Row (${rowNumError}), Column (${field}) has mixed types: ERROR`
+          )
+        }
+
+        headers.push(
+          `${field}:${firstFoundType === 'chars' ? '$char' : ''}${
+            longestValueForField
+              ? longestValueForField
+              : firstFoundType === 'chars'
+              ? '1'
+              : 'best'
+          }.`
+        )
+      }
+    }
   })
 
-  if (invalidString) {
-    return 'ERROR: LARGE STRING LENGTH'
+  if (sasFormats) {
+    headers = headers.sort(
+      (a, b) =>
+        headerFields.indexOf(a.replace(/:.*/, '')) -
+        headerFields.indexOf(b.replace(/:.*/, ''))
+    )
   }
 
+  if (invalidString) return 'ERROR: LARGE STRING LENGTH'
+
   csvTest = data.map((row: any) => {
     const fields = Object.keys(row).map((fieldName, index) => {
       let value
@@ -76,6 +121,17 @@ export const convertToCSV = (data: any) => {
       // stringify with replacer converts null values to empty strings
       value = currentCell === null ? '' : currentCell
 
+      if (formats && formats[fieldName] === 'best.') {
+        if (value && !specialMissingValueRegExp.test(value)) {
+          console.log(`🤖[value]🤖`, value)
+          throw new Error(
+            'Special missing value can only be a single character from A to Z or _'
+          )
+        }
+
+        return `.${value.toLowerCase()}`
+      }
+
       // if there any present, it should have preceding (") for escaping
       value = value.replace(/"/g, `""`)
 

src/utils/formatDataForRequest.ts

@@ -7,9 +7,17 @@ export const formatDataForRequest = (data: any) => {
   const result: any = {}
 
   for (const tableName in data) {
+    if (
+      tableName.match(/^\$.*/) &&
+      Object.keys(data).includes(tableName.replace(/^\$/, ''))
+    ) {
+      continue
+    }
+
     tableCounter++
     sasjsTables.push(tableName)
-    const csv = convertToCSV(data[tableName])
+    const csv = convertToCSV(data[tableName], data[`$${tableName}`])
+
     if (csv === 'ERROR: LARGE STRING LENGTH') {
       throw new Error(
         'The max length of a string value in SASjs is 32765 characters.'
@@ -27,6 +35,7 @@ export const formatDataForRequest = (data: any) => {
       result[`sasjs${tableCounter}data`] = csv
     }
   }
+
   result['sasjs_tables'] = sasjsTables.join(' ')
 
   return result