From 70e26c57d96f599e810ac84401a2a96ce4a98c32 Mon Sep 17 00:00:00 2001
From: Allan Bowe <allnbwe@gmail.com>
Date: Thu, 6 Jan 2022 12:34:01 +0000
Subject: [PATCH] chore(docs): updating readme with new special missing
 support.  Also adding a .gitpod.yml file for a better gitpod experience.

---
 .gitpod.yml |  2 ++
 README.md   | 86 ++++++++++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 81 insertions(+), 7 deletions(-)
 create mode 100644 .gitpod.yml

diff --git a/.gitpod.yml b/.gitpod.yml
new file mode 100644
index 0000000..da5a51b
--- /dev/null
+++ b/.gitpod.yml
@@ -0,0 +1,2 @@
+tasks:
+  - init: npm install && npm run build
\ No newline at end of file
diff --git a/README.md b/README.md
index 3fd4c89..cef8411 100644
--- a/README.md
+++ b/README.md
@@ -142,6 +142,71 @@ The response object will contain returned tables and columns.  Table names are a
 
 The adapter will also cache the logs (if debug enabled) and even the work tables.  For performance, it is best to keep debug mode off.
 
+### Variable Types
+
+The SAS type (char/numeric) of the values is determined according to a set of rules:
+
+* If the values are numeric, the SAS type is numeric
+* If the values are all string, the SAS type is character
+* If the values contain a single character (a-Z + underscore) AND a numeric, then the SAS type is numeric (with special missing values).  
+* `null` is set to either '.' or '' depending on the assigned or derived type per the above rules.  If entire column is `null` then the type will be numeric.
+
+The following table illustrates the formats applied to columns under various scenarios:
+
+|JS Values |SAS Format|
+|---|---|
+|'a', 'a' |$char1.|
+|0, '_' |best.|
+|'Z', 0 |best.|
+|'a', 'aaa' |$char3.|
+|null, 'a', 'aaa' | $char3.|
+|null, 'a', 0 | best.|
+|null, null | best.|
+|null, '' | $char1.|
+|null, 'a' | $char1.|
+|'a' | $char1.|
+|'a', null | $char1.|
+|'a', null, 0 | best.|
+
+Validation is also performed on the values.  The following combinations will throw errors:
+
+|JS Values |SAS Format|
+|---|---|
+|null, 'aaaa', 0 | Error: mixed types. 'aaaa' is not a special missing value.|
+|0, 'a', '!' | Error: mixed types. '!' is not a special missing value|
+|1.1, '.', 0| Error: mixed types.  For regular nulls, use `null`|
+
+### Variable Format Override
+The auto-detect functionality above is thwarted in the following scenarios:
+
+* A character column containing only `null` values (is considered numeric)
+* A numeric column containing only special missing values (is considered character)
+
+To cater for these scenarios, an optional array of formats can be passed along with the data to ensure that SAS will read them in correctly.
+
+To understand these formats, it should be noted that the JSON data is NOT passed directly (as JSON) to SAS. It is first converted into CSV, and the header row is actually an `infile` statement in disguise.  It looks a bit like this:
+
+```csv
+CHARVAR1:$char4. CHARVAR2:$char1. NUMVAR:best.
+LOAD,,0
+ABCD,X,.
+```
+
+To provide overrides to this header row, the tables object can be constructed as follows (with a leading '$' in the table name):
+
+```javascript
+let specialData={
+  "tablewith2cols2rows": [
+    {"col1": "val1","specialMissingsCol": "A"},
+    {"col1": "val2","specialMissingsCol": "_"}
+  ],
+  "$tablewith2cols2rows":{"formats":{"specialMissingsCol":"best."}
+  }
+};
+```
+
+It is not necessary to provide formats for ALL the columns, only the ones that need to be overridden.
+
 ## SAS Inputs / Outputs
 
 The SAS side is handled by a number of macros in the [macro core](https://github.com/sasjs/core) library.
@@ -153,18 +218,25 @@ The following snippet shows the process of SAS tables arriving / leaving:
 %webout(FETCH)
 
 /* some sas code */
-data some sas tables;
+data a b c;
   set from js;
 run;
 
-%webout(OPEN)  /* open the JSON to be returned */
-%webout(OBJ,some) /* `some` table is sent in object format */
-%webout(ARR,sas) /* `sas` table is sent in array format, smaller filesize */
-%webout(OBJ,tables,fmt=N) /* unformatted (raw) data */
-%webout(OBJ,tables,label=newtable) /* rename tables on export */
-%webout(CLOSE) /* close the JSON and send some extra useful variables too */
+%webout(OPEN)  /* Open the JSON to be returned */
+%webout(OBJ,a) /* Rows in table `a` are objects (easy to use) */
+%webout(ARR,b) /* Rows in table `b` are arrays (compact) */
+%webout(OBJ,c,fmt=N) /* Table `c` is sent unformatted (raw) */
+%webout(OBJ,c,label=d) /* Rename as `d` on JS side */
+%webout(CLOSE) /* Close the JSON and add default variables */
 ```
 
+By default, special SAS numeric missings (_a-Z) are converted to `null` in the JSON.  If you'd like to preserve these, use the `nullify=NO` option as follows:
+
+```sas
+%webout(OBJ,a,nullify=NO)
+```
+In this case, special missings (such as `.a`, `.b`) are converted to javascript string values (`'A', 'B'`).
+
 ## Configuration
 
 Configuration on the client side involves passing an object on startup, which can also be passed with each request.  Technical documentation on the SASjsConfig class is available [here](https://adapter.sasjs.io/classes/types.sasjsconfig.html).  The main config items are: