Merge pull request #653 from sasjs/form-data-fix

fix: reverted sasjsJobExecutor no need for that

.github/dependabot.yml

@@ -4,4 +4,4 @@ updates:
     directory: '/'
     schedule:
       interval: monthly
-    open-pull-requests-limit: 10
+    open-pull-requests-limit: 2

.github/reviewer-lottery.yml

@@ -2,11 +2,8 @@ groups:
   - name: SASjs Devs # name of the group
     reviewers: 1 # how many reviewers do you want to assign?
     usernames: # github usernames of the reviewers
-      - krishna-acondy
       - YuryShkoda
-      - saadjutt01
       - medjedovicm
-      - allanbowe
       - sabhas
   - name: SASjs QA
     reviewers: 1

.github/workflows/build.yml

@@ -22,6 +22,8 @@ jobs:
         with:
           node-version: ${{ matrix.node-version }}
           cache: npm
+      - name: Check npm audit
+        run: npm audit --production --audit-level=low
       - name: Install Dependencies
         run: npm ci
       - name: Check code style

.gitpod.yml

@@ -0,0 +1,2 @@
+tasks:
+  - init: npm install && npm run build

README.md

@@ -142,6 +142,71 @@ The response object will contain returned tables and columns.  Table names are a
 
 The adapter will also cache the logs (if debug enabled) and even the work tables.  For performance, it is best to keep debug mode off.
 
+### Variable Types
+
+The SAS type (char/numeric) of the values is determined according to a set of rules:
+
+* If the values are numeric, the SAS type is numeric
+* If the values are all string, the SAS type is character
+* If the values contain a single character (a-Z + underscore) AND a numeric, then the SAS type is numeric (with special missing values).  
+* `null` is set to either '.' or '' depending on the assigned or derived type per the above rules.  If entire column is `null` then the type will be numeric.
+
+The following table illustrates the formats applied to columns under various scenarios:
+
+|JS Values |SAS Format|
+|---|---|
+|'a', 'a' |$char1.|
+|0, '_' |best.|
+|'Z', 0 |best.|
+|'a', 'aaa' |$char3.|
+|null, 'a', 'aaa' | $char3.|
+|null, 'a', 0 | best.|
+|null, null | best.|
+|null, '' | $char1.|
+|null, 'a' | $char1.|
+|'a' | $char1.|
+|'a', null | $char1.|
+|'a', null, 0 | best.|
+
+Validation is also performed on the values.  The following combinations will throw errors:
+
+|JS Values |SAS Format|
+|---|---|
+|null, 'aaaa', 0 | Error: mixed types. 'aaaa' is not a special missing value.|
+|0, 'a', '!' | Error: mixed types. '!' is not a special missing value|
+|1.1, '.', 0| Error: mixed types.  For regular nulls, use `null`|
+
+### Variable Format Override
+The auto-detect functionality above is thwarted in the following scenarios:
+
+* A character column containing only `null` values (is considered numeric)
+* A numeric column containing only special missing values (is considered character)
+
+To cater for these scenarios, an optional array of formats can be passed along with the data to ensure that SAS will read them in correctly.
+
+To understand these formats, it should be noted that the JSON data is NOT passed directly (as JSON) to SAS. It is first converted into CSV, and the header row is actually an `infile` statement in disguise.  It looks a bit like this:
+
+```csv
+CHARVAR1:$char4. CHARVAR2:$char1. NUMVAR:best.
+LOAD,,0
+ABCD,X,.
+```
+
+To provide overrides to this header row, the tables object can be constructed as follows (with a leading '$' in the table name):
+
+```javascript
+let specialData={
+  "tablewith2cols2rows": [
+    {"col1": "val1","specialMissingsCol": "A"},
+    {"col1": "val2","specialMissingsCol": "_"}
+  ],
+  "$tablewith2cols2rows":{"formats":{"specialMissingsCol":"best."}
+  }
+};
+```
+
+It is not necessary to provide formats for ALL the columns, only the ones that need to be overridden.
+
 ## SAS Inputs / Outputs
 
 The SAS side is handled by a number of macros in the [macro core](https://github.com/sasjs/core) library.
@@ -153,16 +218,29 @@ The following snippet shows the process of SAS tables arriving / leaving:
 %webout(FETCH)
 
 /* some sas code */
-data some sas tables;
+data a b c;
   set from js;
 run;
 
-%webout(OPEN)  /* open the JSON to be returned */
-%webout(OBJ,some) /* `some` table is sent in object format */
-%webout(ARR,sas) /* `sas` table is sent in array format, smaller filesize */
-%webout(OBJ,tables,fmt=N) /* unformatted (raw) data */
-%webout(OBJ,tables,label=newtable) /* rename tables on export */
-%webout(CLOSE) /* close the JSON and send some extra useful variables too */
+%webout(OPEN)  /* Open the JSON to be returned */
+%webout(OBJ,a) /* Rows in table `a` are objects (easy to use) */
+%webout(ARR,b) /* Rows in table `b` are arrays (compact) */
+%webout(OBJ,c,fmt=N) /* Table `c` is sent unformatted (raw) */
+%webout(OBJ,c,label=d) /* Rename as `d` on JS side */
+%webout(CLOSE) /* Close the JSON and add default variables */
+```
+
+By default, special SAS numeric missings (_a-Z) are converted to `null` in the JSON.  If you'd like to preserve these, use the `missing=STRING` option as follows:
+
+```sas
+%webout(OBJ,a,missing=STRING)
+```
+In this case, special missings (such as `.a`, `.b`) are converted to javascript string values (`'A', 'B'`).
+
+Where an entire column is made up of special missing numerics, there would be no way to distinguish it from a single-character column by looking at the values.  To cater for this scenario, it is possible to export the variable types (and other attributes such as label and format) by adding a `showmeta` param to the `webout()` macro as follows:
+
+```sas
+%webout(OBJ,a,missing=STRING,showmeta=YES)
 ```
 
 ## Configuration
@@ -170,12 +248,13 @@ run;
 Configuration on the client side involves passing an object on startup, which can also be passed with each request.  Technical documentation on the SASjsConfig class is available [here](https://adapter.sasjs.io/classes/types.sasjsconfig.html).  The main config items are:
 
 * `appLoc` - this is the folder under which the SAS services will be created.
-* `serverType` - either `SAS9` or `SASVIYA`.
+* `serverType` - either `SAS9`, `SASVIYA` or `SASJS`.  The `SASJS` server type is for use with [sasjs/server](https://github.com/sasjs/server).
 * `serverUrl` - the location (including http protocol and port) of the SAS Server. Can be omitted, eg if serving directly from the SAS Web Server, or in streaming mode.
 * `debug` - if `true` then SAS Logs and extra debug information is returned.
 * `LoginMechanism` - either `Default` or `Redirected`.  If `Redirected` then authentication occurs through the injection of an additional screen, which contains the SASLogon prompt.  This allows for more complex authentication flows (such as 2FA) and avoids the need to handle passwords in the application itself.  The styling of the redirect flow can also be modified.  If left at "Default" then the developer must capture the username and password and use these with the `.login()` method.
 * `useComputeApi` - Only relevant when the serverType is `SASVIYA`. If `true` the [Compute API](#using-the-compute-api) is used.  If `false` the [JES API](#using-the-jes-api) is used.  If `null` or `undefined` the [Web](#using-jes-web-app) approach is used.
 * `contextName` - Compute context on which the requests will be called.  If missing or not provided, defaults to `Job Execution Compute context`.
+* `requestHistoryLimit` - Request history limit. Increasing this limit may affect browser performance, especially with debug (logs) enabled.  Default is 10.
 
 The adapter supports a number of approaches for interfacing with Viya (`serverType` is `SASVIYA`).  For maximum performance, be sure to [configure your compute context](https://sasjs.io/guide-viya/#shared-account-and-server-re-use) with `reuseServerProcesses` as `true` and a system account in `runServerAs`.  This functionality is available since Viya 3.5.  This configuration is supported when [creating contexts using the CLI](https://sasjs.io/sasjs-cli-context/#sasjs-context-create).
 

jest.config.js

@@ -127,7 +127,7 @@ module.exports = {
   setupFiles: [],
 
   // A list of paths to modules that run some code to configure or set up the testing framework before each test
-  setupFilesAfterEnv: ['jest-extended'],
+  setupFilesAfterEnv: ['jest-extended/all'],
 
   // A list of paths to snapshot serializer modules Jest should use for snapshot testing
   // snapshotSerializers: [],

package.json

@@ -40,44 +40,44 @@
   },
   "license": "ISC",
   "devDependencies": {
-    "@types/axios": "^0.14.0",
-    "@types/express": "^4.17.13",
-    "@types/form-data": "^2.5.0",
-    "@types/jest": "^27.0.2",
-    "@types/mime": "^2.0.3",
-    "@types/pem": "^1.9.6",
-    "@types/tough-cookie": "^4.0.1",
-    "copyfiles": "^2.4.1",
-    "cp": "^0.2.0",
-    "dotenv": "^10.0.0",
-    "express": "^4.17.1",
-    "jest": "^27.2.0",
-    "jest-extended": "^0.11.5",
-    "node-polyfill-webpack-plugin": "^1.1.4",
-    "path": "^0.12.7",
-    "pem": "^1.14.4",
-    "process": "^0.11.10",
-    "rimraf": "^3.0.2",
-    "semantic-release": "^18.0.0",
-    "terser-webpack-plugin": "^5.2.4",
-    "ts-jest": "^27.0.3",
-    "ts-loader": "^9.2.6",
-    "tslint": "^6.1.3",
-    "tslint-config-prettier": "^1.18.0",
-    "typedoc": "0.19.2",
-    "typedoc-neo-theme": "^1.1.1",
-    "typedoc-plugin-external-module-name": "^4.0.6",
-    "typescript": "4.3.5",
-    "webpack": "^5.56.0",
-    "webpack-cli": "^4.7.2"
+    "@types/axios": "0.14.0",
+    "@types/express": "4.17.13",
+    "@types/form-data": "2.5.0",
+    "@types/jest": "27.0.2",
+    "@types/mime": "2.0.3",
+    "@types/pem": "1.9.6",
+    "@types/tough-cookie": "4.0.1",
+    "copyfiles": "2.4.1",
+    "cp": "0.2.0",
+    "dotenv": "10.0.0",
+    "express": "4.17.1",
+    "jest": "27.4.7",
+    "jest-extended": "2.0.0",
+    "node-polyfill-webpack-plugin": "1.1.4",
+    "path": "0.12.7",
+    "pem": "1.14.4",
+    "process": "0.11.10",
+    "rimraf": "3.0.2",
+    "semantic-release": "18.0.0",
+    "terser-webpack-plugin": "5.3.0",
+    "ts-jest": "27.1.3",
+    "ts-loader": "9.2.6",
+    "tslint": "6.1.3",
+    "tslint-config-prettier": "1.18.0",
+    "typedoc": "0.22.11",
+    "typedoc-neo-theme": "1.1.1",
+    "typedoc-plugin-external-module-name": "4.0.6",
+    "typescript": "4.5.4",
+    "webpack": "5.66.0",
+    "webpack-cli": "4.7.2"
   },
   "main": "index.js",
   "dependencies": {
-    "@sasjs/utils": "^2.32.0",
-    "axios": "^0.21.4",
-    "axios-cookiejar-support": "^1.0.1",
-    "form-data": "^4.0.0",
-    "https": "^1.0.0",
-    "tough-cookie": "^4.0.0"
+    "@sasjs/utils": "2.35.0",
+    "axios": "0.26.0",
+    "axios-cookiejar-support": "1.0.1",
+    "form-data": "4.0.0",
+    "https": "1.0.0",
+    "tough-cookie": "4.0.0"
   }
 }

sasjs-tests/README.md

@@ -47,7 +47,7 @@ npm i -g copyfiles
 ```
 and then run to build:
 ```bash
-npm run update:adapter && npm run build 
+npm run update:adapter && npm run build
 ```
 when it finishes run to deploy:
 ```bash
@@ -70,7 +70,7 @@ parmcards4;
   %webout(FETCH)
   %webout(OPEN)
   %macro x();
-  %do i=1 %to &_webin_file_count; %webout(OBJ,&&_webin_name&i) %end;
+  %do i=1 %to &_webin_file_count; %webout(OBJ,&&_webin_name&i,missing=STRING) %end;
   %mend; %x()
   %webout(CLOSE)
 ;;;;
@@ -79,7 +79,7 @@ parmcards4;
   %webout(FETCH)
   %webout(OPEN)
   %macro x();
-  %do i=1 %to &_webin_file_count; %webout(ARR,&&_webin_name&i) %end;
+  %do i=1 %to &_webin_file_count; %webout(ARR,&&_webin_name&i,missing=STRING) %end;
   %mend; %x()
   %webout(CLOSE)
 ;;;;
@@ -111,7 +111,7 @@ parmcards4;
   %macro x();
   %do i=1 %to %sysfunc(countw(&sasjs_tables));
     %let table=%scan(&sasjs_tables,&i);
-    %webout(OBJ,&table)
+    %webout(OBJ,&table,missing=STRING)
   %end;
   %mend;
   %x()
@@ -125,7 +125,7 @@ parmcards4;
   %macro x();
   %do i=1 %to %sysfunc(countw(&sasjs_tables));
     %let table=%scan(&sasjs_tables,&i);
-    %webout(ARR,&table)
+    %webout(ARR,&table,missing=STRING)
   %end;
   %mend;
   %x()

sasjs-tests/package.json

@@ -5,7 +5,7 @@
   "private": true,
   "dependencies": {
     "@sasjs/adapter": "file:../build/sasjs-adapter-5.0.0.tgz",
-    "@sasjs/test-framework": "^1.4.2",
+    "@sasjs/test-framework": "^1.4.3",
     "@types/jest": "^26.0.20",
     "@types/node": "^14.14.41",
     "@types/react": "^17.0.1",
@@ -22,7 +22,7 @@
     "build": "react-scripts build",
     "test": "react-scripts test",
     "eject": "react-scripts eject",
-    "update:adapter": "cd .. && npm run package:lib && cd sasjs-tests && npm i ../build/sasjs-adapter-5.0.0.tgz",
+    "update:adapter": "cd .. && npm run package:lib && cd sasjs-tests && npm i ../build/sasjs-adapter-5.0.0.tgz --legacy-peer-deps",
     "deploy:tests": "rsync -avhe ssh ./build/* --delete $SSH_ACCOUNT:$DEPLOY_PATH || npm run deploy:tests-win",
     "deploy:tests-win": "scp %DEPLOY_PATH% ./build/*",
     "deploy": "npm run update:adapter && npm run build && npm run deploy:tests"
@@ -43,6 +43,6 @@
     ]
   },
   "devDependencies": {
-    "node-sass": "^5.0.0"
+    "node-sass": "^6.0.1"
   }
 }

sasjs-tests/src/testSuites/Basic.ts

@@ -6,6 +6,7 @@ const stringData: any = { table1: [{ col1: 'first col value' }] }
 
 const defaultConfig: SASjsConfig = {
   serverUrl: window.location.origin,
+  pathSASJS: '/SASjsApi/stp/execute',
   pathSAS9: '/SASStoredProcess/do',
   pathSASViya: '/SASJobExecution',
   appLoc: '/Public/seedapp',

sasjs-tests/src/testSuites/SpecialCases.ts

@@ -79,6 +79,19 @@ const errorAndCsrfData: any = {
   _csrf: [{ col1: 'q', col2: 'w', col3: 'e', col4: 'r' }]
 }
 
+const testTable = 'sometable'
+export const testTableWithNullVars: { [key: string]: any } = {
+  [testTable]: [
+    { var1: 'string', var2: 232, nullvar: 'A' },
+    { var1: 'string', var2: 232, nullvar: 'B' },
+    { var1: 'string', var2: 232, nullvar: '_' },
+    { var1: 'string', var2: 232, nullvar: 0 },
+    { var1: 'string', var2: 232, nullvar: 'z' },
+    { var1: 'string', var2: 232, nullvar: null }
+  ],
+  [`$${testTable}`]: { formats: { var1: '$char12.', nullvar: 'best.' } }
+}
+
 export const specialCaseTests = (adapter: SASjs): TestSuite => ({
   name: 'Special Cases',
   tests: [
@@ -247,6 +260,39 @@ export const specialCaseTests = (adapter: SASjs): TestSuite => ({
           res._csrf[0].COL4 === errorAndCsrfData._csrf[0].col4
         )
       }
+    },
+    {
+      title: 'Special missing values',
+      description: 'Should support special missing values',
+      test: () => {
+        return adapter.request('common/sendObj', testTableWithNullVars)
+      },
+      assertion: (res: any) => {
+        let assertionRes = true
+
+        testTableWithNullVars[testTable].forEach(
+          (row: { [key: string]: any }, i: number) =>
+            Object.keys(row).forEach((col: string) => {
+              const resValue = res[testTable][i][col.toUpperCase()]
+
+              if (
+                typeof row[col] === 'string' &&
+                testTableWithNullVars[`$${testTable}`].formats[col] ===
+                  'best.' &&
+                row[col].toUpperCase() !== resValue
+              ) {
+                assertionRes = false
+              } else if (
+                typeof row[col] !== 'string' &&
+                row[col] !== resValue
+              ) {
+                assertionRes = false
+              }
+            })
+        )
+
+        return assertionRes
+      }
     }
   ]
 })

src/SASViyaApiClient.ts

@@ -22,8 +22,8 @@ import { pollJobState } from './api/viya/pollJobState'
 import { getTokens } from './auth/getTokens'
 import { uploadTables } from './api/viya/uploadTables'
 import { executeScript } from './api/viya/executeScript'
-import { getAccessToken } from './auth/getAccessToken'
-import { refreshTokens } from './auth/refreshTokens'
+import { getAccessTokenForViya } from './auth/getAccessTokenForViya'
+import { refreshTokensForViya } from './auth/refreshTokensForViya'
 
 /**
  * A client for interfacing with the SAS Viya REST API.
@@ -534,21 +534,26 @@ export class SASViyaApiClient {
     clientSecret: string,
     authCode: string
   ): Promise<SasAuthResponse> {
-    return getAccessToken(this.requestClient, clientId, clientSecret, authCode)
+    return getAccessTokenForViya(
+      this.requestClient,
+      clientId,
+      clientSecret,
+      authCode
+    )
   }
 
   /**
    * Exchanges the refresh token for an access token for the given client.
    * @param clientId - the client ID to authenticate with.
    * @param clientSecret - the client secret to authenticate with.
-   * @param authCode - the refresh token received from the server.
+   * @param refreshToken - the refresh token received from the server.
    */
   public async refreshTokens(
     clientId: string,
     clientSecret: string,
     refreshToken: string
   ) {
-    return refreshTokens(
+    return refreshTokensForViya(
       this.requestClient,
       clientId,
       clientSecret,

src/SASjs.ts

@@ -5,21 +5,22 @@ import {
   EditContextInput,
   PollOptions,
   LoginMechanism,
-  FolderMember,
-  ServiceMember,
-  ExecutionQuery
+  ExecutionQuery,
+  FileTree
 } from './types'
 import { SASViyaApiClient } from './SASViyaApiClient'
 import { SAS9ApiClient } from './SAS9ApiClient'
-import { SASjsApiClient } from './SASjsApiClient'
+import { SASjsApiClient, SASjsAuthResponse } from './SASjsApiClient'
 import { AuthManager } from './auth'
 import {
   ServerType,
   MacroVar,
   AuthConfig,
-  ExtraResponseAttributes
+  ExtraResponseAttributes,
+  SasAuthResponse
 } from '@sasjs/utils/types'
 import { RequestClient } from './request/RequestClient'
+import { SasjsRequestClient } from './request/SasjsRequestClient'
 import {
   JobExecutor,
   WebJobExecutor,
@@ -53,7 +54,7 @@ export default class SASjs {
   private jobsPath: string = ''
   private sasViyaApiClient: SASViyaApiClient | null = null
   private sas9ApiClient: SAS9ApiClient | null = null
-  private SASjsApiClient: SASjsApiClient | null = null
+  private sasJSApiClient: SASjsApiClient | null = null
   private fileUploader: FileUploader | null = null
   private authManager: AuthManager | null = null
   private requestClient: RequestClient | null = null
@@ -75,12 +76,19 @@ export default class SASjs {
     return this.requestClient?.getCsrfToken(type)
   }
 
+  /**
+   * Executes code against a SAS 9 server.  Requires a runner to be present in
+   * the users home directory in metadata.
+   * @param linesOfCode - lines of sas code from the file to run.
+   * @param username - a string representing the username.
+   * @param password - a string representing the password.
+   */
   public async executeScriptSAS9(
     linesOfCode: string[],
     userName: string,
     password: string
   ) {
-    this.isMethodSupported('executeScriptSAS9', ServerType.Sas9)
+    this.isMethodSupported('executeScriptSAS9', [ServerType.Sas9])
 
     return await this.sas9ApiClient?.executeScript(
       linesOfCode,
@@ -89,12 +97,44 @@ export default class SASjs {
     )
   }
 
+  /**
+   * Executes sas code in a SAS Viya compute session.
+   * @param fileName - name of the file to run. It will be converted to path to the file being submitted for execution.
+   * @param linesOfCode - lines of sas code from the file to run.
+   * @param contextName - context name on which code will be run on the server.
+   * @param authConfig - (optional) the access token, refresh token, client and secret for authorizing the request.
+   * @param debug - (optional) if true, global debug config will be overriden
+   */
+  public async executeScriptSASViya(
+    fileName: string,
+    linesOfCode: string[],
+    contextName: string,
+    authConfig?: AuthConfig,
+    debug?: boolean
+  ) {
+    this.isMethodSupported('executeScriptSASViya', [ServerType.SasViya])
+    if (!contextName) {
+      throw new Error(
+        'Context name is undefined. Please set a `contextName` in your SASjs or override config.'
+      )
+    }
+
+    return await this.sasViyaApiClient!.executeScript(
+      fileName,
+      linesOfCode,
+      contextName,
+      authConfig,
+      null,
+      debug ? debug : this.sasjsConfig.debug
+    )
+  }
+
   /**
    * Gets compute contexts.
    * @param accessToken - an access token for an authorized user.
    */
   public async getComputeContexts(accessToken: string) {
-    this.isMethodSupported('getComputeContexts', ServerType.SasViya)
+    this.isMethodSupported('getComputeContexts', [ServerType.SasViya])
 
     return await this.sasViyaApiClient!.getComputeContexts(accessToken)
   }
@@ -104,7 +144,7 @@ export default class SASjs {
    * @param accessToken - an access token for an authorized user.
    */
   public async getLauncherContexts(accessToken: string) {
-    this.isMethodSupported('getLauncherContexts', ServerType.SasViya)
+    this.isMethodSupported('getLauncherContexts', [ServerType.SasViya])
 
     return await this.sasViyaApiClient!.getLauncherContexts(accessToken)
   }
@@ -113,7 +153,7 @@ export default class SASjs {
    * Gets default(system) launcher contexts.
    */
   public getDefaultComputeContexts() {
-    this.isMethodSupported('getDefaultComputeContexts', ServerType.SasViya)
+    this.isMethodSupported('getDefaultComputeContexts', [ServerType.SasViya])
 
     return this.sasViyaApiClient!.getDefaultComputeContexts()
   }
@@ -123,7 +163,7 @@ export default class SASjs {
    * @param authConfig - an access token, refresh token, client and secret for an authorized user.
    */
   public async getExecutableContexts(authConfig: AuthConfig) {
-    this.isMethodSupported('getExecutableContexts', ServerType.SasViya)
+    this.isMethodSupported('getExecutableContexts', [ServerType.SasViya])
 
     return await this.sasViyaApiClient!.getExecutableContexts(authConfig)
   }
@@ -145,7 +185,7 @@ export default class SASjs {
     accessToken: string,
     authorizedUsers?: string[]
   ) {
-    this.isMethodSupported('createComputeContext', ServerType.SasViya)
+    this.isMethodSupported('createComputeContext', [ServerType.SasViya])
 
     return await this.sasViyaApiClient!.createComputeContext(
       contextName,
@@ -170,7 +210,7 @@ export default class SASjs {
     launchType: string,
     accessToken: string
   ) {
-    this.isMethodSupported('createLauncherContext', ServerType.SasViya)
+    this.isMethodSupported('createLauncherContext', [ServerType.SasViya])
 
     return await this.sasViyaApiClient!.createLauncherContext(
       contextName,
@@ -191,7 +231,7 @@ export default class SASjs {
     editedContext: EditContextInput,
     accessToken?: string
   ) {
-    this.isMethodSupported('editComputeContext', ServerType.SasViya)
+    this.isMethodSupported('editComputeContext', [ServerType.SasViya])
 
     return await this.sasViyaApiClient!.editComputeContext(
       contextName,
@@ -206,7 +246,7 @@ export default class SASjs {
    * @param accessToken - an access token for an authorized user.
    */
   public async deleteComputeContext(contextName: string, accessToken?: string) {
-    this.isMethodSupported('deleteComputeContext', ServerType.SasViya)
+    this.isMethodSupported('deleteComputeContext', [ServerType.SasViya])
 
     return await this.sasViyaApiClient!.deleteComputeContext(
       contextName,
@@ -224,7 +264,7 @@ export default class SASjs {
     contextName: string,
     accessToken?: string
   ) {
-    this.isMethodSupported('getComputeContextByName', ServerType.SasViya)
+    this.isMethodSupported('getComputeContextByName', [ServerType.SasViya])
 
     return await this.sasViyaApiClient!.getComputeContextByName(
       contextName,
@@ -238,7 +278,7 @@ export default class SASjs {
    * @param accessToken - an access token for an authorized user.
    */
   public async getComputeContextById(contextId: string, accessToken?: string) {
-    this.isMethodSupported('getComputeContextById', ServerType.SasViya)
+    this.isMethodSupported('getComputeContextById', [ServerType.SasViya])
 
     return await this.sasViyaApiClient!.getComputeContextById(
       contextId,
@@ -247,43 +287,11 @@ export default class SASjs {
   }
 
   public async createSession(contextName: string, accessToken: string) {
-    this.isMethodSupported('createSession', ServerType.SasViya)
+    this.isMethodSupported('createSession', [ServerType.SasViya])
 
     return await this.sasViyaApiClient!.createSession(contextName, accessToken)
   }
 
-  /**
-   * Executes the sas code against given sas server
-   * @param fileName - name of the file to run. It will be converted to path to the file being submitted for execution.
-   * @param linesOfCode - lines of sas code from the file to run.
-   * @param contextName - context name on which code will be run on the server.
-   * @param authConfig - (optional) the access token, refresh token, client and secret for authorizing the request.
-   * @param debug - (optional) if true, global debug config will be overriden
-   */
-  public async executeScriptSASViya(
-    fileName: string,
-    linesOfCode: string[],
-    contextName: string,
-    authConfig?: AuthConfig,
-    debug?: boolean
-  ) {
-    this.isMethodSupported('executeScriptSASViya', ServerType.SasViya)
-    if (!contextName) {
-      throw new Error(
-        'Context name is undefined. Please set a `contextName` in your SASjs or override config.'
-      )
-    }
-
-    return await this.sasViyaApiClient!.executeScript(
-      fileName,
-      linesOfCode,
-      contextName,
-      authConfig,
-      null,
-      debug ? debug : this.sasjsConfig.debug
-    )
-  }
-
   /**
    * Creates a folder in the logical SAS folder tree
    * @param folderName - name of the folder to be created.
@@ -357,7 +365,7 @@ export default class SASjs {
    * @param accessToken - the access token to authorize the request.
    */
   public async getFolder(folderPath: string, accessToken?: string) {
-    this.isMethodSupported('getFolder', ServerType.SasViya)
+    this.isMethodSupported('getFolder', [ServerType.SasViya])
     return await this.sasViyaApiClient!.getFolder(folderPath, accessToken)
   }
 
@@ -367,7 +375,7 @@ export default class SASjs {
    * @param accessToken - an access token for authorizing the request.
    */
   public async deleteFolder(folderPath: string, accessToken: string) {
-    this.isMethodSupported('deleteFolder', ServerType.SasViya)
+    this.isMethodSupported('deleteFolder', [ServerType.SasViya])
 
     return await this.sasViyaApiClient?.deleteFolder(folderPath, accessToken)
   }
@@ -382,7 +390,7 @@ export default class SASjs {
     accessToken?: string,
     limit?: number
   ) {
-    this.isMethodSupported('listFolder', ServerType.SasViya)
+    this.isMethodSupported('listFolder', [ServerType.SasViya])
 
     return await this.sasViyaApiClient?.listFolder(
       sourceFolder,
@@ -404,7 +412,7 @@ export default class SASjs {
     targetFolderName: string,
     accessToken: string
   ) {
-    this.isMethodSupported('moveFolder', ServerType.SasViya)
+    this.isMethodSupported('moveFolder', [ServerType.SasViya])
 
     return await this.sasViyaApiClient?.moveFolder(
       sourceFolder,
@@ -422,7 +430,7 @@ export default class SASjs {
     accessToken?: string,
     sasApiClient?: SASViyaApiClient
   ) {
-    this.isMethodSupported('createJobDefinition', ServerType.SasViya)
+    this.isMethodSupported('createJobDefinition', [ServerType.SasViya])
 
     if (sasApiClient)
       return await sasApiClient!.createJobDefinition(
@@ -442,7 +450,7 @@ export default class SASjs {
   }
 
   public async getAuthCode(clientId: string) {
-    this.isMethodSupported('getAuthCode', ServerType.SasViya)
+    this.isMethodSupported('getAuthCode', [ServerType.SasViya])
 
     return await this.sasViyaApiClient!.getAuthCode(clientId)
   }
@@ -457,8 +465,14 @@ export default class SASjs {
     clientId: string,
     clientSecret: string,
     authCode: string
-  ) {
-    this.isMethodSupported('getAccessToken', ServerType.SasViya)
+  ): Promise<SasAuthResponse | SASjsAuthResponse> {
+    this.isMethodSupported('getAccessToken', [
+      ServerType.SasViya,
+      ServerType.Sasjs
+    ])
+
+    if (this.sasjsConfig.serverType === ServerType.Sasjs)
+      return await this.sasJSApiClient!.getAccessToken(clientId, authCode)
 
     return await this.sasViyaApiClient!.getAccessToken(
       clientId,
@@ -467,12 +481,24 @@ export default class SASjs {
     )
   }
 
+  /**
+   * Exchanges the refresh token for an access token for the given client.
+   * @param clientId - the client ID to authenticate with.
+   * @param clientSecret - the client secret to authenticate with.
+   * @param refreshToken - the refresh token received from the server.
+   */
   public async refreshTokens(
     clientId: string,
     clientSecret: string,
     refreshToken: string
-  ) {
-    this.isMethodSupported('refreshTokens', ServerType.SasViya)
+  ): Promise<SasAuthResponse | SASjsAuthResponse> {
+    this.isMethodSupported('refreshTokens', [
+      ServerType.SasViya,
+      ServerType.Sasjs
+    ])
+
+    if (this.sasjsConfig.serverType === ServerType.Sasjs)
+      return await this.sasJSApiClient!.refreshTokens(refreshToken)
 
     return await this.sasViyaApiClient!.refreshTokens(
       clientId,
@@ -482,7 +508,7 @@ export default class SASjs {
   }
 
   public async deleteClient(clientId: string, accessToken: string) {
-    this.isMethodSupported('deleteClient', ServerType.SasViya)
+    this.isMethodSupported('deleteClient', [ServerType.SasViya])
 
     return await this.sasViyaApiClient!.deleteClient(clientId, accessToken)
   }
@@ -528,29 +554,39 @@ export default class SASjs {
 
   /**
    * Checks whether a session is active, or login is required.
-   * @param accessToken - an optional access token is required for SASjs server type.
    * @returns - a promise which resolves with an object containing two values - a boolean `isLoggedIn`, and a string `userName`.
    */
-  public async checkSession(accessToken?: string) {
-    return this.authManager!.checkSession(accessToken)
+  public async checkSession() {
+    return this.authManager!.checkSession()
   }
 
   /**
    * Logs into the SAS server with the supplied credentials.
    * @param username - a string representing the username.
    * @param password - a string representing the password.
+   * @param clientId - a string representing the client ID.
    */
   public async logIn(
     username?: string,
     password?: string,
+    clientId?: string,
     options: LoginOptions = {}
   ): Promise<LoginResult> {
     if (this.sasjsConfig.loginMechanism === LoginMechanism.Default) {
-      if (!username || !password) {
+      if (!username || !password)
         throw new Error(
           'A username and password are required when using the default login mechanism.'
         )
+
+      if (this.sasjsConfig.serverType === ServerType.Sasjs) {
+        if (!clientId)
+          throw new Error(
+            'A username, password and clientId are required when using the default login mechanism with server type SASJS.'
+          )
+
+        return this.authManager!.logInSasjs(username, password, clientId)
       }
+
       return this.authManager!.logIn(username, password)
     }
 
@@ -565,10 +601,9 @@ export default class SASjs {
 
   /**
    * Logs out of the configured SAS server.
-   * @param accessToken - an optional access token is required for SASjs server type.
    */
-  public logOut(accessToken?: string) {
-    return this.authManager!.logOut(accessToken)
+  public logOut() {
+    return this.authManager!.logOut()
   }
 
   /**
@@ -607,7 +642,8 @@ export default class SASjs {
   }
 
   /**
-   * Makes a request to the SAS Service specified in `SASjob`. The response
+   * Makes a request to program specified in `SASjob` (could be a Viya Job, a
+   * SAS 9 Stored Process, or a SASjs Server Stored Program). The response
    * object will always contain table names in lowercase, and column names in
    * uppercase. Values are returned formatted by default, unformatted
    * values can be configured as an option in the `%webout` macro.
@@ -616,7 +652,8 @@ export default class SASjs {
    *  the SAS `_program` parameter to run a Job Definition or SAS 9 Stored
    *  Process). Is prepended at runtime with the value of `appLoc`.
    * @param data - a JSON object containing one or more tables to be sent to
-   * SAS. Can be `null` if no inputs required.
+   * SAS.  For an example of the table structure, see the project README. This
+   * value can be `null` if no inputs are required.
    * @param config - provide any changes to the config here, for instance to
    * enable/disable `debug`. Any change provided will override the global config,
    * for that particular function call.
@@ -646,9 +683,19 @@ export default class SASjs {
 
     const validationResult = this.validateInput(data)
 
+    // status is true if the data passes validation checks above
     if (validationResult.status) {
-      if (
-        config.serverType !== ServerType.Sas9 &&
+      if (config.serverType === ServerType.Sasjs) {
+        return await this.webJobExecutor!.execute(
+          sasJob,
+          data,
+          config,
+          loginRequiredCallback,
+          authConfig,
+          extraResponseAttributes
+        )
+      } else if (
+        config.serverType === ServerType.SasViya &&
         config.useComputeApi !== undefined &&
         config.useComputeApi !== null
       ) {
@@ -707,15 +754,19 @@ export default class SASjs {
     msg: string
   } {
     if (data === null) return { status: true, msg: '' }
+
+    const isSasFormatsTable = (key: string) =>
+      key.match(/^\$.*/) && Object.keys(data).includes(key.replace(/^\$/, ''))
+
     for (const key in data) {
-      if (!key.match(/^[a-zA-Z_]/)) {
+      if (!key.match(/^[a-zA-Z_]/) && !isSasFormatsTable(key)) {
         return {
           status: false,
           msg: 'First letter of table should be alphabet or underscore.'
         }
       }
 
-      if (!key.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/)) {
+      if (!key.match(/^[a-zA-Z_][a-zA-Z0-9_]*$/) && !isSasFormatsTable(key)) {
         return { status: false, msg: 'Table name should be alphanumeric.' }
       }
 
@@ -726,7 +777,7 @@ export default class SASjs {
         }
       }
 
-      if (this.getType(data[key]) !== 'Array') {
+      if (this.getType(data[key]) !== 'Array' && !isSasFormatsTable(key)) {
         return {
           status: false,
           msg: 'Parameter data contains invalid table structure.'
@@ -742,6 +793,7 @@ export default class SASjs {
         }
       }
     }
+
     return { status: true, msg: '' }
   }
 
@@ -777,7 +829,7 @@ export default class SASjs {
     accessToken?: string,
     isForced = false
   ) {
-    this.isMethodSupported('deployServicePack', ServerType.SasViya)
+    this.isMethodSupported('deployServicePack', [ServerType.SasViya])
 
     let sasApiClient: any = null
     if (serverUrl || appLoc) {
@@ -831,12 +883,26 @@ export default class SASjs {
     )
   }
 
-  public async deployToSASjs(members: [FolderMember, ServiceMember]) {
-    return await this.SASjsApiClient?.deploy(members, this.sasjsConfig.appLoc)
+  /**
+   * Creates the folders and services at the given location `appLoc` on the given server `serverUrl`.
+   * @param members - the JSON specifying the folders and services to be created.
+   * @param appLoc - the base folder in which to create the new folders and
+   * services.  If not provided, is taken from SASjsConfig.
+   * @param authConfig - a valid client, secret, refresh and access tokens that are authorised to execute compute jobs.
+   */
+  public async deployToSASjs(
+    members: FileTree,
+    appLoc?: string,
+    authConfig?: AuthConfig
+  ) {
+    if (!appLoc) {
+      appLoc = this.sasjsConfig.appLoc
+    }
+    return await this.sasJSApiClient?.deploy(members, appLoc, authConfig)
   }
 
   public async executeJobSASjs(query: ExecutionQuery) {
-    return await this.SASjsApiClient?.executeJob(query)
+    return await this.sasJSApiClient?.executeJob(query)
   }
 
   /**
@@ -872,7 +938,7 @@ export default class SASjs {
       ...config
     }
 
-    this.isMethodSupported('startComputeJob', ServerType.SasViya)
+    this.isMethodSupported('startComputeJob', [ServerType.SasViya])
     if (!config.contextName) {
       throw new Error(
         'Context name is undefined. Please set a `contextName` in your SASjs or override config.'
@@ -964,9 +1030,14 @@ export default class SASjs {
     }
 
     if (!this.requestClient) {
-      this.requestClient = new RequestClient(
+      const RequestClientClass =
+        this.sasjsConfig.serverType === ServerType.Sasjs
+          ? SasjsRequestClient
+          : RequestClient
+      this.requestClient = new RequestClientClass(
         this.sasjsConfig.serverUrl,
-        this.sasjsConfig.httpsAgentOptions
+        this.sasjsConfig.httpsAgentOptions,
+        this.sasjsConfig.requestHistoryLimit
       )
     } else {
       this.requestClient.setConfig(
@@ -980,7 +1051,7 @@ export default class SASjs {
         ? this.sasjsConfig.pathSASViya
         : this.sasjsConfig.serverType === ServerType.Sas9
         ? this.sasjsConfig.pathSAS9
-        : this.sasjsConfig.pathSASJS || ''
+        : this.sasjsConfig.pathSASJS
 
     this.authManager = new AuthManager(
       this.sasjsConfig.serverUrl,
@@ -1020,10 +1091,10 @@ export default class SASjs {
     }
 
     if (this.sasjsConfig.serverType === ServerType.Sasjs) {
-      if (this.SASjsApiClient) {
-        this.SASjsApiClient.setConfig(this.sasjsConfig.serverUrl)
+      if (this.sasJSApiClient) {
+        this.sasJSApiClient.setConfig(this.sasjsConfig.serverUrl)
       } else {
-        this.SASjsApiClient = new SASjsApiClient(
+        this.sasJSApiClient = new SASjsApiClient(
           this.sasjsConfig.serverUrl,
           this.requestClient
         )
@@ -1117,12 +1188,15 @@ export default class SASjs {
     })
   }
 
-  private isMethodSupported(method: string, serverType: string) {
-    if (this.sasjsConfig.serverType !== serverType) {
+  private isMethodSupported(method: string, serverTypes: ServerType[]) {
+    if (
+      !this.sasjsConfig.serverType ||
+      !serverTypes.includes(this.sasjsConfig.serverType)
+    ) {
       throw new Error(
-        `Method '${method}' is only supported on ${
-          serverType === ServerType.Sas9 ? 'SAS9' : 'SAS Viya'
-        } servers.`
+        `Method '${method}' is only supported on ${serverTypes.join(
+          ', '
+        )} servers.`
       )
     }
   }

src/SASjsApiClient.ts

@@ -1,5 +1,11 @@
-import { FolderMember, ServiceMember, ExecutionQuery } from './types'
+import { AuthConfig, ServerType } from '@sasjs/utils/types'
+import { FileTree, ExecutionQuery } from './types'
 import { RequestClient } from './request/RequestClient'
+import { getAccessTokenForSasjs } from './auth/getAccessTokenForSasjs'
+import { refreshTokensForSasjs } from './auth/refreshTokensForSasjs'
+import { getAuthCodeForSasjs } from './auth/getAuthCodeForSasjs'
+import { parseWeboutResponse } from './utils'
+import { getTokens } from './auth/getTokens'
 
 export class SASjsApiClient {
   constructor(
@@ -11,7 +17,19 @@ export class SASjsApiClient {
     if (serverUrl) this.serverUrl = serverUrl
   }
 
-  public async deploy(members: [FolderMember, ServiceMember], appLoc: string) {
+  public async deploy(
+    members: FileTree,
+    appLoc: string,
+    authConfig?: AuthConfig
+  ) {
+    let access_token = (authConfig || {}).access_token
+    if (authConfig) {
+      ;({ access_token } = await getTokens(
+        this.requestClient,
+        authConfig,
+        ServerType.Sasjs
+      ))
+    }
     const { result } = await this.requestClient.post<{
       status: string
       message: string
@@ -19,7 +37,7 @@ export class SASjsApiClient {
     }>(
       'SASjsApi/drive/deploy',
       { fileTree: members, appLoc: appLoc },
-      undefined
+      access_token
     )
 
     return Promise.resolve(result)
@@ -32,8 +50,53 @@ export class SASjsApiClient {
       log?: string
       logPath?: string
       error?: {}
+      _webout?: string
     }>('SASjsApi/stp/execute', query, undefined)
 
+    if (Object.keys(result).includes('_webout')) {
+      result._webout = parseWeboutResponse(result._webout!)
+    }
+
     return Promise.resolve(result)
   }
+
+  /**
+   * Exchanges the auth code for an access token for the given client.
+   * @param clientId - the client ID to authenticate with.
+   * @param authCode - the auth code received from the server.
+   */
+  public async getAccessToken(
+    clientId: string,
+    authCode: string
+  ): Promise<SASjsAuthResponse> {
+    return getAccessTokenForSasjs(this.requestClient, clientId, authCode)
+  }
+
+  /**
+   * Exchanges the refresh token for an access token.
+   * @param refreshToken - the refresh token received from the server.
+   */
+  public async refreshTokens(refreshToken: string): Promise<SASjsAuthResponse> {
+    return refreshTokensForSasjs(this.requestClient, refreshToken)
+  }
+
+  /**
+   * Performs a login authenticate and returns an auth code for the given client.
+   * @param username - a string representing the username.
+   * @param password - a string representing the password.
+   * @param clientId - the client ID to authenticate with.
+   */
+  public async getAuthCode(
+    username: string,
+    password: string,
+    clientId: string
+  ) {
+    return getAuthCodeForSasjs(this.requestClient, username, password, clientId)
+  }
+}
+
+// todo move to sasjs/utils
+export interface SASjsAuthResponse {
+  access_token: string
+  refresh_token: string
 }

src/SessionManager.ts

@@ -168,7 +168,7 @@ export class SessionManager {
     ) {
       if (stateLink) {
         if (this.debug && !this.printedSessionState.printed) {
-          logger.info('Polling session status...')
+          logger.info(`Polling: ${this.serverUrl + stateLink.href}`)
 
           this.printedSessionState.printed = true
         }

src/api/viya/executeScript.ts

@@ -272,7 +272,13 @@ export async function executeScript(
 
     return { result: jobResult?.result, log }
   } catch (e) {
-    if (e && e.status === 404) {
+    interface HttpError {
+      status: number
+    }
+
+    const error = e as HttpError
+
+    if (error.status === 404) {
       return executeScript(
         requestClient,
         sessionManager,
@@ -287,7 +293,7 @@ export async function executeScript(
         true
       )
     } else {
-      throw prefixMessage(e, 'Error while executing script. ')
+      throw prefixMessage(e as Error, 'Error while executing script. ')
     }
   }
 }

src/api/viya/pollJobState.ts

@@ -206,10 +206,11 @@ const doPoll = async (
 
     pollCount++
 
+    const jobHref = postedJob.links.find((l: Link) => l.rel === 'self')!.href
+
     if (pollOptions?.streamLog) {
-      const jobUrl = postedJob.links.find((l: Link) => l.rel === 'self')
       const { result: job } = await requestClient.get<Job>(
-        jobUrl!.href,
+        jobHref,
         authConfig?.access_token
       )
 
@@ -231,7 +232,7 @@ const doPoll = async (
     }
 
     if (debug && printedState !== state) {
-      logger.info('Polling job status...')
+      logger.info(`Polling: ${requestClient.getBaseUrl() + jobHref}/state`)
       logger.info(`Current job state: ${state}`)
 
       printedState = state

src/api/viya/spec/pollJobState.spec.ts

@@ -9,7 +9,10 @@ import * as isNodeModule from '../../../utils/isNode'
 import { PollOptions } from '../../../types'
 import { WriteStream } from 'fs'
 
+const baseUrl = 'http://localhost'
 const requestClient = new (<jest.Mock<RequestClient>>RequestClient)()
+requestClient['httpClient'].defaults.baseURL = baseUrl
+
 const defaultPollOptions: PollOptions = {
   maxPollCount: 100,
   pollInterval: 500,
@@ -195,7 +198,7 @@ describe('pollJobState', () => {
     expect((process as any).logger.info).toHaveBeenCalledTimes(4)
     expect((process as any).logger.info).toHaveBeenNthCalledWith(
       1,
-      'Polling job status...'
+      `Polling: ${baseUrl}/job/state`
     )
     expect((process as any).logger.info).toHaveBeenNthCalledWith(
       2,
@@ -203,7 +206,7 @@ describe('pollJobState', () => {
     )
     expect((process as any).logger.info).toHaveBeenNthCalledWith(
       3,
-      'Polling job status...'
+      `Polling: ${baseUrl}/job/state`
     )
     expect((process as any).logger.info).toHaveBeenNthCalledWith(
       4,

src/api/viya/spec/writeStream.spec.ts

@@ -1,24 +1,34 @@
 import { WriteStream } from '../../../types'
 import { writeStream } from '../writeStream'
-import 'jest-extended'
+import {
+  createWriteStream,
+  fileExists,
+  readFile,
+  deleteFile
+} from '@sasjs/utils'
 
 describe('writeStream', () => {
-  const stream: WriteStream = {
-    write: jest.fn(),
-    path: 'test'
-  }
+  const filename = 'test.txt'
+  const content = 'test'
+  let stream: WriteStream
+
+  beforeAll(async () => {
+    stream = await createWriteStream(filename)
+  })
 
   it('should resolve when the stream is written successfully', async () => {
-    expect(writeStream(stream, 'test')).toResolve()
+    await expect(writeStream(stream, content)).toResolve()
+    await expect(fileExists(filename)).resolves.toEqual(true)
+    await expect(readFile(filename)).resolves.toEqual(content + '\n')
 
-    expect(stream.write).toHaveBeenCalledWith('test\n', expect.anything())
+    await deleteFile(filename)
   })
 
   it('should reject when the write errors out', async () => {
     jest
       .spyOn(stream, 'write')
       .mockImplementation((_, callback) => callback(new Error('Test Error')))
-    const error = await writeStream(stream, 'test').catch((e) => e)
+    const error = await writeStream(stream, content).catch((e) => e)
 
     expect(error.message).toEqual('Test Error')
   })

src/api/viya/writeStream.ts

@@ -3,13 +3,9 @@ import { WriteStream } from '../../types'
 export const writeStream = async (
   stream: WriteStream,
   content: string
-): Promise<void> => {
-  return new Promise((resolve, reject) => {
-    stream.write(content + '\n', (e) => {
-      if (e) {
-        return reject(e)
-      }
-      return resolve()
-    })
+): Promise<void> =>
+  stream.write(content + '\n', (e) => {
+    if (e) return Promise.reject(e)
+
+    return Promise.resolve()
   })
-}