Merge pull request #310 from sasjs/issue-309

Issue 309

README.md

@@ -6,7 +6,7 @@ SASjs is a open-source framework for building Web Apps on SAS® platforms. You c
 
 1 - `npm install @sasjs/adapter` - for use in a node project
 
-2 - [Download](https://cdn.jsdelivr.net/npm/@sasjs/adapter@1/index.js) and use a copy of the latest JS file
+2 - [Download](https://cdn.jsdelivr.net/npm/@sasjs/adapter@2/index.js) and use a copy of the latest JS file
 
 3 - Reference directly from the CDN - in which case click [here](https://www.jsdelivr.com/package/npm/@sasjs/adapter?tab=collection) and select "SRI" to get the script tag with the integrity hash.
 
@@ -41,8 +41,165 @@ parmcards4;
 
 You now have a simple web app with a backend service!
 
+## Detailed Overview
+
+The SASjs adapter is a JS library and a set of SAS Macros that handle the communication between the frontend app and backend SAS services.
+
+There are three parts to consider:
+
+1. JS request / response
+2. SAS inputs / outputs
+3. Configuration
+
+### JS Request / Response
+
+To install the library you can simply run `npm i @sasjs/adapter` or include a `<script>` tag with a reference to our [CDN](https://www.jsdelivr.com/package/npm/@sasjs/adapter).
+
+Full technical documentation is available [here](https://adapter.sasjs.io).  The main parts are:
+
+### Instantiation
+The following code will instantiate an instance of the adapter:
+
+```javascript
+let sasJs = new SASjs.default(
+  {
+    appLoc: "/Your/SAS/Folder",
+    serverType:"SAS9"
+  }
+);
+```
+If you've installed it via NPM, you can import it as a default import like so:
+```
+  import SASjs from '@sasjs/adapter';
+```
+You can then instantiate it with:
+```
+const sasJs = new SASjs({your config})
+```
+
+More on the config later.
+
+### SAS Logon
+The login process can be handled directly, as below, or as a callback function to a SAS request.
+
+```javascript
+sasJs.logIn('USERNAME','PASSWORD'
+  ).then((response) => {
+  if (response.isLoggedIn === true) {
+    console.log('do stuff')
+  } else {
+    console.log('do other stuff')
+  }
+}
+```
+
+###  Request / Response
+A simple request can be sent to SAS in the following fashion:
+
+```javascript
+sasJs.request("/path/to/my/service", dataObject)
+  .then((response) => {
+    // all tables are in the response object, eg:
+    console.log(response.tablewith2cols1row[0].COL1.value)
+  })
+```
+We supply the path to the SAS service, and a data object.  The data object can be null (for services with no input), or can contain one or more tables in the following format:
+
+```javascript
+let dataObject={
+	"tablewith2cols1row": [{
+		"col1": "val1",
+		"col2": 42
+	}],
+	"tablewith1col2rows": [{
+		"col": "row1"
+	}, {
+		"col": "row2"
+	}]
+};
+```
+
+There are optional parameters such as a config object and a callback login function.
+
+The response object will contain returned tables and columns.  Table names are always lowercase, and column names uppercase.
+
+The adapter will also cache the logs (if debug enabled) and even the work tables.  For performance, it is best to keep debug mode off.
+
+## SAS Inputs / Outputs
+
+The SAS side is handled by a number of macros in the [macro core](https://github.com/sasjs/core) library.
+
+The following snippet shows the process of SAS tables arriving / leaving:
+```sas
+/* fetch all input tables sent from frontend - they arrive as work tables */
+%webout(FETCH)
+
+/* some sas code */
+data some sas tables;
+  set from js;
+run;
+
+%webout(OPEN)  /* open the JSON to be returned */
+%webout(OBJ,some) /* `some` table is sent in object format */
+%webout(ARR,sas) /* `sas` table is sent in array format, smaller filesize */
+%webout(OBJ,tables,fmt=N) /* unformatted (raw) data */
+%webout(OBJ,tables,label=newtable) /* rename tables on export */
+%webout(CLOSE) /* close the JSON and send some extra useful variables too */
+
+```
+
+## Configuration
+
+Configuration on the client side involves passing an object on startup, which can also be passed with each request.  Technical documentation on the SASjsConfig class is available [here](https://adapter.sasjs.io/classes/types.sasjsconfig.html).  The main config items are:
+
+* `appLoc` - this is the folder under which the SAS services will be created.
+* `serverType` - either `SAS9` or `SASVIYA`.
+* `serverUrl` - the location (including http protocol and port) of the SAS Server. Can be omitted, eg if serving directly from the SAS Web Server, or in streaming mode.
+* `debug` - if `true` then SAS Logs and extra debug information is returned.
+* `useComputeApi` - if `true` and the serverType is `SASVIYA` then the REST APIs will be called directly (rather than using the JES web service).
+* `contextName` - if missing or blank, and `useComputeApi` is `true` and `serverType` is `SASVIYA` then the JES API will be used.
+
+The adapter supports a number of approaches for interfacing with Viya (`serverType` is `SASVIYA`).  For maximum performance, be sure to [configure your compute context](https://sasjs.io/guide-viya/#shared-account-and-server-re-use) with `reuseServerProcesses` as `true` and a system account in `runServerAs`.  This functionality is available since Viya 3.5.  This configuration is supported when [creating contexts using the CLI](https://sasjs.io/sasjs-cli-context/#sasjs-context-create).
+
+### Using JES Web App
+
+In this setup, all requests are routed through the JES web app, at `YOURSERVER/SASJobExecution`.  This is the most reliable method, and also the slowest.  One request is made to the JES app, and remaining requests (getting job uri, session spawning, passing parameters, running the program, fetching the log) are made on the SAS server by the JES app.
+
+```
+{
+  appLoc:"/Your/Path",
+  serverType:"SASVIYA"
+}
+```
+
+### Using the JES API
+Here we are running Jobs using the Job Execution Service except this time we are making the requests directly using the REST API instead of through the JES Web App.  This is helpful when we need to call web services outside of a browser (eg with the SASjs CLI or other commandline tools).  To save one network request, the adapter prefetches the JOB URIs and passes them in the `__job` parameter.
+
+```
+{
+  appLoc:"/Your/Path",
+  serverType:"SASVIYA",
+  useComputeApi: true
+}
+```
+
+### Using the Compute API
+This approach is by far the fastest, as a result of the optimisations we have built into the adapter.  With this configuration, in the first sasjs request, we take a URI map of the services in the target folder, and create a session manager - which spawns an extra session.  The next time a request is made, the adapter will use the 'hot' session.  Sessions are deleted after every use, which actually makes this _less_ resource intensive than a typical JES web app, in which all sessions are kept alive by default for 15 minutes.
+
+```
+{
+  appLoc:"/Your/Path",
+  serverType:"SASVIYA",
+  useComputeApi: true,
+  contextName: 'yourComputeContext'
+}
+```
+
+
 # More resources
 
-For more information and examples specific to this adapter you can check out the [user guide](https://sasjs.io/sasjs-adapter/) or the [technical](http://adapter.sasjs.io/) documentation.
+For more information and examples specific to this adapter you can check out the [user guide](https://sasjs.io/sasjs-adapter/) or the [technical](http://adapter.sasjs.io/) documentation. 
 
 For more information on building web apps in general, check out these [resources](https://sasjs.io/training/resources/) or contact the [author](https://www.linkedin.com/in/allanbowe/) directly.
+
+If you are a SAS 9 or SAS Viya customer you can also request a copy of [Data Controller](https://datacontroller.io) - free for up to 5 users, this tool makes use of all parts of the SASjs framework.

src/FileUploader.ts

@@ -1,6 +1,6 @@
 import { isUrl } from './utils'
 import { UploadFile } from './types/UploadFile'
-import { ErrorResponse, LoginRequiredError } from './types'
+import { ErrorResponse, LoginRequiredError } from './types/errors'
 import { RequestClient } from './request/RequestClient'
 
 export class FileUploader {

src/SASViyaApiClient.ts

@@ -1,4 +1,10 @@
-import { convertToCSV, isRelativePath, isUri, isUrl } from './utils'
+import {
+  convertToCSV,
+  isRelativePath,
+  isUri,
+  isUrl,
+  fetchLogByChunks
+} from './utils'
 import * as NodeFormData from 'form-data'
 import {
   Job,
@@ -8,10 +14,13 @@ import {
   Folder,
   EditContextInput,
   JobDefinition,
-  PollOptions,
+  PollOptions
-  ComputeJobExecutionError,
-  JobExecutionError
 } from './types'
+import {
+  ComputeJobExecutionError,
+  JobExecutionError,
+  NotFoundError
+} from './types/errors'
 import { formatDataForRequest } from './utils/formatDataForRequest'
 import { SessionManager } from './SessionManager'
 import { ContextManager } from './ContextManager'
@@ -19,7 +28,6 @@ import { timestampToYYYYMMDDHHMMSS } from '@sasjs/utils/time'
 import { Logger, LogLevel } from '@sasjs/utils/logger'
 import { isAuthorizeFormRequired } from './auth/isAuthorizeFormRequired'
 import { RequestClient } from './request/RequestClient'
-import { NotFoundError } from './types/NotFoundError'
 import { SasAuthResponse } from '@sasjs/utils/types'
 import { prefixMessage } from '@sasjs/utils/error'
 
@@ -418,19 +426,19 @@ export class SASViyaApiClient {
         })
 
       let jobResult
-      let log
+      let log = ''
 
       const logLink = currentJob.links.find((l) => l.rel === 'log')
 
       if (debug && logLink) {
-        log = await this.requestClient
+        const logUrl = `${logLink.href}/content`
-          .get<any>(`${logLink.href}/content?limit=10000`, accessToken)
+        const logCount = currentJob.logStatistics?.lineCount ?? 1000000
-          .then((res: any) =>
+        log = await fetchLogByChunks(
-            res.result.items.map((i: any) => i.line).join('\n')
+          this.requestClient,
-          )
+          accessToken!,
-          .catch((err) => {
+          logUrl,
-            throw prefixMessage(err, 'Error while getting log. ')
+          logCount
-          })
+        )
       }
 
       if (jobStatus === 'failed' || jobStatus === 'error') {
@@ -451,14 +459,14 @@ export class SASViyaApiClient {
           .catch(async (e) => {
             if (e instanceof NotFoundError) {
               if (logLink) {
-                log = await this.requestClient
+                const logUrl = `${logLink.href}/content`
-                  .get<any>(`${logLink.href}/content?limit=10000`, accessToken)
+                const logCount = currentJob.logStatistics?.lineCount ?? 1000000
-                  .then((res: any) =>
+                log = await fetchLogByChunks(
-                    res.result.items.map((i: any) => i.line).join('\n')
+                  this.requestClient,
-                  )
+                  accessToken!,
-                  .catch((err) => {
+                  logUrl,
-                    throw prefixMessage(err, 'Error while getting log. ')
+                  logCount
-                  })
+                )
 
                 return Promise.reject({
                   status: 500,
@@ -1082,7 +1090,9 @@ export class SASViyaApiClient {
       .get<string>(
         `${this.serverUrl}${stateLink.href}?_action=wait&wait=30`,
         accessToken,
-        'text/plain'
+        'text/plain',
+        {},
+        this.debug
       )
       .catch((err) => {
         throw prefixMessage(err, 'Error while getting job state. ')
@@ -1107,7 +1117,9 @@ export class SASViyaApiClient {
               .get<string>(
                 `${this.serverUrl}${stateLink.href}?_action=wait&wait=30`,
                 accessToken,
-                'text/plain'
+                'text/plain',
+                {},
+                this.debug
               )
               .catch((err) => {
                 throw prefixMessage(

src/auth/AuthManager.ts

@@ -82,17 +82,33 @@ export class AuthManager {
    * @returns - a promise which resolves with an object containing two values - a boolean `isLoggedIn`, and a string `userName`.
    */
   public async checkSession() {
-    const { result: loginResponse } = await this.requestClient.get<string>(
+    //For VIYA we will send request on API endpoint. Which is faster then pinging SASJobExecution.
-      this.loginUrl.replace('.do', ''),
+    //For SAS9 we will send request on SASStoredProcess
-      undefined,
+    const url =
-      'text/plain'
+      this.serverType === 'SASVIYA'
-    )
+        ? `${this.serverUrl}/identities`
-    const responseText = loginResponse
+        : `${this.serverUrl}/SASStoredProcess`
-    const isLoggedIn = /<button.+onClick.+logout/gm.test(responseText)
+
-    let loginForm: any = null
+    const { result: loginResponse } = await this.requestClient
+      .get<string>(url, undefined, 'text/plain')
+      .catch((err: any) => {
+        return { result: 'authErr' }
+      })
+
+    const isLoggedIn = loginResponse !== 'authErr'
+    let loginForm = null
 
     if (!isLoggedIn) {
-      loginForm = await this.getLoginForm(responseText)
+      //We will logout to make sure cookies are removed and login form is presented
+      this.logOut()
+
+      const { result: formResponse } = await this.requestClient.get<string>(
+        this.loginUrl.replace('.do', ''),
+        undefined,
+        'text/plain'
+      )
+
+      loginForm = await this.getLoginForm(formResponse)
     }
 
     return Promise.resolve({

src/auth/spec/AuthManager.spec.ts

@@ -176,41 +176,19 @@ describe('AuthManager', () => {
 
     const response = await authManager.checkSession()
     expect(response.isLoggedIn).toBeTruthy()
-    expect(mockedAxios.get).toHaveBeenNthCalledWith(1, `/SASLogon/login`, {
+    expect(mockedAxios.get).toHaveBeenNthCalledWith(
-      withCredentials: true,
+      1,
-      responseType: 'text',
+      `http://test-server.com/identities`,
-      transformResponse: undefined,
+      {
-      headers: {
+        withCredentials: true,
-        Accept: '*/*',
+        responseType: 'text',
-        'Content-Type': 'text/plain'
+        transformResponse: undefined,
+        headers: {
+          Accept: '*/*',
+          'Content-Type': 'text/plain'
+        }
       }
-    })
-
-    done()
-  })
-
-  it('should check and return session information if logged in', async (done) => {
-    const authManager = new AuthManager(
-      serverUrl,
-      serverType,
-      requestClient,
-      authCallback
     )
-    mockedAxios.get.mockImplementation(() =>
-      Promise.resolve({ data: '<button onClick="logout">' })
-    )
-
-    const response = await authManager.checkSession()
-    expect(response.isLoggedIn).toBeTruthy()
-    expect(mockedAxios.get).toHaveBeenNthCalledWith(1, `/SASLogon/login`, {
-      withCredentials: true,
-      responseType: 'text',
-      transformResponse: undefined,
-      headers: {
-        Accept: '*/*',
-        'Content-Type': 'text/plain'
-      }
-    })
 
     done()
   })

src/index.ts

@@ -1,5 +1,6 @@
 import SASjs from './SASjs'
 export * from './types'
+export * from './types/errors'
 export * from './SASViyaApiClient'
 export * from './SAS9ApiClient'
 export default SASjs

src/job-execution/ComputeJobExecutor.ts

@@ -1,7 +1,10 @@
 import { ServerType } from '@sasjs/utils/types'
-import { ErrorResponse } from '..'
 import { SASViyaApiClient } from '../SASViyaApiClient'
-import { ComputeJobExecutionError, LoginRequiredError } from '../types'
+import {
+  ErrorResponse,
+  ComputeJobExecutionError,
+  LoginRequiredError
+} from '../types/errors'
 import { BaseJobExecutor } from './JobExecutor'
 
 export class ComputeJobExecutor extends BaseJobExecutor {

src/job-execution/JesJobExecutor.ts

@@ -1,7 +1,10 @@
 import { ServerType } from '@sasjs/utils/types'
-import { ErrorResponse } from '..'
 import { SASViyaApiClient } from '../SASViyaApiClient'
-import { JobExecutionError, LoginRequiredError } from '../types'
+import {
+  ErrorResponse,
+  JobExecutionError,
+  LoginRequiredError
+} from '../types/errors'
 import { BaseJobExecutor } from './JobExecutor'
 
 export class JesJobExecutor extends BaseJobExecutor {

src/job-execution/WebJobExecutor.ts

@@ -1,5 +1,9 @@
 import { ServerType } from '@sasjs/utils/types'
-import { ErrorResponse, JobExecutionError, LoginRequiredError } from '..'
+import {
+  ErrorResponse,
+  JobExecutionError,
+  LoginRequiredError
+} from '../types/errors'
 import { generateFileUploadForm } from '../file/generateFileUploadForm'
 import { generateTableUploadForm } from '../file/generateTableUploadForm'
 import { RequestClient } from '../request/RequestClient'

src/request/RequestClient.ts

@@ -1,9 +1,13 @@
 import axios, { AxiosInstance, AxiosRequestConfig, AxiosResponse } from 'axios'
-import { CsrfToken, JobExecutionError } from '..'
+import { CsrfToken } from '..'
 import { isAuthorizeFormRequired, isLogInRequired } from '../auth'
-import { LoginRequiredError } from '../types'
+import {
-import { AuthorizeError } from '../types/AuthorizeError'
+  AuthorizeError,
-import { NotFoundError } from '../types/NotFoundError'
+  LoginRequiredError,
+  NotFoundError,
+  InternalServerError,
+  JobExecutionError
+} from '../types/errors'
 import { parseWeboutResponse } from '../utils/parseWeboutResponse'
 import { prefixMessage } from '@sasjs/utils/error'
 
@@ -73,7 +77,8 @@ export class RequestClient implements HttpClient {
     url: string,
     accessToken: string | undefined,
     contentType: string = 'application/json',
-    overrideHeaders: { [key: string]: string | number } = {}
+    overrideHeaders: { [key: string]: string | number } = {},
+    debug: boolean = false
   ): Promise<{ result: T; etag: string }> {
     const headers = {
       ...this.getHeaders(accessToken, contentType),
@@ -93,20 +98,23 @@ export class RequestClient implements HttpClient {
       .get<T>(url, requestConfig)
       .then((response) => {
         throwIfError(response)
+
         return this.parseResponse<T>(response)
       })
       .catch(async (e) => {
-        return await this.handleError(e, () =>
+        return await this.handleError(
-          this.get<T>(url, accessToken, contentType, overrideHeaders).catch(
+          e,
-            (err) => {
+          () =>
-              throw prefixMessage(
+            this.get<T>(url, accessToken, contentType, overrideHeaders).catch(
-                err,
+              (err) => {
-                'Error while executing handle error callback. '
+                throw prefixMessage(
-              )
+                  err,
-            }
+                  'Error while executing handle error callback. '
-          )
+                )
+              }
+            ),
+          debug
         ).catch((err) => {
-          console.log(`[err]`, err)
           throw prefixMessage(err, 'Error while handling error. ')
         })
       })
@@ -340,7 +348,11 @@ export class RequestClient implements HttpClient {
     }
   }
 
-  private handleError = async (e: any, callback: any) => {
+  private handleError = async (
+    e: any,
+    callback: any,
+    debug: boolean = false
+  ) => {
     const response = e.response as AxiosResponse
 
     if (e instanceof AuthorizeError) {
@@ -386,6 +398,9 @@ export class RequestClient implements HttpClient {
       throw e
     } else if (response?.status === 404) {
       throw new NotFoundError(response.config.url!)
+    } else if (response?.status === 502) {
+      if (debug) throw new InternalServerError()
+      else return
     }
 
     throw e
@@ -457,6 +472,7 @@ const throwIfError = (response: AxiosResponse) => {
   }
 
   const error = parseError(response.data as string)
+
   if (error) {
     throw error
   }

src/types/Job.ts

@@ -1,5 +1,6 @@
 import { Link } from './Link'
 import { JobResult } from './JobResult'
+import { LogStatistics } from './LogStatistics'
 
 export interface Job {
   id: string
@@ -10,4 +11,5 @@ export interface Job {
   links: Link[]
   results: JobResult
   error?: any
+  logStatistics: LogStatistics
 }

src/types/LogStatistics.ts

@@ -0,0 +1,4 @@
+export interface LogStatistics {
+  lineCount: number
+  modifiedTimeStamp: string
+}

src/types/errors/ComputeJobExecutionError.ts

@@ -1,4 +1,4 @@
-import { Job } from './Job'
+import { Job } from '../Job'
 
 export class ComputeJobExecutionError extends Error {
   constructor(public job: Job, public log: string) {

src/types/errors/InternalServerError.ts

@@ -0,0 +1,9 @@
+export class InternalServerError extends Error {
+  constructor() {
+    super('Error: Internal server error.')
+
+    this.name = 'InternalServerError'
+
+    Object.setPrototypeOf(this, InternalServerError.prototype)
+  }
+}

src/types/errors/index.ts

@@ -0,0 +1,7 @@
+export * from './AuthorizeError'
+export * from './ComputeJobExecutionError'
+export * from './InternalServerError'
+export * from './JobExecutionError'
+export * from './LoginRequiredError'
+export * from './NotFoundError'
+export * from './ErrorResponse'

src/types/index.ts

@@ -1,14 +1,10 @@
-export * from './ComputeJobExecutionError'
 export * from './Context'
 export * from './CsrfToken'
-export * from './ErrorResponse'
 export * from './Folder'
 export * from './Job'
-export * from './JobExecutionError'
 export * from './JobDefinition'
 export * from './JobResult'
 export * from './Link'
-export * from './LoginRequiredError'
 export * from './SASjsConfig'
 export * from './SASjsRequest'
 export * from './Session'

src/utils/fetchLogByChunks.ts

@@ -0,0 +1,43 @@
+import { RequestClient } from '../request/RequestClient'
+import { prefixMessage } from '@sasjs/utils/error'
+
+/**
+ * Fetches content of the log file
+ * @param {object} requestClient - client object of Request Client.
+ * @param {string} accessToken - an access token for an authorized user.
+ * @param {string} logUrl - url of the log file.
+ * @param {number} logCount- total number of log lines in file.
+ * @returns an string containing log lines.
+ */
+export const fetchLogByChunks = async (
+  requestClient: RequestClient,
+  accessToken: string,
+  logUrl: string,
+  logCount: number
+): Promise<string> => {
+  let log: string = ''
+
+  const loglimit = logCount < 10000 ? logCount : 10000
+  let start = 0
+  do {
+    console.log(
+      `Fetching logs from line no: ${start + 1} to ${
+        start + loglimit
+      } of ${logCount}.`
+    )
+    const logChunkJson = await requestClient!
+      .get<any>(`${logUrl}?start=${start}&limit=${loglimit}`, accessToken)
+      .then((res: any) => res.result)
+      .catch((err) => {
+        throw prefixMessage(err, 'Error while getting log. ')
+      })
+
+    if (logChunkJson.items.length === 0) break
+
+    const logChunk = logChunkJson.items.map((i: any) => i.line).join('\n')
+    log += logChunk
+
+    start += loglimit
+  } while (start < logCount)
+  return log
+}

src/utils/index.ts

@@ -11,3 +11,4 @@ export * from './parseSasViyaLog'
 export * from './serialize'
 export * from './splitChunks'
 export * from './parseWeboutResponse'
+export * from './fetchLogByChunks'