Merge pull request #300 from sasjs/fix-log-by-chunks

fix: after job executing get complete log

.gitignore

@@ -3,4 +3,6 @@ build
 
 .env
 
-/coverage
+/coverage
+
+.DS_Store

README.md

@@ -6,7 +6,7 @@ SASjs is a open-source framework for building Web Apps on SAS® platforms. You c
 
 1 - `npm install @sasjs/adapter` - for use in a node project
 
-2 - [Download](https://cdn.jsdelivr.net/npm/@sasjs/adapter@1/index.js) and use a copy of the latest JS file
+2 - [Download](https://cdn.jsdelivr.net/npm/@sasjs/adapter@2/index.js) and use a copy of the latest JS file
 
 3 - Reference directly from the CDN - in which case click [here](https://www.jsdelivr.com/package/npm/@sasjs/adapter?tab=collection) and select "SRI" to get the script tag with the integrity hash.
 
@@ -41,8 +41,165 @@ parmcards4;
 
 You now have a simple web app with a backend service!
 
+## Detailed Overview
+
+The SASjs adapter is a JS library and a set of SAS Macros that handle the communication between the frontend app and backend SAS services.
+
+There are three parts to consider:
+
+1. JS request / response
+2. SAS inputs / outputs
+3. Configuration
+
+### JS Request / Response
+
+To install the library you can simply run `npm i @sasjs/adapter` or include a `<script>` tag with a reference to our [CDN](https://www.jsdelivr.com/package/npm/@sasjs/adapter).
+
+Full technical documentation is available [here](https://adapter.sasjs.io).  The main parts are:
+
+### Instantiation
+The following code will instantiate an instance of the adapter:
+
+```javascript
+let sasJs = new SASjs.default(
+  {
+    appLoc: "/Your/SAS/Folder",
+    serverType:"SAS9"
+  }
+);
+```
+If you've installed it via NPM, you can import it as a default import like so:
+```
+  import SASjs from '@sasjs/adapter';
+```
+You can then instantiate it with:
+```
+const sasJs = new SASjs({your config})
+```
+
+More on the config later.
+
+### SAS Logon
+The login process can be handled directly, as below, or as a callback function to a SAS request.
+
+```javascript
+sasJs.logIn('USERNAME','PASSWORD'
+  ).then((response) => {
+  if (response.isLoggedIn === true) {
+    console.log('do stuff')
+  } else {
+    console.log('do other stuff')
+  }
+}
+```
+
+###  Request / Response
+A simple request can be sent to SAS in the following fashion:
+
+```javascript
+sasJs.request("/path/to/my/service", dataObject)
+  .then((response) => {
+    // all tables are in the response object, eg:
+    console.log(response.tablewith2cols1row[0].COL1.value)
+  })
+```
+We supply the path to the SAS service, and a data object.  The data object can be null (for services with no input), or can contain one or more tables in the following format:
+
+```javascript
+let dataObject={
+	"tablewith2cols1row": [{
+		"col1": "val1",
+		"col2": 42
+	}],
+	"tablewith1col2rows": [{
+		"col": "row1"
+	}, {
+		"col": "row2"
+	}]
+};
+```
+
+There are optional parameters such as a config object and a callback login function.
+
+The response object will contain returned tables and columns.  Table names are always lowercase, and column names uppercase.
+
+The adapter will also cache the logs (if debug enabled) and even the work tables.  For performance, it is best to keep debug mode off.
+
+## SAS Inputs / Outputs
+
+The SAS side is handled by a number of macros in the [macro core](https://github.com/sasjs/core) library.
+
+The following snippet shows the process of SAS tables arriving / leaving:
+```sas
+/* fetch all input tables sent from frontend - they arrive as work tables */
+%webout(FETCH)
+
+/* some sas code */
+data some sas tables;
+  set from js;
+run;
+
+%webout(OPEN)  /* open the JSON to be returned */
+%webout(OBJ,some) /* `some` table is sent in object format */
+%webout(ARR,sas) /* `sas` table is sent in array format, smaller filesize */
+%webout(OBJ,tables,fmt=N) /* unformatted (raw) data */
+%webout(OBJ,tables,label=newtable) /* rename tables on export */
+%webout(CLOSE) /* close the JSON and send some extra useful variables too */
+
+```
+
+## Configuration
+
+Configuration on the client side involves passing an object on startup, which can also be passed with each request.  Technical documentation on the SASjsConfig class is available [here](https://adapter.sasjs.io/classes/types.sasjsconfig.html).  The main config items are:
+
+* `appLoc` - this is the folder under which the SAS services will be created.
+* `serverType` - either `SAS9` or `SASVIYA`.
+* `serverUrl` - the location (including http protocol and port) of the SAS Server. Can be omitted, eg if serving directly from the SAS Web Server, or in streaming mode.
+* `debug` - if `true` then SAS Logs and extra debug information is returned.
+* `useComputeApi` - if `true` and the serverType is `SASVIYA` then the REST APIs will be called directly (rather than using the JES web service).
+* `contextName` - if missing or blank, and `useComputeApi` is `true` and `serverType` is `SASVIYA` then the JES API will be used.
+
+The adapter supports a number of approaches for interfacing with Viya (`serverType` is `SASVIYA`).  For maximum performance, be sure to [configure your compute context](https://sasjs.io/guide-viya/#shared-account-and-server-re-use) with `reuseServerProcesses` as `true` and a system account in `runServerAs`.  This functionality is available since Viya 3.5.  This configuration is supported when [creating contexts using the CLI](https://sasjs.io/sasjs-cli-context/#sasjs-context-create).
+
+### Using JES Web App
+
+In this setup, all requests are routed through the JES web app, at `YOURSERVER/SASJobExecution`.  This is the most reliable method, and also the slowest.  One request is made to the JES app, and remaining requests (getting job uri, session spawning, passing parameters, running the program, fetching the log) are made on the SAS server by the JES app.
+
+```
+{
+  appLoc:"/Your/Path",
+  serverType:"SASVIYA"
+}
+```
+
+### Using the JES API
+Here we are running Jobs using the Job Execution Service except this time we are making the requests directly using the REST API instead of through the JES Web App.  This is helpful when we need to call web services outside of a browser (eg with the SASjs CLI or other commandline tools).  To save one network request, the adapter prefetches the JOB URIs and passes them in the `__job` parameter.
+
+```
+{
+  appLoc:"/Your/Path",
+  serverType:"SASVIYA",
+  useComputeApi: true
+}
+```
+
+### Using the Compute API
+This approach is by far the fastest, as a result of the optimisations we have built into the adapter.  With this configuration, in the first sasjs request, we take a URI map of the services in the target folder, and create a session manager - which spawns an extra session.  The next time a request is made, the adapter will use the 'hot' session.  Sessions are deleted after every use, which actually makes this _less_ resource intensive than a typical JES web app, in which all sessions are kept alive by default for 15 minutes.
+
+```
+{
+  appLoc:"/Your/Path",
+  serverType:"SASVIYA",
+  useComputeApi: true,
+  contextName: 'yourComputeContext'
+}
+```
+
+
 # More resources
 
-For more information and examples specific to this adapter you can check out the [user guide](https://sasjs.io/sasjs-adapter/) or the [technical](http://adapter.sasjs.io/) documentation.
+For more information and examples specific to this adapter you can check out the [user guide](https://sasjs.io/sasjs-adapter/) or the [technical](http://adapter.sasjs.io/) documentation. 
 
 For more information on building web apps in general, check out these [resources](https://sasjs.io/training/resources/) or contact the [author](https://www.linkedin.com/in/allanbowe/) directly.
+
+If you are a SAS 9 or SAS Viya customer you can also request a copy of [Data Controller](https://datacontroller.io) - free for up to 5 users, this tool makes use of all parts of the SASjs framework.

package.json

@@ -43,23 +43,23 @@
     "jest-extended": "^0.11.5",
     "path": "^0.12.7",
     "rimraf": "^3.0.2",
-    "semantic-release": "^17.3.9",
+    "semantic-release": "^17.4.1",
     "terser-webpack-plugin": "^4.2.3",
     "ts-jest": "^25.5.1",
     "ts-loader": "^8.0.17",
     "tslint": "^6.1.3",
     "tslint-config-prettier": "^1.18.0",
-    "typedoc": "^0.19.2",
+    "typedoc": "^0.20.30",
     "typedoc-neo-theme": "^1.1.0",
     "typedoc-plugin-external-module-name": "^4.0.6",
     "typescript": "^3.9.9",
-    "webpack": "^5.21.2",
+    "webpack": "^5.24.4",
     "webpack-cli": "^4.5.0"
   },
   "main": "index.js",
   "dependencies": {
+    "@sasjs/utils": "^2.6.3",
     "axios": "^0.21.1",
-    "@sasjs/utils": "^2.5.0",
     "form-data": "^3.0.0",
     "https": "^1.0.0"
   }

sasjs-tests/package-lock.json

@@ -2004,21 +2004,19 @@
       }
     },
     "@sasjs/adapter": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-2.1.0.tgz",
-      "integrity": "sha512-FdvxAPzXHHwCurH5B1RxSGeO6h+lVqDGAtrTAQDByT7HhsikAMFi6mW5dJR/1CKcI2rLOdE/479yYqjwfX8nwg==",
+      "version": "file:../build/sasjs-adapter-5.0.0.tgz",
+      "integrity": "sha512-1t+3LIL2BFw8HpZUPI9QM24801+JH4DCAu4eHoLLmytYhN72asMi1aVtgSDb1xiJYgpbTG7EK3qRpHIV8cEN8w==",
       "requires": {
-        "@sasjs/utils": "^2.0.2",
-        "es6-promise": "^4.2.8",
+        "@sasjs/utils": "^2.5.0",
+        "axios": "^0.21.1",
         "form-data": "^3.0.0",
-        "https": "^1.0.0",
-        "isomorphic-fetch": "^2.2.1"
+        "https": "^1.0.0"
       },
       "dependencies": {
         "form-data": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz",
-          "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==",
+          "version": "3.0.1",
+          "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
+          "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==",
           "requires": {
             "asynckit": "^0.4.0",
             "combined-stream": "^1.0.8",
@@ -2048,10 +2046,11 @@
       }
     },
     "@sasjs/utils": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.2.0.tgz",
-      "integrity": "sha512-T01TYAFU+WQ1uAUaunZXg0MotQe5jn3auVuMPPbY5RU5VhIn4oVhr6qz3EHyNsYYKD1aj5pP7w2elnBzkgWhGw==",
+      "version": "2.5.1",
+      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.5.1.tgz",
+      "integrity": "sha512-a3ISiUX8Yz7au4XYxq2KWf9ODT6nsIDbE4FEqS+AQ3McxZkfuAk4v+REXjOmIlcyQd4R4bufEK8XoB6AROn9sA==",
       "requires": {
+        "@types/prompts": "^2.0.9",
         "consola": "^2.15.0",
         "prompts": "^2.4.0",
         "valid-url": "^1.0.9"
@@ -2366,6 +2365,14 @@
       "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-2.1.6.tgz",
       "integrity": "sha512-6gOkRe7OIioWAXfnO/2lFiv+SJichKVSys1mSsgyrYHSEjk8Ctv4tSR/Odvnu+HWlH2C8j53dahU03XmQdd5fA=="
     },
+    "@types/prompts": {
+      "version": "2.0.9",
+      "resolved": "https://registry.npmjs.org/@types/prompts/-/prompts-2.0.9.tgz",
+      "integrity": "sha512-TORZP+FSjTYMWwKadftmqEn6bziN5RnfygehByGsjxoK5ydnClddtv6GikGWPvCm24oI+YBwck5WDxIIyNxUrA==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/prop-types": {
       "version": "15.7.3",
       "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.3.tgz",
@@ -3396,6 +3403,14 @@
       "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.1.1.tgz",
       "integrity": "sha512-5Kgy8Cz6LPC9DJcNb3yjAXTu3XihQgEdnIg50c//zOC/MyLP0Clg+Y8Sh9ZjjnvBrDZU4DgXS9C3T9r4/scGZQ=="
     },
+    "axios": {
+      "version": "0.21.1",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.1.tgz",
+      "integrity": "sha512-dKQiRHxGD9PPRIUNIWvZhPTPpl1rf/OxTYKsqKUDjBwYylTvV7SjSHJb9ratfyzM6wCdLCOYLzs73qpg5c4iGA==",
+      "requires": {
+        "follow-redirects": "^1.10.0"
+      }
+    },
     "axobject-query": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-2.2.0.tgz",
@@ -4677,9 +4692,9 @@
       "integrity": "sha512-e54B99q/OUoH64zYYRf3HBP5z24G38h5D3qXu23JGRoigpX5Ss4r9ZnDk3g0Z8uQC2x2lPaJ+UlWBc1ZWBWdLg=="
     },
     "consola": {
-      "version": "2.15.2",
-      "resolved": "https://registry.npmjs.org/consola/-/consola-2.15.2.tgz",
-      "integrity": "sha512-VxqWw5C8O/mQpZYtfaaSCDJcVK3AxyvQ26rhgvyAI4j/QJISh8DLwFS8GQU+9154u4ngyCsSlnyIAYJme9kQug=="
+      "version": "2.15.3",
+      "resolved": "https://registry.npmjs.org/consola/-/consola-2.15.3.tgz",
+      "integrity": "sha512-9vAdYbHj6x2fLKC4+oPH0kFzY/orMZyG2Aj+kNylHxKGJ/Ed4dpNyAQYwJOdqO4zdM7XpVHmyejQDcQHrnuXbw=="
     },
     "console-browserify": {
       "version": "1.2.0",
@@ -5722,24 +5737,6 @@
       "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
       "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k="
     },
-    "encoding": {
-      "version": "0.1.13",
-      "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz",
-      "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==",
-      "requires": {
-        "iconv-lite": "^0.6.2"
-      },
-      "dependencies": {
-        "iconv-lite": {
-          "version": "0.6.2",
-          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.2.tgz",
-          "integrity": "sha512-2y91h5OpQlolefMPmUlivelittSWy0rP+oYVpn6A7GwVHNE8AWzoYOBNmlwks3LobaJxgHCYZAnyNo2GgpNRNQ==",
-          "requires": {
-            "safer-buffer": ">= 2.1.2 < 3.0.0"
-          }
-        }
-      }
-    },
     "end-of-stream": {
       "version": "1.4.4",
       "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
@@ -5882,11 +5879,6 @@
         "es6-symbol": "^3.1.1"
       }
     },
-    "es6-promise": {
-      "version": "4.2.8",
-      "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz",
-      "integrity": "sha512-HJDGx5daxeIvxdBxvG2cb9g4tEvwIk3i8+nhX0yGrYmZUzbkdg8QbDevheDB8gd0//uPj4c1EQua8Q+MViT0/w=="
-    },
     "es6-symbol": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/es6-symbol/-/es6-symbol-3.1.3.tgz",
@@ -8584,15 +8576,6 @@
       "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
       "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
     },
-    "isomorphic-fetch": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/isomorphic-fetch/-/isomorphic-fetch-2.2.1.tgz",
-      "integrity": "sha1-YRrhrPFPXoH3KVB0coGf6XM1WKk=",
-      "requires": {
-        "node-fetch": "^1.0.1",
-        "whatwg-fetch": ">=0.10.0"
-      }
-    },
     "isstream": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
@@ -11423,15 +11406,6 @@
         }
       }
     },
-    "node-fetch": {
-      "version": "1.7.3",
-      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-1.7.3.tgz",
-      "integrity": "sha512-NhZ4CsKx7cYm2vSrBAr2PvFOe6sWDf0UYLRqA6svUYg7+/TSfVAu49jYC4BvQ4Sms9SZgdqGBgroqfDhJdTyKQ==",
-      "requires": {
-        "encoding": "^0.1.11",
-        "is-stream": "^1.0.1"
-      }
-    },
     "node-forge": {
       "version": "0.10.0",
       "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-0.10.0.tgz",

sasjs-tests/package.json

@@ -4,7 +4,7 @@
   "homepage": ".",
   "private": true,
   "dependencies": {
-    "@sasjs/adapter": "^2.1.0",
+    "@sasjs/adapter": "^2.2.4",
     "@sasjs/test-framework": "^1.4.0",
     "@types/jest": "^26.0.20",
     "@types/node": "^14.14.25",

sasjs-tests/public/config.json

@@ -6,7 +6,7 @@
     "appLoc": "/Public/app",
     "serverType": "SASVIYA",
     "debug": false,
-    "contextName": "SharedCompute",
+    "contextName": "sasjs adapter compute context",
     "useComputeApi": true
   }
 }

sasjs-tests/src/testSuites/Basic.ts

@@ -2,6 +2,8 @@ import SASjs, { SASjsConfig } from "@sasjs/adapter";
 import { TestSuite } from "@sasjs/test-framework";
 import { ServerType } from "@sasjs/utils/types";
 
+const stringData: any = { table1: [{ col1: "first col value" }] };
+
 const defaultConfig: SASjsConfig = {
   serverUrl: window.location.origin,
   pathSAS9: "/SASStoredProcess/do",
@@ -51,6 +53,36 @@ export const basicTests = (
       assertion: (response: any) =>
         response && response.isLoggedIn && response.userName === userName
     },
+    {
+      title: "Trigger login callback",
+      description:
+        "Should trigger required login callback and after successful login, it should finish the request",
+      test: async () => {
+        await adapter.logOut();
+        
+        return await adapter.request("common/sendArr", stringData, null, () => {
+          adapter.logIn(userName, password);
+        });
+      },
+      assertion: (response: any) => {
+        return response.table1[0][0] === stringData.table1[0].col1;
+      }
+    },
+    {
+      title: "Request with debug on",
+      description:
+        "Should complete successful request with debugging switched on",
+      test: async () => {
+        const config = {
+          debug: true
+        }
+
+        return await adapter.request("common/sendArr", stringData, config)
+      },
+      assertion: (response: any) => {
+        return response.table1[0][0] === stringData.table1[0].col1;
+      }
+    },
     {
       title: "Default config",
       description:

src/FileUploader.ts

@@ -1,6 +1,6 @@
 import { isUrl } from './utils'
 import { UploadFile } from './types/UploadFile'
-import { ErrorResponse, LoginRequiredError } from './types'
+import { ErrorResponse, LoginRequiredError } from './types/errors'
 import { RequestClient } from './request/RequestClient'
 
 export class FileUploader {

src/SASViyaApiClient.ts

@@ -1,4 +1,10 @@
-import { convertToCSV, isRelativePath, isUri, isUrl } from './utils'
+import {
+  convertToCSV,
+  isRelativePath,
+  isUri,
+  isUrl,
+  fetchLogByChunks
+} from './utils'
 import * as NodeFormData from 'form-data'
 import {
   Job,
@@ -8,10 +14,13 @@ import {
   Folder,
   EditContextInput,
   JobDefinition,
-  PollOptions,
-  ComputeJobExecutionError,
-  JobExecutionError
+  PollOptions
 } from './types'
+import {
+  ComputeJobExecutionError,
+  JobExecutionError,
+  NotFoundError
+} from './types/errors'
 import { formatDataForRequest } from './utils/formatDataForRequest'
 import { SessionManager } from './SessionManager'
 import { ContextManager } from './ContextManager'
@@ -19,7 +28,6 @@ import { timestampToYYYYMMDDHHMMSS } from '@sasjs/utils/time'
 import { Logger, LogLevel } from '@sasjs/utils/logger'
 import { isAuthorizeFormRequired } from './auth/isAuthorizeFormRequired'
 import { RequestClient } from './request/RequestClient'
-import { NotFoundError } from './types/NotFoundError'
 import { SasAuthResponse } from '@sasjs/utils/types'
 import { prefixMessage } from '@sasjs/utils/error'
 
@@ -418,19 +426,19 @@ export class SASViyaApiClient {
         })
 
       let jobResult
-      let log
+      let log = ''
 
       const logLink = currentJob.links.find((l) => l.rel === 'log')
 
       if (debug && logLink) {
-        log = await this.requestClient
-          .get<any>(`${logLink.href}/content?limit=10000`, accessToken)
-          .then((res: any) =>
-            res.result.items.map((i: any) => i.line).join('\n')
-          )
-          .catch((err) => {
-            throw prefixMessage(err, 'Error while getting log. ')
-          })
+        const logUrl = `${logLink.href}/content`
+        const logCount = currentJob.logStatistics?.lineCount ?? 1000000
+        log = await fetchLogByChunks(
+          this.requestClient,
+          accessToken!,
+          logUrl,
+          logCount
+        )
       }
 
       if (jobStatus === 'failed' || jobStatus === 'error') {
@@ -451,14 +459,14 @@ export class SASViyaApiClient {
           .catch(async (e) => {
             if (e instanceof NotFoundError) {
               if (logLink) {
-                log = await this.requestClient
-                  .get<any>(`${logLink.href}/content?limit=10000`, accessToken)
-                  .then((res: any) =>
-                    res.result.items.map((i: any) => i.line).join('\n')
-                  )
-                  .catch((err) => {
-                    throw prefixMessage(err, 'Error while getting log. ')
-                  })
+                const logUrl = `${logLink.href}/content`
+                const logCount = currentJob.logStatistics?.lineCount ?? 1000000
+                log = await fetchLogByChunks(
+                  this.requestClient,
+                  accessToken!,
+                  logUrl,
+                  logCount
+                )
 
                 return Promise.reject({
                   status: 500,
@@ -884,9 +892,7 @@ export class SASViyaApiClient {
       waitForResult,
       pollOptions,
       printPid
-    ).catch((err) => {
-      throw prefixMessage(err, 'Error while executing script. ')
-    })
+    )
   }
 
   /**
@@ -979,7 +985,13 @@ export class SASViyaApiClient {
       postJobRequestBody,
       accessToken
     )
-    const jobStatus = await this.pollJobState(postedJob, etag, accessToken)
+    const jobStatus = await this.pollJobState(
+      postedJob,
+      etag,
+      accessToken
+    ).catch((err) => {
+      throw prefixMessage(err, 'Error while polling job status. ')
+    })
     const { result: currentJob } = await this.requestClient.get<Job>(
       `${this.serverUrl}/jobExecution/jobs/${postedJob.id}`,
       accessToken
@@ -1078,7 +1090,9 @@ export class SASViyaApiClient {
       .get<string>(
         `${this.serverUrl}${stateLink.href}?_action=wait&wait=30`,
         accessToken,
-        'text/plain'
+        'text/plain',
+        {},
+        this.debug
       )
       .catch((err) => {
         throw prefixMessage(err, 'Error while getting job state. ')
@@ -1103,10 +1117,15 @@ export class SASViyaApiClient {
               .get<string>(
                 `${this.serverUrl}${stateLink.href}?_action=wait&wait=30`,
                 accessToken,
-                'text/plain'
+                'text/plain',
+                {},
+                this.debug
               )
               .catch((err) => {
-                throw prefixMessage(err, 'Error while getting job state. ')
+                throw prefixMessage(
+                  err,
+                  'Error while getting job state after interval. '
+                )
               })
 
             postedJobState = jobState.trim()

src/index.ts

@@ -1,5 +1,6 @@
 import SASjs from './SASjs'
 export * from './types'
+export * from './types/errors'
 export * from './SASViyaApiClient'
 export * from './SAS9ApiClient'
 export default SASjs

src/job-execution/ComputeJobExecutor.ts

@@ -1,7 +1,10 @@
 import { ServerType } from '@sasjs/utils/types'
-import { ErrorResponse } from '..'
 import { SASViyaApiClient } from '../SASViyaApiClient'
-import { ComputeJobExecutionError, LoginRequiredError } from '../types'
+import {
+  ErrorResponse,
+  ComputeJobExecutionError,
+  LoginRequiredError
+} from '../types/errors'
 import { BaseJobExecutor } from './JobExecutor'
 
 export class ComputeJobExecutor extends BaseJobExecutor {
@@ -20,35 +23,52 @@ export class ComputeJobExecutor extends BaseJobExecutor {
     const waitForResult = true
     const expectWebout = true
 
-    return this.sasViyaApiClient
-      ?.executeComputeJob(
-        sasJob,
-        config.contextName,
-        config.debug,
-        data,
-        accessToken,
-        waitForResult,
-        expectWebout
-      )
-      .then((response) => {
-        this.appendRequest(response, sasJob, config.debug)
-        let responseJson
+    const requestPromise = new Promise((resolve, reject) => {
+      this.sasViyaApiClient
+        ?.executeComputeJob(
+          sasJob,
+          config.contextName,
+          config.debug,
+          data,
+          accessToken,
+          waitForResult,
+          expectWebout
+        )
+        .then((response) => {
+          this.appendRequest(response, sasJob, config.debug)
 
-        return response.result
+          resolve(response.result)
+        })
+        .catch(async (e: Error) => {
+          if (e instanceof ComputeJobExecutionError) {
+            this.appendRequest(e, sasJob, config.debug)
 
-        return responseJson
-      })
-      .catch(async (e: Error) => {
-        if (e instanceof ComputeJobExecutionError) {
-          this.appendRequest(e, sasJob, config.debug)
-        }
-        if (e instanceof LoginRequiredError) {
-          await loginCallback()
-          this.appendWaitingRequest(() =>
-            this.execute(sasJob, data, config, loginRequiredCallback)
-          )
-        }
-        return Promise.reject(new ErrorResponse(e?.message, e))
-      })
+            reject(new ErrorResponse(e?.message, e))
+          }
+
+          if (e instanceof LoginRequiredError) {
+            await loginCallback()
+            this.appendWaitingRequest(() => {
+              return this.execute(
+                sasJob,
+                data,
+                config,
+                loginRequiredCallback
+              ).then(
+                (res: any) => {
+                  resolve(res)
+                },
+                (err: any) => {
+                  reject(err)
+                }
+              )
+            })
+          } else {
+            reject(new ErrorResponse(e?.message, e))
+          }
+        })
+    })
+
+    return requestPromise
   }
 }

src/job-execution/JesJobExecutor.ts

@@ -1,7 +1,10 @@
 import { ServerType } from '@sasjs/utils/types'
-import { ErrorResponse } from '..'
 import { SASViyaApiClient } from '../SASViyaApiClient'
-import { JobExecutionError, LoginRequiredError } from '../types'
+import {
+  ErrorResponse,
+  JobExecutionError,
+  LoginRequiredError
+} from '../types/errors'
 import { BaseJobExecutor } from './JobExecutor'
 
 export class JesJobExecutor extends BaseJobExecutor {
@@ -17,24 +20,52 @@ export class JesJobExecutor extends BaseJobExecutor {
     accessToken?: string
   ) {
     const loginCallback = loginRequiredCallback || (() => Promise.resolve())
-    return await this.sasViyaApiClient
-      ?.executeJob(sasJob, config.contextName, config.debug, data, accessToken)
-      .then((response) => {
-        this.appendRequest(response, sasJob, config.debug)
 
-        return response.result
-      })
-      .catch(async (e: Error) => {
-        if (e instanceof JobExecutionError) {
-          this.appendRequest(e, sasJob, config.debug)
-        }
-        if (e instanceof LoginRequiredError) {
-          await loginCallback()
-          this.appendWaitingRequest(() =>
-            this.execute(sasJob, data, config, loginRequiredCallback)
-          )
-        }
-        return Promise.reject(new ErrorResponse(e?.message, e))
-      })
+    const requestPromise = new Promise((resolve, reject) => {
+      this.sasViyaApiClient
+        ?.executeJob(
+          sasJob,
+          config.contextName,
+          config.debug,
+          data,
+          accessToken
+        )
+        .then((response) => {
+          this.appendRequest(response, sasJob, config.debug)
+
+          resolve(response.result)
+        })
+        .catch(async (e: Error) => {
+          if (e instanceof JobExecutionError) {
+            this.appendRequest(e, sasJob, config.debug)
+
+            reject(new ErrorResponse(e?.message, e))
+          }
+
+          if (e instanceof LoginRequiredError) {
+            await loginCallback()
+
+            this.appendWaitingRequest(() => {
+              return this.execute(
+                sasJob,
+                data,
+                config,
+                loginRequiredCallback
+              ).then(
+                (res: any) => {
+                  resolve(res)
+                },
+                (err: any) => {
+                  reject(err)
+                }
+              )
+            })
+          } else {
+            reject(new ErrorResponse(e?.message, e))
+          }
+        })
+    })
+
+    return requestPromise
   }
 }

src/job-execution/WebJobExecutor.ts

@@ -1,5 +1,9 @@
 import { ServerType } from '@sasjs/utils/types'
-import { ErrorResponse, JobExecutionError, LoginRequiredError } from '..'
+import {
+  ErrorResponse,
+  JobExecutionError,
+  LoginRequiredError
+} from '../types/errors'
 import { generateFileUploadForm } from '../file/generateFileUploadForm'
 import { generateTableUploadForm } from '../file/generateTableUploadForm'
 import { RequestClient } from '../request/RequestClient'
@@ -7,6 +11,11 @@ import { SASViyaApiClient } from '../SASViyaApiClient'
 import { isRelativePath } from '../utils'
 import { BaseJobExecutor } from './JobExecutor'
 
+export interface WaitingRequstPromise {
+  promise: Promise<any> | null
+  resolve: any
+  reject: any
+}
 export class WebJobExecutor extends BaseJobExecutor {
   constructor(
     serverUrl: string,
@@ -80,30 +89,51 @@ export class WebJobExecutor extends BaseJobExecutor {
       }
     }
 
-    return this.requestClient!.post(apiUrl, formData, undefined)
-      .then(async (res) => {
-        if (this.serverType === ServerType.SasViya && config.debug) {
-          const jsonResponse = await this.parseSasViyaDebugResponse(
-            res.result as string
-          )
+    const requestPromise = new Promise((resolve, reject) => {
+      this.requestClient!.post(apiUrl, formData, undefined)
+        .then(async (res) => {
+          if (this.serverType === ServerType.SasViya && config.debug) {
+            const jsonResponse = await this.parseSasViyaDebugResponse(
+              res.result as string
+            )
+            this.appendRequest(res, sasJob, config.debug)
+            resolve(jsonResponse)
+          }
           this.appendRequest(res, sasJob, config.debug)
-          return jsonResponse
-        }
-        this.appendRequest(res, sasJob, config.debug)
-        return res.result
-      })
-      .catch(async (e: Error) => {
-        if (e instanceof JobExecutionError) {
-          this.appendRequest(e, sasJob, config.debug)
-        }
-        if (e instanceof LoginRequiredError) {
-          await loginCallback()
-          this.appendWaitingRequest(() =>
-            this.execute(sasJob, data, config, loginRequiredCallback)
-          )
-        }
-        return Promise.reject(new ErrorResponse(e?.message, e))
-      })
+          resolve(res.result)
+        })
+        .catch(async (e: Error) => {
+          if (e instanceof JobExecutionError) {
+            this.appendRequest(e, sasJob, config.debug)
+
+            reject(new ErrorResponse(e?.message, e))
+          }
+
+          if (e instanceof LoginRequiredError) {
+            await loginCallback()
+
+            this.appendWaitingRequest(() => {
+              return this.execute(
+                sasJob,
+                data,
+                config,
+                loginRequiredCallback
+              ).then(
+                (res: any) => {
+                  resolve(res)
+                },
+                (err: any) => {
+                  reject(err)
+                }
+              )
+            })
+          } else {
+            reject(new ErrorResponse(e?.message, e))
+          }
+        })
+    })
+
+    return requestPromise
   }
 
   private parseSasViyaDebugResponse = async (response: string) => {

src/request/RequestClient.ts

@@ -1,10 +1,15 @@
 import axios, { AxiosInstance, AxiosRequestConfig, AxiosResponse } from 'axios'
-import { CsrfToken, JobExecutionError } from '..'
+import { CsrfToken } from '..'
 import { isAuthorizeFormRequired, isLogInRequired } from '../auth'
-import { LoginRequiredError } from '../types'
-import { AuthorizeError } from '../types/AuthorizeError'
-import { NotFoundError } from '../types/NotFoundError'
+import {
+  AuthorizeError,
+  LoginRequiredError,
+  NotFoundError,
+  InternalServerError,
+  JobExecutionError
+} from '../types/errors'
 import { parseWeboutResponse } from '../utils/parseWeboutResponse'
+import { prefixMessage } from '@sasjs/utils/error'
 
 export interface HttpClient {
   get<T>(
@@ -72,7 +77,8 @@ export class RequestClient implements HttpClient {
     url: string,
     accessToken: string | undefined,
     contentType: string = 'application/json',
-    overrideHeaders: { [key: string]: string | number } = {}
+    overrideHeaders: { [key: string]: string | number } = {},
+    debug: boolean = false
   ): Promise<{ result: T; etag: string }> {
     const headers = {
       ...this.getHeaders(accessToken, contentType),
@@ -92,12 +98,25 @@ export class RequestClient implements HttpClient {
       .get<T>(url, requestConfig)
       .then((response) => {
         throwIfError(response)
+
         return this.parseResponse<T>(response)
       })
       .catch(async (e) => {
-        return await this.handleError(e, () =>
-          this.get<T>(url, accessToken, contentType, overrideHeaders)
-        )
+        return await this.handleError(
+          e,
+          () =>
+            this.get<T>(url, accessToken, contentType, overrideHeaders).catch(
+              (err) => {
+                throw prefixMessage(
+                  err,
+                  'Error while executing handle error callback. '
+                )
+              }
+            ),
+          debug
+        ).catch((err) => {
+          throw prefixMessage(err, 'Error while handling error. ')
+        })
       })
   }
 
@@ -329,37 +348,68 @@ export class RequestClient implements HttpClient {
     }
   }
 
-  private handleError = async (e: any, callback: any) => {
+  private handleError = async (
+    e: any,
+    callback: any,
+    debug: boolean = false
+  ) => {
     const response = e.response as AxiosResponse
+
     if (e instanceof AuthorizeError) {
-      const res = await this.httpClient.get(e.confirmUrl, {
-        responseType: 'text',
-        headers: { 'Content-Type': 'text/plain', Accept: '*/*' }
-      })
+      const res = await this.httpClient
+        .get(e.confirmUrl, {
+          responseType: 'text',
+          headers: { 'Content-Type': 'text/plain', Accept: '*/*' }
+        })
+        .catch((err) => {
+          throw prefixMessage(err, 'Error while getting error confirmUrl. ')
+        })
 
       if (isAuthorizeFormRequired(res?.data as string)) {
-        await this.authorize(res.data as string)
+        await this.authorize(res.data as string).catch((err) => {
+          throw prefixMessage(err, 'Error while authorizing request. ')
+        })
       }
-      return await callback()
+
+      return await callback().catch((err: any) => {
+        throw prefixMessage(
+          err,
+          'Error while executing callback in handleError. '
+        )
+      })
     }
+
     if (e instanceof LoginRequiredError) {
       this.clearCsrfTokens()
     }
+
     if (response?.status === 403 || response?.status === 449) {
       this.parseAndSetCsrfToken(response)
+
       if (this.csrfToken.headerName && this.csrfToken.value) {
-        return await callback()
+        return await callback().catch((err: any) => {
+          throw prefixMessage(
+            err,
+            'Error while executing callback in handleError. '
+          )
+        })
       }
+
       throw e
     } else if (response?.status === 404) {
       throw new NotFoundError(response.config.url!)
+    } else if (response?.status === 502) {
+      if (debug) throw new InternalServerError()
+      else return
     }
+
     throw e
   }
 
-  private async parseResponse<T>(response: AxiosResponse<any>) {
+  private parseResponse<T>(response: AxiosResponse<any>) {
     const etag = response?.headers ? response.headers['etag'] : ''
     let parsedResponse
+    let includeSAS9Log: boolean = false
 
     try {
       if (typeof response.data === 'string') {
@@ -373,11 +423,20 @@ export class RequestClient implements HttpClient {
       } catch {
         parsedResponse = response.data
       }
+
+      includeSAS9Log = true
     }
-    return {
+
+    let responseToReturn: { result: T; etag: any; log?: string } = {
       result: parsedResponse as T,
       etag
     }
+
+    if (includeSAS9Log) {
+      responseToReturn.log = response.data
+    }
+
+    return responseToReturn
   }
 }
 
@@ -413,6 +472,7 @@ const throwIfError = (response: AxiosResponse) => {
   }
 
   const error = parseError(response.data as string)
+
   if (error) {
     throw error
   }

src/test/SessionManager.spec.ts

@@ -29,7 +29,10 @@ describe('SessionManager', () => {
         Promise.resolve({ data: sampleResponse })
       )
 
-      const expectedResponse = { etag: '', result: sampleResponse }
+      const expectedResponse = {
+        etag: '',
+        result: sampleResponse
+      }
 
       await expect(
         sessionManager.getVariable(

src/types/Job.ts

@@ -1,5 +1,6 @@
 import { Link } from './Link'
 import { JobResult } from './JobResult'
+import { LogStatistics } from './LogStatistics'
 
 export interface Job {
   id: string
@@ -10,4 +11,5 @@ export interface Job {
   links: Link[]
   results: JobResult
   error?: any
+  logStatistics: LogStatistics
 }

src/types/LogStatistics.ts

@@ -0,0 +1,4 @@
+export interface LogStatistics {
+  lineCount: number
+  modifiedTimeStamp: string
+}

src/types/errors/ComputeJobExecutionError.ts

@@ -1,4 +1,4 @@
-import { Job } from './Job'
+import { Job } from '../Job'
 
 export class ComputeJobExecutionError extends Error {
   constructor(public job: Job, public log: string) {

src/types/errors/InternalServerError.ts

@@ -0,0 +1,9 @@
+export class InternalServerError extends Error {
+  constructor() {
+    super('Error: Internal server error.')
+
+    this.name = 'InternalServerError'
+
+    Object.setPrototypeOf(this, InternalServerError.prototype)
+  }
+}

src/types/errors/index.ts

@@ -0,0 +1,7 @@
+export * from './AuthorizeError'
+export * from './ComputeJobExecutionError'
+export * from './InternalServerError'
+export * from './JobExecutionError'
+export * from './LoginRequiredError'
+export * from './NotFoundError'
+export * from './ErrorResponse'

src/types/index.ts

@@ -1,14 +1,10 @@
-export * from './ComputeJobExecutionError'
 export * from './Context'
 export * from './CsrfToken'
-export * from './ErrorResponse'
 export * from './Folder'
 export * from './Job'
-export * from './JobExecutionError'
 export * from './JobDefinition'
 export * from './JobResult'
 export * from './Link'
-export * from './LoginRequiredError'
 export * from './SASjsConfig'
 export * from './SASjsRequest'
 export * from './Session'

src/utils/fetchLogByChunks.ts

@@ -0,0 +1,43 @@
+import { RequestClient } from '../request/RequestClient'
+import { prefixMessage } from '@sasjs/utils/error'
+
+/**
+ * Fetches content of the log file
+ * @param {object} requestClient - client object of Request Client.
+ * @param {string} accessToken - an access token for an authorized user.
+ * @param {string} logUrl - url of the log file.
+ * @param {number} logCount- total number of log lines in file.
+ * @returns an string containing log lines.
+ */
+export const fetchLogByChunks = async (
+  requestClient: RequestClient,
+  accessToken: string,
+  logUrl: string,
+  logCount: number
+): Promise<string> => {
+  let log: string = ''
+
+  const loglimit = logCount < 10000 ? logCount : 10000
+  let start = 0
+  do {
+    console.log(
+      `Fetching logs from line no: ${start + 1} to ${
+        start + loglimit
+      } of ${logCount}.`
+    )
+    const logChunkJson = await requestClient!
+      .get<any>(`${logUrl}?start=${start}&limit=${loglimit}`, accessToken)
+      .then((res: any) => res.result)
+      .catch((err) => {
+        throw prefixMessage(err, 'Error while getting log. ')
+      })
+
+    if (logChunkJson.items.length === 0) break
+
+    const logChunk = logChunkJson.items.map((i: any) => i.line).join('\n')
+    log += logChunk
+
+    start += loglimit
+  } while (start < logCount)
+  return log
+}

src/utils/index.ts

@@ -11,3 +11,4 @@ export * from './parseSasViyaLog'
 export * from './serialize'
 export * from './splitChunks'
 export * from './parseWeboutResponse'
+export * from './fetchLogByChunks'