feat(sas9-support): add support for SAS9 via username/password login

package.json

@@ -37,6 +37,7 @@
   "license": "ISC",
   "devDependencies": {
     "@types/jest": "^26.0.22",
+    "@types/tough-cookie": "^4.0.0",
     "cp": "^0.2.0",
     "dotenv": "^8.2.0",
     "jest": "^26.6.3",
@@ -60,13 +61,12 @@
   "main": "index.js",
   "dependencies": {
     "@sasjs/utils": "^2.10.2",
-    "assert": "^2.0.0",
     "axios": "^0.21.1",
+    "axios-cookiejar-support": "^1.0.1",
     "buffer": "^6.0.3",
     "form-data": "^4.0.0",
     "https": "^1.0.0",
-    "node-libcurl": "^2.3.3",
+    "tough-cookie": "^4.0.0",
-    "stream": "0.0.2",
+    "url": "^0.11.0"
-    "stream-browserify": "^3.0.0"
   }
 }

src/SASjs.ts

@@ -839,8 +839,7 @@ export default class SASjs {
     this.sas9JobExecutor = new Sas9JobExecutor(
       this.sasjsConfig.serverUrl,
       this.sasjsConfig.serverType!,
-      this.jobsPath,
+      this.jobsPath
-      this.requestClient
     )
 
     this.computeJobExecutor = new ComputeJobExecutor(

src/job-execution/Sas9JobExecutor.ts

@@ -1,47 +1,29 @@
 import { ServerType } from '@sasjs/utils/types'
-import { Curl } from 'node-libcurl'
 import * as NodeFormData from 'form-data'
-import {
+import { ErrorResponse } from '../types/errors'
-  ErrorResponse,
+import { convertToCSV, isRelativePath } from '../utils'
-  JobExecutionError,
-  LoginRequiredError
-} from '../types/errors'
-import { generateFileUploadForm } from '../file/generateFileUploadForm'
-import { generateTableUploadForm } from '../file/generateTableUploadForm'
-import { RequestClient } from '../request/RequestClient'
-import { SASViyaApiClient } from '../SASViyaApiClient'
-import { isRelativePath } from '../utils'
 import { BaseJobExecutor } from './JobExecutor'
+import { Sas9RequestClient } from '../request/Sas9RequestClient'
 
-interface WaitingRequstPromise {
-  promise: Promise<any> | null
-  resolve: any
-  reject: any
-}
 export class Sas9JobExecutor extends BaseJobExecutor {
+  private requestClient: Sas9RequestClient
   constructor(
     serverUrl: string,
     serverType: ServerType,
-    private jobsPath: string,
+    private jobsPath: string
-    private requestClient: RequestClient
   ) {
     super(serverUrl, serverType)
+    this.requestClient = new Sas9RequestClient(serverUrl, false)
   }
 
-  async execute(
+  async execute(sasJob: string, data: any, config: any) {
-    sasJob: string,
-    data: any,
-    config: any,
-    loginRequiredCallback?: any
-  ) {
-    const loginCallback = loginRequiredCallback || (() => Promise.resolve())
     const program = isRelativePath(sasJob)
       ? config.appLoc
         ? config.appLoc.replace(/\/?$/, '/') + sasJob.replace(/^\//, '')
         : sasJob
       : sasJob
     const jobUri = ''
-    let apiUrl = `${config.serverUrl}${this.jobsPath}/?${
+    let apiUrl = `${config.serverUrl}${this.jobsPath}?${
       jobUri.length > 0
         ? '__program=' + program + '&_job=' + jobUri
         : '_program=' + program
@@ -56,34 +38,13 @@ export class Sas9JobExecutor extends BaseJobExecutor {
       ...this.getRequestParams(config)
     }
 
-    let formData =
+    let formData = new NodeFormData()
-      typeof FormData === 'undefined' ? new NodeFormData() : new FormData()
 
     if (data) {
-      const stringifiedData = JSON.stringify(data)
+      try {
-      if (
+        formData = generateFileUploadForm(formData, data)
-        config.serverType === ServerType.Sas9 ||
+      } catch (e) {
-        stringifiedData.length > 500000 ||
+        return Promise.reject(new ErrorResponse(e?.message, e))
-        stringifiedData.includes(';')
-      ) {
-        // file upload approach
-        try {
-          formData = generateFileUploadForm(formData as FormData, data)
-        } catch (e) {
-          return Promise.reject(new ErrorResponse(e?.message, e))
-        }
-      } else {
-        // param based approach
-        try {
-          const {
-            formData: newFormData,
-            requestParams: params
-          } = generateTableUploadForm(formData as FormData, data)
-          formData = newFormData
-          requestParams = { ...requestParams, ...params }
-        } catch (e) {
-          return Promise.reject(new ErrorResponse(e?.message, e))
-        }
       }
     }
 
@@ -93,89 +54,21 @@ export class Sas9JobExecutor extends BaseJobExecutor {
       }
     }
 
-    const requestPromise = new Promise((resolve, reject) => {
+    await this.requestClient.login(config.username, config.password)
-      const curl = new Curl()
+    const contentType =
-      curl.setOpt('URL', apiUrl)
+      data && Object.keys(data).length
-      curl.setOpt('USERAGENT', 'curl/7.64.1')
+        ? 'multipart/form-data; boundary=' + (formData as any)._boundary
-      curl.setOpt('FOLLOWLOCATION', true)
+        : 'text/plain'
-      curl.setOpt('SSL_VERIFYPEER', false)
+    return await this.requestClient!.post(
-      curl.setOpt('COOKIEFILE', 'cookiefile')
+      apiUrl,
-      curl.on('end', (statusCode, res, headers) => {
+      formData,
-        console.log('res', res)
+      undefined,
-        console.log('statusCode', statusCode)
+      contentType,
-        console.log('statusCode', statusCode)
+      {
-        curl.on('end', (statusCode, res, headers) => {
+        Accept: '*/*',
-          console.log('res', res)
+        Connection: 'Keep-Alive'
-          console.log('statusCode', statusCode)
+      }
-          console.log('statusCode', statusCode)
+    )
-          resolve(res)
-        })
-        curl.perform()
-      })
-      curl.on('error', (error) => {
-        console.log('error', error)
-        reject(error)
-      })
-
-      curl.perform()
-      // this.requestClient!.post(apiUrl, formData, undefined)
-      //   .then(async (res) => {
-      //     if (this.serverType === ServerType.SasViya && config.debug) {
-      //       const jsonResponse = await this.parseSasViyaDebugResponse(
-      //         res.result as string
-      //       )
-      //       this.appendRequest(res, sasJob, config.debug)
-      //       resolve(jsonResponse)
-      //     }
-      //     this.appendRequest(res, sasJob, config.debug)
-      //     resolve(res.result)
-      //   })
-      //   .catch(async (e: Error) => {
-      //     if (e instanceof JobExecutionError) {
-      //       this.appendRequest(e, sasJob, config.debug)
-
-      //       reject(new ErrorResponse(e?.message, e))
-      //     }
-
-      //     if (e instanceof LoginRequiredError) {
-      //       await loginCallback()
-
-      //       this.appendWaitingRequest(() => {
-      //         return this.execute(
-      //           sasJob,
-      //           data,
-      //           config,
-      //           loginRequiredCallback
-      //         ).then(
-      //           (res: any) => {
-      //             resolve(res)
-      //           },
-      //           (err: any) => {
-      //             reject(err)
-      //           }
-      //         )
-      //       })
-      //     } else {
-      //       reject(new ErrorResponse(e?.message, e))
-      //     }
-      //   })
-    })
-
-    return requestPromise
-  }
-
-  private parseSasViyaDebugResponse = async (response: string) => {
-    const iframeStart = response.split(
-      '<iframe style="width: 99%; height: 500px" src="'
-    )[1]
-    const jsonUrl = iframeStart ? iframeStart.split('"></iframe>')[0] : null
-    if (!jsonUrl) {
-      throw new Error('Unable to find webout file URL.')
-    }
-
-    return this.requestClient
-      .get(this.serverUrl + jsonUrl, undefined)
-      .then((res) => res.result)
   }
 
   private getRequestParams(config: any): any {
@@ -190,26 +83,26 @@ export class Sas9JobExecutor extends BaseJobExecutor {
 
     return requestParams
   }
+}
 
-  private parseSAS9ErrorResponse(response: string) {
+const generateFileUploadForm = (
-    const logLines = response.split('\n')
+  formData: NodeFormData,
-    const parsedLines: string[] = []
+  data: any
-    let firstErrorLineIndex: number = -1
+): NodeFormData => {
-
+  for (const tableName in data) {
-    logLines.map((line: string, index: number) => {
+    const name = tableName
-      if (
+    const csv = convertToCSV(data[tableName])
-        line.toLowerCase().includes('error') &&
+    if (csv === 'ERROR: LARGE STRING LENGTH') {
-        !line.toLowerCase().includes('this request completed with errors.') &&
+      throw new Error(
-        firstErrorLineIndex === -1
+        'The max length of a string value in SASjs is 32765 characters.'
-      ) {
+      )
-        firstErrorLineIndex = index
-      }
-    })
-
-    for (let i = firstErrorLineIndex - 10; i <= firstErrorLineIndex + 10; i++) {
-      parsedLines.push(logLines[i])
     }
 
-    return parsedLines.join(', ')
+    formData.append(name, csv, {
+      filename: `${name}.csv`,
+      contentType: 'application/csv'
+    })
   }
+
+  return formData
 }

src/request/RequestClient.ts

@@ -44,11 +44,11 @@ export interface HttpClient {
 }
 
 export class RequestClient implements HttpClient {
-  private csrfToken: CsrfToken = { headerName: '', value: '' }
+  protected csrfToken: CsrfToken = { headerName: '', value: '' }
-  private fileUploadCsrfToken: CsrfToken | undefined
+  protected fileUploadCsrfToken: CsrfToken | undefined
-  private httpClient: AxiosInstance
+  protected httpClient: AxiosInstance
 
-  constructor(private baseUrl: string, allowInsecure = false) {
+  constructor(protected baseUrl: string, allowInsecure = false) {
     const https = require('https')
     if (allowInsecure && https.Agent) {
       this.httpClient = axios.create({
@@ -214,9 +214,8 @@ export class RequestClient implements HttpClient {
     const headers = this.getHeaders(accessToken, 'application/json')
 
     if (this.fileUploadCsrfToken?.value) {
-      headers[
+      headers[this.fileUploadCsrfToken.headerName] =
-        this.fileUploadCsrfToken.headerName
+        this.fileUploadCsrfToken.value
-      ] = this.fileUploadCsrfToken.value
     }
 
     try {
@@ -291,7 +290,7 @@ export class RequestClient implements HttpClient {
       })
   }
 
-  private getHeaders = (
+  protected getHeaders = (
     accessToken: string | undefined,
     contentType: string
   ) => {
@@ -316,7 +315,7 @@ export class RequestClient implements HttpClient {
     return headers
   }
 
-  private parseAndSetFileUploadCsrfToken = (response: AxiosResponse) => {
+  protected parseAndSetFileUploadCsrfToken = (response: AxiosResponse) => {
     const token = this.parseCsrfToken(response)
 
     if (token) {
@@ -324,7 +323,7 @@ export class RequestClient implements HttpClient {
     }
   }
 
-  private parseAndSetCsrfToken = (response: AxiosResponse) => {
+  protected parseAndSetCsrfToken = (response: AxiosResponse) => {
     const token = this.parseCsrfToken(response)
 
     if (token) {
@@ -333,9 +332,9 @@ export class RequestClient implements HttpClient {
   }
 
   private parseCsrfToken = (response: AxiosResponse): CsrfToken | undefined => {
-    const tokenHeader = (response.headers[
+    const tokenHeader = (
-      'x-csrf-header'
+      response.headers['x-csrf-header'] as string
-    ] as string)?.toLowerCase()
+    )?.toLowerCase()
 
     if (tokenHeader) {
       const token = response.headers[tokenHeader]
@@ -348,7 +347,7 @@ export class RequestClient implements HttpClient {
     }
   }
 
-  private handleError = async (
+  protected handleError = async (
     e: any,
     callback: any,
     debug: boolean = false
@@ -406,7 +405,7 @@ export class RequestClient implements HttpClient {
     throw e
   }
 
-  private parseResponse<T>(response: AxiosResponse<any>) {
+  protected parseResponse<T>(response: AxiosResponse<any>) {
     const etag = response?.headers ? response.headers['etag'] : ''
     let parsedResponse
     let includeSAS9Log: boolean = false
@@ -440,7 +439,7 @@ export class RequestClient implements HttpClient {
   }
 }
 
-const throwIfError = (response: AxiosResponse) => {
+export const throwIfError = (response: AxiosResponse) => {
   if (response.status === 401) {
     throw new LoginRequiredError()
   }

src/request/Sas9RequestClient.ts

@@ -0,0 +1,117 @@
+import { AxiosRequestConfig } from 'axios'
+import axiosCookieJarSupport from 'axios-cookiejar-support'
+import * as tough from 'tough-cookie'
+import { prefixMessage } from '@sasjs/utils/error'
+import { RequestClient, throwIfError } from './RequestClient'
+
+export class Sas9RequestClient extends RequestClient {
+  constructor(baseUrl: string, allowInsecure = false) {
+    super(baseUrl, allowInsecure)
+    this.httpClient.defaults.maxRedirects = 0
+    this.httpClient.defaults.validateStatus = (status) =>
+      status >= 200 && status < 303
+
+    if (axiosCookieJarSupport) {
+      axiosCookieJarSupport(this.httpClient)
+      this.httpClient.defaults.jar = new tough.CookieJar()
+    }
+  }
+
+  public async login(username: string, password: string) {
+    const codeInjectorPath = `/User Folders/${username}/My Folder/sasjs/runner`
+    if (this.httpClient.defaults.jar) {
+      ;(this.httpClient.defaults.jar as tough.CookieJar).removeAllCookies()
+      await this.get(
+        `/SASStoredProcess/do?_program=${codeInjectorPath}&_username=${username}&_password=${password}`,
+        undefined,
+        'text/plain'
+      )
+    }
+  }
+
+  public async get<T>(
+    url: string,
+    accessToken: string | undefined,
+    contentType: string = 'application/json',
+    overrideHeaders: { [key: string]: string | number } = {},
+    debug: boolean = false
+  ): Promise<{ result: T; etag: string }> {
+    const headers = {
+      ...this.getHeaders(accessToken, contentType),
+      ...overrideHeaders
+    }
+
+    const requestConfig: AxiosRequestConfig = {
+      headers,
+      responseType: contentType === 'text/plain' ? 'text' : 'json',
+      withCredentials: true
+    }
+    if (contentType === 'text/plain') {
+      requestConfig.transformResponse = undefined
+    }
+
+    return this.httpClient
+      .get<T>(url, requestConfig)
+      .then((response) => {
+        if (response.status === 302) {
+          return this.get(
+            response.headers['location'],
+            accessToken,
+            contentType
+          )
+        }
+        throwIfError(response)
+        return this.parseResponse<T>(response)
+      })
+      .catch(async (e) => {
+        return await this.handleError(
+          e,
+          () =>
+            this.get<T>(url, accessToken, contentType, overrideHeaders).catch(
+              (err) => {
+                throw prefixMessage(
+                  err,
+                  'Error while executing handle error callback. '
+                )
+              }
+            ),
+          debug
+        ).catch((err) => {
+          throw prefixMessage(err, 'Error while handling error. ')
+        })
+      })
+  }
+
+  public post<T>(
+    url: string,
+    data: any,
+    accessToken: string | undefined,
+    contentType = 'application/json',
+    overrideHeaders: { [key: string]: string | number } = {}
+  ): Promise<{ result: T; etag: string }> {
+    const headers = {
+      ...this.getHeaders(accessToken, contentType),
+      ...overrideHeaders
+    }
+
+    return this.httpClient
+      .post<T>(url, data, { headers, withCredentials: true })
+      .then(async (response) => {
+        if (response.status === 302) {
+          return await this.get(
+            response.headers['location'],
+            undefined,
+            contentType,
+            overrideHeaders
+          )
+        }
+        throwIfError(response)
+        return this.parseResponse<T>(response)
+      })
+      .catch(async (e) => {
+        return await this.handleError(e, () =>
+          this.post<T>(url, data, accessToken, contentType, overrideHeaders)
+        )
+      })
+  }
+}