Merge pull request #641 from sasjs/issue-624

fix(special-missing): fix convertToCSV format object

jest.config.js

@@ -127,7 +127,7 @@ module.exports = {
   setupFiles: [],
 
   // A list of paths to modules that run some code to configure or set up the testing framework before each test
-  setupFilesAfterEnv: ['jest-extended'],
+  setupFilesAfterEnv: ['jest-extended/all'],
 
   // A list of paths to snapshot serializer modules Jest should use for snapshot testing
   // snapshotSerializers: [],

package.json

@@ -51,30 +51,30 @@
     "cp": "0.2.0",
     "dotenv": "10.0.0",
     "express": "4.17.1",
-    "jest": "27.2.0",
+    "jest": "27.4.7",
-    "jest-extended": "0.11.5",
+    "jest-extended": "2.0.0",
     "node-polyfill-webpack-plugin": "1.1.4",
     "path": "0.12.7",
     "pem": "1.14.4",
     "process": "0.11.10",
     "rimraf": "3.0.2",
     "semantic-release": "18.0.0",
-    "terser-webpack-plugin": "5.2.4",
+    "terser-webpack-plugin": "5.3.0",
-    "ts-jest": "27.0.3",
+    "ts-jest": "27.1.3",
     "ts-loader": "9.2.6",
     "tslint": "6.1.3",
     "tslint-config-prettier": "1.18.0",
-    "typedoc": "0.19.2",
+    "typedoc": "0.22.11",
     "typedoc-neo-theme": "1.1.1",
     "typedoc-plugin-external-module-name": "4.0.6",
-    "typescript": "4.3.5",
+    "typescript": "4.5.4",
-    "webpack": "5.56.0",
+    "webpack": "5.66.0",
     "webpack-cli": "4.7.2"
   },
   "main": "index.js",
   "dependencies": {
-    "@sasjs/utils": "2.32.0",
+    "@sasjs/utils": "2.35.0",
-    "axios": "0.25.0",
+    "axios": "0.26.0",
     "axios-cookiejar-support": "1.0.1",
     "form-data": "4.0.0",
     "https": "1.0.0",

sasjs-tests/src/testSuites/Basic.ts

@@ -6,6 +6,7 @@ const stringData: any = { table1: [{ col1: 'first col value' }] }
 
 const defaultConfig: SASjsConfig = {
   serverUrl: window.location.origin,
+  pathSASJS: '/SASjsApi/stp/execute',
   pathSAS9: '/SASStoredProcess/do',
   pathSASViya: '/SASJobExecution',
   appLoc: '/Public/seedapp',

sasjs-tests/src/testSuites/SpecialCases.ts

@@ -80,7 +80,7 @@ const errorAndCsrfData: any = {
 }
 
 const testTable = 'sometable'
-const testTableWithNullVars: { [key: string]: any } = {
+export const testTableWithNullVars: { [key: string]: any } = {
   [testTable]: [
     { var1: 'string', var2: 232, nullvar: 'A' },
     { var1: 'string', var2: 232, nullvar: 'B' },

src/SASjs.ts

@@ -5,9 +5,8 @@ import {
   EditContextInput,
   PollOptions,
   LoginMechanism,
-  FolderMember,
+  ExecutionQuery,
-  ServiceMember,
+  FileTree
-  ExecutionQuery
 } from './types'
 import { SASViyaApiClient } from './SASViyaApiClient'
 import { SAS9ApiClient } from './SAS9ApiClient'
@@ -28,6 +27,7 @@ import {
   ComputeJobExecutor,
   JesJobExecutor,
   Sas9JobExecutor,
+  SasJsJobExecutor,
   FileUploader
 } from './job-execution'
 import { ErrorResponse } from './types/errors'
@@ -63,6 +63,7 @@ export default class SASjs {
   private computeJobExecutor: JobExecutor | null = null
   private jesJobExecutor: JobExecutor | null = null
   private sas9JobExecutor: JobExecutor | null = null
+  private sasJsJobExecutor: JobExecutor | null = null
 
   constructor(config?: Partial<SASjsConfig>) {
     this.sasjsConfig = {
@@ -77,6 +78,12 @@ export default class SASjs {
     return this.requestClient?.getCsrfToken(type)
   }
 
+  /**
+   * Executes the sas code against SAS9 server
+   * @param linesOfCode - lines of sas code from the file to run.
+   * @param username - a string representing the username.
+   * @param password - a string representing the password.
+   */
   public async executeScriptSAS9(
     linesOfCode: string[],
     userName: string,
@@ -91,6 +98,38 @@ export default class SASjs {
     )
   }
 
+  /**
+   * Executes the sas code against SASViya server
+   * @param fileName - name of the file to run. It will be converted to path to the file being submitted for execution.
+   * @param linesOfCode - lines of sas code from the file to run.
+   * @param contextName - context name on which code will be run on the server.
+   * @param authConfig - (optional) the access token, refresh token, client and secret for authorizing the request.
+   * @param debug - (optional) if true, global debug config will be overriden
+   */
+  public async executeScriptSASViya(
+    fileName: string,
+    linesOfCode: string[],
+    contextName: string,
+    authConfig?: AuthConfig,
+    debug?: boolean
+  ) {
+    this.isMethodSupported('executeScriptSASViya', [ServerType.SasViya])
+    if (!contextName) {
+      throw new Error(
+        'Context name is undefined. Please set a `contextName` in your SASjs or override config.'
+      )
+    }
+
+    return await this.sasViyaApiClient!.executeScript(
+      fileName,
+      linesOfCode,
+      contextName,
+      authConfig,
+      null,
+      debug ? debug : this.sasjsConfig.debug
+    )
+  }
+
   /**
    * Gets compute contexts.
    * @param accessToken - an access token for an authorized user.
@@ -254,38 +293,6 @@ export default class SASjs {
     return await this.sasViyaApiClient!.createSession(contextName, accessToken)
   }
 
-  /**
-   * Executes the sas code against given sas server
-   * @param fileName - name of the file to run. It will be converted to path to the file being submitted for execution.
-   * @param linesOfCode - lines of sas code from the file to run.
-   * @param contextName - context name on which code will be run on the server.
-   * @param authConfig - (optional) the access token, refresh token, client and secret for authorizing the request.
-   * @param debug - (optional) if true, global debug config will be overriden
-   */
-  public async executeScriptSASViya(
-    fileName: string,
-    linesOfCode: string[],
-    contextName: string,
-    authConfig?: AuthConfig,
-    debug?: boolean
-  ) {
-    this.isMethodSupported('executeScriptSASViya', [ServerType.SasViya])
-    if (!contextName) {
-      throw new Error(
-        'Context name is undefined. Please set a `contextName` in your SASjs or override config.'
-      )
-    }
-
-    return await this.sasViyaApiClient!.executeScript(
-      fileName,
-      linesOfCode,
-      contextName,
-      authConfig,
-      null,
-      debug ? debug : this.sasjsConfig.debug
-    )
-  }
-
   /**
    * Creates a folder in the logical SAS folder tree
    * @param folderName - name of the folder to be created.
@@ -676,7 +683,16 @@ export default class SASjs {
     const validationResult = this.validateInput(data)
 
     if (validationResult.status) {
-      if (
+      if (config.serverType === ServerType.Sasjs) {
+        return await this.sasJsJobExecutor!.execute(
+          sasJob,
+          data,
+          config,
+          loginRequiredCallback,
+          authConfig,
+          extraResponseAttributes
+        )
+      } else if (
         config.serverType !== ServerType.Sas9 &&
         config.useComputeApi !== undefined &&
         config.useComputeApi !== null
@@ -865,8 +881,22 @@ export default class SASjs {
     )
   }
 
-  public async deployToSASjs(members: [FolderMember, ServiceMember]) {
+  /**
-    return await this.sasJSApiClient?.deploy(members, this.sasjsConfig.appLoc)
+   * Creates the folders and services at the given location `appLoc` on the given server `serverUrl`.
+   * @param members - the JSON specifying the folders and services to be created.
+   * @param appLoc - the base folder in which to create the new folders and
+   * services.  If not provided, is taken from SASjsConfig.
+   * @param authConfig - a valid client, secret, refresh and access tokens that are authorised to execute compute jobs.
+   */
+  public async deployToSASjs(
+    members: FileTree,
+    appLoc?: string,
+    authConfig?: AuthConfig
+  ) {
+    if (!appLoc) {
+      appLoc = this.sasjsConfig.appLoc
+    }
+    return await this.sasJSApiClient?.deploy(members, appLoc, authConfig)
   }
 
   public async executeJobSASjs(query: ExecutionQuery) {
@@ -1018,7 +1048,7 @@ export default class SASjs {
         ? this.sasjsConfig.pathSASViya
         : this.sasjsConfig.serverType === ServerType.Sas9
         ? this.sasjsConfig.pathSAS9
-        : this.sasjsConfig.pathSASJS || ''
+        : this.sasjsConfig.pathSASJS
 
     this.authManager = new AuthManager(
       this.sasjsConfig.serverUrl,
@@ -1083,6 +1113,13 @@ export default class SASjs {
       this.sasViyaApiClient!
     )
 
+    this.sasJsJobExecutor = new SasJsJobExecutor(
+      this.sasjsConfig.serverUrl,
+      this.sasjsConfig.serverType!,
+      this.jobsPath,
+      this.requestClient
+    )
+
     this.sas9JobExecutor = new Sas9JobExecutor(
       this.sasjsConfig.serverUrl,
       this.sasjsConfig.serverType!,

src/SASjsApiClient.ts

@@ -1,8 +1,11 @@
-import { FolderMember, ServiceMember, ExecutionQuery } from './types'
+import { AuthConfig, ServerType } from '@sasjs/utils/types'
+import { FileTree, ExecutionQuery } from './types'
 import { RequestClient } from './request/RequestClient'
 import { getAccessTokenForSasjs } from './auth/getAccessTokenForSasjs'
 import { refreshTokensForSasjs } from './auth/refreshTokensForSasjs'
 import { getAuthCodeForSasjs } from './auth/getAuthCodeForSasjs'
+import { parseWeboutResponse } from './utils'
+import { getTokens } from './auth/getTokens'
 
 export class SASjsApiClient {
   constructor(
@@ -14,7 +17,19 @@ export class SASjsApiClient {
     if (serverUrl) this.serverUrl = serverUrl
   }
 
-  public async deploy(members: [FolderMember, ServiceMember], appLoc: string) {
+  public async deploy(
+    members: FileTree,
+    appLoc: string,
+    authConfig?: AuthConfig
+  ) {
+    let access_token = (authConfig || {}).access_token
+    if (authConfig) {
+      ;({ access_token } = await getTokens(
+        this.requestClient,
+        authConfig,
+        ServerType.Sasjs
+      ))
+    }
     const { result } = await this.requestClient.post<{
       status: string
       message: string
@@ -22,7 +37,7 @@ export class SASjsApiClient {
     }>(
       'SASjsApi/drive/deploy',
       { fileTree: members, appLoc: appLoc },
-      undefined
+      access_token
     )
 
     return Promise.resolve(result)
@@ -35,8 +50,13 @@ export class SASjsApiClient {
       log?: string
       logPath?: string
       error?: {}
+      _webout?: string
     }>('SASjsApi/stp/execute', query, undefined)
 
+    if (Object.keys(result).includes('_webout')) {
+      result._webout = parseWeboutResponse(result._webout!)
+    }
+
     return Promise.resolve(result)
   }
 

src/api/viya/executeScript.ts

@@ -272,7 +272,13 @@ export async function executeScript(
 
     return { result: jobResult?.result, log }
   } catch (e) {
-    if (e && e.status === 404) {
+    interface HttpError {
+      status: number
+    }
+
+    const error = e as HttpError
+
+    if (error.status === 404) {
       return executeScript(
         requestClient,
         sessionManager,
@@ -287,7 +293,7 @@ export async function executeScript(
         true
       )
     } else {
-      throw prefixMessage(e, 'Error while executing script. ')
+      throw prefixMessage(e as Error, 'Error while executing script. ')
     }
   }
 }

src/api/viya/spec/writeStream.spec.ts

@@ -1,24 +1,34 @@
 import { WriteStream } from '../../../types'
 import { writeStream } from '../writeStream'
-import 'jest-extended'
+import {
+  createWriteStream,
+  fileExists,
+  readFile,
+  deleteFile
+} from '@sasjs/utils'
 
 describe('writeStream', () => {
-  const stream: WriteStream = {
+  const filename = 'test.txt'
-    write: jest.fn(),
+  const content = 'test'
-    path: 'test'
+  let stream: WriteStream
-  }
+
+  beforeAll(async () => {
+    stream = await createWriteStream(filename)
+  })
 
   it('should resolve when the stream is written successfully', async () => {
-    expect(writeStream(stream, 'test')).toResolve()
+    await expect(writeStream(stream, content)).toResolve()
+    await expect(fileExists(filename)).resolves.toEqual(true)
+    await expect(readFile(filename)).resolves.toEqual(content + '\n')
 
-    expect(stream.write).toHaveBeenCalledWith('test\n', expect.anything())
+    await deleteFile(filename)
   })
 
   it('should reject when the write errors out', async () => {
     jest
       .spyOn(stream, 'write')
       .mockImplementation((_, callback) => callback(new Error('Test Error')))
-    const error = await writeStream(stream, 'test').catch((e) => e)
+    const error = await writeStream(stream, content).catch((e) => e)
 
     expect(error.message).toEqual('Test Error')
   })

src/api/viya/writeStream.ts

@@ -3,13 +3,9 @@ import { WriteStream } from '../../types'
 export const writeStream = async (
   stream: WriteStream,
   content: string
-): Promise<void> => {
+): Promise<void> =>
-  return new Promise((resolve, reject) => {
   stream.write(content + '\n', (e) => {
-      if (e) {
+    if (e) return Promise.reject(e)
-        return reject(e)
+
-      }
+    return Promise.resolve()
-      return resolve()
   })
-  })
-}

src/file/generateFileUploadForm.ts

@@ -5,8 +5,11 @@ export const generateFileUploadForm = (
   data: any
 ): FormData => {
   for (const tableName in data) {
+    if (!Array.isArray(data[tableName])) continue
+
     const name = tableName
     const csv = convertToCSV(data[tableName])
+
     if (csv === 'ERROR: LARGE STRING LENGTH') {
       throw new Error(
         'The max length of a string value in SASjs is 32765 characters.'

src/file/spec/generateFileUploadForm.spec.ts

@@ -0,0 +1,55 @@
+import { generateFileUploadForm } from '../generateFileUploadForm'
+
+describe('generateFileUploadForm', () => {
+  beforeAll(() => {
+    function FormDataMock(this: any) {
+      this.append = () => {}
+    }
+
+    const BlobMock = jest.fn()
+
+    ;(global as any).FormData = FormDataMock
+    ;(global as any).Blob = BlobMock
+  })
+
+  it('should generate file upload form from data', () => {
+    const formData = new FormData()
+    const testTable = 'sometable'
+    const testTableWithNullVars: { [key: string]: any } = {
+      [testTable]: [
+        { var1: 'string', var2: 232, nullvar: 'A' },
+        { var1: 'string', var2: 232, nullvar: 'B' },
+        { var1: 'string', var2: 232, nullvar: '_' },
+        { var1: 'string', var2: 232, nullvar: 0 },
+        { var1: 'string', var2: 232, nullvar: 'z' },
+        { var1: 'string', var2: 232, nullvar: null }
+      ],
+      [`$${testTable}`]: { formats: { var1: '$char12.', nullvar: 'best.' } }
+    }
+    const tableName = Object.keys(testTableWithNullVars).filter((key: string) =>
+      Array.isArray(testTableWithNullVars[key])
+    )[0]
+
+    jest.spyOn(formData, 'append').mockImplementation(() => {})
+
+    generateFileUploadForm(formData, testTableWithNullVars)
+
+    expect(formData.append).toHaveBeenCalledOnce()
+    expect(formData.append).toHaveBeenCalledWith(
+      tableName,
+      {},
+      `${tableName}.csv`
+    )
+  })
+
+  it('should throw an error if too large string was provided', () => {
+    const formData = new FormData()
+    const data = { testTable: [{ var1: 'z'.repeat(32765 + 1) }] }
+
+    expect(() => generateFileUploadForm(formData, data)).toThrow(
+      new Error(
+        'The max length of a string value in SASjs is 32765 characters.'
+      )
+    )
+  })
+})

src/job-execution/SasJsJobExecutor.ts

@@ -0,0 +1,123 @@
+import {
+  AuthConfig,
+  ExtraResponseAttributes,
+  ServerType
+} from '@sasjs/utils/types'
+import {
+  ErrorResponse,
+  JobExecutionError,
+  LoginRequiredError
+} from '../types/errors'
+import { RequestClient } from '../request/RequestClient'
+import {
+  isRelativePath,
+  appendExtraResponseAttributes,
+  getValidJson
+} from '../utils'
+import { BaseJobExecutor } from './JobExecutor'
+import { parseWeboutResponse } from '../utils/parseWeboutResponse'
+
+export class SasJsJobExecutor extends BaseJobExecutor {
+  constructor(
+    serverUrl: string,
+    serverType: ServerType,
+    private jobsPath: string,
+    private requestClient: RequestClient
+  ) {
+    super(serverUrl, serverType)
+  }
+
+  async execute(
+    sasJob: string,
+    data: any,
+    config: any,
+    loginRequiredCallback?: any,
+    authConfig?: AuthConfig,
+    extraResponseAttributes: ExtraResponseAttributes[] = []
+  ) {
+    const loginCallback = loginRequiredCallback || (() => Promise.resolve())
+    const program = isRelativePath(sasJob)
+      ? config.appLoc
+        ? config.appLoc.replace(/\/?$/, '/') + sasJob.replace(/^\//, '')
+        : sasJob
+      : sasJob
+    let apiUrl = `${config.serverUrl}${this.jobsPath}/?${'_program=' + program}`
+
+    const requestParams = this.getRequestParams(config)
+
+    const requestPromise = new Promise((resolve, reject) => {
+      this.requestClient!.post(
+        apiUrl,
+        { ...requestParams, ...data },
+        authConfig?.access_token
+      )
+        .then(async (res: any) => {
+          const resObj = {
+            result: res.result._webout,
+            log: res.result.log
+          }
+          this.requestClient!.appendRequest(resObj, sasJob, config.debug)
+
+          let jsonResponse = res.result
+
+          if (config.debug) {
+            const webout = parseWeboutResponse(res.result._webout, apiUrl)
+            jsonResponse = getValidJson(webout)
+          } else {
+            jsonResponse = getValidJson(res.result._webout)
+          }
+
+          const responseObject = appendExtraResponseAttributes(
+            { result: jsonResponse },
+            extraResponseAttributes
+          )
+          resolve(responseObject)
+        })
+        .catch(async (e: Error) => {
+          if (e instanceof JobExecutionError) {
+            this.requestClient!.appendRequest(e, sasJob, config.debug)
+            reject(new ErrorResponse(e?.message, e))
+          }
+
+          if (e instanceof LoginRequiredError) {
+            this.appendWaitingRequest(() => {
+              return this.execute(
+                sasJob,
+                data,
+                config,
+                loginRequiredCallback,
+                authConfig,
+                extraResponseAttributes
+              ).then(
+                (res: any) => {
+                  resolve(res)
+                },
+                (err: any) => {
+                  reject(err)
+                }
+              )
+            })
+
+            await loginCallback()
+          } else {
+            reject(new ErrorResponse(e?.message, e))
+          }
+        })
+    })
+
+    return requestPromise
+  }
+
+  private getRequestParams(config: any): any {
+    const requestParams: any = {}
+
+    if (config.debug) {
+      requestParams['_omittextlog'] = 'false'
+      requestParams['_omitsessionresults'] = 'false'
+
+      requestParams['_debug'] = 131
+    }
+
+    return requestParams
+  }
+}

src/job-execution/index.ts

@@ -1,6 +1,7 @@
 export * from './ComputeJobExecutor'
+export * from './FileUploader'
 export * from './JesJobExecutor'
 export * from './JobExecutor'
 export * from './Sas9JobExecutor'
+export * from './SasJsJobExecutor'
 export * from './WebJobExecutor'
-export * from './FileUploader'

src/test/ContextManager.spec.ts

@@ -423,7 +423,7 @@ describe('ContextManager', () => {
           true
         )
       } catch (error) {
-        editError = error
+        editError = error as Error
       }
 
       await expect(
@@ -542,7 +542,7 @@ describe('ContextManager', () => {
           true
         )
       } catch (error) {
-        deleteError = error
+        deleteError = error as Error
       }
 
       await expect(

src/types/SASjsConfig.ts

@@ -16,7 +16,7 @@ export class SASjsConfig {
    * The location of the STP Process Web Application.  By default the adapter
    * will use '/SASjsApi/stp/execute' on SAS JS.
    */
-  pathSASJS?: string = ''
+  pathSASJS: string = ''
   /**
    * The location of the Stored Process Web Application.  By default the adapter
    * will use '/SASStoredProcess/do' on SAS 9.

src/types/system/global.d.ts

@@ -0,0 +1 @@
+import 'jest-extended'

src/utils/convertToCsv.ts

@@ -3,7 +3,7 @@
  * @param data - the array of JSON objects to convert.
  */
 export const convertToCSV = (
-  data: any,
+  data: any[],
   sasFormats?: { formats: { [key: string]: string } }
 ) => {
   let formats = sasFormats?.formats
@@ -76,7 +76,7 @@ export const convertToCSV = (
               return byteSize
             }
           })
-          .sort((a: number, b: number) => b - a)[0]
+          .sort((a: any, b: any) => b - a)[0]
 
         if (longestValueForField && longestValueForField > 32765) {
           invalidString = true

src/utils/parseWeboutResponse.ts

@@ -10,6 +10,7 @@ export const parseWeboutResponse = (response: string, url?: string): string => {
         .split('>>weboutEND<<')[0]
     } catch (e) {
       if (url) throw new WeboutResponseError(url)
+
       sasResponse = ''
       console.error(e)
     }

tsconfig.json

@@ -6,7 +6,8 @@
     "declaration": true,
     "outDir": "./build",
     "strict": true,
-    "sourceMap": true
+    "sourceMap": true,
+    "typeRoots": ["./node_modules/@types", "./src/types/system"]
   },
   "include": ["src"],
   "exclude": ["node_modules"]