chore(streamlog): optimise polling mechanism

src/api/viya/spec/executeScript.spec.ts

@@ -344,7 +344,6 @@ describe('executeScript', () => {
       requestClient,
       mockJob,
       false,
-      '',
       mockAuthConfig,
       defaultPollOptions
     )
@@ -546,7 +545,7 @@ describe('executeScript', () => {
       if (url.includes('_webout')) {
         return Promise.reject(new NotFoundError(url))
       }
-      return Promise.resolve({ result: mockJob, etag: '' })
+      return Promise.resolve({ result: mockJob, etag: '', status: 200 })
     })
 
     const error = await executeScript(
@@ -645,7 +644,9 @@ const setupMocks = () => {
     .mockImplementation(() => Promise.resolve({ result: mockJob, etag: '' }))
   jest
     .spyOn(requestClient, 'get')
-    .mockImplementation(() => Promise.resolve({ result: mockJob, etag: '' }))
+    .mockImplementation(() =>
+      Promise.resolve({ result: mockJob, etag: '', status: 200 })
+    )
   jest
     .spyOn(requestClient, 'delete')
     .mockImplementation(() => Promise.resolve({ result: {}, etag: '' }))
@@ -658,7 +659,7 @@ const setupMocks = () => {
   jest
     .spyOn(sessionManager, 'getVariable')
     .mockImplementation(() =>
-      Promise.resolve({ result: { value: 'test' }, etag: 'test' })
+      Promise.resolve({ result: { value: 'test' }, etag: 'test', status: 200 })
     )
   jest
     .spyOn(sessionManager, 'getSession')

src/api/viya/spec/mockResponses.ts

@@ -31,6 +31,13 @@ export const mockJob: Job = {
       type: 'log',
       uri: 'log'
     },
+    {
+      rel: 'self',
+      href: '/job',
+      method: 'GET',
+      type: 'job',
+      uri: 'job'
+    },
     {
       rel: 'state',
       href: '/state',
@@ -54,3 +61,13 @@ export const mockAuthConfig: AuthConfig = {
   access_token: 'acc355',
   refresh_token: 'r3fr35h'
 }
+
+export class MockStream {
+  _write(chunk: string, _: any, next: Function) {
+    next()
+  }
+
+  reset() {}
+
+  destroy() {}
+}

src/api/viya/spec/pollJobState.spec.ts

@@ -1,10 +1,11 @@
+import * as fs from 'fs'
+import { Logger, LogLevel } from '@sasjs/utils'
 import { RequestClient } from '../../../request/RequestClient'
 import { mockAuthConfig, mockJob } from './mockResponses'
 import { pollJobState } from '../pollJobState'
 import * as getTokensModule from '../../../auth/getTokens'
 import * as saveLogModule from '../saveLog'
 import { PollOptions } from '../../../types'
-import { Logger, LogLevel } from '@sasjs/utils'
 
 const requestClient = new (<jest.Mock<RequestClient>>RequestClient)()
 const defaultPollOptions: PollOptions = {
@@ -24,7 +25,6 @@ describe('pollJobState', () => {
       requestClient,
       mockJob,
       false,
-      'test',
       mockAuthConfig,
       defaultPollOptions
     )
@@ -40,7 +40,6 @@ describe('pollJobState', () => {
       requestClient,
       mockJob,
       false,
-      'test',
       undefined,
       defaultPollOptions
     )
@@ -53,7 +52,6 @@ describe('pollJobState', () => {
       requestClient,
       { ...mockJob, links: mockJob.links.filter((l) => l.rel !== 'state') },
       false,
-      'test',
       undefined,
       defaultPollOptions
     ).catch((e) => e)
@@ -62,23 +60,12 @@ describe('pollJobState', () => {
   })
 
   it('should attempt to refresh tokens before each poll', async () => {
-    jest
-      .spyOn(requestClient, 'get')
-      .mockImplementationOnce(() =>
-        Promise.resolve({ result: 'pending', etag: '' })
-      )
-      .mockImplementationOnce(() =>
-        Promise.resolve({ result: 'running', etag: '' })
-      )
-      .mockImplementation(() =>
-        Promise.resolve({ result: 'completed', etag: '' })
-      )
+    mockSimplePoll()
 
     await pollJobState(
       requestClient,
       mockJob,
       false,
-      'test',
       mockAuthConfig,
       defaultPollOptions
     )
@@ -87,23 +74,12 @@ describe('pollJobState', () => {
   })
 
   it('should attempt to fetch and save the log after each poll', async () => {
-    jest
-      .spyOn(requestClient, 'get')
-      .mockImplementationOnce(() =>
-        Promise.resolve({ result: 'pending', etag: '' })
-      )
-      .mockImplementationOnce(() =>
-        Promise.resolve({ result: 'running', etag: '' })
-      )
-      .mockImplementation(() =>
-        Promise.resolve({ result: 'completed', etag: '' })
-      )
+    mockSimplePoll()
 
     await pollJobState(
       requestClient,
       mockJob,
       false,
-      'test',
       mockAuthConfig,
       defaultPollOptions
     )
@@ -112,20 +88,12 @@ describe('pollJobState', () => {
   })
 
   it('should return the current status when the max poll count is reached', async () => {
-    jest
-      .spyOn(requestClient, 'get')
-      .mockImplementationOnce(() =>
-        Promise.resolve({ result: 'pending', etag: '' })
-      )
-      .mockImplementationOnce(() =>
-        Promise.resolve({ result: 'running', etag: '' })
-      )
+    mockRunningPoll()
 
     const state = await pollJobState(
       requestClient,
       mockJob,
       false,
-      'test',
       mockAuthConfig,
       {
         ...defaultPollOptions,
@@ -136,51 +104,47 @@ describe('pollJobState', () => {
     expect(state).toEqual('running')
   })
 
-  it('should continue polling until the job completes or errors', async () => {
-    jest
-      .spyOn(requestClient, 'get')
-      .mockImplementationOnce(() =>
-        Promise.resolve({ result: 'pending', etag: '' })
-      )
-      .mockImplementationOnce(() =>
-        Promise.resolve({ result: 'running', etag: '' })
-      )
-      .mockImplementation(() =>
-        Promise.resolve({ result: 'completed', etag: '' })
-      )
+  it('should poll with a larger interval for longer running jobs', async () => {
+    mockLongPoll()
+
+    const state = await pollJobState(
+      requestClient,
+      mockJob,
+      false,
+      mockAuthConfig,
+      {
+        ...defaultPollOptions,
+        maxPollCount: 200,
+        pollInterval: 10
+      }
+    )
+
+    expect(state).toEqual('completed')
+  }, 200000)
+
+  it('should continue polling until the job completes or errors', async () => {
+    mockSimplePoll(1)
 
     const state = await pollJobState(
       requestClient,
       mockJob,
       false,
-      'test',
       undefined,
       defaultPollOptions
     )
 
-    expect(requestClient.get).toHaveBeenCalledTimes(4)
+    expect(requestClient.get).toHaveBeenCalledTimes(3)
     expect(state).toEqual('completed')
   })
 
   it('should print the state to the console when debug is on', async () => {
     jest.spyOn((process as any).logger, 'info')
-    jest
-      .spyOn(requestClient, 'get')
-      .mockImplementationOnce(() =>
-        Promise.resolve({ result: 'pending', etag: '' })
-      )
-      .mockImplementationOnce(() =>
-        Promise.resolve({ result: 'running', etag: '' })
-      )
-      .mockImplementation(() =>
-        Promise.resolve({ result: 'completed', etag: '' })
-      )
+    mockSimplePoll()
 
     await pollJobState(
       requestClient,
       mockJob,
       true,
-      'test',
       undefined,
       defaultPollOptions
     )
@@ -205,21 +169,12 @@ describe('pollJobState', () => {
   })
 
   it('should continue polling when there is a single error in between', async () => {
-    jest
-      .spyOn(requestClient, 'get')
-      .mockImplementationOnce(() =>
-        Promise.resolve({ result: 'pending', etag: '' })
-      )
-      .mockImplementationOnce(() => Promise.reject('Status Error'))
-      .mockImplementationOnce(() =>
-        Promise.resolve({ result: 'completed', etag: '' })
-      )
+    mockPollWithSingleError()
 
     const state = await pollJobState(
       requestClient,
       mockJob,
       false,
-      'test',
       undefined,
       defaultPollOptions
     )
@@ -229,20 +184,19 @@ describe('pollJobState', () => {
   })
 
   it('should throw an error when the error count exceeds the set value of 5', async () => {
-    jest
-      .spyOn(requestClient, 'get')
-      .mockImplementation(() => Promise.reject('Status Error'))
+    mockErroredPoll()
 
     const error = await pollJobState(
       requestClient,
       mockJob,
       false,
-      'test',
       undefined,
       defaultPollOptions
     ).catch((e) => e)
 
-    expect(error).toContain('Error while getting job state after interval.')
+    expect(error.message).toEqual(
+      'Error while polling job state for job j0b: Status Error'
+    )
   })
 })
 
@@ -251,11 +205,12 @@ const setupMocks = () => {
   jest.mock('../../../request/RequestClient')
   jest.mock('../../../auth/getTokens')
   jest.mock('../saveLog')
+  jest.mock('fs')
 
   jest
     .spyOn(requestClient, 'get')
     .mockImplementation(() =>
-      Promise.resolve({ result: 'completed', etag: '' })
+      Promise.resolve({ result: 'completed', etag: '', status: 200 })
     )
   jest
     .spyOn(getTokensModule, 'getTokens')
@@ -263,4 +218,84 @@ const setupMocks = () => {
   jest
     .spyOn(saveLogModule, 'saveLog')
     .mockImplementation(() => Promise.resolve())
+  jest
+    .spyOn(fs, 'createWriteStream')
+    .mockImplementation(() => ({} as unknown as fs.WriteStream))
+}
+
+const mockSimplePoll = (runningCount = 2) => {
+  let count = 0
+  jest.spyOn(requestClient, 'get').mockImplementation((url) => {
+    count++
+    if (url.includes('job')) {
+      return Promise.resolve({ result: mockJob, etag: '', status: 200 })
+    }
+    return Promise.resolve({
+      result:
+        count === 0
+          ? 'pending'
+          : count <= runningCount
+          ? 'running'
+          : 'completed',
+      etag: '',
+      status: 200
+    })
+  })
+}
+
+const mockRunningPoll = () => {
+  let count = 0
+  jest.spyOn(requestClient, 'get').mockImplementation((url) => {
+    count++
+    if (url.includes('job')) {
+      return Promise.resolve({ result: mockJob, etag: '', status: 200 })
+    }
+    return Promise.resolve({
+      result: count === 0 ? 'pending' : 'running',
+      etag: '',
+      status: 200
+    })
+  })
+}
+
+const mockLongPoll = () => {
+  let count = 0
+  jest.spyOn(requestClient, 'get').mockImplementation((url) => {
+    count++
+    if (url.includes('job')) {
+      return Promise.resolve({ result: mockJob, etag: '', status: 200 })
+    }
+    return Promise.resolve({
+      result: count <= 101 ? 'running' : 'completed',
+      etag: '',
+      status: 200
+    })
+  })
+}
+
+const mockPollWithSingleError = () => {
+  let count = 0
+  jest.spyOn(requestClient, 'get').mockImplementation((url) => {
+    count++
+    if (url.includes('job')) {
+      return Promise.resolve({ result: mockJob, etag: '', status: 200 })
+    }
+    if (count === 1) {
+      return Promise.reject('Status Error')
+    }
+    return Promise.resolve({
+      result: count === 0 ? 'pending' : 'completed',
+      etag: '',
+      status: 200
+    })
+  })
+}
+
+const mockErroredPoll = () => {
+  jest.spyOn(requestClient, 'get').mockImplementation((url) => {
+    if (url.includes('job')) {
+      return Promise.resolve({ result: mockJob, etag: '', status: 200 })
+    }
+    return Promise.reject('Status Error')
+  })
 }

src/api/viya/spec/saveLog.spec.ts

@@ -1,11 +1,13 @@
 import { Logger, LogLevel } from '@sasjs/utils'
-import * as fileModule from '@sasjs/utils/file'
 import { RequestClient } from '../../../request/RequestClient'
 import * as fetchLogsModule from '../../../utils/fetchLogByChunks'
+import * as writeStreamModule from '../writeStream'
 import { saveLog } from '../saveLog'
 import { mockJob } from './mockResponses'
+import { WriteStream } from 'fs'
 
 const requestClient = new (<jest.Mock<RequestClient>>RequestClient)()
+const stream = {} as unknown as WriteStream
 
 describe('saveLog', () => {
   beforeEach(() => {
@@ -14,16 +16,21 @@ describe('saveLog', () => {
   })
 
   it('should return immediately if shouldSaveLog is false', async () => {
-    await saveLog(mockJob, requestClient, false, '/test', 't0k3n')
+    await saveLog(mockJob, requestClient, false, 0, 100, stream, 't0k3n')
 
-    expect(fetchLogsModule.fetchLogByChunks).not.toHaveBeenCalled()
-    expect(fileModule.createFile).not.toHaveBeenCalled()
+    expect(fetchLogsModule.fetchLog).not.toHaveBeenCalled()
+    expect(writeStreamModule.writeStream).not.toHaveBeenCalled()
   })
 
   it('should throw an error when a valid access token is not provided', async () => {
-    const error = await saveLog(mockJob, requestClient, true, '/test').catch(
-      (e) => e
-    )
+    const error = await saveLog(
+      mockJob,
+      requestClient,
+      true,
+      0,
+      100,
+      stream
+    ).catch((e) => e)
 
     expect(error.message).toContain(
       `Logs for job ${mockJob.id} cannot be fetched without a valid access token.`
@@ -35,7 +42,9 @@ describe('saveLog', () => {
       { ...mockJob, links: mockJob.links.filter((l) => l.rel !== 'log') },
       requestClient,
       true,
-      '/test',
+      0,
+      100,
+      stream,
       't0k3n'
     ).catch((e) => e)
 
@@ -45,15 +54,19 @@ describe('saveLog', () => {
   })
 
   it('should fetch and save logs to the given path', async () => {
-    await saveLog(mockJob, requestClient, true, '/test', 't0k3n')
+    await saveLog(mockJob, requestClient, true, 0, 100, stream, 't0k3n')
 
-    expect(fetchLogsModule.fetchLogByChunks).toHaveBeenCalledWith(
+    expect(fetchLogsModule.fetchLog).toHaveBeenCalledWith(
       requestClient,
       't0k3n',
       '/log/content',
+      0,
       100
     )
-    expect(fileModule.createFile).toHaveBeenCalledWith('/test', 'Test Log')
+    expect(writeStreamModule.writeStream).toHaveBeenCalledWith(
+      stream,
+      'Test Log'
+    )
   })
 })
 
@@ -62,11 +75,12 @@ const setupMocks = () => {
   jest.mock('../../../request/RequestClient')
   jest.mock('../../../utils/fetchLogByChunks')
   jest.mock('@sasjs/utils')
+  jest.mock('../writeStream')
 
   jest
-    .spyOn(fetchLogsModule, 'fetchLogByChunks')
+    .spyOn(fetchLogsModule, 'fetchLog')
     .mockImplementation(() => Promise.resolve('Test Log'))
   jest
-    .spyOn(fileModule, 'createFile')
+    .spyOn(writeStreamModule, 'writeStream')
     .mockImplementation(() => Promise.resolve())
 }