sasjs/adapter
1
0
Fork 0
mirror of https://github.com/sasjs/adapter.git synced 2026-01-04 11:10:05 +00:00
Code Issues Projects Releases Wiki Activity

chore(poll-job-status): renamed PollOptions to PollStrategy and added docs

Browse Source
This commit is contained in:
Yury Shkoda
2023-05-15 16:32:07 +03:00
parent f9f4aa5aa6
commit 153b285670
8 changed files with 112 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/SASViyaApiClient.ts
View File

@@ -9,7 +9,7 @@ import {
File,
EditContextInput,
JobDefinition,
PollOptions
PollStrategy
} from './types'
import {
CertificateError,
@@ -276,7 +276,7 @@ export class SASViyaApiClient {
* @param debug - when set to true, the log will be returned.
* @param expectWebout - when set to true, the automatic _webout fileref will be checked for content, and that content returned. This fileref is used when the Job contains a SASjs web request (as opposed to executing arbitrary SAS code).
* @param waitForResult - when set to true, function will return the session
* @param pollOptions - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: { MAX_POLL_COUNT: 24 * 60 * 60, POLL_INTERVAL: 1000 }.
* @param pollStrategy - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: { maxPollCount: 24 * 60 * 60, pollInterval: 1000 }.
* @param printPid - a boolean that indicates whether the function should print (PID) of the started job.
* @param variables - an object that represents macro variables.
*/
@@ -289,7 +289,7 @@ export class SASViyaApiClient {
debug: boolean = false,
expectWebout = false,
waitForResult = true,
pollOptions?: PollOptions,
pollStrategy?: PollStrategy,
printPid = false,
variables?: MacroVar
): Promise<any> {
@@ -305,7 +305,7 @@ export class SASViyaApiClient {
debug,
expectWebout,
waitForResult,
pollOptions,
pollStrategy,
printPid,
variables
)
@@ -627,7 +627,7 @@ export class SASViyaApiClient {
* @param accessToken - an optional access token for an authorized user.
* @param waitForResult - a boolean indicating if the function should wait for a result.
* @param expectWebout - a boolean indicating whether to expect a _webout response.
* @param pollOptions - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: { MAX_POLL_COUNT: 24 * 60 * 60, POLL_INTERVAL: 1000 }.
* @param pollStrategy - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: { maxPollCount: 24 * 60 * 60, pollInterval: 1000 }.
* @param printPid - a boolean that indicates whether the function should print (PID) of the started job.
* @param variables - an object that represents macro variables.
*/
@@ -639,7 +639,7 @@ export class SASViyaApiClient {
authConfig?: AuthConfig,
waitForResult = true,
expectWebout = false,
pollOptions?: PollOptions,
pollStrategy?: PollStrategy,
printPid = false,
variables?: MacroVar
) {
@@ -718,7 +718,7 @@ export class SASViyaApiClient {
debug,
expectWebout,
waitForResult,
pollOptions,
pollStrategy,
printPid,
variables
)
@@ -910,14 +910,14 @@ export class SASViyaApiClient {
private async pollJobState(
postedJob: Job,
authConfig?: AuthConfig,
pollOptions?: PollOptions
pollStrategy?: PollStrategy
) {
return pollJobState(
this.requestClient,
postedJob,
this.debug,
authConfig,
pollOptions
pollStrategy
)
}

8
src/SASjs.ts
View File

@@ -3,7 +3,7 @@ import {
SASjsConfig,
UploadFile,
EditContextInput,
PollOptions,
PollStrategy,
LoginMechanism
} from './types'
import { SASViyaApiClient } from './SASViyaApiClient'
@@ -851,7 +851,7 @@ export default class SASjs {
* @param authConfig - a valid client, secret, refresh and access tokens that are authorised to execute compute jobs.
* The access token is not required when the user is authenticated via the browser.
* @param waitForResult - a boolean that indicates whether the function needs to wait for execution to complete.
* @param pollOptions - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: { MAX_POLL_COUNT: 24 * 60 * 60, POLL_INTERVAL: 1000 }.
* @param pollStrategy - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: { maxPollCount: 24 * 60 * 60, pollInterval: 1000 }.
* @param printPid - a boolean that indicates whether the function should print (PID) of the started job.
* @param variables - an object that represents macro variables.
*/
@@ -861,7 +861,7 @@ export default class SASjs {
config: any = {},
authConfig?: AuthConfig,
waitForResult?: boolean,
pollOptions?: PollOptions,
pollStrategy?: PollStrategy,
printPid = false,
variables?: MacroVar
) {
@@ -885,7 +885,7 @@ export default class SASjs {
authConfig,
!!waitForResult,
false,
pollOptions,
pollStrategy,
printPid,
variables
)

8
src/api/viya/executeScript.ts
View File

@@ -2,7 +2,7 @@ import { timestampToYYYYMMDDHHMMSS } from '@sasjs/utils/time'
import { AuthConfig, MacroVar } from '@sasjs/utils/types'
import { prefixMessage } from '@sasjs/utils/error'
import {
PollOptions,
PollStrategy,
Job,
ComputeJobExecutionError,
NotFoundError
@@ -25,7 +25,7 @@ import { uploadTables } from './uploadTables'
* @param debug - when set to true, the log will be returned.
* @param expectWebout - when set to true, the automatic _webout fileref will be checked for content, and that content returned. This fileref is used when the Job contains a SASjs web request (as opposed to executing arbitrary SAS code).
* @param waitForResult - when set to true, function will return the session
* @param pollOptions - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: { MAX_POLL_COUNT: 24 * 60 * 60, POLL_INTERVAL: 1000 }.
* @param pollStrategy - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: { maxPollCount: 24 * 60 * 60, pollInterval: 1000 }.
* @param printPid - a boolean that indicates whether the function should print (PID) of the started job.
* @param variables - an object that represents macro variables.
*/
@@ -41,7 +41,7 @@ export async function executeScript(
debug: boolean = false,
expectWebout = false,
waitForResult = true,
pollOptions?: PollOptions,
pollStrategy?: PollStrategy,
printPid = false,
variables?: MacroVar
): Promise<any> {
@@ -179,7 +179,7 @@ export async function executeScript(
postedJob,
debug,
authConfig,
pollOptions
pollStrategy
).catch(async (err) => {
const error = err?.response?.data
const result = /err=[0-9]*,/.exec(error)

74
src/api/viya/pollJobState.ts
View File

@@ -1,5 +1,5 @@
import { AuthConfig } from '@sasjs/utils/types'
import { Job, PollOptions } from '../..'
import { Job, PollStrategy, PollStrategies } from '../..'
import { getTokens } from '../../auth/getTokens'
import { RequestClient } from '../../request/RequestClient'
import { JobStatePollError } from '../../types/errors'
@@ -16,31 +16,55 @@ export enum JobState {
Error = 'error'
}
type PollStrategies = PollOptions[]
/**
* Polls job status using default or provided poll strategies.
* @param requestClient - the pre-configured HTTP request client.
* @param postedJob - the relative or absolute path to the job.
* @param debug - sets the _debug flag in the job arguments.
* @param authConfig - an access token, refresh token, client and secret for an authorized user.
* @param pollStrategy - an object containing maxPollCount, pollInterval, streamLog and logFolderPath. It will override the first default poll strategy if provided.
* Example:
* { maxPollCount: 200, pollInterval: 300, streamLog: false }
* @param pollStrategies - an array of poll strategies. It will override default poll strategies if provided.
* Example:
* [
* { maxPollCount: 200, pollInterval: 300, streamLog: false },
* { maxPollCount: 300, pollInterval: 3000, streamLog: false },
* { maxPollCount: 500, pollInterval: 30000, streamLog: false }
* ]
* Default poll strategies:
* [
* { maxPollCount: 200, pollInterval: 300, streamLog: false }, // INFO: approximately ~2 mins (including time to get response (~300ms))
* { maxPollCount: 300, pollInterval: 3000, streamLog: false }, // INFO: approximately ~5.5 mins (including time to get response (~300ms))
* { maxPollCount: 400, pollInterval: 30000, streamLog: false }, // INFO: approximately ~50.5 mins (including time to get response (~300ms))
* { maxPollCount: 3400, pollInterval: 60000, streamLog: false } // INFO: approximately ~3015 mins (~125 hours) (including time to get response (~300ms))
* ]
* @returns - a promise which resolves with a job state
*/
export async function pollJobState(
requestClient: RequestClient,
postedJob: Job,
debug: boolean,
authConfig?: AuthConfig,
pollOptions?: PollOptions,
pollStrategy?: PollStrategy,
pollStrategies?: PollStrategies
) {
): Promise<JobState> {
const logger = process.logger || console
const defaultStreamLog = pollStrategy ? pollStrategy.streamLog : false
const defaultPollStrategies: PollStrategies = [
{ maxPollCount: 200, pollInterval: 300, streamLog: false }, // INFO: approximately ~2 mins (including time to get response (~300ms))
{ maxPollCount: 300, pollInterval: 3000, streamLog: false }, // INFO: approximately ~5.5 mins (including time to get response (~300ms))
{ maxPollCount: 400, pollInterval: 30000, streamLog: false }, // INFO: approximately ~50.5 mins (including time to get response (~300ms))
{ maxPollCount: 3400, pollInterval: 60000, streamLog: false } // INFO: approximately ~3015 mins (~125 hours) (including time to get response (~300ms))
{ maxPollCount: 200, pollInterval: 300, streamLog: defaultStreamLog },
{ maxPollCount: 300, pollInterval: 3000, streamLog: defaultStreamLog },
{ maxPollCount: 400, pollInterval: 30000, streamLog: defaultStreamLog },
{ maxPollCount: 3400, pollInterval: 60000, streamLog: defaultStreamLog }
]
if (pollStrategies === undefined) pollStrategies = defaultPollStrategies
else validatePollStrategies(pollStrategies)
let defaultPollOptions: PollOptions = pollStrategies.splice(0, 1)[0]
let defaultPollStrategy: PollStrategy = pollStrategies.splice(0, 1)[0]
pollOptions = { ...defaultPollOptions, ...(pollOptions || {}) }
pollStrategy = { ...defaultPollStrategy, ...(pollStrategy || {}) }
const stateLink = postedJob.links.find((l: any) => l.rel === 'state')
if (!stateLink) {
@@ -69,9 +93,9 @@ export async function pollJobState(
}
let logFileStream
if (pollOptions.streamLog && isNode()) {
if (pollStrategy.streamLog && isNode()) {
const { getFileStream } = require('./getFileStream')
logFileStream = await getFileStream(postedJob, pollOptions.logFolderPath)
logFileStream = await getFileStream(postedJob, pollStrategy.logFolderPath)
}
let result = await doPoll(
@@ -80,7 +104,7 @@ export async function pollJobState(
currentState,
debug,
pollCount,
pollOptions,
pollStrategy,
authConfig,
logFileStream
)
@@ -90,7 +114,7 @@ export async function pollJobState(
if (
!needsRetry(currentState) ||
(pollCount >= pollOptions.maxPollCount && !pollStrategies.length)
(pollCount >= pollStrategy.maxPollCount && !pollStrategies.length)
) {
return currentState
}
@@ -98,11 +122,11 @@ export async function pollJobState(
// INFO: If we get to this point, this is a long-running job that needs longer polling.
// We will resume polling with a bigger interval according to the next polling strategy
while (pollStrategies.length && needsRetry(currentState)) {
defaultPollOptions = pollStrategies.splice(0, 1)[0]
defaultPollStrategy = pollStrategies.splice(0, 1)[0]
if (pollOptions) {
defaultPollOptions.streamLog = pollOptions.streamLog
defaultPollOptions.logFolderPath = pollOptions.logFolderPath
if (pollStrategy) {
defaultPollStrategy.streamLog = pollStrategy.streamLog
defaultPollStrategy.logFolderPath = pollStrategy.logFolderPath
}
result = await doPoll(
@@ -111,7 +135,7 @@ export async function pollJobState(
currentState,
debug,
pollCount,
defaultPollOptions,
defaultPollStrategy,
authConfig,
logFileStream
)
@@ -169,11 +193,11 @@ const doPoll = async (
currentState: JobState,
debug: boolean,
pollCount: number,
pollOptions: PollOptions,
pollStrategy: PollStrategy,
authConfig?: AuthConfig,
logStream?: WriteStream
): Promise<{ state: JobState; pollCount: number }> => {
const { maxPollCount, pollInterval } = pollOptions
const { maxPollCount, pollInterval } = pollStrategy
const logger = process.logger || console
const stateLink = postedJob.links.find((l: Link) => l.rel === 'state')!
let maxErrorCount = 5
@@ -208,7 +232,7 @@ const doPoll = async (
const jobHref = postedJob.links.find((l: Link) => l.rel === 'self')!.href
if (pollOptions?.streamLog) {
if (pollStrategy?.streamLog) {
const { result: job } = await requestClient.get<Job>(
jobHref,
authConfig?.access_token
@@ -251,7 +275,7 @@ const doPoll = async (
}
const validatePollStrategies = (strategies: PollStrategies) => {
const throwError = (message?: string, strategy?: PollOptions) => {
const throwError = (message?: string, strategy?: PollStrategy) => {
throw new Error(
`Poll strategies are not valid.${message ? ` ${message}` : ''}${
strategy
@@ -263,7 +287,7 @@ const validatePollStrategies = (strategies: PollStrategies) => {
if (!strategies.length) throwError('No strategies provided.')
strategies.forEach((strategy: PollOptions, i: number) => {
strategies.forEach((strategy: PollStrategy, i: number) => {
const { maxPollCount, pollInterval } = strategy
if (maxPollCount < 1) {

44
src/api/viya/spec/executeScript.spec.ts
View File

@@ -7,13 +7,13 @@ import * as uploadTablesModule from '../uploadTables'
import * as getTokensModule from '../../../auth/getTokens'
import * as formatDataModule from '../../../utils/formatDataForRequest'
import * as fetchLogsModule from '../../../utils/fetchLogByChunks'
import { PollOptions } from '../../../types'
import { PollStrategy } from '../../../types'
import { ComputeJobExecutionError, NotFoundError } from '../../../types/errors'
import { Logger, LogLevel } from '@sasjs/utils'
const sessionManager = new (<jest.Mock<SessionManager>>SessionManager)()
const requestClient = new (<jest.Mock<RequestClient>>RequestClient)()
const defaultPollOptions: PollOptions = {
const defaultPollStrategy: PollStrategy = {
maxPollCount: 100,
pollInterval: 500,
streamLog: false
@@ -98,7 +98,7 @@ describe('executeScript', () => {
false,
false,
false,
defaultPollOptions,
defaultPollStrategy,
true
)
@@ -126,7 +126,7 @@ describe('executeScript', () => {
false,
false,
false,
defaultPollOptions,
defaultPollStrategy,
true
).catch((e: any) => e)
@@ -152,7 +152,7 @@ describe('executeScript', () => {
false,
false,
false,
defaultPollOptions,
defaultPollStrategy,
true
)
@@ -176,7 +176,7 @@ describe('executeScript', () => {
false,
false,
false,
defaultPollOptions,
defaultPollStrategy,
true
)
@@ -202,7 +202,7 @@ describe('executeScript', () => {
false,
false,
false,
defaultPollOptions,
defaultPollStrategy,
true
)
@@ -248,7 +248,7 @@ describe('executeScript', () => {
true,
false,
false,
defaultPollOptions,
defaultPollStrategy,
true
)
@@ -295,7 +295,7 @@ describe('executeScript', () => {
true,
false,
false,
defaultPollOptions,
defaultPollStrategy,
true
).catch((e: any) => e)
@@ -315,7 +315,7 @@ describe('executeScript', () => {
true,
false,
false,
defaultPollOptions,
defaultPollStrategy,
true
)
@@ -335,7 +335,7 @@ describe('executeScript', () => {
false,
false,
true,
defaultPollOptions,
defaultPollStrategy,
true
)
@@ -344,7 +344,7 @@ describe('executeScript', () => {
mockJob,
false,
mockAuthConfig,
defaultPollOptions
defaultPollStrategy
)
})
@@ -365,7 +365,7 @@ describe('executeScript', () => {
false,
false,
true,
defaultPollOptions,
defaultPollStrategy,
true
).catch((e: any) => e)
@@ -391,7 +391,7 @@ describe('executeScript', () => {
false,
false,
true,
defaultPollOptions,
defaultPollStrategy,
true
).catch((e: any) => e)
@@ -417,7 +417,7 @@ describe('executeScript', () => {
true,
false,
true,
defaultPollOptions,
defaultPollStrategy,
true
)
@@ -442,7 +442,7 @@ describe('executeScript', () => {
false,
false,
true,
defaultPollOptions,
defaultPollStrategy,
true
)
@@ -468,7 +468,7 @@ describe('executeScript', () => {
true,
false,
true,
defaultPollOptions,
defaultPollStrategy,
true
).catch((e: any) => e)
@@ -503,7 +503,7 @@ describe('executeScript', () => {
true,
false,
true,
defaultPollOptions,
defaultPollStrategy,
true
).catch((e: any) => e)
@@ -532,7 +532,7 @@ describe('executeScript', () => {
false,
true,
true,
defaultPollOptions,
defaultPollStrategy,
true
)
@@ -563,7 +563,7 @@ describe('executeScript', () => {
false,
true,
true,
defaultPollOptions,
defaultPollStrategy,
true
).catch((e: any) => e)
@@ -597,7 +597,7 @@ describe('executeScript', () => {
false,
true,
true,
defaultPollOptions,
defaultPollStrategy,
true
)
@@ -624,7 +624,7 @@ describe('executeScript', () => {
false,
true,
true,
defaultPollOptions,
defaultPollStrategy,
true
).catch((e: any) => e)

40
src/api/viya/spec/pollJobState.spec.ts
View File

@@ -7,14 +7,14 @@ import * as saveLogModule from '../saveLog'
import * as getFileStreamModule from '../getFileStream'
import * as isNodeModule from '../../../utils/isNode'
import * as delayModule from '../../../utils/delay'
import { PollOptions } from '../../../types'
import { PollStrategy, PollStrategies } from '../../../types'
import { WriteStream } from 'fs'
const baseUrl = 'http://localhost'
const requestClient = new (<jest.Mock<RequestClient>>RequestClient)()
requestClient['httpClient'].defaults.baseURL = baseUrl
const defaultPollOptions: PollOptions = {
const defaultPollStrategy: PollStrategy = {
maxPollCount: 100,
pollInterval: 500,
streamLog: false
@@ -32,7 +32,7 @@ describe('pollJobState', () => {
mockJob,
false,
mockAuthConfig,
defaultPollOptions
defaultPollStrategy
)
expect(getTokensModule.getTokens).toHaveBeenCalledWith(
@@ -47,7 +47,7 @@ describe('pollJobState', () => {
mockJob,
false,
undefined,
defaultPollOptions
defaultPollStrategy
)
expect(getTokensModule.getTokens).not.toHaveBeenCalled()
@@ -59,7 +59,7 @@ describe('pollJobState', () => {
{ ...mockJob, links: mockJob.links.filter((l) => l.rel !== 'state') },
false,
undefined,
defaultPollOptions
defaultPollStrategy
).catch((e: any) => e)
expect((error as Error).message).toContain('Job state link was not found.')
@@ -73,7 +73,7 @@ describe('pollJobState', () => {
mockJob,
false,
mockAuthConfig,
defaultPollOptions
defaultPollStrategy
)
expect(getTokensModule.getTokens).toHaveBeenCalledTimes(3)
@@ -84,7 +84,7 @@ describe('pollJobState', () => {
const { saveLog } = require('../saveLog')
await pollJobState(requestClient, mockJob, false, mockAuthConfig, {
...defaultPollOptions,
...defaultPollStrategy,
streamLog: true
})
@@ -97,7 +97,7 @@ describe('pollJobState', () => {
const { saveLog } = require('../saveLog')
await pollJobState(requestClient, mockJob, false, mockAuthConfig, {
...defaultPollOptions,
...defaultPollStrategy,
streamLog: true
})
@@ -112,7 +112,7 @@ describe('pollJobState', () => {
const { getFileStream } = require('../getFileStream')
await pollJobState(requestClient, mockJob, false, mockAuthConfig, {
...defaultPollOptions,
...defaultPollStrategy,
streamLog: true
})
@@ -128,7 +128,7 @@ describe('pollJobState', () => {
mockJob,
false,
mockAuthConfig,
defaultPollOptions
defaultPollStrategy
)
expect(saveLogModule.saveLog).not.toHaveBeenCalled()
@@ -137,18 +137,18 @@ describe('pollJobState', () => {
it('should return the current status when the max poll count is reached', async () => {
mockRunningPoll()
const pollOptions = {
...defaultPollOptions,
const pollStrategy = {
...defaultPollStrategy,
maxPollCount: 1
}
const pollStrategies = [pollOptions]
const pollStrategies = [pollStrategy]
const state = await pollJobState(
requestClient,
mockJob,
false,
mockAuthConfig,
pollOptions,
pollStrategy,
pollStrategies
)
@@ -164,7 +164,7 @@ describe('pollJobState', () => {
false,
mockAuthConfig,
{
...defaultPollOptions,
...defaultPollStrategy,
maxPollCount: 200,
pollInterval: 10
}
@@ -181,7 +181,7 @@ describe('pollJobState', () => {
mockJob,
false,
undefined,
defaultPollOptions
defaultPollStrategy
)
expect(requestClient.get).toHaveBeenCalledTimes(2)
@@ -197,7 +197,7 @@ describe('pollJobState', () => {
mockJob,
true,
undefined,
defaultPollOptions
defaultPollStrategy
)
expect((process as any).logger.info).toHaveBeenCalledTimes(4)
@@ -227,7 +227,7 @@ describe('pollJobState', () => {
mockJob,
false,
undefined,
defaultPollOptions
defaultPollStrategy
)
expect(requestClient.get).toHaveBeenCalledTimes(2)
@@ -242,7 +242,7 @@ describe('pollJobState', () => {
mockJob,
false,
undefined,
defaultPollOptions
defaultPollStrategy
).catch((e: any) => e)
expect(error.message).toEqual(
@@ -288,7 +288,7 @@ describe('pollJobState', () => {
'Poll strategies are not valid. No strategies provided.'
)
let pollStrategies: PollOptions[] = []
let pollStrategies: PollStrategies = []
await expect(
pollJobState(

4
src/types/PollOptions.ts → src/types/PollStrategy.ts
View File

@@ -1,6 +1,8 @@
export interface PollOptions {
export interface PollStrategy {
maxPollCount: number
pollInterval: number // milliseconds
streamLog: boolean
logFolderPath?: string
}
export type PollStrategies = PollStrategy[]

2
src/types/index.ts
View File

@@ -10,6 +10,6 @@ export * from './SASjsConfig'
export * from './SASjsRequest'
export * from './Session'
export * from './UploadFile'
export * from './PollOptions'
export * from './PollStrategy'
export * from './WriteStream'
export * from './ExecuteScript'