From 04ccbf68437a107a35f998bebb39e3b2256ebf60 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Wed, 7 Jul 2021 10:02:14 +0100 Subject: [PATCH 01/15] feat(log): write logs to file when polling for job status --- src/SASViyaApiClient.ts | 541 +++------------------------------- src/api/viya/executeScript.ts | 303 +++++++++++++++++++ src/api/viya/pollJobState.ts | 178 +++++++++++ src/api/viya/uploadTables.ts | 35 +++ src/auth/tokens.ts | 122 ++++++++ src/request/RequestClient.ts | 5 + src/types/PollOptions.ts | 6 +- 7 files changed, 689 insertions(+), 501 deletions(-) create mode 100644 src/api/viya/executeScript.ts create mode 100644 src/api/viya/pollJobState.ts create mode 100644 src/api/viya/uploadTables.ts create mode 100644 src/auth/tokens.ts diff --git a/src/SASViyaApiClient.ts b/src/SASViyaApiClient.ts index 068e780..3c1b324 100644 --- a/src/SASViyaApiClient.ts +++ b/src/SASViyaApiClient.ts @@ -1,10 +1,4 @@ -import { - convertToCSV, - isRelativePath, - isUri, - isUrl, - fetchLogByChunks -} from './utils' +import { isRelativePath, isUri, isUrl } from './utils' import * as NodeFormData from 'form-data' import { Job, @@ -17,28 +11,18 @@ import { JobDefinition, PollOptions } from './types' -import { - ComputeJobExecutionError, - JobExecutionError, - NotFoundError -} from './types/errors' -import { formatDataForRequest } from './utils/formatDataForRequest' +import { JobExecutionError } from './types/errors' import { SessionManager } from './SessionManager' import { ContextManager } from './ContextManager' -import { - timestampToYYYYMMDDHHMMSS, - isAccessTokenExpiring, - isRefreshTokenExpiring, - Logger, - LogLevel, - SasAuthResponse, - MacroVar, - AuthConfig -} from '@sasjs/utils' +import { SasAuthResponse, MacroVar, AuthConfig } from '@sasjs/utils' import { isAuthorizeFormRequired } from './auth/isAuthorizeFormRequired' import { RequestClient } from './request/RequestClient' import { prefixMessage } from '@sasjs/utils/error' import * as mime from 'mime' +import { pollJobState } from './api/viya/pollJobState' +import { getAccessToken, getTokens, refreshTokens } from './auth/tokens' +import { uploadTables } from './api/viya/uploadTables' +import { executeScript } from './api/viya/executeScript' /** * A client for interfacing with the SAS Viya REST API. @@ -174,13 +158,6 @@ export class SASViyaApiClient { throw new Error(`Execution context ${contextName} not found.`) } - const createSessionRequest = { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json' - } - } const { result: createdSession } = await this.requestClient.post( `/compute/contexts/${executionContext.id}/sessions`, {}, @@ -295,249 +272,22 @@ export class SASViyaApiClient { printPid = false, variables?: MacroVar ): Promise { - let access_token = (authConfig || {}).access_token - if (authConfig) { - ;({ access_token } = await this.getTokens(authConfig)) - } - - const logger = process.logger || console - - try { - let executionSessionId: string - - const session = await this.sessionManager - .getSession(access_token) - .catch((err) => { - throw prefixMessage(err, 'Error while getting session. ') - }) - - executionSessionId = session!.id - - if (printPid) { - const { result: jobIdVariable } = await this.sessionManager - .getVariable(executionSessionId, 'SYSJOBID', access_token) - .catch((err) => { - throw prefixMessage(err, 'Error while getting session variable. ') - }) - - if (jobIdVariable && jobIdVariable.value) { - const relativeJobPath = this.rootFolderName - ? jobPath.split(this.rootFolderName).join('').replace(/^\//, '') - : jobPath - - const logger = new Logger(debug ? LogLevel.Debug : LogLevel.Info) - - logger.info( - `Triggered '${relativeJobPath}' with PID ${ - jobIdVariable.value - } at ${timestampToYYYYMMDDHHMMSS()}` - ) - } - } - - const jobArguments: { [key: string]: any } = { - _contextName: contextName, - _OMITJSONLISTING: true, - _OMITJSONLOG: true, - _OMITSESSIONRESULTS: true, - _OMITTEXTLISTING: true, - _OMITTEXTLOG: true - } - - if (debug) { - jobArguments['_OMITTEXTLOG'] = false - jobArguments['_OMITSESSIONRESULTS'] = false - } - - let fileName - - if (isRelativePath(jobPath)) { - fileName = `exec-${ - jobPath.includes('/') ? jobPath.split('/')[1] : jobPath - }` - } else { - const jobPathParts = jobPath.split('/') - fileName = jobPathParts.pop() - } - - let jobVariables: any = { - SYS_JES_JOB_URI: '', - _program: isRelativePath(jobPath) - ? this.rootFolderName + '/' + jobPath - : jobPath - } - - if (variables) jobVariables = { ...jobVariables, ...variables } - - if (debug) jobVariables = { ...jobVariables, _DEBUG: 131 } - - let files: any[] = [] - - if (data) { - if (JSON.stringify(data).includes(';')) { - files = await this.uploadTables(data, access_token).catch((err) => { - throw prefixMessage(err, 'Error while uploading tables. ') - }) - - jobVariables['_webin_file_count'] = files.length - - files.forEach((fileInfo, index) => { - jobVariables[ - `_webin_fileuri${index + 1}` - ] = `/files/files/${fileInfo.file.id}` - jobVariables[`_webin_name${index + 1}`] = fileInfo.tableName - }) - } else { - jobVariables = { ...jobVariables, ...formatDataForRequest(data) } - } - } - - // Execute job in session - const jobRequestBody = { - name: fileName, - description: 'Powered by SASjs', - code: linesOfCode, - variables: jobVariables, - arguments: jobArguments - } - - const { result: postedJob, etag } = await this.requestClient - .post( - `/compute/sessions/${executionSessionId}/jobs`, - jobRequestBody, - access_token - ) - .catch((err) => { - throw prefixMessage(err, 'Error while posting job. ') - }) - - if (!waitForResult) return session - - if (debug) { - logger.info(`Job has been submitted for '${fileName}'.`) - logger.info( - `You can monitor the job progress at '${this.serverUrl}${ - postedJob.links.find((l: any) => l.rel === 'state')!.href - }'.` - ) - } - - const jobStatus = await this.pollJobState( - postedJob, - etag, - authConfig, - pollOptions - ).catch(async (err) => { - const error = err?.response?.data - const result = /err=[0-9]*,/.exec(error) - - const errorCode = '5113' - if (result?.[0]?.slice(4, -1) === errorCode) { - const sessionLogUrl = - postedJob.links.find((l: any) => l.rel === 'up')!.href + '/log' - const logCount = 1000000 - err.log = await fetchLogByChunks( - this.requestClient, - access_token!, - sessionLogUrl, - logCount - ) - } - throw prefixMessage(err, 'Error while polling job status. ') - }) - - if (authConfig) { - ;({ access_token } = await this.getTokens(authConfig)) - } - - const { result: currentJob } = await this.requestClient - .get( - `/compute/sessions/${executionSessionId}/jobs/${postedJob.id}`, - access_token - ) - .catch((err) => { - throw prefixMessage(err, 'Error while getting job. ') - }) - - let jobResult - let log = '' - - const logLink = currentJob.links.find((l) => l.rel === 'log') - - if (debug && logLink) { - const logUrl = `${logLink.href}/content` - const logCount = currentJob.logStatistics?.lineCount ?? 1000000 - log = await fetchLogByChunks( - this.requestClient, - access_token!, - logUrl, - logCount - ) - } - - if (jobStatus === 'failed' || jobStatus === 'error') { - return Promise.reject(new ComputeJobExecutionError(currentJob, log)) - } - - let resultLink - - if (expectWebout) { - resultLink = `/compute/sessions/${executionSessionId}/filerefs/_webout/content` - } else { - return { job: currentJob, log } - } - - if (resultLink) { - jobResult = await this.requestClient - .get(resultLink, access_token, 'text/plain') - .catch(async (e) => { - if (e instanceof NotFoundError) { - if (logLink) { - const logUrl = `${logLink.href}/content` - const logCount = currentJob.logStatistics?.lineCount ?? 1000000 - log = await fetchLogByChunks( - this.requestClient, - access_token!, - logUrl, - logCount - ) - - return Promise.reject({ - status: 500, - log - }) - } - } - - return { - result: JSON.stringify(e) - } - }) - } - - await this.sessionManager - .clearSession(executionSessionId, access_token) - .catch((err) => { - throw prefixMessage(err, 'Error while clearing session. ') - }) - - return { result: jobResult?.result, log } - } catch (e) { - if (e && e.status === 404) { - return this.executeScript( - jobPath, - linesOfCode, - contextName, - authConfig, - data, - debug, - false, - true - ) - } else { - throw prefixMessage(e, 'Error while executing script. ') - } - } + return executeScript( + this.requestClient, + this.sessionManager, + this.rootFolderName, + jobPath, + linesOfCode, + contextName, + authConfig, + data, + debug, + expectWebout, + waitForResult, + pollOptions, + printPid, + variables + ) } /** @@ -772,37 +522,7 @@ export class SASViyaApiClient { clientSecret: string, authCode: string ): Promise { - const url = this.serverUrl + '/SASLogon/oauth/token' - let token - if (typeof Buffer === 'undefined') { - token = btoa(clientId + ':' + clientSecret) - } else { - token = Buffer.from(clientId + ':' + clientSecret).toString('base64') - } - const headers = { - Authorization: 'Basic ' + token - } - - let formData - if (typeof FormData === 'undefined') { - formData = new NodeFormData() - } else { - formData = new FormData() - } - formData.append('grant_type', 'authorization_code') - formData.append('code', authCode) - - const authResponse = await this.requestClient - .post( - url, - formData, - undefined, - 'multipart/form-data; boundary=' + (formData as any)._boundary, - headers - ) - .then((res) => res.result as SasAuthResponse) - - return authResponse + return getAccessToken(this.requestClient, clientId, clientSecret, authCode) } /** @@ -816,39 +536,12 @@ export class SASViyaApiClient { clientSecret: string, refreshToken: string ) { - const url = this.serverUrl + '/SASLogon/oauth/token' - let token - if (typeof Buffer === 'undefined') { - token = btoa(clientId + ':' + clientSecret) - } else { - token = Buffer.from(clientId + ':' + clientSecret).toString('base64') - } - const headers = { - Authorization: 'Basic ' + token - } - - let formData - if (typeof FormData === 'undefined') { - formData = new NodeFormData() - formData.append('grant_type', 'refresh_token') - formData.append('refresh_token', refreshToken) - } else { - formData = new FormData() - formData.append('grant_type', 'refresh_token') - formData.append('refresh_token', refreshToken) - } - - const authResponse = await this.requestClient - .post( - url, - formData, - undefined, - 'multipart/form-data; boundary=' + (formData as any)._boundary, - headers - ) - .then((res) => res.result) - - return authResponse + return refreshTokens( + this.requestClient, + clientId, + clientSecret, + refreshToken + ) } /** @@ -895,7 +588,7 @@ export class SASViyaApiClient { ) { let access_token = (authConfig || {}).access_token if (authConfig) { - ;({ access_token } = await this.getTokens(authConfig)) + ;({ access_token } = await getTokens(this.requestClient, authConfig)) } if (isRelativePath(sasJob) && !this.rootFolderName) { @@ -991,7 +684,7 @@ export class SASViyaApiClient { ) { let access_token = (authConfig || {}).access_token if (authConfig) { - ;({ access_token } = await this.getTokens(authConfig)) + ;({ access_token } = await getTokens(this.requestClient, authConfig)) } if (isRelativePath(sasJob) && !this.rootFolderName) { throw new Error( @@ -1140,157 +833,24 @@ export class SASViyaApiClient { this.folderMap.set(path, itemsAtRoot) } - // REFACTOR: set default value for 'pollOptions' attribute private async pollJobState( - postedJob: any, + postedJob: Job, etag: string | null, authConfig?: AuthConfig, pollOptions?: PollOptions ) { - const logger = process.logger || console - - let POLL_INTERVAL = 300 - let MAX_POLL_COUNT = 1000 - let MAX_ERROR_COUNT = 5 - let access_token = (authConfig || {}).access_token - if (authConfig) { - ;({ access_token } = await this.getTokens(authConfig)) - } - - if (pollOptions) { - POLL_INTERVAL = pollOptions.POLL_INTERVAL || POLL_INTERVAL - MAX_POLL_COUNT = pollOptions.MAX_POLL_COUNT || MAX_POLL_COUNT - } - - let postedJobState = '' - let pollCount = 0 - let errorCount = 0 - const headers: any = { - 'Content-Type': 'application/json', - 'If-None-Match': etag - } - if (access_token) { - headers.Authorization = `Bearer ${access_token}` - } - const stateLink = postedJob.links.find((l: any) => l.rel === 'state') - if (!stateLink) { - Promise.reject(`Job state link was not found.`) - } - - const { result: state } = await this.requestClient - .get( - `${this.serverUrl}${stateLink.href}?_action=wait&wait=300`, - access_token, - 'text/plain', - {}, - this.debug - ) - .catch((err) => { - console.error( - `Error fetching job state from ${this.serverUrl}${stateLink.href}. Starting poll, assuming job to be running.`, - err - ) - return { result: 'unavailable' } - }) - - const currentState = state.trim() - if (currentState === 'completed') { - return Promise.resolve(currentState) - } - - return new Promise(async (resolve, _) => { - let printedState = '' - - const interval = setInterval(async () => { - if ( - postedJobState === 'running' || - postedJobState === '' || - postedJobState === 'pending' || - postedJobState === 'unavailable' - ) { - if (authConfig) { - ;({ access_token } = await this.getTokens(authConfig)) - } - - if (stateLink) { - const { result: jobState } = await this.requestClient - .get( - `${this.serverUrl}${stateLink.href}?_action=wait&wait=300`, - access_token, - 'text/plain', - {}, - this.debug - ) - .catch((err) => { - errorCount++ - if ( - pollCount >= MAX_POLL_COUNT || - errorCount >= MAX_ERROR_COUNT - ) { - throw prefixMessage( - err, - 'Error while getting job state after interval. ' - ) - } - console.error( - `Error fetching job state from ${this.serverUrl}${stateLink.href}. Resuming poll, assuming job to be running.`, - err - ) - return { result: 'unavailable' } - }) - - postedJobState = jobState.trim() - if (postedJobState != 'unavailable' && errorCount > 0) { - errorCount = 0 - } - - if (this.debug && printedState !== postedJobState) { - logger.info('Polling job status...') - logger.info(`Current job state: ${postedJobState}`) - - printedState = postedJobState - } - - pollCount++ - - if (pollCount >= MAX_POLL_COUNT) { - resolve(postedJobState) - } - } - } else { - clearInterval(interval) - resolve(postedJobState) - } - }, POLL_INTERVAL) - }) + return pollJobState( + this.requestClient, + postedJob, + this.debug, + etag, + authConfig, + pollOptions + ) } private async uploadTables(data: any, accessToken?: string) { - const uploadedFiles = [] - const headers: any = { - 'Content-Type': 'application/json' - } - if (accessToken) { - headers.Authorization = `Bearer ${accessToken}` - } - - for (const tableName in data) { - const csv = convertToCSV(data[tableName]) - if (csv === 'ERROR: LARGE STRING LENGTH') { - throw new Error( - 'The max length of a string value in SASjs is 32765 characters.' - ) - } - - const uploadResponse = await this.requestClient - .uploadFile(`${this.serverUrl}/files/files#rawUpload`, csv, accessToken) - .catch((err) => { - throw prefixMessage(err, 'Error while uploading file. ') - }) - - uploadedFiles.push({ tableName, file: uploadResponse.result }) - } - return uploadedFiles + return uploadTables(this.requestClient, data, accessToken) } private async getFolderDetails( @@ -1493,21 +1053,4 @@ export class SASViyaApiClient { return movedFolder } - - private async getTokens(authConfig: AuthConfig): Promise { - const logger = process.logger || console - let { access_token, refresh_token, client, secret } = authConfig - if ( - isAccessTokenExpiring(access_token) || - isRefreshTokenExpiring(refresh_token) - ) { - logger.info('Refreshing access and refresh tokens.') - ;({ access_token, refresh_token } = await this.refreshTokens( - client, - secret, - refresh_token - )) - } - return { access_token, refresh_token, client, secret } - } } diff --git a/src/api/viya/executeScript.ts b/src/api/viya/executeScript.ts new file mode 100644 index 0000000..0ea25c6 --- /dev/null +++ b/src/api/viya/executeScript.ts @@ -0,0 +1,303 @@ +import { + AuthConfig, + MacroVar, + Logger, + LogLevel, + timestampToYYYYMMDDHHMMSS +} from '@sasjs/utils' +import { prefixMessage } from '@sasjs/utils/error' +import { + PollOptions, + Job, + ComputeJobExecutionError, + NotFoundError +} from '../..' +import { getTokens } from '../../auth/tokens' +import { RequestClient } from '../../request/RequestClient' +import { SessionManager } from '../../SessionManager' +import { isRelativePath, fetchLogByChunks } from '../../utils' +import { formatDataForRequest } from '../../utils/formatDataForRequest' +import { pollJobState } from './pollJobState' +import { uploadTables } from './uploadTables' + +/** + * Executes code on the current SAS Viya server. + * @param jobPath - the path to the file being submitted for execution. + * @param linesOfCode - an array of code lines to execute. + * @param contextName - the context to execute the code in. + * @param authConfig - an object containing an access token, refresh token, client ID and secret. + * @param data - execution data. + * @param debug - when set to true, the log will be returned. + * @param expectWebout - when set to true, the automatic _webout fileref will be checked for content, and that content returned. This fileref is used when the Job contains a SASjs web request (as opposed to executing arbitrary SAS code). + * @param waitForResult - when set to true, function will return the session + * @param pollOptions - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: { MAX_POLL_COUNT: 24 * 60 * 60, POLL_INTERVAL: 1000 }. + * @param printPid - a boolean that indicates whether the function should print (PID) of the started job. + * @param variables - an object that represents macro variables. + */ +export async function executeScript( + requestClient: RequestClient, + sessionManager: SessionManager, + rootFolderName: string, + jobPath: string, + linesOfCode: string[], + contextName: string, + authConfig?: AuthConfig, + data = null, + debug: boolean = false, + expectWebout = false, + waitForResult = true, + pollOptions?: PollOptions, + printPid = false, + variables?: MacroVar +): Promise { + let access_token = (authConfig || {}).access_token + if (authConfig) { + ;({ access_token } = await getTokens(requestClient, authConfig)) + } + + const logger = process.logger || console + + try { + let executionSessionId: string + + const session = await sessionManager + .getSession(access_token) + .catch((err) => { + throw prefixMessage(err, 'Error while getting session. ') + }) + + executionSessionId = session!.id + + if (printPid) { + const { result: jobIdVariable } = await sessionManager + .getVariable(executionSessionId, 'SYSJOBID', access_token) + .catch((err) => { + throw prefixMessage(err, 'Error while getting session variable. ') + }) + + if (jobIdVariable && jobIdVariable.value) { + const relativeJobPath = rootFolderName + ? jobPath.split(rootFolderName).join('').replace(/^\//, '') + : jobPath + + const logger = new Logger(debug ? LogLevel.Debug : LogLevel.Info) + + logger.info( + `Triggered '${relativeJobPath}' with PID ${ + jobIdVariable.value + } at ${timestampToYYYYMMDDHHMMSS()}` + ) + } + } + + const jobArguments: { [key: string]: any } = { + _contextName: contextName, + _OMITJSONLISTING: true, + _OMITJSONLOG: true, + _OMITSESSIONRESULTS: true, + _OMITTEXTLISTING: true, + _OMITTEXTLOG: true + } + + if (debug) { + jobArguments['_OMITTEXTLOG'] = false + jobArguments['_OMITSESSIONRESULTS'] = false + } + + let fileName + + if (isRelativePath(jobPath)) { + fileName = `exec-${ + jobPath.includes('/') ? jobPath.split('/')[1] : jobPath + }` + } else { + const jobPathParts = jobPath.split('/') + fileName = jobPathParts.pop() + } + + let jobVariables: any = { + SYS_JES_JOB_URI: '', + _program: isRelativePath(jobPath) + ? rootFolderName + '/' + jobPath + : jobPath + } + + if (variables) jobVariables = { ...jobVariables, ...variables } + + if (debug) jobVariables = { ...jobVariables, _DEBUG: 131 } + + let files: any[] = [] + + if (data) { + if (JSON.stringify(data).includes(';')) { + files = await uploadTables(requestClient, data, access_token).catch( + (err) => { + throw prefixMessage(err, 'Error while uploading tables. ') + } + ) + + jobVariables['_webin_file_count'] = files.length + + files.forEach((fileInfo, index) => { + jobVariables[ + `_webin_fileuri${index + 1}` + ] = `/files/files/${fileInfo.file.id}` + jobVariables[`_webin_name${index + 1}`] = fileInfo.tableName + }) + } else { + jobVariables = { ...jobVariables, ...formatDataForRequest(data) } + } + } + + // Execute job in session + const jobRequestBody = { + name: fileName, + description: 'Powered by SASjs', + code: linesOfCode, + variables: jobVariables, + arguments: jobArguments + } + + const { result: postedJob, etag } = await requestClient + .post( + `/compute/sessions/${executionSessionId}/jobs`, + jobRequestBody, + access_token + ) + .catch((err) => { + throw prefixMessage(err, 'Error while posting job. ') + }) + + if (!waitForResult) return session + + if (debug) { + logger.info(`Job has been submitted for '${fileName}'.`) + logger.info( + `You can monitor the job progress at '${requestClient.getBaseUrl()}${ + postedJob.links.find((l: any) => l.rel === 'state')!.href + }'.` + ) + } + + const jobStatus = await pollJobState( + requestClient, + postedJob, + debug, + etag, + authConfig, + pollOptions + ).catch(async (err) => { + const error = err?.response?.data + const result = /err=[0-9]*,/.exec(error) + + const errorCode = '5113' + if (result?.[0]?.slice(4, -1) === errorCode) { + const sessionLogUrl = + postedJob.links.find((l: any) => l.rel === 'up')!.href + '/log' + const logCount = 1000000 + err.log = await fetchLogByChunks( + requestClient, + access_token!, + sessionLogUrl, + logCount + ) + } + throw prefixMessage(err, 'Error while polling job status. ') + }) + + if (authConfig) { + ;({ access_token } = await getTokens(requestClient, authConfig)) + } + + const { result: currentJob } = await requestClient + .get( + `/compute/sessions/${executionSessionId}/jobs/${postedJob.id}`, + access_token + ) + .catch((err) => { + throw prefixMessage(err, 'Error while getting job. ') + }) + + let jobResult + let log = '' + + const logLink = currentJob.links.find((l) => l.rel === 'log') + + if (debug && logLink) { + const logUrl = `${logLink.href}/content` + const logCount = currentJob.logStatistics?.lineCount ?? 1000000 + log = await fetchLogByChunks( + requestClient, + access_token!, + logUrl, + logCount + ) + } + + if (jobStatus === 'failed' || jobStatus === 'error') { + return Promise.reject(new ComputeJobExecutionError(currentJob, log)) + } + + let resultLink + + if (expectWebout) { + resultLink = `/compute/sessions/${executionSessionId}/filerefs/_webout/content` + } else { + return { job: currentJob, log } + } + + if (resultLink) { + jobResult = await requestClient + .get(resultLink, access_token, 'text/plain') + .catch(async (e) => { + if (e instanceof NotFoundError) { + if (logLink) { + const logUrl = `${logLink.href}/content` + const logCount = currentJob.logStatistics?.lineCount ?? 1000000 + log = await fetchLogByChunks( + requestClient, + access_token!, + logUrl, + logCount + ) + + return Promise.reject({ + status: 500, + log + }) + } + } + + return { + result: JSON.stringify(e) + } + }) + } + + await sessionManager + .clearSession(executionSessionId, access_token) + .catch((err) => { + throw prefixMessage(err, 'Error while clearing session. ') + }) + + return { result: jobResult?.result, log } + } catch (e) { + if (e && e.status === 404) { + return executeScript( + requestClient, + sessionManager, + rootFolderName, + jobPath, + linesOfCode, + contextName, + authConfig, + data, + debug, + false, + true + ) + } else { + throw prefixMessage(e, 'Error while executing script. ') + } + } +} diff --git a/src/api/viya/pollJobState.ts b/src/api/viya/pollJobState.ts new file mode 100644 index 0000000..1bd85f8 --- /dev/null +++ b/src/api/viya/pollJobState.ts @@ -0,0 +1,178 @@ +import { AuthConfig, createFile, generateTimestamp } from '@sasjs/utils' +import { prefixMessage } from '@sasjs/utils/error' +import { Job, PollOptions } from '../..' +import { getTokens } from '../../auth/tokens' +import { RequestClient } from '../../request/RequestClient' +import { fetchLogByChunks } from '../../utils' + +export async function pollJobState( + requestClient: RequestClient, + postedJob: Job, + debug: boolean, + etag: string | null, + authConfig?: AuthConfig, + pollOptions?: PollOptions +) { + const logger = process.logger || console + + let POLL_INTERVAL = 300 + let MAX_POLL_COUNT = 1000 + let MAX_ERROR_COUNT = 5 + let access_token = (authConfig || {}).access_token + if (authConfig) { + ;({ access_token } = await getTokens(requestClient, authConfig)) + } + + if (pollOptions) { + POLL_INTERVAL = pollOptions.pollInterval || POLL_INTERVAL + MAX_POLL_COUNT = pollOptions.maxPollCount || MAX_POLL_COUNT + } + + let postedJobState = '' + let pollCount = 0 + let errorCount = 0 + const headers: any = { + 'Content-Type': 'application/json', + 'If-None-Match': etag + } + if (access_token) { + headers.Authorization = `Bearer ${access_token}` + } + const stateLink = postedJob.links.find((l: any) => l.rel === 'state') + if (!stateLink) { + return Promise.reject(`Job state link was not found.`) + } + + const { result: state } = await requestClient + .get( + `${stateLink.href}?_action=wait&wait=300`, + access_token, + 'text/plain', + {}, + debug + ) + .catch((err) => { + logger.error( + `Error fetching job state from ${stateLink.href}. Starting poll, assuming job to be running.`, + err + ) + return { result: 'unavailable' } + }) + + const currentState = state.trim() + if (currentState === 'completed') { + return Promise.resolve(currentState) + } + + return new Promise(async (resolve, _) => { + let printedState = '' + + const interval = setInterval(async () => { + if ( + postedJobState === 'running' || + postedJobState === '' || + postedJobState === 'pending' || + postedJobState === 'unavailable' + ) { + if (authConfig) { + ;({ access_token } = await getTokens(requestClient, authConfig)) + } + + if (stateLink) { + const { result: jobState } = await requestClient + .get( + `${stateLink.href}?_action=wait&wait=300`, + access_token, + 'text/plain', + {}, + debug + ) + .catch((err) => { + errorCount++ + if ( + pollCount >= MAX_POLL_COUNT || + errorCount >= MAX_ERROR_COUNT + ) { + throw prefixMessage( + err, + 'Error while getting job state after interval. ' + ) + } + logger.error( + `Error fetching job state from ${stateLink.href}. Resuming poll, assuming job to be running.`, + err + ) + return { result: 'unavailable' } + }) + + postedJobState = jobState.trim() + if (postedJobState != 'unavailable' && errorCount > 0) { + errorCount = 0 + } + + if (debug && printedState !== postedJobState) { + logger.info('Polling job status...') + logger.info(`Current job state: ${postedJobState}`) + + printedState = postedJobState + } + + pollCount++ + + await saveLog( + postedJob, + requestClient, + pollOptions?.streamLog || false, + pollOptions?.logFilePath, + access_token + ) + + if (pollCount >= MAX_POLL_COUNT) { + resolve(postedJobState) + } + } + } else { + clearInterval(interval) + resolve(postedJobState) + } + }, POLL_INTERVAL) + }) +} + +async function saveLog( + job: Job, + requestClient: RequestClient, + shouldSaveLog: boolean, + logFilePath?: string, + accessToken?: string +) { + if (!shouldSaveLog) { + return + } + + if (!accessToken) { + throw new Error( + `Logs for job ${job.id} cannot be fetched without a valid access token.` + ) + } + + const logger = process.logger || console + const logFileName = `${job.name || 'job'}-${generateTimestamp()}.log` + const logPath = `${logFilePath || process.cwd()}/${logFileName}` + const jobLogUrl = job.links.find((l) => l.rel === 'log') + + if (!jobLogUrl) { + throw new Error(`Log URL for job ${job.id} was not found.`) + } + + const logCount = job.logStatistics?.lineCount ?? 1000000 + const log = await fetchLogByChunks( + requestClient, + accessToken, + `${jobLogUrl.href}/content`, + logCount + ) + + logger.info(`Writing logs to ${logPath}`) + await createFile(logPath, log) +} diff --git a/src/api/viya/uploadTables.ts b/src/api/viya/uploadTables.ts new file mode 100644 index 0000000..13bf7b8 --- /dev/null +++ b/src/api/viya/uploadTables.ts @@ -0,0 +1,35 @@ +import { prefixMessage } from '@sasjs/utils/error' +import { RequestClient } from '../../request/RequestClient' +import { convertToCSV } from '../../utils/convertToCsv' + +export async function uploadTables( + requestClient: RequestClient, + data: any, + accessToken?: string +) { + const uploadedFiles = [] + const headers: any = { + 'Content-Type': 'application/json' + } + if (accessToken) { + headers.Authorization = `Bearer ${accessToken}` + } + + for (const tableName in data) { + const csv = convertToCSV(data[tableName]) + if (csv === 'ERROR: LARGE STRING LENGTH') { + throw new Error( + 'The max length of a string value in SASjs is 32765 characters.' + ) + } + + const uploadResponse = await requestClient + .uploadFile(`/files/files#rawUpload`, csv, accessToken) + .catch((err) => { + throw prefixMessage(err, 'Error while uploading file. ') + }) + + uploadedFiles.push({ tableName, file: uploadResponse.result }) + } + return uploadedFiles +} diff --git a/src/auth/tokens.ts b/src/auth/tokens.ts new file mode 100644 index 0000000..bc940e2 --- /dev/null +++ b/src/auth/tokens.ts @@ -0,0 +1,122 @@ +import { + AuthConfig, + isAccessTokenExpiring, + isRefreshTokenExpiring, + SasAuthResponse +} from '@sasjs/utils' +import * as NodeFormData from 'form-data' +import { RequestClient } from '../request/RequestClient' + +/** + * Exchanges the auth code for an access token for the given client. + * @param requestClient - the pre-configured HTTP request client + * @param clientId - the client ID to authenticate with. + * @param clientSecret - the client secret to authenticate with. + * @param authCode - the auth code received from the server. + */ +export async function getAccessToken( + requestClient: RequestClient, + clientId: string, + clientSecret: string, + authCode: string +): Promise { + const url = '/SASLogon/oauth/token' + let token + if (typeof Buffer === 'undefined') { + token = btoa(clientId + ':' + clientSecret) + } else { + token = Buffer.from(clientId + ':' + clientSecret).toString('base64') + } + const headers = { + Authorization: 'Basic ' + token + } + + let formData + if (typeof FormData === 'undefined') { + formData = new NodeFormData() + } else { + formData = new FormData() + } + formData.append('grant_type', 'authorization_code') + formData.append('code', authCode) + + const authResponse = await requestClient + .post( + url, + formData, + undefined, + 'multipart/form-data; boundary=' + (formData as any)._boundary, + headers + ) + .then((res) => res.result as SasAuthResponse) + + return authResponse +} + +/** + * Returns the auth configuration, refreshing the tokens if necessary. + * @param requestClient - the pre-configured HTTP request client + * @param authConfig - an object containing a client ID, secret, access token and refresh token + */ +export async function getTokens( + requestClient: RequestClient, + authConfig: AuthConfig +): Promise { + const logger = process.logger || console + let { access_token, refresh_token, client, secret } = authConfig + if ( + isAccessTokenExpiring(access_token) || + isRefreshTokenExpiring(refresh_token) + ) { + logger.info('Refreshing access and refresh tokens.') + ;({ access_token, refresh_token } = await refreshTokens( + requestClient, + client, + secret, + refresh_token + )) + } + return { access_token, refresh_token, client, secret } +} + +/** + * Exchanges the refresh token for an access token for the given client. + * @param requestClient - the pre-configured HTTP request client + * @param clientId - the client ID to authenticate with. + * @param clientSecret - the client secret to authenticate with. + * @param authCode - the refresh token received from the server. + */ +export async function refreshTokens( + requestClient: RequestClient, + clientId: string, + clientSecret: string, + refreshToken: string +) { + const url = '/SASLogon/oauth/token' + let token + if (typeof Buffer === 'undefined') { + token = btoa(clientId + ':' + clientSecret) + } else { + token = Buffer.from(clientId + ':' + clientSecret).toString('base64') + } + const headers = { + Authorization: 'Basic ' + token + } + + const formData = + typeof FormData === 'undefined' ? new NodeFormData() : new FormData() + formData.append('grant_type', 'refresh_token') + formData.append('refresh_token', refreshToken) + + const authResponse = await requestClient + .post( + url, + formData, + undefined, + 'multipart/form-data; boundary=' + (formData as any)._boundary, + headers + ) + .then((res) => res.result) + + return authResponse +} diff --git a/src/request/RequestClient.ts b/src/request/RequestClient.ts index eb6aac2..2701986 100644 --- a/src/request/RequestClient.ts +++ b/src/request/RequestClient.ts @@ -43,6 +43,7 @@ export interface HttpClient { getCsrfToken(type: 'general' | 'file'): CsrfToken | undefined clearCsrfTokens(): void + getBaseUrl(): string } export class RequestClient implements HttpClient { @@ -78,6 +79,10 @@ export class RequestClient implements HttpClient { this.fileUploadCsrfToken = { headerName: '', value: '' } } + public getBaseUrl() { + return this.httpClient.defaults.baseURL || '' + } + public async get( url: string, accessToken: string | undefined, diff --git a/src/types/PollOptions.ts b/src/types/PollOptions.ts index 0472f0c..6daa1b7 100644 --- a/src/types/PollOptions.ts +++ b/src/types/PollOptions.ts @@ -1,4 +1,6 @@ export interface PollOptions { - MAX_POLL_COUNT?: number - POLL_INTERVAL?: number + maxPollCount: number + pollInterval: number + streamLog: boolean + logFilePath?: string } From 13be2f9c701ba2ce05f80a43ff57c4c252a4291b Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Fri, 9 Jul 2021 09:17:26 +0100 Subject: [PATCH 02/15] chore(*): remove unused dependencies and variables, fix imports --- package-lock.json | 46 ++++++++++++++++++++--- package.json | 3 +- src/SASViyaApiClient.ts | 12 ------ src/SASjs.ts | 8 +++- src/api/viya/executeScript.ts | 69 +++++++++++++++-------------------- src/api/viya/pollJobState.ts | 39 +++++++++++--------- tsconfig.json | 2 +- 7 files changed, 100 insertions(+), 79 deletions(-) diff --git a/package-lock.json b/package-lock.json index 059a16c..b59f89d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,8 @@ "url": "^0.11.0" }, "devDependencies": { + "@types/axios": "^0.14.0", + "@types/form-data": "^2.5.0", "@types/jest": "^26.0.23", "@types/mime": "^2.0.3", "@types/tough-cookie": "^4.0.0", @@ -24,7 +26,6 @@ "dotenv": "^10.0.0", "jest": "^27.0.6", "jest-extended": "^0.11.5", - "mime": "^2.5.2", "node-polyfill-webpack-plugin": "^1.1.4", "path": "^0.12.7", "process": "^0.11.10", @@ -1435,6 +1436,16 @@ "node": ">= 6" } }, + "node_modules/@types/axios": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@types/axios/-/axios-0.14.0.tgz", + "integrity": "sha1-7CMA++fX3d1+udOr+HmZlkyvzkY=", + "deprecated": "This is a stub types definition for axios (https://github.com/mzabriskie/axios). axios provides its own type definitions, so you don't need @types/axios installed!", + "dev": true, + "dependencies": { + "axios": "*" + } + }, "node_modules/@types/babel__core": { "version": "7.1.14", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.14.tgz", @@ -1502,6 +1513,16 @@ "integrity": "sha512-LfZwXoGUDo0C3me81HXgkBg5CTQYb6xzEl+fNmbO4JdRiSKQ8A0GD1OBBvKAIsbCUgoyAty7m99GqqMQe784ew==", "dev": true }, + "node_modules/@types/form-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.5.0.tgz", + "integrity": "sha512-23/wYiuckYYtFpL+4RPWiWmRQH2BjFuqCUi2+N3amB1a1Drv+i/byTrGvlLwRVLFNAZbwpbQ7JvTK+VCAPMbcg==", + "deprecated": "This is a stub types definition. form-data provides its own type definitions, so you do not need this installed.", + "dev": true, + "dependencies": { + "form-data": "*" + } + }, "node_modules/@types/graceful-fs": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz", @@ -11155,11 +11176,6 @@ "safer-buffer": "^2.0.2", "tweetnacl": "~0.14.0" }, - "bin": { - "sshpk-conv": "bin/sshpk-conv", - "sshpk-sign": "bin/sshpk-sign", - "sshpk-verify": "bin/sshpk-verify" - }, "engines": { "node": ">=0.10.0" } @@ -15848,6 +15864,15 @@ "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", "dev": true }, + "@types/axios": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@types/axios/-/axios-0.14.0.tgz", + "integrity": "sha1-7CMA++fX3d1+udOr+HmZlkyvzkY=", + "dev": true, + "requires": { + "axios": "*" + } + }, "@types/babel__core": { "version": "7.1.14", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.14.tgz", @@ -15915,6 +15940,15 @@ "integrity": "sha512-LfZwXoGUDo0C3me81HXgkBg5CTQYb6xzEl+fNmbO4JdRiSKQ8A0GD1OBBvKAIsbCUgoyAty7m99GqqMQe784ew==", "dev": true }, + "@types/form-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.5.0.tgz", + "integrity": "sha512-23/wYiuckYYtFpL+4RPWiWmRQH2BjFuqCUi2+N3amB1a1Drv+i/byTrGvlLwRVLFNAZbwpbQ7JvTK+VCAPMbcg==", + "dev": true, + "requires": { + "form-data": "*" + } + }, "@types/graceful-fs": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz", diff --git a/package.json b/package.json index 65fd231..471ca36 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,8 @@ }, "license": "ISC", "devDependencies": { + "@types/axios": "^0.14.0", + "@types/form-data": "^2.5.0", "@types/jest": "^26.0.23", "@types/mime": "^2.0.3", "@types/tough-cookie": "^4.0.0", @@ -46,7 +48,6 @@ "dotenv": "^10.0.0", "jest": "^27.0.6", "jest-extended": "^0.11.5", - "mime": "^2.5.2", "node-polyfill-webpack-plugin": "^1.1.4", "path": "^0.12.7", "process": "^0.11.10", diff --git a/src/SASViyaApiClient.ts b/src/SASViyaApiClient.ts index 8d33de0..78d92b5 100644 --- a/src/SASViyaApiClient.ts +++ b/src/SASViyaApiClient.ts @@ -18,7 +18,6 @@ import { SasAuthResponse, MacroVar, AuthConfig } from '@sasjs/utils/types' import { isAuthorizeFormRequired } from './auth/isAuthorizeFormRequired' import { RequestClient } from './request/RequestClient' import { prefixMessage } from '@sasjs/utils/error' -import * as mime from 'mime' import { pollJobState } from './api/viya/pollJobState' import { getAccessToken, getTokens, refreshTokens } from './auth/tokens' import { uploadTables } from './api/viya/uploadTables' @@ -334,9 +333,6 @@ export class SASViyaApiClient { const formData = new NodeFormData() formData.append('file', contentBuffer, fileName) - const mimeType = - mime.getType(fileName.match(/\.[0-9a-z]+$/i)?.[0] || '') ?? 'text/plain' - return ( await this.requestClient.post( `/files/files?parentFolderUri=${parentFolderUri}&typeDefName=file#rawUpload`, @@ -939,14 +935,6 @@ export class SASViyaApiClient { ? sourceFolder : await this.getFolderUri(sourceFolder, accessToken) - const requestInfo = { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer ' + accessToken - } - } - const { result: members } = await this.requestClient.get<{ items: any[] }>( `${this.serverUrl}${sourceFolderUri}/members?limit=${limit}`, accessToken diff --git a/src/SASjs.ts b/src/SASjs.ts index d60ad27..b91e034 100644 --- a/src/SASjs.ts +++ b/src/SASjs.ts @@ -4,7 +4,12 @@ import { SASViyaApiClient } from './SASViyaApiClient' import { SAS9ApiClient } from './SAS9ApiClient' import { FileUploader } from './FileUploader' import { AuthManager } from './auth' -import { ServerType, MacroVar, AuthConfig } from '@sasjs/utils/types' +import { + ServerType, + MacroVar, + AuthConfig, + ExtraResponseAttributes +} from '@sasjs/utils/types' import { RequestClient } from './request/RequestClient' import { JobExecutor, @@ -14,7 +19,6 @@ import { Sas9JobExecutor } from './job-execution' import { ErrorResponse } from './types/errors' -import { ExtraResponseAttributes } from '@sasjs/utils/types' const defaultConfig: SASjsConfig = { serverUrl: '', diff --git a/src/api/viya/executeScript.ts b/src/api/viya/executeScript.ts index 0ea25c6..0abe466 100644 --- a/src/api/viya/executeScript.ts +++ b/src/api/viya/executeScript.ts @@ -1,10 +1,5 @@ -import { - AuthConfig, - MacroVar, - Logger, - LogLevel, - timestampToYYYYMMDDHHMMSS -} from '@sasjs/utils' +import { timestampToYYYYMMDDHHMMSS } from '@sasjs/utils/time' +import { AuthConfig, MacroVar } from '@sasjs/utils/types' import { prefixMessage } from '@sasjs/utils/error' import { PollOptions, @@ -42,7 +37,7 @@ export async function executeScript( linesOfCode: string[], contextName: string, authConfig?: AuthConfig, - data = null, + data: any = null, debug: boolean = false, expectWebout = false, waitForResult = true, @@ -80,7 +75,7 @@ export async function executeScript( ? jobPath.split(rootFolderName).join('').replace(/^\//, '') : jobPath - const logger = new Logger(debug ? LogLevel.Debug : LogLevel.Info) + const logger = process.logger || console logger.info( `Triggered '${relativeJobPath}' with PID ${ @@ -235,44 +230,40 @@ export async function executeScript( } if (jobStatus === 'failed' || jobStatus === 'error') { - return Promise.reject(new ComputeJobExecutionError(currentJob, log)) + throw new ComputeJobExecutionError(currentJob, log) } - let resultLink - - if (expectWebout) { - resultLink = `/compute/sessions/${executionSessionId}/filerefs/_webout/content` - } else { + if (!expectWebout) { return { job: currentJob, log } } - if (resultLink) { - jobResult = await requestClient - .get(resultLink, access_token, 'text/plain') - .catch(async (e) => { - if (e instanceof NotFoundError) { - if (logLink) { - const logUrl = `${logLink.href}/content` - const logCount = currentJob.logStatistics?.lineCount ?? 1000000 - log = await fetchLogByChunks( - requestClient, - access_token!, - logUrl, - logCount - ) + const resultLink = `/compute/sessions/${executionSessionId}/filerefs/_webout/content` - return Promise.reject({ - status: 500, - log - }) - } - } + jobResult = await requestClient + .get(resultLink, access_token, 'text/plain') + .catch(async (e) => { + if (e instanceof NotFoundError) { + if (logLink) { + const logUrl = `${logLink.href}/content` + const logCount = currentJob.logStatistics?.lineCount ?? 1000000 + log = await fetchLogByChunks( + requestClient, + access_token!, + logUrl, + logCount + ) - return { - result: JSON.stringify(e) + return Promise.reject({ + status: 500, + log + }) } - }) - } + } + + return { + result: JSON.stringify(e) + } + }) await sessionManager .clearSession(executionSessionId, access_token) diff --git a/src/api/viya/pollJobState.ts b/src/api/viya/pollJobState.ts index 1bd85f8..63796ce 100644 --- a/src/api/viya/pollJobState.ts +++ b/src/api/viya/pollJobState.ts @@ -1,5 +1,7 @@ -import { AuthConfig, createFile, generateTimestamp } from '@sasjs/utils' +import { AuthConfig } from '@sasjs/utils' import { prefixMessage } from '@sasjs/utils/error' +import { generateTimestamp } from '@sasjs/utils/time' +import { createFile } from '@sasjs/utils/file' import { Job, PollOptions } from '../..' import { getTokens } from '../../auth/tokens' import { RequestClient } from '../../request/RequestClient' @@ -15,17 +17,23 @@ export async function pollJobState( ) { const logger = process.logger || console - let POLL_INTERVAL = 300 - let MAX_POLL_COUNT = 1000 - let MAX_ERROR_COUNT = 5 + let pollInterval = 300 + let maxPollCount = 1000 + let maxErrorCount = 5 let access_token = (authConfig || {}).access_token + + const logFileName = `${postedJob.name || 'job'}-${generateTimestamp()}.log` + const logFilePath = `${ + pollOptions?.logFilePath || process.cwd() + }/${logFileName}` + if (authConfig) { ;({ access_token } = await getTokens(requestClient, authConfig)) } if (pollOptions) { - POLL_INTERVAL = pollOptions.pollInterval || POLL_INTERVAL - MAX_POLL_COUNT = pollOptions.maxPollCount || MAX_POLL_COUNT + pollInterval = pollOptions.pollInterval || pollInterval + maxPollCount = pollOptions.maxPollCount || maxPollCount } let postedJobState = '' @@ -89,10 +97,7 @@ export async function pollJobState( ) .catch((err) => { errorCount++ - if ( - pollCount >= MAX_POLL_COUNT || - errorCount >= MAX_ERROR_COUNT - ) { + if (pollCount >= maxPollCount || errorCount >= maxErrorCount) { throw prefixMessage( err, 'Error while getting job state after interval. ' @@ -123,11 +128,11 @@ export async function pollJobState( postedJob, requestClient, pollOptions?.streamLog || false, - pollOptions?.logFilePath, + logFilePath, access_token ) - if (pollCount >= MAX_POLL_COUNT) { + if (pollCount >= maxPollCount) { resolve(postedJobState) } } @@ -135,7 +140,7 @@ export async function pollJobState( clearInterval(interval) resolve(postedJobState) } - }, POLL_INTERVAL) + }, pollInterval) }) } @@ -143,7 +148,7 @@ async function saveLog( job: Job, requestClient: RequestClient, shouldSaveLog: boolean, - logFilePath?: string, + logFilePath: string, accessToken?: string ) { if (!shouldSaveLog) { @@ -157,8 +162,6 @@ async function saveLog( } const logger = process.logger || console - const logFileName = `${job.name || 'job'}-${generateTimestamp()}.log` - const logPath = `${logFilePath || process.cwd()}/${logFileName}` const jobLogUrl = job.links.find((l) => l.rel === 'log') if (!jobLogUrl) { @@ -173,6 +176,6 @@ async function saveLog( logCount ) - logger.info(`Writing logs to ${logPath}`) - await createFile(logPath, log) + logger.info(`Writing logs to ${logFilePath}`) + await createFile(logFilePath, log) } diff --git a/tsconfig.json b/tsconfig.json index e3dc154..36161db 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,5 +9,5 @@ "sourceMap": true }, "include": ["src"], - "exclude": ["node_modules", "**/*.spec.ts"] + "exclude": ["node_modules"] } From 0114a80e382c6d41d760360e31fc0951656ea3a2 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Fri, 9 Jul 2021 09:17:49 +0100 Subject: [PATCH 03/15] chore(execute): add tests for executeScript --- src/api/viya/spec/executeScript.spec.ts | 678 ++++++++++++++++++++++++ src/api/viya/spec/mockResponses.ts | 56 ++ 2 files changed, 734 insertions(+) create mode 100644 src/api/viya/spec/executeScript.spec.ts create mode 100644 src/api/viya/spec/mockResponses.ts diff --git a/src/api/viya/spec/executeScript.spec.ts b/src/api/viya/spec/executeScript.spec.ts new file mode 100644 index 0000000..03af686 --- /dev/null +++ b/src/api/viya/spec/executeScript.spec.ts @@ -0,0 +1,678 @@ +import { RequestClient } from '../../../request/RequestClient' +import { SessionManager } from '../../../SessionManager' +import { executeScript } from '../executeScript' +import { mockSession, mockAuthConfig, mockJob } from './mockResponses' +import * as pollJobStateModule from '../pollJobState' +import * as uploadTablesModule from '../uploadTables' +import * as tokensModule from '../../../auth/tokens' +import * as formatDataModule from '../../../utils/formatDataForRequest' +import * as fetchLogsModule from '../../../utils/fetchLogByChunks' +import { PollOptions } from '../../../types' +import { ComputeJobExecutionError, NotFoundError } from '../../../types/errors' +import { Logger, LogLevel } from '@sasjs/utils' + +const sessionManager = new (>SessionManager)() +const requestClient = new (>RequestClient)() +const defaultPollOptions: PollOptions = { + maxPollCount: 100, + pollInterval: 500, + streamLog: false +} + +describe('executeScript', () => { + beforeEach(() => { + ;(process as any).logger = new Logger(LogLevel.Off) + setupMocks() + }) + + it('should not try to get fresh tokens if an authConfig is not provided', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put hello'], + 'test context' + ) + + expect(tokensModule.getTokens).not.toHaveBeenCalled() + }) + + it('should try to get fresh tokens if an authConfig is provided', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put hello'], + 'test context', + mockAuthConfig + ) + + expect(tokensModule.getTokens).toHaveBeenCalledWith( + requestClient, + mockAuthConfig + ) + }) + + it('should get a session from the session manager before executing', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put hello'], + 'test context' + ) + + expect(sessionManager.getSession).toHaveBeenCalledWith(undefined) + }) + + it('should handle errors while getting a session', async () => { + jest + .spyOn(sessionManager, 'getSession') + .mockImplementation(() => Promise.reject('Test Error')) + + const error = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put hello'], + 'test context' + ).catch((e) => e) + + expect(error.includes('Error while getting session.')).toBeTruthy() + }) + + it('should fetch the PID when printPid is true', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put hello'], + 'test context', + mockAuthConfig, + null, + false, + false, + false, + defaultPollOptions, + true + ) + + expect(sessionManager.getVariable).toHaveBeenCalledWith( + mockSession.id, + 'SYSJOBID', + mockAuthConfig.access_token + ) + }) + + it('should handle errors while getting the job PID', async () => { + jest + .spyOn(sessionManager, 'getVariable') + .mockImplementation(() => Promise.reject('Test Error')) + + const error = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put hello'], + 'test context', + mockAuthConfig, + null, + false, + false, + false, + defaultPollOptions, + true + ).catch((e) => e) + + expect(error.includes('Error while getting session variable.')).toBeTruthy() + }) + + it('should use the file upload approach when data contains semicolons', async () => { + jest + .spyOn(uploadTablesModule, 'uploadTables') + .mockImplementation(() => + Promise.resolve([{ tableName: 'test', file: { id: 1 } }]) + ) + + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put hello'], + 'test context', + mockAuthConfig, + { foo: 'bar;' }, + false, + false, + false, + defaultPollOptions, + true + ) + + expect(uploadTablesModule.uploadTables).toHaveBeenCalledWith( + requestClient, + { foo: 'bar;' }, + mockAuthConfig.access_token + ) + }) + + it('should format data as CSV when it does not contain semicolons', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put hello'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + false, + false, + defaultPollOptions, + true + ) + + expect(formatDataModule.formatDataForRequest).toHaveBeenCalledWith({ + foo: 'bar' + }) + }) + + it('should submit a job for execution via the compute API', async () => { + jest + .spyOn(formatDataModule, 'formatDataForRequest') + .mockImplementation(() => ({ sasjs_tables: 'foo', sasjs0data: 'bar' })) + + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + false, + false, + defaultPollOptions, + true + ) + + expect(requestClient.post).toHaveBeenCalledWith( + `/compute/sessions/${mockSession.id}/jobs`, + { + name: 'exec-test', + description: 'Powered by SASjs', + code: ['%put "hello";'], + variables: { + SYS_JES_JOB_URI: '', + _program: 'test/test', + sasjs_tables: 'foo', + sasjs0data: 'bar' + }, + arguments: { + _contextName: 'test context', + _OMITJSONLISTING: true, + _OMITJSONLOG: true, + _OMITSESSIONRESULTS: true, + _OMITTEXTLISTING: true, + _OMITTEXTLOG: true + } + }, + mockAuthConfig.access_token + ) + }) + + it('should set the correct variables when debug is true', async () => { + jest + .spyOn(formatDataModule, 'formatDataForRequest') + .mockImplementation(() => ({ sasjs_tables: 'foo', sasjs0data: 'bar' })) + + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + true, + false, + false, + defaultPollOptions, + true + ) + + expect(requestClient.post).toHaveBeenCalledWith( + `/compute/sessions/${mockSession.id}/jobs`, + { + name: 'exec-test', + description: 'Powered by SASjs', + code: ['%put "hello";'], + variables: { + SYS_JES_JOB_URI: '', + _program: 'test/test', + sasjs_tables: 'foo', + sasjs0data: 'bar', + _DEBUG: 131 + }, + arguments: { + _contextName: 'test context', + _OMITJSONLISTING: true, + _OMITJSONLOG: true, + _OMITSESSIONRESULTS: false, + _OMITTEXTLISTING: true, + _OMITTEXTLOG: false + } + }, + mockAuthConfig.access_token + ) + }) + + it('should handle errors during job submission', async () => { + jest + .spyOn(requestClient, 'post') + .mockImplementation(() => Promise.reject('Test Error')) + + const error = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + true, + false, + false, + defaultPollOptions, + true + ).catch((e) => e) + + console.log(error) + expect(error.includes('Error while posting job')).toBeTruthy() + }) + + it('should immediately return the session when waitForResult is false', async () => { + const result = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + true, + false, + false, + defaultPollOptions, + true + ) + + expect(result).toEqual(mockSession) + }) + + it('should poll for job completion when waitForResult is true', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + false, + true, + defaultPollOptions, + true + ) + + expect(pollJobStateModule.pollJobState).toHaveBeenCalledWith( + requestClient, + mockJob, + false, + '', + mockAuthConfig, + defaultPollOptions + ) + }) + + it('should handle general errors when polling for job status', async () => { + jest + .spyOn(pollJobStateModule, 'pollJobState') + .mockImplementation(() => Promise.reject('Poll Error')) + + const error = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + false, + true, + defaultPollOptions, + true + ).catch((e) => e) + + expect(error.includes('Error while polling job status.')).toBeTruthy() + }) + + it('should fetch the log and append it to the error in case of a 5113 error code', async () => { + jest + .spyOn(pollJobStateModule, 'pollJobState') + .mockImplementation(() => + Promise.reject({ response: { data: 'err=5113,' } }) + ) + + const error = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + false, + true, + defaultPollOptions, + true + ).catch((e) => e) + + expect(fetchLogsModule.fetchLogByChunks).toHaveBeenCalledWith( + requestClient, + mockAuthConfig.access_token, + mockJob.links.find((l) => l.rel === 'up')!.href + '/log', + 1000000 + ) + expect(error.log).toEqual('Test Log') + }) + + it('should fetch the logs for the job if debug is true and a log URL is available', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + true, + false, + true, + defaultPollOptions, + true + ) + + expect(fetchLogsModule.fetchLogByChunks).toHaveBeenCalledWith( + requestClient, + mockAuthConfig.access_token, + mockJob.links.find((l) => l.rel === 'log')!.href + '/content', + mockJob.logStatistics.lineCount + ) + }) + + it('should not fetch the logs for the job if debug is false', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + false, + true, + defaultPollOptions, + true + ) + + expect(fetchLogsModule.fetchLogByChunks).not.toHaveBeenCalled() + }) + + it('should throw a ComputeJobExecutionError if the job has failed', async () => { + jest + .spyOn(pollJobStateModule, 'pollJobState') + .mockImplementation(() => Promise.resolve('failed')) + + const error: ComputeJobExecutionError = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + true, + false, + true, + defaultPollOptions, + true + ).catch((e) => e) + + expect(fetchLogsModule.fetchLogByChunks).toHaveBeenCalledWith( + requestClient, + mockAuthConfig.access_token, + mockJob.links.find((l) => l.rel === 'log')!.href + '/content', + mockJob.logStatistics.lineCount + ) + + expect(error).toBeInstanceOf(ComputeJobExecutionError) + expect(error.log).toEqual('Test Log') + expect(error.job).toEqual(mockJob) + }) + + it('should throw a ComputeJobExecutionError if the job has errored out', async () => { + jest + .spyOn(pollJobStateModule, 'pollJobState') + .mockImplementation(() => Promise.resolve('error')) + + const error: ComputeJobExecutionError = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + true, + false, + true, + defaultPollOptions, + true + ).catch((e) => e) + + expect(fetchLogsModule.fetchLogByChunks).toHaveBeenCalledWith( + requestClient, + mockAuthConfig.access_token, + mockJob.links.find((l) => l.rel === 'log')!.href + '/content', + mockJob.logStatistics.lineCount + ) + + expect(error).toBeInstanceOf(ComputeJobExecutionError) + expect(error.log).toEqual('Test Log') + expect(error.job).toEqual(mockJob) + }) + + it('should fetch the result if expectWebout is true', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + true, + true, + defaultPollOptions, + true + ) + + expect(requestClient.get).toHaveBeenCalledWith( + `/compute/sessions/${mockSession.id}/filerefs/_webout/content`, + mockAuthConfig.access_token, + 'text/plain' + ) + }) + + it('should fetch the logs if the webout file was not found', async () => { + jest.spyOn(requestClient, 'get').mockImplementation((url, ...rest) => { + if (url.includes('_webout')) { + return Promise.reject(new NotFoundError(url)) + } + return Promise.resolve({ result: mockJob, etag: '' }) + }) + + const error = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + true, + true, + defaultPollOptions, + true + ).catch((e) => e) + + expect(requestClient.get).toHaveBeenCalledWith( + `/compute/sessions/${mockSession.id}/filerefs/_webout/content`, + mockAuthConfig.access_token, + 'text/plain' + ) + + expect(fetchLogsModule.fetchLogByChunks).toHaveBeenCalledWith( + requestClient, + mockAuthConfig.access_token, + mockJob.links.find((l) => l.rel === 'log')!.href + '/content', + mockJob.logStatistics.lineCount + ) + + expect(error.status).toEqual(500) + expect(error.log).toEqual('Test Log') + }) + + it('should clear the session after execution is complete', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + true, + true, + defaultPollOptions, + true + ) + + expect(sessionManager.clearSession).toHaveBeenCalledWith( + mockSession.id, + mockAuthConfig.access_token + ) + }) + + it('should handle errors while clearing a session', async () => { + jest + .spyOn(sessionManager, 'clearSession') + .mockImplementation(() => Promise.reject('Clear Session Error')) + + const error = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + true, + true, + defaultPollOptions, + true + ).catch((e) => { + console.log(e) + return e + }) + + expect(error.includes('Error while clearing session.')).toBeTruthy() + }) +}) + +const setupMocks = () => { + jest.restoreAllMocks() + jest.mock('../../../request/RequestClient') + jest.mock('../../../SessionManager') + jest.mock('../../../auth/tokens') + jest.mock('../pollJobState') + jest.mock('../uploadTables') + jest.mock('../../../utils/formatDataForRequest') + jest.mock('../../../utils/fetchLogByChunks') + + jest + .spyOn(requestClient, 'post') + .mockImplementation(() => Promise.resolve({ result: mockJob, etag: '' })) + jest + .spyOn(requestClient, 'get') + .mockImplementation(() => Promise.resolve({ result: mockJob, etag: '' })) + jest + .spyOn(requestClient, 'delete') + .mockImplementation(() => Promise.resolve({ result: {}, etag: '' })) + jest + .spyOn(tokensModule, 'getTokens') + .mockImplementation(() => Promise.resolve(mockAuthConfig)) + jest + .spyOn(pollJobStateModule, 'pollJobState') + .mockImplementation(() => Promise.resolve('completed')) + jest + .spyOn(sessionManager, 'getVariable') + .mockImplementation(() => + Promise.resolve({ result: { value: 'test' }, etag: 'test' }) + ) + jest + .spyOn(sessionManager, 'getSession') + .mockImplementation(() => Promise.resolve(mockSession)) + jest + .spyOn(sessionManager, 'clearSession') + .mockImplementation(() => Promise.resolve()) + jest + .spyOn(formatDataModule, 'formatDataForRequest') + .mockImplementation(() => ({ sasjs_tables: 'test', sasjs0data: 'test' })) + jest + .spyOn(fetchLogsModule, 'fetchLogByChunks') + .mockImplementation(() => Promise.resolve('Test Log')) +} diff --git a/src/api/viya/spec/mockResponses.ts b/src/api/viya/spec/mockResponses.ts new file mode 100644 index 0000000..e85028c --- /dev/null +++ b/src/api/viya/spec/mockResponses.ts @@ -0,0 +1,56 @@ +import { AuthConfig } from '@sasjs/utils/types' +import { Job, Session } from '../../../types' + +export const mockSession: Session = { + id: 's35510n', + state: 'idle', + links: [], + attributes: { + sessionInactiveTimeout: 1 + }, + creationTimeStamp: new Date().valueOf().toString() +} + +export const mockJob: Job = { + id: 'j0b', + name: 'test job', + uri: '/j0b', + createdBy: 'test user', + results: { + '_webout.json': 'test' + }, + logStatistics: { + lineCount: 100, + modifiedTimeStamp: new Date().valueOf().toString() + }, + links: [ + { + rel: 'log', + href: '/log', + method: 'GET', + type: 'log', + uri: 'log' + }, + { + rel: 'state', + href: '/state', + method: 'GET', + type: 'state', + uri: 'state' + }, + { + rel: 'up', + href: '/job', + method: 'GET', + type: 'up', + uri: 'job' + } + ] +} + +export const mockAuthConfig: AuthConfig = { + client: 'cl13nt', + secret: '53cr3t', + access_token: 'acc355', + refresh_token: 'r3fr35h' +} From 1c90f4f455bea77d410e93372d269e6a72bab00c Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Fri, 9 Jul 2021 09:29:57 +0100 Subject: [PATCH 04/15] chore(*): remove log --- src/api/viya/spec/executeScript.spec.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/api/viya/spec/executeScript.spec.ts b/src/api/viya/spec/executeScript.spec.ts index 03af686..bca8fdb 100644 --- a/src/api/viya/spec/executeScript.spec.ts +++ b/src/api/viya/spec/executeScript.spec.ts @@ -624,10 +624,7 @@ describe('executeScript', () => { true, defaultPollOptions, true - ).catch((e) => { - console.log(e) - return e - }) + ).catch((e) => e) expect(error.includes('Error while clearing session.')).toBeTruthy() }) From f57c7b8f7d097e0d36e16956b46d84223a1cfee1 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Mon, 12 Jul 2021 20:30:42 +0100 Subject: [PATCH 05/15] chore(deps): up utils version --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index b59f89d..970d08e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,7 @@ "name": "@sasjs/adapter", "license": "ISC", "dependencies": { - "@sasjs/utils": "^2.23.2", + "@sasjs/utils": "^2.24.0", "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", "form-data": "^4.0.0", @@ -1272,9 +1272,9 @@ } }, "node_modules/@sasjs/utils": { - "version": "2.23.2", - "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.23.2.tgz", - "integrity": "sha512-PGHrqLgi/QajseksqD/2CSL+b45tLLQQUZrBW/PpHzvx2qcwXxfrWvnSo6v3UwUigxgyu+xkPK5AIlEJ81Tndw==", + "version": "2.24.0", + "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.24.0.tgz", + "integrity": "sha512-tWCqY8u/1So7mcUOgwzaePdpJaCQ60ihNlvTNHSzlJP5WIR2LKraoyinMfocWrinf/FwrR6qNFuefKZ3d5M8LA==", "dependencies": { "@types/prompts": "^2.0.13", "chalk": "^4.1.1", @@ -15720,9 +15720,9 @@ } }, "@sasjs/utils": { - "version": "2.23.2", - "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.23.2.tgz", - "integrity": "sha512-PGHrqLgi/QajseksqD/2CSL+b45tLLQQUZrBW/PpHzvx2qcwXxfrWvnSo6v3UwUigxgyu+xkPK5AIlEJ81Tndw==", + "version": "2.24.0", + "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.24.0.tgz", + "integrity": "sha512-tWCqY8u/1So7mcUOgwzaePdpJaCQ60ihNlvTNHSzlJP5WIR2LKraoyinMfocWrinf/FwrR6qNFuefKZ3d5M8LA==", "requires": { "@types/prompts": "^2.0.13", "chalk": "^4.1.1", diff --git a/package.json b/package.json index 471ca36..5e217df 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ }, "main": "index.js", "dependencies": { - "@sasjs/utils": "^2.23.2", + "@sasjs/utils": "^2.24.0", "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", "form-data": "^4.0.0", From 123b9fb5356ee2147c66040fbf1bde98a9cc5f65 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Mon, 12 Jul 2021 20:31:17 +0100 Subject: [PATCH 06/15] chore(refactor): split up and add tests for core functionality --- src/SASViyaApiClient.ts | 4 +- src/api/viya/executeScript.ts | 2 +- src/api/viya/pollJobState.ts | 56 +---- src/api/viya/saveLog.ts | 40 ++++ src/api/viya/spec/executeScript.spec.ts | 20 +- src/api/viya/spec/pollJobState.spec.ts | 266 ++++++++++++++++++++++++ src/api/viya/spec/saveLog.spec.ts | 72 +++++++ src/auth/getAccessToken.ts | 49 +++++ src/auth/getTokens.ts | 40 ++++ src/auth/refreshTokens.ts | 49 +++++ src/auth/spec/getTokens.spec.ts | 79 +++++++ src/auth/spec/mockResponses.ts | 22 ++ src/auth/spec/refreshTokens.spec.ts | 75 +++++++ src/auth/tokens.ts | 122 ----------- 14 files changed, 717 insertions(+), 179 deletions(-) create mode 100644 src/api/viya/saveLog.ts create mode 100644 src/api/viya/spec/pollJobState.spec.ts create mode 100644 src/api/viya/spec/saveLog.spec.ts create mode 100644 src/auth/getAccessToken.ts create mode 100644 src/auth/getTokens.ts create mode 100644 src/auth/refreshTokens.ts create mode 100644 src/auth/spec/getTokens.spec.ts create mode 100644 src/auth/spec/refreshTokens.spec.ts delete mode 100644 src/auth/tokens.ts diff --git a/src/SASViyaApiClient.ts b/src/SASViyaApiClient.ts index 78d92b5..ee439f2 100644 --- a/src/SASViyaApiClient.ts +++ b/src/SASViyaApiClient.ts @@ -19,9 +19,11 @@ import { isAuthorizeFormRequired } from './auth/isAuthorizeFormRequired' import { RequestClient } from './request/RequestClient' import { prefixMessage } from '@sasjs/utils/error' import { pollJobState } from './api/viya/pollJobState' -import { getAccessToken, getTokens, refreshTokens } from './auth/tokens' +import { getTokens } from './auth/getTokens' import { uploadTables } from './api/viya/uploadTables' import { executeScript } from './api/viya/executeScript' +import { getAccessToken } from './auth/getAccessToken' +import { refreshTokens } from './auth/refreshTokens' /** * A client for interfacing with the SAS Viya REST API. diff --git a/src/api/viya/executeScript.ts b/src/api/viya/executeScript.ts index 0abe466..c75a3e5 100644 --- a/src/api/viya/executeScript.ts +++ b/src/api/viya/executeScript.ts @@ -7,7 +7,7 @@ import { ComputeJobExecutionError, NotFoundError } from '../..' -import { getTokens } from '../../auth/tokens' +import { getTokens } from '../../auth/getTokens' import { RequestClient } from '../../request/RequestClient' import { SessionManager } from '../../SessionManager' import { isRelativePath, fetchLogByChunks } from '../../utils' diff --git a/src/api/viya/pollJobState.ts b/src/api/viya/pollJobState.ts index 63796ce..8d95c42 100644 --- a/src/api/viya/pollJobState.ts +++ b/src/api/viya/pollJobState.ts @@ -1,11 +1,10 @@ -import { AuthConfig } from '@sasjs/utils' +import { AuthConfig } from '@sasjs/utils/types' import { prefixMessage } from '@sasjs/utils/error' import { generateTimestamp } from '@sasjs/utils/time' -import { createFile } from '@sasjs/utils/file' import { Job, PollOptions } from '../..' -import { getTokens } from '../../auth/tokens' +import { getTokens } from '../../auth/getTokens' import { RequestClient } from '../../request/RequestClient' -import { fetchLogByChunks } from '../../utils' +import { saveLog } from './saveLog' export async function pollJobState( requestClient: RequestClient, @@ -48,7 +47,7 @@ export async function pollJobState( } const stateLink = postedJob.links.find((l: any) => l.rel === 'state') if (!stateLink) { - return Promise.reject(`Job state link was not found.`) + throw new Error(`Job state link was not found.`) } const { result: state } = await requestClient @@ -72,7 +71,7 @@ export async function pollJobState( return Promise.resolve(currentState) } - return new Promise(async (resolve, _) => { + return new Promise(async (resolve, reject) => { let printedState = '' const interval = setInterval(async () => { @@ -98,9 +97,12 @@ export async function pollJobState( .catch((err) => { errorCount++ if (pollCount >= maxPollCount || errorCount >= maxErrorCount) { - throw prefixMessage( - err, - 'Error while getting job state after interval. ' + clearInterval(interval) + reject( + prefixMessage( + err, + 'Error while getting job state after interval. ' + ) ) } logger.error( @@ -143,39 +145,3 @@ export async function pollJobState( }, pollInterval) }) } - -async function saveLog( - job: Job, - requestClient: RequestClient, - shouldSaveLog: boolean, - logFilePath: string, - accessToken?: string -) { - if (!shouldSaveLog) { - return - } - - if (!accessToken) { - throw new Error( - `Logs for job ${job.id} cannot be fetched without a valid access token.` - ) - } - - const logger = process.logger || console - const jobLogUrl = job.links.find((l) => l.rel === 'log') - - if (!jobLogUrl) { - throw new Error(`Log URL for job ${job.id} was not found.`) - } - - const logCount = job.logStatistics?.lineCount ?? 1000000 - const log = await fetchLogByChunks( - requestClient, - accessToken, - `${jobLogUrl.href}/content`, - logCount - ) - - logger.info(`Writing logs to ${logFilePath}`) - await createFile(logFilePath, log) -} diff --git a/src/api/viya/saveLog.ts b/src/api/viya/saveLog.ts new file mode 100644 index 0000000..9200930 --- /dev/null +++ b/src/api/viya/saveLog.ts @@ -0,0 +1,40 @@ +import { createFile } from '@sasjs/utils/file' +import { Job } from '../..' +import { RequestClient } from '../../request/RequestClient' +import { fetchLogByChunks } from '../../utils' + +export async function saveLog( + job: Job, + requestClient: RequestClient, + shouldSaveLog: boolean, + logFilePath: string, + accessToken?: string +) { + if (!shouldSaveLog) { + return + } + + if (!accessToken) { + throw new Error( + `Logs for job ${job.id} cannot be fetched without a valid access token.` + ) + } + + const logger = process.logger || console + const jobLogUrl = job.links.find((l) => l.rel === 'log') + + if (!jobLogUrl) { + throw new Error(`Log URL for job ${job.id} was not found.`) + } + + const logCount = job.logStatistics?.lineCount ?? 1000000 + const log = await fetchLogByChunks( + requestClient, + accessToken, + `${jobLogUrl.href}/content`, + logCount + ) + + logger.info(`Writing logs to ${logFilePath}`) + await createFile(logFilePath, log) +} diff --git a/src/api/viya/spec/executeScript.spec.ts b/src/api/viya/spec/executeScript.spec.ts index bca8fdb..314c70b 100644 --- a/src/api/viya/spec/executeScript.spec.ts +++ b/src/api/viya/spec/executeScript.spec.ts @@ -4,7 +4,7 @@ import { executeScript } from '../executeScript' import { mockSession, mockAuthConfig, mockJob } from './mockResponses' import * as pollJobStateModule from '../pollJobState' import * as uploadTablesModule from '../uploadTables' -import * as tokensModule from '../../../auth/tokens' +import * as getTokensModule from '../../../auth/getTokens' import * as formatDataModule from '../../../utils/formatDataForRequest' import * as fetchLogsModule from '../../../utils/fetchLogByChunks' import { PollOptions } from '../../../types' @@ -35,7 +35,7 @@ describe('executeScript', () => { 'test context' ) - expect(tokensModule.getTokens).not.toHaveBeenCalled() + expect(getTokensModule.getTokens).not.toHaveBeenCalled() }) it('should try to get fresh tokens if an authConfig is provided', async () => { @@ -49,7 +49,7 @@ describe('executeScript', () => { mockAuthConfig ) - expect(tokensModule.getTokens).toHaveBeenCalledWith( + expect(getTokensModule.getTokens).toHaveBeenCalledWith( requestClient, mockAuthConfig ) @@ -82,7 +82,7 @@ describe('executeScript', () => { 'test context' ).catch((e) => e) - expect(error.includes('Error while getting session.')).toBeTruthy() + expect(error).toContain('Error while getting session.') }) it('should fetch the PID when printPid is true', async () => { @@ -130,7 +130,7 @@ describe('executeScript', () => { true ).catch((e) => e) - expect(error.includes('Error while getting session variable.')).toBeTruthy() + expect(error).toContain('Error while getting session variable.') }) it('should use the file upload approach when data contains semicolons', async () => { @@ -300,7 +300,7 @@ describe('executeScript', () => { ).catch((e) => e) console.log(error) - expect(error.includes('Error while posting job')).toBeTruthy() + expect(error).toContain('Error while posting job') }) it('should immediately return the session when waitForResult is false', async () => { @@ -371,7 +371,7 @@ describe('executeScript', () => { true ).catch((e) => e) - expect(error.includes('Error while polling job status.')).toBeTruthy() + expect(error).toContain('Error while polling job status.') }) it('should fetch the log and append it to the error in case of a 5113 error code', async () => { @@ -626,7 +626,7 @@ describe('executeScript', () => { true ).catch((e) => e) - expect(error.includes('Error while clearing session.')).toBeTruthy() + expect(error).toContain('Error while clearing session.') }) }) @@ -634,7 +634,7 @@ const setupMocks = () => { jest.restoreAllMocks() jest.mock('../../../request/RequestClient') jest.mock('../../../SessionManager') - jest.mock('../../../auth/tokens') + jest.mock('../../../auth/getTokens') jest.mock('../pollJobState') jest.mock('../uploadTables') jest.mock('../../../utils/formatDataForRequest') @@ -650,7 +650,7 @@ const setupMocks = () => { .spyOn(requestClient, 'delete') .mockImplementation(() => Promise.resolve({ result: {}, etag: '' })) jest - .spyOn(tokensModule, 'getTokens') + .spyOn(getTokensModule, 'getTokens') .mockImplementation(() => Promise.resolve(mockAuthConfig)) jest .spyOn(pollJobStateModule, 'pollJobState') diff --git a/src/api/viya/spec/pollJobState.spec.ts b/src/api/viya/spec/pollJobState.spec.ts new file mode 100644 index 0000000..23ac560 --- /dev/null +++ b/src/api/viya/spec/pollJobState.spec.ts @@ -0,0 +1,266 @@ +import { RequestClient } from '../../../request/RequestClient' +import { mockAuthConfig, mockJob } from './mockResponses' +import { pollJobState } from '../pollJobState' +import * as getTokensModule from '../../../auth/getTokens' +import * as saveLogModule from '../saveLog' +import { PollOptions } from '../../../types' +import { Logger, LogLevel } from '@sasjs/utils' + +const requestClient = new (>RequestClient)() +const defaultPollOptions: PollOptions = { + maxPollCount: 100, + pollInterval: 500, + streamLog: false +} + +describe('pollJobState', () => { + beforeEach(() => { + ;(process as any).logger = new Logger(LogLevel.Off) + setupMocks() + }) + + it('should get valid tokens if the authConfig has been provided', async () => { + await pollJobState( + requestClient, + mockJob, + false, + 'test', + mockAuthConfig, + defaultPollOptions + ) + + expect(getTokensModule.getTokens).toHaveBeenCalledWith( + requestClient, + mockAuthConfig + ) + }) + + it('should not attempt to get tokens if the authConfig has not been provided', async () => { + await pollJobState( + requestClient, + mockJob, + false, + 'test', + undefined, + defaultPollOptions + ) + + expect(getTokensModule.getTokens).not.toHaveBeenCalled() + }) + + it('should throw an error if the job does not have a state link', async () => { + const error = await pollJobState( + requestClient, + { ...mockJob, links: mockJob.links.filter((l) => l.rel !== 'state') }, + false, + 'test', + undefined, + defaultPollOptions + ).catch((e) => e) + + expect((error as Error).message).toContain('Job state link was not found.') + }) + + it('should attempt to refresh tokens before each poll', async () => { + jest + .spyOn(requestClient, 'get') + .mockImplementationOnce(() => + Promise.resolve({ result: 'pending', etag: '' }) + ) + .mockImplementationOnce(() => + Promise.resolve({ result: 'running', etag: '' }) + ) + .mockImplementation(() => + Promise.resolve({ result: 'completed', etag: '' }) + ) + + await pollJobState( + requestClient, + mockJob, + false, + 'test', + mockAuthConfig, + defaultPollOptions + ) + + expect(getTokensModule.getTokens).toHaveBeenCalledTimes(3) + }) + + it('should attempt to fetch and save the log after each poll', async () => { + jest + .spyOn(requestClient, 'get') + .mockImplementationOnce(() => + Promise.resolve({ result: 'pending', etag: '' }) + ) + .mockImplementationOnce(() => + Promise.resolve({ result: 'running', etag: '' }) + ) + .mockImplementation(() => + Promise.resolve({ result: 'completed', etag: '' }) + ) + + await pollJobState( + requestClient, + mockJob, + false, + 'test', + mockAuthConfig, + defaultPollOptions + ) + + expect(saveLogModule.saveLog).toHaveBeenCalledTimes(2) + }) + + it('should return the current status when the max poll count is reached', async () => { + jest + .spyOn(requestClient, 'get') + .mockImplementationOnce(() => + Promise.resolve({ result: 'pending', etag: '' }) + ) + .mockImplementationOnce(() => + Promise.resolve({ result: 'running', etag: '' }) + ) + + const state = await pollJobState( + requestClient, + mockJob, + false, + 'test', + mockAuthConfig, + { + ...defaultPollOptions, + maxPollCount: 1 + } + ) + + expect(state).toEqual('running') + }) + + it('should continue polling until the job completes or errors', async () => { + jest + .spyOn(requestClient, 'get') + .mockImplementationOnce(() => + Promise.resolve({ result: 'pending', etag: '' }) + ) + .mockImplementationOnce(() => + Promise.resolve({ result: 'running', etag: '' }) + ) + .mockImplementation(() => + Promise.resolve({ result: 'completed', etag: '' }) + ) + + const state = await pollJobState( + requestClient, + mockJob, + false, + 'test', + undefined, + defaultPollOptions + ) + + expect(requestClient.get).toHaveBeenCalledTimes(4) + expect(state).toEqual('completed') + }) + + it('should print the state to the console when debug is on', async () => { + jest.spyOn((process as any).logger, 'info') + jest + .spyOn(requestClient, 'get') + .mockImplementationOnce(() => + Promise.resolve({ result: 'pending', etag: '' }) + ) + .mockImplementationOnce(() => + Promise.resolve({ result: 'running', etag: '' }) + ) + .mockImplementation(() => + Promise.resolve({ result: 'completed', etag: '' }) + ) + + await pollJobState( + requestClient, + mockJob, + true, + 'test', + undefined, + defaultPollOptions + ) + + expect((process as any).logger.info).toHaveBeenCalledTimes(4) + expect((process as any).logger.info).toHaveBeenNthCalledWith( + 1, + 'Polling job status...' + ) + expect((process as any).logger.info).toHaveBeenNthCalledWith( + 2, + 'Current job state: running' + ) + expect((process as any).logger.info).toHaveBeenNthCalledWith( + 3, + 'Polling job status...' + ) + expect((process as any).logger.info).toHaveBeenNthCalledWith( + 4, + 'Current job state: completed' + ) + }) + + it('should continue polling when there is a single error in between', async () => { + jest + .spyOn(requestClient, 'get') + .mockImplementationOnce(() => + Promise.resolve({ result: 'pending', etag: '' }) + ) + .mockImplementationOnce(() => Promise.reject('Status Error')) + .mockImplementationOnce(() => + Promise.resolve({ result: 'completed', etag: '' }) + ) + + const state = await pollJobState( + requestClient, + mockJob, + false, + 'test', + undefined, + defaultPollOptions + ) + + expect(requestClient.get).toHaveBeenCalledTimes(3) + expect(state).toEqual('completed') + }) + + it('should throw an error when the error count exceeds the set value of 5', async () => { + jest + .spyOn(requestClient, 'get') + .mockImplementation(() => Promise.reject('Status Error')) + + const error = await pollJobState( + requestClient, + mockJob, + false, + 'test', + undefined, + defaultPollOptions + ).catch((e) => e) + + expect(error).toContain('Error while getting job state after interval.') + }) +}) + +const setupMocks = () => { + jest.restoreAllMocks() + jest.mock('../../../request/RequestClient') + jest.mock('../../../auth/getTokens') + jest.mock('../saveLog') + + jest + .spyOn(requestClient, 'get') + .mockImplementation(() => + Promise.resolve({ result: 'completed', etag: '' }) + ) + jest + .spyOn(getTokensModule, 'getTokens') + .mockImplementation(() => Promise.resolve(mockAuthConfig)) + jest + .spyOn(saveLogModule, 'saveLog') + .mockImplementation(() => Promise.resolve()) +} diff --git a/src/api/viya/spec/saveLog.spec.ts b/src/api/viya/spec/saveLog.spec.ts new file mode 100644 index 0000000..c4b8b9d --- /dev/null +++ b/src/api/viya/spec/saveLog.spec.ts @@ -0,0 +1,72 @@ +import { Logger, LogLevel } from '@sasjs/utils' +import * as fileModule from '@sasjs/utils/file' +import { RequestClient } from '../../../request/RequestClient' +import * as fetchLogsModule from '../../../utils/fetchLogByChunks' +import { saveLog } from '../saveLog' +import { mockJob } from './mockResponses' + +const requestClient = new (>RequestClient)() + +describe('saveLog', () => { + beforeEach(() => { + ;(process as any).logger = new Logger(LogLevel.Off) + setupMocks() + }) + + it('should return immediately if shouldSaveLog is false', async () => { + await saveLog(mockJob, requestClient, false, '/test', 't0k3n') + + expect(fetchLogsModule.fetchLogByChunks).not.toHaveBeenCalled() + expect(fileModule.createFile).not.toHaveBeenCalled() + }) + + it('should throw an error when a valid access token is not provided', async () => { + const error = await saveLog(mockJob, requestClient, true, '/test').catch( + (e) => e + ) + + expect(error.message).toContain( + `Logs for job ${mockJob.id} cannot be fetched without a valid access token.` + ) + }) + + it('should throw an error when the log URL is not available', async () => { + const error = await saveLog( + { ...mockJob, links: mockJob.links.filter((l) => l.rel !== 'log') }, + requestClient, + true, + '/test', + 't0k3n' + ).catch((e) => e) + + expect(error.message).toContain( + `Log URL for job ${mockJob.id} was not found.` + ) + }) + + it('should fetch and save logs to the given path', async () => { + await saveLog(mockJob, requestClient, true, '/test', 't0k3n') + + expect(fetchLogsModule.fetchLogByChunks).toHaveBeenCalledWith( + requestClient, + 't0k3n', + '/log/content', + 100 + ) + expect(fileModule.createFile).toHaveBeenCalledWith('/test', 'Test Log') + }) +}) + +const setupMocks = () => { + jest.restoreAllMocks() + jest.mock('../../../request/RequestClient') + jest.mock('../../../utils/fetchLogByChunks') + jest.mock('@sasjs/utils') + + jest + .spyOn(fetchLogsModule, 'fetchLogByChunks') + .mockImplementation(() => Promise.resolve('Test Log')) + jest + .spyOn(fileModule, 'createFile') + .mockImplementation(() => Promise.resolve()) +} diff --git a/src/auth/getAccessToken.ts b/src/auth/getAccessToken.ts new file mode 100644 index 0000000..0b11340 --- /dev/null +++ b/src/auth/getAccessToken.ts @@ -0,0 +1,49 @@ +import { SasAuthResponse } from '@sasjs/utils' +import * as NodeFormData from 'form-data' +import { RequestClient } from '../request/RequestClient' + +/** + * Exchanges the auth code for an access token for the given client. + * @param requestClient - the pre-configured HTTP request client + * @param clientId - the client ID to authenticate with. + * @param clientSecret - the client secret to authenticate with. + * @param authCode - the auth code received from the server. + */ +export async function getAccessToken( + requestClient: RequestClient, + clientId: string, + clientSecret: string, + authCode: string +): Promise { + const url = '/SASLogon/oauth/token' + let token + if (typeof Buffer === 'undefined') { + token = btoa(clientId + ':' + clientSecret) + } else { + token = Buffer.from(clientId + ':' + clientSecret).toString('base64') + } + const headers = { + Authorization: 'Basic ' + token + } + + let formData + if (typeof FormData === 'undefined') { + formData = new NodeFormData() + } else { + formData = new FormData() + } + formData.append('grant_type', 'authorization_code') + formData.append('code', authCode) + + const authResponse = await requestClient + .post( + url, + formData, + undefined, + 'multipart/form-data; boundary=' + (formData as any)._boundary, + headers + ) + .then((res) => res.result as SasAuthResponse) + + return authResponse +} diff --git a/src/auth/getTokens.ts b/src/auth/getTokens.ts new file mode 100644 index 0000000..031c6a3 --- /dev/null +++ b/src/auth/getTokens.ts @@ -0,0 +1,40 @@ +import { + AuthConfig, + isAccessTokenExpiring, + isRefreshTokenExpiring, + hasTokenExpired +} from '@sasjs/utils' +import { RequestClient } from '../request/RequestClient' +import { refreshTokens } from './refreshTokens' + +/** + * Returns the auth configuration, refreshing the tokens if necessary. + * @param requestClient - the pre-configured HTTP request client + * @param authConfig - an object containing a client ID, secret, access token and refresh token + */ +export async function getTokens( + requestClient: RequestClient, + authConfig: AuthConfig +): Promise { + const logger = process.logger || console + let { access_token, refresh_token, client, secret } = authConfig + if ( + isAccessTokenExpiring(access_token) || + isRefreshTokenExpiring(refresh_token) + ) { + if (hasTokenExpired(refresh_token)) { + const error = + 'Unable to obtain new access token. Your refresh token has expired.' + logger.error(error) + throw new Error(error) + } + logger.info('Refreshing access and refresh tokens.') + ;({ access_token, refresh_token } = await refreshTokens( + requestClient, + client, + secret, + refresh_token + )) + } + return { access_token, refresh_token, client, secret } +} diff --git a/src/auth/refreshTokens.ts b/src/auth/refreshTokens.ts new file mode 100644 index 0000000..5871d63 --- /dev/null +++ b/src/auth/refreshTokens.ts @@ -0,0 +1,49 @@ +import { SasAuthResponse } from '@sasjs/utils/types' +import { prefixMessage } from '@sasjs/utils/error' +import * as NodeFormData from 'form-data' +import { RequestClient } from '../request/RequestClient' + +/** + * Exchanges the refresh token for an access token for the given client. + * @param requestClient - the pre-configured HTTP request client + * @param clientId - the client ID to authenticate with. + * @param clientSecret - the client secret to authenticate with. + * @param authCode - the refresh token received from the server. + */ +export async function refreshTokens( + requestClient: RequestClient, + clientId: string, + clientSecret: string, + refreshToken: string +) { + const url = '/SASLogon/oauth/token' + let token + token = + typeof Buffer === 'undefined' + ? btoa(clientId + ':' + clientSecret) + : Buffer.from(clientId + ':' + clientSecret).toString('base64') + + const headers = { + Authorization: 'Basic ' + token + } + + const formData = + typeof FormData === 'undefined' ? new NodeFormData() : new FormData() + formData.append('grant_type', 'refresh_token') + formData.append('refresh_token', refreshToken) + + const authResponse = await requestClient + .post( + url, + formData, + undefined, + 'multipart/form-data; boundary=' + (formData as any)._boundary, + headers + ) + .then((res) => res.result) + .catch((err) => { + throw prefixMessage(err, 'Error while refreshing tokens') + }) + + return authResponse +} diff --git a/src/auth/spec/getTokens.spec.ts b/src/auth/spec/getTokens.spec.ts new file mode 100644 index 0000000..de4397c --- /dev/null +++ b/src/auth/spec/getTokens.spec.ts @@ -0,0 +1,79 @@ +import { AuthConfig } from '@sasjs/utils' +import * as refreshTokensModule from '../refreshTokens' +import { generateToken, mockAuthResponse } from './mockResponses' +import { getTokens } from '../getTokens' +import { RequestClient } from '../../request/RequestClient' + +const requestClient = new (>RequestClient)() + +describe('getTokens', () => { + it('should attempt to refresh tokens if the access token is expiring', async () => { + setupMocks() + const access_token = generateToken(30) + const refresh_token = generateToken(86400000) + const authConfig: AuthConfig = { + access_token, + refresh_token, + client: 'cl13nt', + secret: 's3cr3t' + } + + await getTokens(requestClient, authConfig) + + expect(refreshTokensModule.refreshTokens).toHaveBeenCalledWith( + requestClient, + authConfig.client, + authConfig.secret, + authConfig.refresh_token + ) + }) + + it('should attempt to refresh tokens if the refresh token is expiring', async () => { + setupMocks() + const access_token = generateToken(86400000) + const refresh_token = generateToken(30) + const authConfig: AuthConfig = { + access_token, + refresh_token, + client: 'cl13nt', + secret: 's3cr3t' + } + + await getTokens(requestClient, authConfig) + + expect(refreshTokensModule.refreshTokens).toHaveBeenCalledWith( + requestClient, + authConfig.client, + authConfig.secret, + authConfig.refresh_token + ) + }) + + it('should throw an error if the refresh token has already expired', async () => { + setupMocks() + const access_token = generateToken(86400000) + const refresh_token = generateToken(-36000) + const authConfig: AuthConfig = { + access_token, + refresh_token, + client: 'cl13nt', + secret: 's3cr3t' + } + const expectedError = + 'Unable to obtain new access token. Your refresh token has expired.' + + const error = await getTokens(requestClient, authConfig).catch((e) => e) + + expect(error.message).toEqual(expectedError) + }) +}) + +const setupMocks = () => { + jest.restoreAllMocks() + jest.mock('../../request/RequestClient') + jest.mock('../refreshTokens') + + jest + .spyOn(refreshTokensModule, 'refreshTokens') + .mockImplementation(() => Promise.resolve(mockAuthResponse)) +} diff --git a/src/auth/spec/mockResponses.ts b/src/auth/spec/mockResponses.ts index 4ffcfb2..e15391a 100644 --- a/src/auth/spec/mockResponses.ts +++ b/src/auth/spec/mockResponses.ts @@ -1,2 +1,24 @@ +import { SasAuthResponse } from '@sasjs/utils/types' + export const mockLoginAuthoriseRequiredResponse = `
` export const mockLoginSuccessResponse = `You have signed in` + +export const mockAuthResponse: SasAuthResponse = { + access_token: 'acc355', + refresh_token: 'r3fr35h', + id_token: 'id', + token_type: 'bearer', + expires_in: new Date().valueOf(), + scope: 'default', + jti: 'test' +} + +export const generateToken = (timeToLiveSeconds: number): string => { + const exp = + new Date(new Date().getTime() + timeToLiveSeconds * 1000).getTime() / 1000 + const header = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9' + const payload = Buffer.from(JSON.stringify({ exp })).toString('base64') + const signature = '4-iaDojEVl0pJQMjrbM1EzUIfAZgsbK_kgnVyVxFSVo' + const token = `${header}.${payload}.${signature}` + return token +} diff --git a/src/auth/spec/refreshTokens.spec.ts b/src/auth/spec/refreshTokens.spec.ts new file mode 100644 index 0000000..8f88f1d --- /dev/null +++ b/src/auth/spec/refreshTokens.spec.ts @@ -0,0 +1,75 @@ +import { AuthConfig } from '@sasjs/utils' +import * as NodeFormData from 'form-data' +import { generateToken, mockAuthResponse } from './mockResponses' +import { RequestClient } from '../../request/RequestClient' +import { refreshTokens } from '../refreshTokens' + +const requestClient = new (>RequestClient)() + +describe('refreshTokens', () => { + it('should attempt to refresh tokens', async () => { + setupMocks() + const access_token = generateToken(30) + const refresh_token = generateToken(30) + const authConfig: AuthConfig = { + access_token, + refresh_token, + client: 'cl13nt', + secret: 's3cr3t' + } + jest + .spyOn(requestClient, 'post') + .mockImplementation(() => + Promise.resolve({ result: mockAuthResponse, etag: '' }) + ) + const token = Buffer.from( + authConfig.client + ':' + authConfig.secret + ).toString('base64') + + await refreshTokens( + requestClient, + authConfig.client, + authConfig.secret, + authConfig.refresh_token + ) + + expect(requestClient.post).toHaveBeenCalledWith( + '/SASLogon/oauth/token', + expect.any(NodeFormData), + undefined, + expect.stringContaining('multipart/form-data; boundary='), + { + Authorization: 'Basic ' + token + } + ) + }) + + it('should handle errors while refreshing tokens', async () => { + setupMocks() + const access_token = generateToken(30) + const refresh_token = generateToken(30) + const authConfig: AuthConfig = { + access_token, + refresh_token, + client: 'cl13nt', + secret: 's3cr3t' + } + jest + .spyOn(requestClient, 'post') + .mockImplementation(() => Promise.reject('Token Error')) + + const error = await refreshTokens( + requestClient, + authConfig.client, + authConfig.secret, + authConfig.refresh_token + ).catch((e) => e) + + expect(error).toContain('Error while refreshing tokens') + }) +}) + +const setupMocks = () => { + jest.restoreAllMocks() + jest.mock('../../request/RequestClient') +} diff --git a/src/auth/tokens.ts b/src/auth/tokens.ts deleted file mode 100644 index bc940e2..0000000 --- a/src/auth/tokens.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { - AuthConfig, - isAccessTokenExpiring, - isRefreshTokenExpiring, - SasAuthResponse -} from '@sasjs/utils' -import * as NodeFormData from 'form-data' -import { RequestClient } from '../request/RequestClient' - -/** - * Exchanges the auth code for an access token for the given client. - * @param requestClient - the pre-configured HTTP request client - * @param clientId - the client ID to authenticate with. - * @param clientSecret - the client secret to authenticate with. - * @param authCode - the auth code received from the server. - */ -export async function getAccessToken( - requestClient: RequestClient, - clientId: string, - clientSecret: string, - authCode: string -): Promise { - const url = '/SASLogon/oauth/token' - let token - if (typeof Buffer === 'undefined') { - token = btoa(clientId + ':' + clientSecret) - } else { - token = Buffer.from(clientId + ':' + clientSecret).toString('base64') - } - const headers = { - Authorization: 'Basic ' + token - } - - let formData - if (typeof FormData === 'undefined') { - formData = new NodeFormData() - } else { - formData = new FormData() - } - formData.append('grant_type', 'authorization_code') - formData.append('code', authCode) - - const authResponse = await requestClient - .post( - url, - formData, - undefined, - 'multipart/form-data; boundary=' + (formData as any)._boundary, - headers - ) - .then((res) => res.result as SasAuthResponse) - - return authResponse -} - -/** - * Returns the auth configuration, refreshing the tokens if necessary. - * @param requestClient - the pre-configured HTTP request client - * @param authConfig - an object containing a client ID, secret, access token and refresh token - */ -export async function getTokens( - requestClient: RequestClient, - authConfig: AuthConfig -): Promise { - const logger = process.logger || console - let { access_token, refresh_token, client, secret } = authConfig - if ( - isAccessTokenExpiring(access_token) || - isRefreshTokenExpiring(refresh_token) - ) { - logger.info('Refreshing access and refresh tokens.') - ;({ access_token, refresh_token } = await refreshTokens( - requestClient, - client, - secret, - refresh_token - )) - } - return { access_token, refresh_token, client, secret } -} - -/** - * Exchanges the refresh token for an access token for the given client. - * @param requestClient - the pre-configured HTTP request client - * @param clientId - the client ID to authenticate with. - * @param clientSecret - the client secret to authenticate with. - * @param authCode - the refresh token received from the server. - */ -export async function refreshTokens( - requestClient: RequestClient, - clientId: string, - clientSecret: string, - refreshToken: string -) { - const url = '/SASLogon/oauth/token' - let token - if (typeof Buffer === 'undefined') { - token = btoa(clientId + ':' + clientSecret) - } else { - token = Buffer.from(clientId + ':' + clientSecret).toString('base64') - } - const headers = { - Authorization: 'Basic ' + token - } - - const formData = - typeof FormData === 'undefined' ? new NodeFormData() : new FormData() - formData.append('grant_type', 'refresh_token') - formData.append('refresh_token', refreshToken) - - const authResponse = await requestClient - .post( - url, - formData, - undefined, - 'multipart/form-data; boundary=' + (formData as any)._boundary, - headers - ) - .then((res) => res.result) - - return authResponse -} From a0fbe1a74088de10227ca1d9ea4c325718e804ba Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Mon, 12 Jul 2021 20:42:49 +0100 Subject: [PATCH 07/15] chore(ci): add coverage report action --- .github/workflows/coverage.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/coverage.yml diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..7fc938f --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,25 @@ +name: SASjs Adapter Coverage Report + +on: + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [12.x] + + steps: + - uses: actions/checkout@v2 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Install Dependencies + run: npm ci + - name: Generate coverage report + uses: artiomtr/jest-coverage-report-action@v2.0-rc.2 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} From 4257ec78aac6a829adcfe5e78ce7fb0898724ea5 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Mon, 12 Jul 2021 20:45:09 +0100 Subject: [PATCH 08/15] chore(ci): add coverage report to build workflow --- .github/workflows/build.yml | 4 ++++ .github/workflows/coverage.yml | 25 ------------------------- 2 files changed, 4 insertions(+), 25 deletions(-) delete mode 100644 .github/workflows/coverage.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 188d999..10d8ea1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,6 +27,10 @@ jobs: run: npm run lint - name: Run unit tests run: npm test + - name: Generate coverage report + uses: artiomtr/jest-coverage-report-action@v2.0-rc.2 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Package run: npm run package:lib env: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml deleted file mode 100644 index 7fc938f..0000000 --- a/.github/workflows/coverage.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: SASjs Adapter Coverage Report - -on: - pull_request: - -jobs: - build: - runs-on: ubuntu-latest - - strategy: - matrix: - node-version: [12.x] - - steps: - - uses: actions/checkout@v2 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - name: Install Dependencies - run: npm ci - - name: Generate coverage report - uses: artiomtr/jest-coverage-report-action@v2.0-rc.2 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} From b9f368193d5be7abb1c6c9897b3f7bae13587927 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Tue, 13 Jul 2021 08:12:15 +0100 Subject: [PATCH 09/15] chore(refactor): add more tests --- src/api/viya/spec/uploadTables.spec.ts | 67 +++++++++++++++++++++++ src/api/viya/uploadTables.ts | 6 --- src/auth/getAccessToken.ts | 6 ++- src/auth/spec/getAccessToken.spec.ts | 75 ++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 7 deletions(-) create mode 100644 src/api/viya/spec/uploadTables.spec.ts create mode 100644 src/auth/spec/getAccessToken.spec.ts diff --git a/src/api/viya/spec/uploadTables.spec.ts b/src/api/viya/spec/uploadTables.spec.ts new file mode 100644 index 0000000..529c6e2 --- /dev/null +++ b/src/api/viya/spec/uploadTables.spec.ts @@ -0,0 +1,67 @@ +import { RequestClient } from '../../../request/RequestClient' +import * as convertToCsvModule from '../../../utils/convertToCsv' +import { uploadTables } from '../uploadTables' + +const requestClient = new (>RequestClient)() + +describe('uploadTables', () => { + beforeEach(() => { + setupMocks() + }) + + it('should return a list of uploaded files', async () => { + const data = { foo: 'bar' } + + const files = await uploadTables(requestClient, data, 't0k3n') + + expect(files).toEqual([{ tableName: 'foo', file: 'test-file' }]) + expect(requestClient.uploadFile).toHaveBeenCalledTimes(1) + expect(requestClient.uploadFile).toHaveBeenCalledWith( + '/files/files#rawUpload', + 'Test CSV', + 't0k3n' + ) + }) + + it('should throw an error when the CSV exceeds the maximum length', async () => { + const data = { foo: 'bar' } + jest + .spyOn(convertToCsvModule, 'convertToCSV') + .mockImplementation(() => 'ERROR: LARGE STRING LENGTH') + + const error = await uploadTables(requestClient, data, 't0k3n').catch( + (e) => e + ) + + expect(requestClient.uploadFile).not.toHaveBeenCalled() + expect(error.message).toEqual( + 'The max length of a string value in SASjs is 32765 characters.' + ) + }) + + it('should throw an error when the file upload fails', async () => { + const data = { foo: 'bar' } + jest + .spyOn(requestClient, 'uploadFile') + .mockImplementation(() => Promise.reject('Upload Error')) + + const error = await uploadTables(requestClient, data, 't0k3n').catch( + (e) => e + ) + + expect(error).toContain('Error while uploading file.') + }) +}) + +const setupMocks = () => { + jest.restoreAllMocks() + jest.mock('../../../utils/convertToCsv') + jest + .spyOn(convertToCsvModule, 'convertToCSV') + .mockImplementation(() => 'Test CSV') + jest + .spyOn(requestClient, 'uploadFile') + .mockImplementation(() => + Promise.resolve({ result: 'test-file', etag: '' }) + ) +} diff --git a/src/api/viya/uploadTables.ts b/src/api/viya/uploadTables.ts index 13bf7b8..e7f5b66 100644 --- a/src/api/viya/uploadTables.ts +++ b/src/api/viya/uploadTables.ts @@ -8,12 +8,6 @@ export async function uploadTables( accessToken?: string ) { const uploadedFiles = [] - const headers: any = { - 'Content-Type': 'application/json' - } - if (accessToken) { - headers.Authorization = `Bearer ${accessToken}` - } for (const tableName in data) { const csv = convertToCSV(data[tableName]) diff --git a/src/auth/getAccessToken.ts b/src/auth/getAccessToken.ts index 0b11340..d51833e 100644 --- a/src/auth/getAccessToken.ts +++ b/src/auth/getAccessToken.ts @@ -1,4 +1,5 @@ -import { SasAuthResponse } from '@sasjs/utils' +import { SasAuthResponse } from '@sasjs/utils/types' +import { prefixMessage } from '@sasjs/utils/error' import * as NodeFormData from 'form-data' import { RequestClient } from '../request/RequestClient' @@ -44,6 +45,9 @@ export async function getAccessToken( headers ) .then((res) => res.result as SasAuthResponse) + .catch((err) => { + throw prefixMessage(err, 'Error while getting access token') + }) return authResponse } diff --git a/src/auth/spec/getAccessToken.spec.ts b/src/auth/spec/getAccessToken.spec.ts new file mode 100644 index 0000000..e4fa00f --- /dev/null +++ b/src/auth/spec/getAccessToken.spec.ts @@ -0,0 +1,75 @@ +import { AuthConfig } from '@sasjs/utils' +import * as NodeFormData from 'form-data' +import { generateToken, mockAuthResponse } from './mockResponses' +import { RequestClient } from '../../request/RequestClient' +import { getAccessToken } from '../getAccessToken' + +const requestClient = new (>RequestClient)() + +describe('getAccessToken', () => { + it('should attempt to refresh tokens', async () => { + setupMocks() + const access_token = generateToken(30) + const refresh_token = generateToken(30) + const authConfig: AuthConfig = { + access_token, + refresh_token, + client: 'cl13nt', + secret: 's3cr3t' + } + jest + .spyOn(requestClient, 'post') + .mockImplementation(() => + Promise.resolve({ result: mockAuthResponse, etag: '' }) + ) + const token = Buffer.from( + authConfig.client + ':' + authConfig.secret + ).toString('base64') + + await getAccessToken( + requestClient, + authConfig.client, + authConfig.secret, + authConfig.refresh_token + ) + + expect(requestClient.post).toHaveBeenCalledWith( + '/SASLogon/oauth/token', + expect.any(NodeFormData), + undefined, + expect.stringContaining('multipart/form-data; boundary='), + { + Authorization: 'Basic ' + token + } + ) + }) + + it('should handle errors while refreshing tokens', async () => { + setupMocks() + const access_token = generateToken(30) + const refresh_token = generateToken(30) + const authConfig: AuthConfig = { + access_token, + refresh_token, + client: 'cl13nt', + secret: 's3cr3t' + } + jest + .spyOn(requestClient, 'post') + .mockImplementation(() => Promise.reject('Token Error')) + + const error = await getAccessToken( + requestClient, + authConfig.client, + authConfig.secret, + authConfig.refresh_token + ).catch((e) => e) + + expect(error).toContain('Error while getting access token') + }) +}) + +const setupMocks = () => { + jest.restoreAllMocks() + jest.mock('../../request/RequestClient') +} From d4725d2e54bb0f885ff74da4abb79e15e06f63d4 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Wed, 14 Jul 2021 07:50:25 +0100 Subject: [PATCH 10/15] chore(refactor): change property name in PollOptions --- src/api/viya/pollJobState.ts | 2 +- src/types/PollOptions.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/viya/pollJobState.ts b/src/api/viya/pollJobState.ts index 8d95c42..5e74dda 100644 --- a/src/api/viya/pollJobState.ts +++ b/src/api/viya/pollJobState.ts @@ -23,7 +23,7 @@ export async function pollJobState( const logFileName = `${postedJob.name || 'job'}-${generateTimestamp()}.log` const logFilePath = `${ - pollOptions?.logFilePath || process.cwd() + pollOptions?.logFolderPath || process.cwd() }/${logFileName}` if (authConfig) { diff --git a/src/types/PollOptions.ts b/src/types/PollOptions.ts index 6daa1b7..82daf9e 100644 --- a/src/types/PollOptions.ts +++ b/src/types/PollOptions.ts @@ -2,5 +2,5 @@ export interface PollOptions { maxPollCount: number pollInterval: number streamLog: boolean - logFilePath?: string + logFolderPath?: string } From 1ff3937d117c763bef435b850d3fa20dddd91371 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Wed, 14 Jul 2021 08:03:54 +0100 Subject: [PATCH 11/15] chore(deps): update dependencies --- package-lock.json | 440 +++++++++++++++++++++++----------------------- package.json | 10 +- 2 files changed, 224 insertions(+), 226 deletions(-) diff --git a/package-lock.json b/package-lock.json index 970d08e..08531a8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,9 +18,9 @@ "devDependencies": { "@types/axios": "^0.14.0", "@types/form-data": "^2.5.0", - "@types/jest": "^26.0.23", + "@types/jest": "^26.0.24", "@types/mime": "^2.0.3", - "@types/tough-cookie": "^4.0.0", + "@types/tough-cookie": "^4.0.1", "copyfiles": "^2.4.1", "cp": "^0.2.0", "dotenv": "^10.0.0", @@ -36,11 +36,11 @@ "ts-loader": "^9.2.2", "tslint": "^6.1.3", "tslint-config-prettier": "^1.18.0", - "typedoc": "^0.21.2", + "typedoc": "^0.21.4", "typedoc-neo-theme": "^1.1.1", "typedoc-plugin-external-module-name": "^4.0.6", - "typescript": "^4.3.4", - "webpack": "^5.41.1", + "typescript": "^4.3.5", + "webpack": "^5.44.0", "webpack-cli": "^4.7.2" } }, @@ -1508,9 +1508,9 @@ } }, "node_modules/@types/estree": { - "version": "0.0.48", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.48.tgz", - "integrity": "sha512-LfZwXoGUDo0C3me81HXgkBg5CTQYb6xzEl+fNmbO4JdRiSKQ8A0GD1OBBvKAIsbCUgoyAty7m99GqqMQe784ew==", + "version": "0.0.50", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.50.tgz", + "integrity": "sha512-C6N5s2ZFtuZRj54k2/zyRhNDjJwwcViAM3Nbm8zjBpbqAdZ00mr0CFxvSKeO8Y/e03WVFLpQMdHYVfUd6SB+Hw==", "dev": true }, "node_modules/@types/form-data": { @@ -1557,9 +1557,9 @@ } }, "node_modules/@types/jest": { - "version": "26.0.23", - "resolved": "https://registry.npmjs.org/@types/jest/-/jest-26.0.23.tgz", - "integrity": "sha512-ZHLmWMJ9jJ9PTiT58juykZpL7KjwJywFN3Rr2pTSkyQfydf/rk22yS7W8p5DaVUMQ2BQC7oYiU3FjbTM/mYrOA==", + "version": "26.0.24", + "resolved": "https://registry.npmjs.org/@types/jest/-/jest-26.0.24.tgz", + "integrity": "sha512-E/X5Vib8BWqZNRlDxj9vYXhsDwPYbPINqKF9BsnSoon4RQ0D9moEuLD8txgyypFLH7J4+Lho9Nr/c8H0Fi+17w==", "dev": true, "dependencies": { "jest-diff": "^26.0.0", @@ -1628,9 +1628,9 @@ "dev": true }, "node_modules/@types/tough-cookie": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.0.tgz", - "integrity": "sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.1.tgz", + "integrity": "sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==", "dev": true }, "node_modules/@types/yargs": { @@ -1649,148 +1649,148 @@ "dev": true }, "node_modules/@webassemblyjs/ast": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.0.tgz", - "integrity": "sha512-kX2W49LWsbthrmIRMbQZuQDhGtjyqXfEmmHyEi4XWnSZtPmxY0+3anPIzsnRb45VH/J55zlOfWvZuY47aJZTJg==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.1.tgz", + "integrity": "sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw==", "dev": true, "dependencies": { - "@webassemblyjs/helper-numbers": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0" + "@webassemblyjs/helper-numbers": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1" } }, "node_modules/@webassemblyjs/floating-point-hex-parser": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.0.tgz", - "integrity": "sha512-Q/aVYs/VnPDVYvsCBL/gSgwmfjeCb4LW8+TMrO3cSzJImgv8lxxEPM2JA5jMrivE7LSz3V+PFqtMbls3m1exDA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.1.tgz", + "integrity": "sha512-iGRfyc5Bq+NnNuX8b5hwBrRjzf0ocrJPI6GWFodBFzmFnyvrQ83SHKhmilCU/8Jv67i4GJZBMhEzltxzcNagtQ==", "dev": true }, "node_modules/@webassemblyjs/helper-api-error": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.0.tgz", - "integrity": "sha512-baT/va95eXiXb2QflSx95QGT5ClzWpGaa8L7JnJbgzoYeaA27FCvuBXU758l+KXWRndEmUXjP0Q5fibhavIn8w==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.1.tgz", + "integrity": "sha512-RlhS8CBCXfRUR/cwo2ho9bkheSXG0+NwooXcc3PAILALf2QLdFyj7KGsKRbVc95hZnhnERon4kW/D3SZpp6Tcg==", "dev": true }, "node_modules/@webassemblyjs/helper-buffer": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.0.tgz", - "integrity": "sha512-u9HPBEl4DS+vA8qLQdEQ6N/eJQ7gT7aNvMIo8AAWvAl/xMrcOSiI2M0MAnMCy3jIFke7bEee/JwdX1nUpCtdyA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.1.tgz", + "integrity": "sha512-gwikF65aDNeeXa8JxXa2BAk+REjSyhrNC9ZwdT0f8jc4dQQeDQ7G4m0f2QCLPJiMTTO6wfDmRmj/pW0PsUvIcA==", "dev": true }, "node_modules/@webassemblyjs/helper-numbers": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.0.tgz", - "integrity": "sha512-DhRQKelIj01s5IgdsOJMKLppI+4zpmcMQ3XboFPLwCpSNH6Hqo1ritgHgD0nqHeSYqofA6aBN/NmXuGjM1jEfQ==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.1.tgz", + "integrity": "sha512-vDkbxiB8zfnPdNK9Rajcey5C0w+QJugEglN0of+kmO8l7lDb77AnlKYQF7aarZuCrv+l0UvqL+68gSDr3k9LPQ==", "dev": true, "dependencies": { - "@webassemblyjs/floating-point-hex-parser": "1.11.0", - "@webassemblyjs/helper-api-error": "1.11.0", + "@webassemblyjs/floating-point-hex-parser": "1.11.1", + "@webassemblyjs/helper-api-error": "1.11.1", "@xtuc/long": "4.2.2" } }, "node_modules/@webassemblyjs/helper-wasm-bytecode": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.0.tgz", - "integrity": "sha512-MbmhvxXExm542tWREgSFnOVo07fDpsBJg3sIl6fSp9xuu75eGz5lz31q7wTLffwL3Za7XNRCMZy210+tnsUSEA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.1.tgz", + "integrity": "sha512-PvpoOGiJwXeTrSf/qfudJhwlvDQxFgelbMqtq52WWiXC6Xgg1IREdngmPN3bs4RoO83PnL/nFrxucXj1+BX62Q==", "dev": true }, "node_modules/@webassemblyjs/helper-wasm-section": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.0.tgz", - "integrity": "sha512-3Eb88hcbfY/FCukrg6i3EH8H2UsD7x8Vy47iVJrP967A9JGqgBVL9aH71SETPx1JrGsOUVLo0c7vMCN22ytJew==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.1.tgz", + "integrity": "sha512-10P9No29rYX1j7F3EVPX3JvGPQPae+AomuSTPiF9eBQeChHI6iqjMIwR9JmOJXwpnn/oVGDk7I5IlskuMwU/pg==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1" } }, "node_modules/@webassemblyjs/ieee754": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.0.tgz", - "integrity": "sha512-KXzOqpcYQwAfeQ6WbF6HXo+0udBNmw0iXDmEK5sFlmQdmND+tr773Ti8/5T/M6Tl/413ArSJErATd8In3B+WBA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.1.tgz", + "integrity": "sha512-hJ87QIPtAMKbFq6CGTkZYJivEwZDbQUgYd3qKSadTNOhVY7p+gfP6Sr0lLRVTaG1JjFj+r3YchoqRYxNH3M0GQ==", "dev": true, "dependencies": { "@xtuc/ieee754": "^1.2.0" } }, "node_modules/@webassemblyjs/leb128": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.0.tgz", - "integrity": "sha512-aqbsHa1mSQAbeeNcl38un6qVY++hh8OpCOzxhixSYgbRfNWcxJNJQwe2rezK9XEcssJbbWIkblaJRwGMS9zp+g==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.1.tgz", + "integrity": "sha512-BJ2P0hNZ0u+Th1YZXJpzW6miwqQUGcIHT1G/sf72gLVD9DZ5AdYTqPNbHZh6K1M5VmKvFXwGSWZADz+qBWxeRw==", "dev": true, "dependencies": { "@xtuc/long": "4.2.2" } }, "node_modules/@webassemblyjs/utf8": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.0.tgz", - "integrity": "sha512-A/lclGxH6SpSLSyFowMzO/+aDEPU4hvEiooCMXQPcQFPPJaYcPQNKGOCLUySJsYJ4trbpr+Fs08n4jelkVTGVw==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.1.tgz", + "integrity": "sha512-9kqcxAEdMhiwQkHpkNiorZzqpGrodQQ2IGrHHxCy+Ozng0ofyMA0lTqiLkVs1uzTRejX+/O0EOT7KxqVPuXosQ==", "dev": true }, "node_modules/@webassemblyjs/wasm-edit": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.0.tgz", - "integrity": "sha512-JHQ0damXy0G6J9ucyKVXO2j08JVJ2ntkdJlq1UTiUrIgfGMmA7Ik5VdC/L8hBK46kVJgujkBIoMtT8yVr+yVOQ==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.1.tgz", + "integrity": "sha512-g+RsupUC1aTHfR8CDgnsVRVZFJqdkFHpsHMfJuWQzWU3tvnLC07UqHICfP+4XyL2tnr1amvl1Sdp06TnYCmVkA==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/helper-wasm-section": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0", - "@webassemblyjs/wasm-opt": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0", - "@webassemblyjs/wast-printer": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/helper-wasm-section": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1", + "@webassemblyjs/wasm-opt": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1", + "@webassemblyjs/wast-printer": "1.11.1" } }, "node_modules/@webassemblyjs/wasm-gen": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.0.tgz", - "integrity": "sha512-BEUv1aj0WptCZ9kIS30th5ILASUnAPEvE3tVMTrItnZRT9tXCLW2LEXT8ezLw59rqPP9klh9LPmpU+WmRQmCPQ==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.1.tgz", + "integrity": "sha512-F7QqKXwwNlMmsulj6+O7r4mmtAlCWfO/0HdgOxSklZfQcDu0TpLiD1mRt/zF25Bk59FIjEuGAIyn5ei4yMfLhA==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/ieee754": "1.11.0", - "@webassemblyjs/leb128": "1.11.0", - "@webassemblyjs/utf8": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/ieee754": "1.11.1", + "@webassemblyjs/leb128": "1.11.1", + "@webassemblyjs/utf8": "1.11.1" } }, "node_modules/@webassemblyjs/wasm-opt": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.0.tgz", - "integrity": "sha512-tHUSP5F4ywyh3hZ0+fDQuWxKx3mJiPeFufg+9gwTpYp324mPCQgnuVKwzLTZVqj0duRDovnPaZqDwoyhIO8kYg==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz", + "integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1" } }, "node_modules/@webassemblyjs/wasm-parser": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.0.tgz", - "integrity": "sha512-6L285Sgu9gphrcpDXINvm0M9BskznnzJTE7gYkjDbxET28shDqp27wpruyx3C2S/dvEwiigBwLA1cz7lNUi0kw==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.1.tgz", + "integrity": "sha512-rrBujw+dJu32gYB7/Lup6UhdkPx9S9SnobZzRVL7VcBH9Bt9bCBLEuX/YXOOtBsOZ4NQrRykKhffRWHvigQvOA==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-api-error": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/ieee754": "1.11.0", - "@webassemblyjs/leb128": "1.11.0", - "@webassemblyjs/utf8": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-api-error": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/ieee754": "1.11.1", + "@webassemblyjs/leb128": "1.11.1", + "@webassemblyjs/utf8": "1.11.1" } }, "node_modules/@webassemblyjs/wast-printer": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.0.tgz", - "integrity": "sha512-Fg5OX46pRdTgB7rKIUojkh9vXaVN6sGYCnEiJN1GYkb0RPwShZXp6KTDqmoMdQPKhcroOXh3fEzmkWmCYaKYhQ==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.1.tgz", + "integrity": "sha512-IQboUWM4eKzWW+N/jij2sRatKMh99QEelo3Eb2q0qXkvPRISAj8Qxtmw5itwqK+TTkBuUIE45AxYPToqPtL5gg==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", + "@webassemblyjs/ast": "1.11.1", "@xtuc/long": "4.2.2" } }, @@ -1834,9 +1834,9 @@ "dev": true }, "node_modules/acorn": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.3.0.tgz", - "integrity": "sha512-tqPKHZ5CaBJw0Xmy0ZZvLs1qTV+BNFSyvn77ASXkpBNfIRk8ev26fKrD9iLGwGA9zedPao52GSHzq8lyZG0NUw==", + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.4.1.tgz", + "integrity": "sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA==", "dev": true, "bin": { "acorn": "bin/acorn" @@ -3753,9 +3753,9 @@ } }, "node_modules/es-module-lexer": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.6.0.tgz", - "integrity": "sha512-f8kcHX1ArhllUtb/wVSyvygoKCznIjnxhLxy7TCvIiMdT7fL4ZDTIKaadMe6eLvOXg6Wk02UeoFgUoZ2EKZZUA==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.7.1.tgz", + "integrity": "sha512-MgtWFl5No+4S3TmhDmCz2ObFGm6lEpTnzbQi+Dd+pw4mlTIZTmM2iAs5gRlmx5zS9luzobCSBSI90JM/1/JgOw==", "dev": true }, "node_modules/es-to-primitive": { @@ -13879,14 +13879,13 @@ } }, "node_modules/typedoc": { - "version": "0.21.2", - "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.21.2.tgz", - "integrity": "sha512-SR1ByJB3USg+jxoxwzMRP07g/0f/cQUE5t7gOh1iTUyjTPyJohu9YSKRlK+MSXXqlhIq+m0jkEHEG5HoY7/Adg==", + "version": "0.21.4", + "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.21.4.tgz", + "integrity": "sha512-slZQhvD9U0d9KacktYAyuNMMOXJRFNHy+Gd8xY2Qrqq3eTTTv3frv3N4au/cFnab9t3T5WA0Orb6QUjMc+1bDA==", "dev": true, "dependencies": { "glob": "^7.1.7", "handlebars": "^4.7.7", - "lodash": "^4.17.21", "lunr": "^2.3.9", "marked": "^2.1.1", "minimatch": "^3.0.0", @@ -14030,9 +14029,9 @@ } }, "node_modules/typescript": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz", - "integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew==", + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.5.tgz", + "integrity": "sha512-DqQgihaQ9cUrskJo9kIyW/+g0Vxsk8cDtZ52a3NGh0YNTfpUSArXSohyUGnvbPazEPLu398C0UxmKSOrPumUzA==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -14342,21 +14341,21 @@ } }, "node_modules/webpack": { - "version": "5.41.1", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.41.1.tgz", - "integrity": "sha512-AJZIIsqJ/MVTmegEq9Tlw5mk5EHdGiJbDdz9qP15vmUH+oxI1FdWcL0E9EO8K/zKaRPWqEs7G/OPxq1P61u5Ug==", + "version": "5.44.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.44.0.tgz", + "integrity": "sha512-I1S1w4QLoKmH19pX6YhYN0NiSXaWY8Ou00oA+aMcr9IUGeF5azns+IKBkfoAAG9Bu5zOIzZt/mN35OffBya8AQ==", "dev": true, "dependencies": { "@types/eslint-scope": "^3.7.0", - "@types/estree": "^0.0.48", - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/wasm-edit": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0", - "acorn": "^8.2.1", + "@types/estree": "^0.0.50", + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/wasm-edit": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1", + "acorn": "^8.4.1", "browserslist": "^4.14.5", "chrome-trace-event": "^1.0.2", "enhanced-resolve": "^5.8.0", - "es-module-lexer": "^0.6.0", + "es-module-lexer": "^0.7.1", "eslint-scope": "5.1.1", "events": "^3.2.0", "glob-to-regexp": "^0.4.1", @@ -15935,9 +15934,9 @@ } }, "@types/estree": { - "version": "0.0.48", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.48.tgz", - "integrity": "sha512-LfZwXoGUDo0C3me81HXgkBg5CTQYb6xzEl+fNmbO4JdRiSKQ8A0GD1OBBvKAIsbCUgoyAty7m99GqqMQe784ew==", + "version": "0.0.50", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.50.tgz", + "integrity": "sha512-C6N5s2ZFtuZRj54k2/zyRhNDjJwwcViAM3Nbm8zjBpbqAdZ00mr0CFxvSKeO8Y/e03WVFLpQMdHYVfUd6SB+Hw==", "dev": true }, "@types/form-data": { @@ -15983,9 +15982,9 @@ } }, "@types/jest": { - "version": "26.0.23", - "resolved": "https://registry.npmjs.org/@types/jest/-/jest-26.0.23.tgz", - "integrity": "sha512-ZHLmWMJ9jJ9PTiT58juykZpL7KjwJywFN3Rr2pTSkyQfydf/rk22yS7W8p5DaVUMQ2BQC7oYiU3FjbTM/mYrOA==", + "version": "26.0.24", + "resolved": "https://registry.npmjs.org/@types/jest/-/jest-26.0.24.tgz", + "integrity": "sha512-E/X5Vib8BWqZNRlDxj9vYXhsDwPYbPINqKF9BsnSoon4RQ0D9moEuLD8txgyypFLH7J4+Lho9Nr/c8H0Fi+17w==", "dev": true, "requires": { "jest-diff": "^26.0.0", @@ -16054,9 +16053,9 @@ "dev": true }, "@types/tough-cookie": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.0.tgz", - "integrity": "sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.1.tgz", + "integrity": "sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==", "dev": true }, "@types/yargs": { @@ -16075,148 +16074,148 @@ "dev": true }, "@webassemblyjs/ast": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.0.tgz", - "integrity": "sha512-kX2W49LWsbthrmIRMbQZuQDhGtjyqXfEmmHyEi4XWnSZtPmxY0+3anPIzsnRb45VH/J55zlOfWvZuY47aJZTJg==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.1.tgz", + "integrity": "sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw==", "dev": true, "requires": { - "@webassemblyjs/helper-numbers": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0" + "@webassemblyjs/helper-numbers": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1" } }, "@webassemblyjs/floating-point-hex-parser": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.0.tgz", - "integrity": "sha512-Q/aVYs/VnPDVYvsCBL/gSgwmfjeCb4LW8+TMrO3cSzJImgv8lxxEPM2JA5jMrivE7LSz3V+PFqtMbls3m1exDA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.1.tgz", + "integrity": "sha512-iGRfyc5Bq+NnNuX8b5hwBrRjzf0ocrJPI6GWFodBFzmFnyvrQ83SHKhmilCU/8Jv67i4GJZBMhEzltxzcNagtQ==", "dev": true }, "@webassemblyjs/helper-api-error": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.0.tgz", - "integrity": "sha512-baT/va95eXiXb2QflSx95QGT5ClzWpGaa8L7JnJbgzoYeaA27FCvuBXU758l+KXWRndEmUXjP0Q5fibhavIn8w==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.1.tgz", + "integrity": "sha512-RlhS8CBCXfRUR/cwo2ho9bkheSXG0+NwooXcc3PAILALf2QLdFyj7KGsKRbVc95hZnhnERon4kW/D3SZpp6Tcg==", "dev": true }, "@webassemblyjs/helper-buffer": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.0.tgz", - "integrity": "sha512-u9HPBEl4DS+vA8qLQdEQ6N/eJQ7gT7aNvMIo8AAWvAl/xMrcOSiI2M0MAnMCy3jIFke7bEee/JwdX1nUpCtdyA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.1.tgz", + "integrity": "sha512-gwikF65aDNeeXa8JxXa2BAk+REjSyhrNC9ZwdT0f8jc4dQQeDQ7G4m0f2QCLPJiMTTO6wfDmRmj/pW0PsUvIcA==", "dev": true }, "@webassemblyjs/helper-numbers": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.0.tgz", - "integrity": "sha512-DhRQKelIj01s5IgdsOJMKLppI+4zpmcMQ3XboFPLwCpSNH6Hqo1ritgHgD0nqHeSYqofA6aBN/NmXuGjM1jEfQ==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.1.tgz", + "integrity": "sha512-vDkbxiB8zfnPdNK9Rajcey5C0w+QJugEglN0of+kmO8l7lDb77AnlKYQF7aarZuCrv+l0UvqL+68gSDr3k9LPQ==", "dev": true, "requires": { - "@webassemblyjs/floating-point-hex-parser": "1.11.0", - "@webassemblyjs/helper-api-error": "1.11.0", + "@webassemblyjs/floating-point-hex-parser": "1.11.1", + "@webassemblyjs/helper-api-error": "1.11.1", "@xtuc/long": "4.2.2" } }, "@webassemblyjs/helper-wasm-bytecode": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.0.tgz", - "integrity": "sha512-MbmhvxXExm542tWREgSFnOVo07fDpsBJg3sIl6fSp9xuu75eGz5lz31q7wTLffwL3Za7XNRCMZy210+tnsUSEA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.1.tgz", + "integrity": "sha512-PvpoOGiJwXeTrSf/qfudJhwlvDQxFgelbMqtq52WWiXC6Xgg1IREdngmPN3bs4RoO83PnL/nFrxucXj1+BX62Q==", "dev": true }, "@webassemblyjs/helper-wasm-section": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.0.tgz", - "integrity": "sha512-3Eb88hcbfY/FCukrg6i3EH8H2UsD7x8Vy47iVJrP967A9JGqgBVL9aH71SETPx1JrGsOUVLo0c7vMCN22ytJew==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.1.tgz", + "integrity": "sha512-10P9No29rYX1j7F3EVPX3JvGPQPae+AomuSTPiF9eBQeChHI6iqjMIwR9JmOJXwpnn/oVGDk7I5IlskuMwU/pg==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1" } }, "@webassemblyjs/ieee754": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.0.tgz", - "integrity": "sha512-KXzOqpcYQwAfeQ6WbF6HXo+0udBNmw0iXDmEK5sFlmQdmND+tr773Ti8/5T/M6Tl/413ArSJErATd8In3B+WBA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.1.tgz", + "integrity": "sha512-hJ87QIPtAMKbFq6CGTkZYJivEwZDbQUgYd3qKSadTNOhVY7p+gfP6Sr0lLRVTaG1JjFj+r3YchoqRYxNH3M0GQ==", "dev": true, "requires": { "@xtuc/ieee754": "^1.2.0" } }, "@webassemblyjs/leb128": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.0.tgz", - "integrity": "sha512-aqbsHa1mSQAbeeNcl38un6qVY++hh8OpCOzxhixSYgbRfNWcxJNJQwe2rezK9XEcssJbbWIkblaJRwGMS9zp+g==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.1.tgz", + "integrity": "sha512-BJ2P0hNZ0u+Th1YZXJpzW6miwqQUGcIHT1G/sf72gLVD9DZ5AdYTqPNbHZh6K1M5VmKvFXwGSWZADz+qBWxeRw==", "dev": true, "requires": { "@xtuc/long": "4.2.2" } }, "@webassemblyjs/utf8": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.0.tgz", - "integrity": "sha512-A/lclGxH6SpSLSyFowMzO/+aDEPU4hvEiooCMXQPcQFPPJaYcPQNKGOCLUySJsYJ4trbpr+Fs08n4jelkVTGVw==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.1.tgz", + "integrity": "sha512-9kqcxAEdMhiwQkHpkNiorZzqpGrodQQ2IGrHHxCy+Ozng0ofyMA0lTqiLkVs1uzTRejX+/O0EOT7KxqVPuXosQ==", "dev": true }, "@webassemblyjs/wasm-edit": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.0.tgz", - "integrity": "sha512-JHQ0damXy0G6J9ucyKVXO2j08JVJ2ntkdJlq1UTiUrIgfGMmA7Ik5VdC/L8hBK46kVJgujkBIoMtT8yVr+yVOQ==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.1.tgz", + "integrity": "sha512-g+RsupUC1aTHfR8CDgnsVRVZFJqdkFHpsHMfJuWQzWU3tvnLC07UqHICfP+4XyL2tnr1amvl1Sdp06TnYCmVkA==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/helper-wasm-section": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0", - "@webassemblyjs/wasm-opt": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0", - "@webassemblyjs/wast-printer": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/helper-wasm-section": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1", + "@webassemblyjs/wasm-opt": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1", + "@webassemblyjs/wast-printer": "1.11.1" } }, "@webassemblyjs/wasm-gen": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.0.tgz", - "integrity": "sha512-BEUv1aj0WptCZ9kIS30th5ILASUnAPEvE3tVMTrItnZRT9tXCLW2LEXT8ezLw59rqPP9klh9LPmpU+WmRQmCPQ==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.1.tgz", + "integrity": "sha512-F7QqKXwwNlMmsulj6+O7r4mmtAlCWfO/0HdgOxSklZfQcDu0TpLiD1mRt/zF25Bk59FIjEuGAIyn5ei4yMfLhA==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/ieee754": "1.11.0", - "@webassemblyjs/leb128": "1.11.0", - "@webassemblyjs/utf8": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/ieee754": "1.11.1", + "@webassemblyjs/leb128": "1.11.1", + "@webassemblyjs/utf8": "1.11.1" } }, "@webassemblyjs/wasm-opt": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.0.tgz", - "integrity": "sha512-tHUSP5F4ywyh3hZ0+fDQuWxKx3mJiPeFufg+9gwTpYp324mPCQgnuVKwzLTZVqj0duRDovnPaZqDwoyhIO8kYg==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz", + "integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1" } }, "@webassemblyjs/wasm-parser": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.0.tgz", - "integrity": "sha512-6L285Sgu9gphrcpDXINvm0M9BskznnzJTE7gYkjDbxET28shDqp27wpruyx3C2S/dvEwiigBwLA1cz7lNUi0kw==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.1.tgz", + "integrity": "sha512-rrBujw+dJu32gYB7/Lup6UhdkPx9S9SnobZzRVL7VcBH9Bt9bCBLEuX/YXOOtBsOZ4NQrRykKhffRWHvigQvOA==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-api-error": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/ieee754": "1.11.0", - "@webassemblyjs/leb128": "1.11.0", - "@webassemblyjs/utf8": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-api-error": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/ieee754": "1.11.1", + "@webassemblyjs/leb128": "1.11.1", + "@webassemblyjs/utf8": "1.11.1" } }, "@webassemblyjs/wast-printer": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.0.tgz", - "integrity": "sha512-Fg5OX46pRdTgB7rKIUojkh9vXaVN6sGYCnEiJN1GYkb0RPwShZXp6KTDqmoMdQPKhcroOXh3fEzmkWmCYaKYhQ==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.1.tgz", + "integrity": "sha512-IQboUWM4eKzWW+N/jij2sRatKMh99QEelo3Eb2q0qXkvPRISAj8Qxtmw5itwqK+TTkBuUIE45AxYPToqPtL5gg==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", + "@webassemblyjs/ast": "1.11.1", "@xtuc/long": "4.2.2" } }, @@ -16260,9 +16259,9 @@ "dev": true }, "acorn": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.3.0.tgz", - "integrity": "sha512-tqPKHZ5CaBJw0Xmy0ZZvLs1qTV+BNFSyvn77ASXkpBNfIRk8ev26fKrD9iLGwGA9zedPao52GSHzq8lyZG0NUw==", + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.4.1.tgz", + "integrity": "sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA==", "dev": true }, "acorn-globals": { @@ -17837,9 +17836,9 @@ } }, "es-module-lexer": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.6.0.tgz", - "integrity": "sha512-f8kcHX1ArhllUtb/wVSyvygoKCznIjnxhLxy7TCvIiMdT7fL4ZDTIKaadMe6eLvOXg6Wk02UeoFgUoZ2EKZZUA==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.7.1.tgz", + "integrity": "sha512-MgtWFl5No+4S3TmhDmCz2ObFGm6lEpTnzbQi+Dd+pw4mlTIZTmM2iAs5gRlmx5zS9luzobCSBSI90JM/1/JgOw==", "dev": true }, "es-to-primitive": { @@ -25588,14 +25587,13 @@ } }, "typedoc": { - "version": "0.21.2", - "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.21.2.tgz", - "integrity": "sha512-SR1ByJB3USg+jxoxwzMRP07g/0f/cQUE5t7gOh1iTUyjTPyJohu9YSKRlK+MSXXqlhIq+m0jkEHEG5HoY7/Adg==", + "version": "0.21.4", + "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.21.4.tgz", + "integrity": "sha512-slZQhvD9U0d9KacktYAyuNMMOXJRFNHy+Gd8xY2Qrqq3eTTTv3frv3N4au/cFnab9t3T5WA0Orb6QUjMc+1bDA==", "dev": true, "requires": { "glob": "^7.1.7", "handlebars": "^4.7.7", - "lodash": "^4.17.21", "lunr": "^2.3.9", "marked": "^2.1.1", "minimatch": "^3.0.0", @@ -25694,9 +25692,9 @@ } }, "typescript": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz", - "integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew==", + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.5.tgz", + "integrity": "sha512-DqQgihaQ9cUrskJo9kIyW/+g0Vxsk8cDtZ52a3NGh0YNTfpUSArXSohyUGnvbPazEPLu398C0UxmKSOrPumUzA==", "dev": true }, "uglify-js": { @@ -25958,21 +25956,21 @@ "dev": true }, "webpack": { - "version": "5.41.1", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.41.1.tgz", - "integrity": "sha512-AJZIIsqJ/MVTmegEq9Tlw5mk5EHdGiJbDdz9qP15vmUH+oxI1FdWcL0E9EO8K/zKaRPWqEs7G/OPxq1P61u5Ug==", + "version": "5.44.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.44.0.tgz", + "integrity": "sha512-I1S1w4QLoKmH19pX6YhYN0NiSXaWY8Ou00oA+aMcr9IUGeF5azns+IKBkfoAAG9Bu5zOIzZt/mN35OffBya8AQ==", "dev": true, "requires": { "@types/eslint-scope": "^3.7.0", - "@types/estree": "^0.0.48", - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/wasm-edit": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0", - "acorn": "^8.2.1", + "@types/estree": "^0.0.50", + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/wasm-edit": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1", + "acorn": "^8.4.1", "browserslist": "^4.14.5", "chrome-trace-event": "^1.0.2", "enhanced-resolve": "^5.8.0", - "es-module-lexer": "^0.6.0", + "es-module-lexer": "^0.7.1", "eslint-scope": "5.1.1", "events": "^3.2.0", "glob-to-regexp": "^0.4.1", diff --git a/package.json b/package.json index 5e217df..95e7deb 100644 --- a/package.json +++ b/package.json @@ -40,9 +40,9 @@ "devDependencies": { "@types/axios": "^0.14.0", "@types/form-data": "^2.5.0", - "@types/jest": "^26.0.23", + "@types/jest": "^26.0.24", "@types/mime": "^2.0.3", - "@types/tough-cookie": "^4.0.0", + "@types/tough-cookie": "^4.0.1", "copyfiles": "^2.4.1", "cp": "^0.2.0", "dotenv": "^10.0.0", @@ -58,11 +58,11 @@ "ts-loader": "^9.2.2", "tslint": "^6.1.3", "tslint-config-prettier": "^1.18.0", - "typedoc": "^0.21.2", + "typedoc": "^0.21.4", "typedoc-neo-theme": "^1.1.1", "typedoc-plugin-external-module-name": "^4.0.6", - "typescript": "^4.3.4", - "webpack": "^5.41.1", + "typescript": "^4.3.5", + "webpack": "^5.44.0", "webpack-cli": "^4.7.2" }, "main": "index.js", From 5c8d311ae887daa71f3735fa6202bee117640159 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Tue, 20 Jul 2021 09:25:39 +0100 Subject: [PATCH 12/15] chore(streamlog): optimise polling mechanism --- package-lock.json | 188 ++++++++++++++- package.json | 2 +- src/SASViyaApiClient.ts | 16 +- src/api/viya/executeScript.ts | 1 - src/api/viya/pollJobState.ts | 307 ++++++++++++++++-------- src/api/viya/saveLog.ts | 28 ++- src/api/viya/spec/executeScript.spec.ts | 9 +- src/api/viya/spec/mockResponses.ts | 17 ++ src/api/viya/spec/pollJobState.spec.ts | 193 +++++++++------ src/api/viya/spec/saveLog.spec.ts | 40 ++- src/api/viya/writeStream.ts | 15 ++ src/types/errors/JobStatePollError.ts | 11 + src/types/errors/index.ts | 1 + src/utils/fetchLogByChunks.ts | 26 +- 14 files changed, 624 insertions(+), 230 deletions(-) create mode 100644 src/api/viya/writeStream.ts create mode 100644 src/types/errors/JobStatePollError.ts diff --git a/package-lock.json b/package-lock.json index 928b41e..b62d148 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,7 @@ "name": "@sasjs/adapter", "license": "ISC", "dependencies": { - "@sasjs/utils": "^2.24.0", + "@sasjs/utils": "^2.25.1", "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", "form-data": "^4.0.0", @@ -1188,8 +1188,9 @@ } }, "node_modules/@sasjs/utils": { - "version": "2.24.0", - "license": "ISC", + "version": "2.25.1", + "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.25.1.tgz", + "integrity": "sha512-lwkPE+QsB81b8/1M8X2zLdhpuiA8pIjgOwJH57zhcsliuDnNs4uijSYu40aYSc8tH98jtSuqWMjfGq8CT9o1Dw==", "dependencies": { "@types/prompts": "^2.0.13", "chalk": "^4.1.1", @@ -7932,7 +7933,182 @@ "treeverse", "validate-npm-package-name", "which", - "write-file-atomic" + "write-file-atomic", + "@npmcli/disparity-colors", + "@npmcli/git", + "@npmcli/installed-package-contents", + "@npmcli/map-workspaces", + "@npmcli/metavuln-calculator", + "@npmcli/move-file", + "@npmcli/name-from-folder", + "@npmcli/node-gyp", + "@npmcli/promise-spawn", + "@tootallnate/once", + "agent-base", + "agentkeepalive", + "aggregate-error", + "ajv", + "ansi-regex", + "ansi-styles", + "aproba", + "are-we-there-yet", + "asap", + "asn1", + "assert-plus", + "asynckit", + "aws-sign2", + "aws4", + "balanced-match", + "bcrypt-pbkdf", + "bin-links", + "binary-extensions", + "brace-expansion", + "builtins", + "caseless", + "cidr-regex", + "clean-stack", + "clone", + "cmd-shim", + "code-point-at", + "color-convert", + "color-name", + "colors", + "combined-stream", + "common-ancestor-path", + "concat-map", + "console-control-strings", + "core-util-is", + "dashdash", + "debug", + "debuglog", + "defaults", + "delayed-stream", + "delegates", + "depd", + "dezalgo", + "diff", + "ecc-jsbn", + "emoji-regex", + "encoding", + "env-paths", + "err-code", + "extend", + "extsprintf", + "fast-deep-equal", + "fast-json-stable-stringify", + "forever-agent", + "fs-minipass", + "fs.realpath", + "function-bind", + "gauge", + "getpass", + "har-schema", + "har-validator", + "has", + "has-flag", + "has-unicode", + "http-cache-semantics", + "http-proxy-agent", + "http-signature", + "https-proxy-agent", + "humanize-ms", + "iconv-lite", + "ignore-walk", + "imurmurhash", + "indent-string", + "infer-owner", + "inflight", + "inherits", + "ip", + "ip-regex", + "is-core-module", + "is-fullwidth-code-point", + "is-lambda", + "is-typedarray", + "isarray", + "isexe", + "isstream", + "jsbn", + "json-schema", + "json-schema-traverse", + "json-stringify-nice", + "json-stringify-safe", + "jsonparse", + "jsprim", + "just-diff", + "just-diff-apply", + "lru-cache", + "mime-db", + "mime-types", + "minimatch", + "minipass-collect", + "minipass-fetch", + "minipass-flush", + "minipass-json-stream", + "minipass-sized", + "minizlib", + "mute-stream", + "negotiator", + "normalize-package-data", + "npm-bundled", + "npm-install-checks", + "npm-normalize-package-bin", + "npm-packlist", + "number-is-nan", + "oauth-sign", + "object-assign", + "once", + "p-map", + "path-is-absolute", + "path-parse", + "performance-now", + "proc-log", + "process-nextick-args", + "promise-all-reject-late", + "promise-call-limit", + "promise-inflight", + "promise-retry", + "promzard", + "psl", + "punycode", + "qs", + "read-cmd-shim", + "readable-stream", + "request", + "resolve", + "retry", + "safe-buffer", + "safer-buffer", + "set-blocking", + "signal-exit", + "smart-buffer", + "socks", + "socks-proxy-agent", + "spdx-correct", + "spdx-exceptions", + "spdx-expression-parse", + "spdx-license-ids", + "sshpk", + "string_decoder", + "string-width", + "stringify-package", + "strip-ansi", + "supports-color", + "tunnel-agent", + "tweetnacl", + "typedarray-to-buffer", + "unique-filename", + "unique-slug", + "uri-js", + "util-deprecate", + "uuid", + "validate-npm-package-license", + "verror", + "walk-up-path", + "wcwidth", + "wide-align", + "wrappy", + "yallist" ], "dev": true, "license": "Artistic-2.0", @@ -14604,7 +14780,9 @@ } }, "@sasjs/utils": { - "version": "2.24.0", + "version": "2.25.1", + "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.25.1.tgz", + "integrity": "sha512-lwkPE+QsB81b8/1M8X2zLdhpuiA8pIjgOwJH57zhcsliuDnNs4uijSYu40aYSc8tH98jtSuqWMjfGq8CT9o1Dw==", "requires": { "@types/prompts": "^2.0.13", "chalk": "^4.1.1", diff --git a/package.json b/package.json index 95e7deb..332b564 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ }, "main": "index.js", "dependencies": { - "@sasjs/utils": "^2.24.0", + "@sasjs/utils": "^2.25.1", "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", "form-data": "^4.0.0", diff --git a/src/SASViyaApiClient.ts b/src/SASViyaApiClient.ts index ee439f2..e68082c 100644 --- a/src/SASViyaApiClient.ts +++ b/src/SASViyaApiClient.ts @@ -754,18 +754,16 @@ export class SASViyaApiClient { jobDefinition, arguments: jobArguments } - const { result: postedJob, etag } = await this.requestClient.post( + const { result: postedJob } = await this.requestClient.post( `${this.serverUrl}/jobExecution/jobs?_action=wait`, postJobRequestBody, access_token ) - const jobStatus = await this.pollJobState( - postedJob, - etag, - authConfig - ).catch((err) => { - throw prefixMessage(err, 'Error while polling job status. ') - }) + const jobStatus = await this.pollJobState(postedJob, authConfig).catch( + (err) => { + throw prefixMessage(err, 'Error while polling job status. ') + } + ) const { result: currentJob } = await this.requestClient.get( `${this.serverUrl}/jobExecution/jobs/${postedJob.id}`, access_token @@ -833,7 +831,6 @@ export class SASViyaApiClient { private async pollJobState( postedJob: Job, - etag: string | null, authConfig?: AuthConfig, pollOptions?: PollOptions ) { @@ -841,7 +838,6 @@ export class SASViyaApiClient { this.requestClient, postedJob, this.debug, - etag, authConfig, pollOptions ) diff --git a/src/api/viya/executeScript.ts b/src/api/viya/executeScript.ts index c75a3e5..e54143f 100644 --- a/src/api/viya/executeScript.ts +++ b/src/api/viya/executeScript.ts @@ -178,7 +178,6 @@ export async function executeScript( requestClient, postedJob, debug, - etag, authConfig, pollOptions ).catch(async (err) => { diff --git a/src/api/viya/pollJobState.ts b/src/api/viya/pollJobState.ts index 5e74dda..b777382 100644 --- a/src/api/viya/pollJobState.ts +++ b/src/api/viya/pollJobState.ts @@ -1,16 +1,19 @@ import { AuthConfig } from '@sasjs/utils/types' -import { prefixMessage } from '@sasjs/utils/error' -import { generateTimestamp } from '@sasjs/utils/time' import { Job, PollOptions } from '../..' import { getTokens } from '../../auth/getTokens' import { RequestClient } from '../../request/RequestClient' +import { JobStatePollError } from '../../types/errors' +import { generateTimestamp } from '@sasjs/utils/time' import { saveLog } from './saveLog' +import { createWriteStream } from '@sasjs/utils/file' +import { WriteStream } from 'fs' +import { Link } from '../../types' +import { prefixMessage } from '@sasjs/utils/error' export async function pollJobState( requestClient: RequestClient, postedJob: Job, debug: boolean, - etag: string | null, authConfig?: AuthConfig, pollOptions?: PollOptions ) { @@ -18,130 +21,226 @@ export async function pollJobState( let pollInterval = 300 let maxPollCount = 1000 - let maxErrorCount = 5 - let access_token = (authConfig || {}).access_token - - const logFileName = `${postedJob.name || 'job'}-${generateTimestamp()}.log` - const logFilePath = `${ - pollOptions?.logFolderPath || process.cwd() - }/${logFileName}` - - if (authConfig) { - ;({ access_token } = await getTokens(requestClient, authConfig)) - } if (pollOptions) { pollInterval = pollOptions.pollInterval || pollInterval maxPollCount = pollOptions.maxPollCount || maxPollCount } - let postedJobState = '' - let pollCount = 0 - let errorCount = 0 - const headers: any = { - 'Content-Type': 'application/json', - 'If-None-Match': etag - } - if (access_token) { - headers.Authorization = `Bearer ${access_token}` - } const stateLink = postedJob.links.find((l: any) => l.rel === 'state') if (!stateLink) { throw new Error(`Job state link was not found.`) } - const { result: state } = await requestClient - .get( - `${stateLink.href}?_action=wait&wait=300`, - access_token, - 'text/plain', - {}, - debug + let currentState = await getJobState( + requestClient, + postedJob, + '', + debug, + authConfig + ).catch((err) => { + logger.error( + `Error fetching job state from ${stateLink.href}. Starting poll, assuming job to be running.`, + err ) - .catch((err) => { - logger.error( - `Error fetching job state from ${stateLink.href}. Starting poll, assuming job to be running.`, - err - ) - return { result: 'unavailable' } - }) + return 'unavailable' + }) + + let pollCount = 0 - const currentState = state.trim() if (currentState === 'completed') { return Promise.resolve(currentState) } - return new Promise(async (resolve, reject) => { - let printedState = '' + let logFileStream + if (pollOptions?.streamLog) { + const logFileName = `${postedJob.name || 'job'}-${generateTimestamp()}.log` + const logFilePath = `${ + pollOptions?.logFolderPath || process.cwd() + }/${logFileName}` - const interval = setInterval(async () => { - if ( - postedJobState === 'running' || - postedJobState === '' || - postedJobState === 'pending' || - postedJobState === 'unavailable' - ) { - if (authConfig) { - ;({ access_token } = await getTokens(requestClient, authConfig)) - } + logFileStream = await createWriteStream(logFilePath) + } - if (stateLink) { - const { result: jobState } = await requestClient - .get( - `${stateLink.href}?_action=wait&wait=300`, - access_token, - 'text/plain', - {}, - debug - ) - .catch((err) => { - errorCount++ - if (pollCount >= maxPollCount || errorCount >= maxErrorCount) { - clearInterval(interval) - reject( - prefixMessage( - err, - 'Error while getting job state after interval. ' - ) - ) - } - logger.error( - `Error fetching job state from ${stateLink.href}. Resuming poll, assuming job to be running.`, - err - ) - return { result: 'unavailable' } - }) + let result = await doPoll( + requestClient, + postedJob, + currentState, + debug, + pollCount, + authConfig, + pollOptions, + logFileStream + ) - postedJobState = jobState.trim() - if (postedJobState != 'unavailable' && errorCount > 0) { - errorCount = 0 - } + currentState = result.state + pollCount = result.pollCount - if (debug && printedState !== postedJobState) { - logger.info('Polling job status...') - logger.info(`Current job state: ${postedJobState}`) + if (!needsRetry(currentState) || pollCount >= maxPollCount) { + return currentState + } - printedState = postedJobState - } + // If we get to this point, this is a long-running job that needs longer polling. + // We will resume polling with a bigger interval of 1 minute + let longJobPollOptions: PollOptions = { + maxPollCount: 24 * 60, + pollInterval: 60000, + streamLog: false + } + if (pollOptions) { + longJobPollOptions.streamLog = pollOptions.streamLog + longJobPollOptions.logFolderPath = pollOptions.logFolderPath + } - pollCount++ + result = await doPoll( + requestClient, + postedJob, + currentState, + debug, + pollCount, + authConfig, + longJobPollOptions, + logFileStream + ) - await saveLog( - postedJob, - requestClient, - pollOptions?.streamLog || false, - logFilePath, - access_token - ) + currentState = result.state + pollCount = result.pollCount - if (pollCount >= maxPollCount) { - resolve(postedJobState) - } - } - } else { - clearInterval(interval) - resolve(postedJobState) - } - }, pollInterval) - }) + if (logFileStream) { + logFileStream.end() + } + + return currentState } + +const getJobState = async ( + requestClient: RequestClient, + job: Job, + currentState: string, + debug: boolean, + authConfig?: AuthConfig +) => { + const stateLink = job.links.find((l: any) => l.rel === 'state') + if (!stateLink) { + throw new Error(`Job state link was not found.`) + } + + if (needsRetry(currentState)) { + let tokens + if (authConfig) { + tokens = await getTokens(requestClient, authConfig) + } + + const { result: jobState } = await requestClient + .get( + `${stateLink.href}?_action=wait&wait=300`, + tokens?.access_token, + 'text/plain', + {}, + debug + ) + .catch((err) => { + throw new JobStatePollError(job.id, err) + }) + + return jobState.trim() + } else { + return currentState + } +} + +const needsRetry = (state: string) => + state === 'running' || + state === '' || + state === 'pending' || + state === 'unavailable' + +const doPoll = async ( + requestClient: RequestClient, + postedJob: Job, + currentState: string, + debug: boolean, + pollCount: number, + authConfig?: AuthConfig, + pollOptions?: PollOptions, + logStream?: WriteStream +): Promise<{ state: string; pollCount: number }> => { + let pollInterval = 300 + let maxPollCount = 1000 + let maxErrorCount = 5 + let errorCount = 0 + let state = currentState + let printedState = '' + let startLogLine = 0 + + const logger = process.logger || console + + if (pollOptions) { + pollInterval = pollOptions.pollInterval || pollInterval + maxPollCount = pollOptions.maxPollCount || maxPollCount + } + + const stateLink = postedJob.links.find((l: Link) => l.rel === 'state') + if (!stateLink) { + throw new Error(`Job state link was not found.`) + } + + while (needsRetry(state) && pollCount <= 100 && pollCount <= maxPollCount) { + state = await getJobState( + requestClient, + postedJob, + state, + debug, + authConfig + ).catch((err) => { + errorCount++ + if (pollCount >= maxPollCount || errorCount >= maxErrorCount) { + throw err + } + logger.error( + `Error fetching job state from ${stateLink.href}. Resuming poll, assuming job to be running.`, + err + ) + return 'unavailable' + }) + + pollCount++ + + const jobUrl = postedJob.links.find((l: Link) => l.rel === 'self') + const { result: job } = await requestClient.get( + jobUrl!.href, + authConfig?.access_token + ) + + const endLogLine = job.logStatistics?.lineCount ?? 1000000 + + await saveLog( + postedJob, + requestClient, + pollOptions?.streamLog || false, + startLogLine, + endLogLine, + logStream, + authConfig?.access_token + ) + + startLogLine += job.logStatistics.lineCount + + if (debug && printedState !== state) { + logger.info('Polling job status...') + logger.info(`Current job state: ${state}`) + + printedState = state + } + + if (state != 'unavailable' && errorCount > 0) { + errorCount = 0 + } + + await delay(pollInterval) + } + + return { state, pollCount } +} + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) diff --git a/src/api/viya/saveLog.ts b/src/api/viya/saveLog.ts index 9200930..94fbbfd 100644 --- a/src/api/viya/saveLog.ts +++ b/src/api/viya/saveLog.ts @@ -1,15 +1,19 @@ -import { createFile } from '@sasjs/utils/file' import { Job } from '../..' import { RequestClient } from '../../request/RequestClient' -import { fetchLogByChunks } from '../../utils' +import { fetchLog } from '../../utils' +import { WriteStream } from 'fs' +import { writeStream } from './writeStream' export async function saveLog( job: Job, requestClient: RequestClient, shouldSaveLog: boolean, - logFilePath: string, + startLine: number, + endLine: number, + logFileStream?: WriteStream, accessToken?: string ) { + console.log('startLine: ', startLine, ' endLine: ', endLine) if (!shouldSaveLog) { return } @@ -20,6 +24,12 @@ export async function saveLog( ) } + if (!logFileStream) { + throw new Error( + `Logs for job ${job.id} cannot be written without a valid write stream.` + ) + } + const logger = process.logger || console const jobLogUrl = job.links.find((l) => l.rel === 'log') @@ -27,14 +37,14 @@ export async function saveLog( throw new Error(`Log URL for job ${job.id} was not found.`) } - const logCount = job.logStatistics?.lineCount ?? 1000000 - const log = await fetchLogByChunks( + const log = await fetchLog( requestClient, accessToken, `${jobLogUrl.href}/content`, - logCount - ) + startLine, + endLine + ).catch((e) => console.log(e)) - logger.info(`Writing logs to ${logFilePath}`) - await createFile(logFilePath, log) + logger.info(`Writing logs to ${logFileStream.path}`) + await writeStream(logFileStream, log || '').catch((e) => console.log(e)) } diff --git a/src/api/viya/spec/executeScript.spec.ts b/src/api/viya/spec/executeScript.spec.ts index 314c70b..5e2fb11 100644 --- a/src/api/viya/spec/executeScript.spec.ts +++ b/src/api/viya/spec/executeScript.spec.ts @@ -344,7 +344,6 @@ describe('executeScript', () => { requestClient, mockJob, false, - '', mockAuthConfig, defaultPollOptions ) @@ -546,7 +545,7 @@ describe('executeScript', () => { if (url.includes('_webout')) { return Promise.reject(new NotFoundError(url)) } - return Promise.resolve({ result: mockJob, etag: '' }) + return Promise.resolve({ result: mockJob, etag: '', status: 200 }) }) const error = await executeScript( @@ -645,7 +644,9 @@ const setupMocks = () => { .mockImplementation(() => Promise.resolve({ result: mockJob, etag: '' })) jest .spyOn(requestClient, 'get') - .mockImplementation(() => Promise.resolve({ result: mockJob, etag: '' })) + .mockImplementation(() => + Promise.resolve({ result: mockJob, etag: '', status: 200 }) + ) jest .spyOn(requestClient, 'delete') .mockImplementation(() => Promise.resolve({ result: {}, etag: '' })) @@ -658,7 +659,7 @@ const setupMocks = () => { jest .spyOn(sessionManager, 'getVariable') .mockImplementation(() => - Promise.resolve({ result: { value: 'test' }, etag: 'test' }) + Promise.resolve({ result: { value: 'test' }, etag: 'test', status: 200 }) ) jest .spyOn(sessionManager, 'getSession') diff --git a/src/api/viya/spec/mockResponses.ts b/src/api/viya/spec/mockResponses.ts index e85028c..22580f7 100644 --- a/src/api/viya/spec/mockResponses.ts +++ b/src/api/viya/spec/mockResponses.ts @@ -31,6 +31,13 @@ export const mockJob: Job = { type: 'log', uri: 'log' }, + { + rel: 'self', + href: '/job', + method: 'GET', + type: 'job', + uri: 'job' + }, { rel: 'state', href: '/state', @@ -54,3 +61,13 @@ export const mockAuthConfig: AuthConfig = { access_token: 'acc355', refresh_token: 'r3fr35h' } + +export class MockStream { + _write(chunk: string, _: any, next: Function) { + next() + } + + reset() {} + + destroy() {} +} diff --git a/src/api/viya/spec/pollJobState.spec.ts b/src/api/viya/spec/pollJobState.spec.ts index 23ac560..dcbd2b9 100644 --- a/src/api/viya/spec/pollJobState.spec.ts +++ b/src/api/viya/spec/pollJobState.spec.ts @@ -1,10 +1,11 @@ +import * as fs from 'fs' +import { Logger, LogLevel } from '@sasjs/utils' import { RequestClient } from '../../../request/RequestClient' import { mockAuthConfig, mockJob } from './mockResponses' import { pollJobState } from '../pollJobState' import * as getTokensModule from '../../../auth/getTokens' import * as saveLogModule from '../saveLog' import { PollOptions } from '../../../types' -import { Logger, LogLevel } from '@sasjs/utils' const requestClient = new (>RequestClient)() const defaultPollOptions: PollOptions = { @@ -24,7 +25,6 @@ describe('pollJobState', () => { requestClient, mockJob, false, - 'test', mockAuthConfig, defaultPollOptions ) @@ -40,7 +40,6 @@ describe('pollJobState', () => { requestClient, mockJob, false, - 'test', undefined, defaultPollOptions ) @@ -53,7 +52,6 @@ describe('pollJobState', () => { requestClient, { ...mockJob, links: mockJob.links.filter((l) => l.rel !== 'state') }, false, - 'test', undefined, defaultPollOptions ).catch((e) => e) @@ -62,23 +60,12 @@ describe('pollJobState', () => { }) it('should attempt to refresh tokens before each poll', async () => { - jest - .spyOn(requestClient, 'get') - .mockImplementationOnce(() => - Promise.resolve({ result: 'pending', etag: '' }) - ) - .mockImplementationOnce(() => - Promise.resolve({ result: 'running', etag: '' }) - ) - .mockImplementation(() => - Promise.resolve({ result: 'completed', etag: '' }) - ) + mockSimplePoll() await pollJobState( requestClient, mockJob, false, - 'test', mockAuthConfig, defaultPollOptions ) @@ -87,23 +74,12 @@ describe('pollJobState', () => { }) it('should attempt to fetch and save the log after each poll', async () => { - jest - .spyOn(requestClient, 'get') - .mockImplementationOnce(() => - Promise.resolve({ result: 'pending', etag: '' }) - ) - .mockImplementationOnce(() => - Promise.resolve({ result: 'running', etag: '' }) - ) - .mockImplementation(() => - Promise.resolve({ result: 'completed', etag: '' }) - ) + mockSimplePoll() await pollJobState( requestClient, mockJob, false, - 'test', mockAuthConfig, defaultPollOptions ) @@ -112,20 +88,12 @@ describe('pollJobState', () => { }) it('should return the current status when the max poll count is reached', async () => { - jest - .spyOn(requestClient, 'get') - .mockImplementationOnce(() => - Promise.resolve({ result: 'pending', etag: '' }) - ) - .mockImplementationOnce(() => - Promise.resolve({ result: 'running', etag: '' }) - ) + mockRunningPoll() const state = await pollJobState( requestClient, mockJob, false, - 'test', mockAuthConfig, { ...defaultPollOptions, @@ -136,51 +104,47 @@ describe('pollJobState', () => { expect(state).toEqual('running') }) - it('should continue polling until the job completes or errors', async () => { - jest - .spyOn(requestClient, 'get') - .mockImplementationOnce(() => - Promise.resolve({ result: 'pending', etag: '' }) - ) - .mockImplementationOnce(() => - Promise.resolve({ result: 'running', etag: '' }) - ) - .mockImplementation(() => - Promise.resolve({ result: 'completed', etag: '' }) - ) + it('should poll with a larger interval for longer running jobs', async () => { + mockLongPoll() + + const state = await pollJobState( + requestClient, + mockJob, + false, + mockAuthConfig, + { + ...defaultPollOptions, + maxPollCount: 200, + pollInterval: 10 + } + ) + + expect(state).toEqual('completed') + }, 200000) + + it('should continue polling until the job completes or errors', async () => { + mockSimplePoll(1) const state = await pollJobState( requestClient, mockJob, false, - 'test', undefined, defaultPollOptions ) - expect(requestClient.get).toHaveBeenCalledTimes(4) + expect(requestClient.get).toHaveBeenCalledTimes(3) expect(state).toEqual('completed') }) it('should print the state to the console when debug is on', async () => { jest.spyOn((process as any).logger, 'info') - jest - .spyOn(requestClient, 'get') - .mockImplementationOnce(() => - Promise.resolve({ result: 'pending', etag: '' }) - ) - .mockImplementationOnce(() => - Promise.resolve({ result: 'running', etag: '' }) - ) - .mockImplementation(() => - Promise.resolve({ result: 'completed', etag: '' }) - ) + mockSimplePoll() await pollJobState( requestClient, mockJob, true, - 'test', undefined, defaultPollOptions ) @@ -205,21 +169,12 @@ describe('pollJobState', () => { }) it('should continue polling when there is a single error in between', async () => { - jest - .spyOn(requestClient, 'get') - .mockImplementationOnce(() => - Promise.resolve({ result: 'pending', etag: '' }) - ) - .mockImplementationOnce(() => Promise.reject('Status Error')) - .mockImplementationOnce(() => - Promise.resolve({ result: 'completed', etag: '' }) - ) + mockPollWithSingleError() const state = await pollJobState( requestClient, mockJob, false, - 'test', undefined, defaultPollOptions ) @@ -229,20 +184,19 @@ describe('pollJobState', () => { }) it('should throw an error when the error count exceeds the set value of 5', async () => { - jest - .spyOn(requestClient, 'get') - .mockImplementation(() => Promise.reject('Status Error')) + mockErroredPoll() const error = await pollJobState( requestClient, mockJob, false, - 'test', undefined, defaultPollOptions ).catch((e) => e) - expect(error).toContain('Error while getting job state after interval.') + expect(error.message).toEqual( + 'Error while polling job state for job j0b: Status Error' + ) }) }) @@ -251,11 +205,12 @@ const setupMocks = () => { jest.mock('../../../request/RequestClient') jest.mock('../../../auth/getTokens') jest.mock('../saveLog') + jest.mock('fs') jest .spyOn(requestClient, 'get') .mockImplementation(() => - Promise.resolve({ result: 'completed', etag: '' }) + Promise.resolve({ result: 'completed', etag: '', status: 200 }) ) jest .spyOn(getTokensModule, 'getTokens') @@ -263,4 +218,84 @@ const setupMocks = () => { jest .spyOn(saveLogModule, 'saveLog') .mockImplementation(() => Promise.resolve()) + jest + .spyOn(fs, 'createWriteStream') + .mockImplementation(() => ({} as unknown as fs.WriteStream)) +} + +const mockSimplePoll = (runningCount = 2) => { + let count = 0 + jest.spyOn(requestClient, 'get').mockImplementation((url) => { + count++ + if (url.includes('job')) { + return Promise.resolve({ result: mockJob, etag: '', status: 200 }) + } + return Promise.resolve({ + result: + count === 0 + ? 'pending' + : count <= runningCount + ? 'running' + : 'completed', + etag: '', + status: 200 + }) + }) +} + +const mockRunningPoll = () => { + let count = 0 + jest.spyOn(requestClient, 'get').mockImplementation((url) => { + count++ + if (url.includes('job')) { + return Promise.resolve({ result: mockJob, etag: '', status: 200 }) + } + return Promise.resolve({ + result: count === 0 ? 'pending' : 'running', + etag: '', + status: 200 + }) + }) +} + +const mockLongPoll = () => { + let count = 0 + jest.spyOn(requestClient, 'get').mockImplementation((url) => { + count++ + if (url.includes('job')) { + return Promise.resolve({ result: mockJob, etag: '', status: 200 }) + } + return Promise.resolve({ + result: count <= 101 ? 'running' : 'completed', + etag: '', + status: 200 + }) + }) +} + +const mockPollWithSingleError = () => { + let count = 0 + jest.spyOn(requestClient, 'get').mockImplementation((url) => { + count++ + if (url.includes('job')) { + return Promise.resolve({ result: mockJob, etag: '', status: 200 }) + } + if (count === 1) { + return Promise.reject('Status Error') + } + return Promise.resolve({ + result: count === 0 ? 'pending' : 'completed', + etag: '', + status: 200 + }) + }) +} + +const mockErroredPoll = () => { + jest.spyOn(requestClient, 'get').mockImplementation((url) => { + if (url.includes('job')) { + return Promise.resolve({ result: mockJob, etag: '', status: 200 }) + } + return Promise.reject('Status Error') + }) } diff --git a/src/api/viya/spec/saveLog.spec.ts b/src/api/viya/spec/saveLog.spec.ts index c4b8b9d..4d35c6f 100644 --- a/src/api/viya/spec/saveLog.spec.ts +++ b/src/api/viya/spec/saveLog.spec.ts @@ -1,11 +1,13 @@ import { Logger, LogLevel } from '@sasjs/utils' -import * as fileModule from '@sasjs/utils/file' import { RequestClient } from '../../../request/RequestClient' import * as fetchLogsModule from '../../../utils/fetchLogByChunks' +import * as writeStreamModule from '../writeStream' import { saveLog } from '../saveLog' import { mockJob } from './mockResponses' +import { WriteStream } from 'fs' const requestClient = new (>RequestClient)() +const stream = {} as unknown as WriteStream describe('saveLog', () => { beforeEach(() => { @@ -14,16 +16,21 @@ describe('saveLog', () => { }) it('should return immediately if shouldSaveLog is false', async () => { - await saveLog(mockJob, requestClient, false, '/test', 't0k3n') + await saveLog(mockJob, requestClient, false, 0, 100, stream, 't0k3n') - expect(fetchLogsModule.fetchLogByChunks).not.toHaveBeenCalled() - expect(fileModule.createFile).not.toHaveBeenCalled() + expect(fetchLogsModule.fetchLog).not.toHaveBeenCalled() + expect(writeStreamModule.writeStream).not.toHaveBeenCalled() }) it('should throw an error when a valid access token is not provided', async () => { - const error = await saveLog(mockJob, requestClient, true, '/test').catch( - (e) => e - ) + const error = await saveLog( + mockJob, + requestClient, + true, + 0, + 100, + stream + ).catch((e) => e) expect(error.message).toContain( `Logs for job ${mockJob.id} cannot be fetched without a valid access token.` @@ -35,7 +42,9 @@ describe('saveLog', () => { { ...mockJob, links: mockJob.links.filter((l) => l.rel !== 'log') }, requestClient, true, - '/test', + 0, + 100, + stream, 't0k3n' ).catch((e) => e) @@ -45,15 +54,19 @@ describe('saveLog', () => { }) it('should fetch and save logs to the given path', async () => { - await saveLog(mockJob, requestClient, true, '/test', 't0k3n') + await saveLog(mockJob, requestClient, true, 0, 100, stream, 't0k3n') - expect(fetchLogsModule.fetchLogByChunks).toHaveBeenCalledWith( + expect(fetchLogsModule.fetchLog).toHaveBeenCalledWith( requestClient, 't0k3n', '/log/content', + 0, 100 ) - expect(fileModule.createFile).toHaveBeenCalledWith('/test', 'Test Log') + expect(writeStreamModule.writeStream).toHaveBeenCalledWith( + stream, + 'Test Log' + ) }) }) @@ -62,11 +75,12 @@ const setupMocks = () => { jest.mock('../../../request/RequestClient') jest.mock('../../../utils/fetchLogByChunks') jest.mock('@sasjs/utils') + jest.mock('../writeStream') jest - .spyOn(fetchLogsModule, 'fetchLogByChunks') + .spyOn(fetchLogsModule, 'fetchLog') .mockImplementation(() => Promise.resolve('Test Log')) jest - .spyOn(fileModule, 'createFile') + .spyOn(writeStreamModule, 'writeStream') .mockImplementation(() => Promise.resolve()) } diff --git a/src/api/viya/writeStream.ts b/src/api/viya/writeStream.ts new file mode 100644 index 0000000..dc09885 --- /dev/null +++ b/src/api/viya/writeStream.ts @@ -0,0 +1,15 @@ +import { WriteStream } from 'fs' + +export const writeStream = async ( + stream: WriteStream, + content: string +): Promise => { + return new Promise((resolve, reject) => { + stream.write(content + '\n\nnext chunk\n\n', (e) => { + if (e) { + return reject(e) + } + return resolve() + }) + }) +} diff --git a/src/types/errors/JobStatePollError.ts b/src/types/errors/JobStatePollError.ts new file mode 100644 index 0000000..2584b1c --- /dev/null +++ b/src/types/errors/JobStatePollError.ts @@ -0,0 +1,11 @@ +export class JobStatePollError extends Error { + constructor(id: string, public originalError: Error) { + super( + `Error while polling job state for job ${id}: ${ + originalError.message || originalError + }` + ) + this.name = 'JobStatePollError' + Object.setPrototypeOf(this, JobStatePollError.prototype) + } +} diff --git a/src/types/errors/index.ts b/src/types/errors/index.ts index 72b63cc..f8595d4 100644 --- a/src/types/errors/index.ts +++ b/src/types/errors/index.ts @@ -2,6 +2,7 @@ export * from './AuthorizeError' export * from './ComputeJobExecutionError' export * from './InternalServerError' export * from './JobExecutionError' +export * from './JobStatePollError' export * from './LoginRequiredError' export * from './NotFoundError' export * from './ErrorResponse' diff --git a/src/utils/fetchLogByChunks.ts b/src/utils/fetchLogByChunks.ts index a149c4c..22e888a 100644 --- a/src/utils/fetchLogByChunks.ts +++ b/src/utils/fetchLogByChunks.ts @@ -14,18 +14,36 @@ export const fetchLogByChunks = async ( accessToken: string, logUrl: string, logCount: number +): Promise => { + return await fetchLog(requestClient, accessToken, logUrl, 0, logCount) +} + +/** + * Fetches a section of the log file delineated by start and end lines + * @param {object} requestClient - client object of Request Client. + * @param {string} accessToken - an access token for an authorized user. + * @param {string} logUrl - url of the log file. + * @param {number} start - the line at which to start fetching the log. + * @param {number} end - the line at which to stop fetching the log. + * @returns an string containing log lines. + */ +export const fetchLog = async ( + requestClient: RequestClient, + accessToken: string, + logUrl: string, + start: number, + end: number ): Promise => { const logger = process.logger || console let log: string = '' - const loglimit = logCount < 10000 ? logCount : 10000 - let start = 0 + const loglimit = end < 10000 ? end : 10000 do { logger.info( `Fetching logs from line no: ${start + 1} to ${ start + loglimit - } of ${logCount}.` + } of ${end}.` ) const logChunkJson = await requestClient! .get(`${logUrl}?start=${start}&limit=${loglimit}`, accessToken) @@ -40,6 +58,6 @@ export const fetchLogByChunks = async ( log += logChunk start += loglimit - } while (start < logCount) + } while (start < end) return log } From cfa0c8b9aff3b99639ac6075a392a9d2bed84308 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Wed, 21 Jul 2021 08:12:34 +0100 Subject: [PATCH 13/15] chore(refactor): only fetch job if streaming logs, fix tests, add JSDoc comments --- src/api/viya/pollJobState.ts | 34 +++++++++++++------------- src/api/viya/saveLog.ts | 17 ++++++++----- src/api/viya/spec/pollJobState.spec.ts | 28 +++++++++++++++------ src/api/viya/spec/saveLog.spec.ts | 21 +++------------- src/api/viya/uploadTables.ts | 8 ++++++ 5 files changed, 60 insertions(+), 48 deletions(-) diff --git a/src/api/viya/pollJobState.ts b/src/api/viya/pollJobState.ts index b777382..6d66244 100644 --- a/src/api/viya/pollJobState.ts +++ b/src/api/viya/pollJobState.ts @@ -8,7 +8,6 @@ import { saveLog } from './saveLog' import { createWriteStream } from '@sasjs/utils/file' import { WriteStream } from 'fs' import { Link } from '../../types' -import { prefixMessage } from '@sasjs/utils/error' export async function pollJobState( requestClient: RequestClient, @@ -206,25 +205,26 @@ const doPoll = async ( pollCount++ - const jobUrl = postedJob.links.find((l: Link) => l.rel === 'self') - const { result: job } = await requestClient.get( - jobUrl!.href, - authConfig?.access_token - ) + if (pollOptions?.streamLog) { + const jobUrl = postedJob.links.find((l: Link) => l.rel === 'self') + const { result: job } = await requestClient.get( + jobUrl!.href, + authConfig?.access_token + ) - const endLogLine = job.logStatistics?.lineCount ?? 1000000 + const endLogLine = job.logStatistics?.lineCount ?? 1000000 - await saveLog( - postedJob, - requestClient, - pollOptions?.streamLog || false, - startLogLine, - endLogLine, - logStream, - authConfig?.access_token - ) + await saveLog( + postedJob, + requestClient, + startLogLine, + endLogLine, + logStream, + authConfig?.access_token + ) - startLogLine += job.logStatistics.lineCount + startLogLine += endLogLine + } if (debug && printedState !== state) { logger.info('Polling job status...') diff --git a/src/api/viya/saveLog.ts b/src/api/viya/saveLog.ts index 94fbbfd..a659acf 100644 --- a/src/api/viya/saveLog.ts +++ b/src/api/viya/saveLog.ts @@ -4,20 +4,25 @@ import { fetchLog } from '../../utils' import { WriteStream } from 'fs' import { writeStream } from './writeStream' +/** + * Appends logs to a supplied write stream. + * This is useful for getting quick feedback on longer running jobs. + * @param job - the job to fetch logs for + * @param requestClient - the pre-configured HTTP request client + * @param startLine - the line at which to start fetching the log + * @param endLine - the line at which to stop fetching the log + * @param logFileStream - the write stream to which the log is appended + * @accessToken - an optional access token for authentication/authorization + * The access token is not required when fetching logs from the browser. + */ export async function saveLog( job: Job, requestClient: RequestClient, - shouldSaveLog: boolean, startLine: number, endLine: number, logFileStream?: WriteStream, accessToken?: string ) { - console.log('startLine: ', startLine, ' endLine: ', endLine) - if (!shouldSaveLog) { - return - } - if (!accessToken) { throw new Error( `Logs for job ${job.id} cannot be fetched without a valid access token.` diff --git a/src/api/viya/spec/pollJobState.spec.ts b/src/api/viya/spec/pollJobState.spec.ts index dcbd2b9..a415b98 100644 --- a/src/api/viya/spec/pollJobState.spec.ts +++ b/src/api/viya/spec/pollJobState.spec.ts @@ -1,11 +1,12 @@ -import * as fs from 'fs' import { Logger, LogLevel } from '@sasjs/utils' +import * as fileModule from '@sasjs/utils/file' import { RequestClient } from '../../../request/RequestClient' import { mockAuthConfig, mockJob } from './mockResponses' import { pollJobState } from '../pollJobState' import * as getTokensModule from '../../../auth/getTokens' import * as saveLogModule from '../saveLog' import { PollOptions } from '../../../types' +import { WriteStream } from 'fs' const requestClient = new (>RequestClient)() const defaultPollOptions: PollOptions = { @@ -73,7 +74,18 @@ describe('pollJobState', () => { expect(getTokensModule.getTokens).toHaveBeenCalledTimes(3) }) - it('should attempt to fetch and save the log after each poll', async () => { + it('should attempt to fetch and save the log after each poll when streamLog is true', async () => { + mockSimplePoll() + + await pollJobState(requestClient, mockJob, false, mockAuthConfig, { + ...defaultPollOptions, + streamLog: true + }) + + expect(saveLogModule.saveLog).toHaveBeenCalledTimes(2) + }) + + it('should not attempt to fetch and save the log after each poll when streamLog is false', async () => { mockSimplePoll() await pollJobState( @@ -84,7 +96,7 @@ describe('pollJobState', () => { defaultPollOptions ) - expect(saveLogModule.saveLog).toHaveBeenCalledTimes(2) + expect(saveLogModule.saveLog).not.toHaveBeenCalled() }) it('should return the current status when the max poll count is reached', async () => { @@ -133,7 +145,7 @@ describe('pollJobState', () => { defaultPollOptions ) - expect(requestClient.get).toHaveBeenCalledTimes(3) + expect(requestClient.get).toHaveBeenCalledTimes(2) expect(state).toEqual('completed') }) @@ -179,7 +191,7 @@ describe('pollJobState', () => { defaultPollOptions ) - expect(requestClient.get).toHaveBeenCalledTimes(3) + expect(requestClient.get).toHaveBeenCalledTimes(2) expect(state).toEqual('completed') }) @@ -205,7 +217,7 @@ const setupMocks = () => { jest.mock('../../../request/RequestClient') jest.mock('../../../auth/getTokens') jest.mock('../saveLog') - jest.mock('fs') + jest.mock('@sasjs/utils/file') jest .spyOn(requestClient, 'get') @@ -219,8 +231,8 @@ const setupMocks = () => { .spyOn(saveLogModule, 'saveLog') .mockImplementation(() => Promise.resolve()) jest - .spyOn(fs, 'createWriteStream') - .mockImplementation(() => ({} as unknown as fs.WriteStream)) + .spyOn(fileModule, 'createWriteStream') + .mockImplementation(() => Promise.resolve({} as unknown as WriteStream)) } const mockSimplePoll = (runningCount = 2) => { diff --git a/src/api/viya/spec/saveLog.spec.ts b/src/api/viya/spec/saveLog.spec.ts index 4d35c6f..a6c662b 100644 --- a/src/api/viya/spec/saveLog.spec.ts +++ b/src/api/viya/spec/saveLog.spec.ts @@ -15,22 +15,10 @@ describe('saveLog', () => { setupMocks() }) - it('should return immediately if shouldSaveLog is false', async () => { - await saveLog(mockJob, requestClient, false, 0, 100, stream, 't0k3n') - - expect(fetchLogsModule.fetchLog).not.toHaveBeenCalled() - expect(writeStreamModule.writeStream).not.toHaveBeenCalled() - }) - it('should throw an error when a valid access token is not provided', async () => { - const error = await saveLog( - mockJob, - requestClient, - true, - 0, - 100, - stream - ).catch((e) => e) + const error = await saveLog(mockJob, requestClient, 0, 100, stream).catch( + (e) => e + ) expect(error.message).toContain( `Logs for job ${mockJob.id} cannot be fetched without a valid access token.` @@ -41,7 +29,6 @@ describe('saveLog', () => { const error = await saveLog( { ...mockJob, links: mockJob.links.filter((l) => l.rel !== 'log') }, requestClient, - true, 0, 100, stream, @@ -54,7 +41,7 @@ describe('saveLog', () => { }) it('should fetch and save logs to the given path', async () => { - await saveLog(mockJob, requestClient, true, 0, 100, stream, 't0k3n') + await saveLog(mockJob, requestClient, 0, 100, stream, 't0k3n') expect(fetchLogsModule.fetchLog).toHaveBeenCalledWith( requestClient, diff --git a/src/api/viya/uploadTables.ts b/src/api/viya/uploadTables.ts index e7f5b66..b9e4402 100644 --- a/src/api/viya/uploadTables.ts +++ b/src/api/viya/uploadTables.ts @@ -2,6 +2,14 @@ import { prefixMessage } from '@sasjs/utils/error' import { RequestClient } from '../../request/RequestClient' import { convertToCSV } from '../../utils/convertToCsv' +/** + * Uploads tables to SAS as specially formatted CSVs. + * This is more compact than JSON, and easier to read within SAS. + * @param requestClient - the pre-configured HTTP request client + * @param data - the JSON representation of the data to be uploaded + * @param accessToken - an optional access token for authentication/authorization + * The access token is not required when uploading tables from the browser. + */ export async function uploadTables( requestClient: RequestClient, data: any, From 7bd2e31f3b5bca20be35e8464efd42ec0452438e Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Wed, 21 Jul 2021 08:13:45 +0100 Subject: [PATCH 14/15] chore(cleanup): remove console logs --- src/api/viya/saveLog.ts | 4 ++-- src/api/viya/spec/executeScript.spec.ts | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/api/viya/saveLog.ts b/src/api/viya/saveLog.ts index a659acf..3bbd498 100644 --- a/src/api/viya/saveLog.ts +++ b/src/api/viya/saveLog.ts @@ -48,8 +48,8 @@ export async function saveLog( `${jobLogUrl.href}/content`, startLine, endLine - ).catch((e) => console.log(e)) + ) logger.info(`Writing logs to ${logFileStream.path}`) - await writeStream(logFileStream, log || '').catch((e) => console.log(e)) + await writeStream(logFileStream, log || '') } diff --git a/src/api/viya/spec/executeScript.spec.ts b/src/api/viya/spec/executeScript.spec.ts index 5e2fb11..9eb80c4 100644 --- a/src/api/viya/spec/executeScript.spec.ts +++ b/src/api/viya/spec/executeScript.spec.ts @@ -299,7 +299,6 @@ describe('executeScript', () => { true ).catch((e) => e) - console.log(error) expect(error).toContain('Error while posting job') }) From c8e029cff49ace43d5566bc61a3a2dd774a0ea88 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Wed, 21 Jul 2021 08:37:45 +0100 Subject: [PATCH 15/15] chore(deps): bump utils --- package-lock.json | 42 +++++++++++++++++++++++++++++++++--------- package.json | 2 +- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8f8364f..e241b7f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,7 @@ "name": "@sasjs/adapter", "license": "ISC", "dependencies": { - "@sasjs/utils": "^2.25.1", + "@sasjs/utils": "^2.25.4", "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", "form-data": "^4.0.0", @@ -1187,10 +1187,11 @@ } }, "node_modules/@sasjs/utils": { - "version": "2.25.1", - "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.25.1.tgz", - "integrity": "sha512-lwkPE+QsB81b8/1M8X2zLdhpuiA8pIjgOwJH57zhcsliuDnNs4uijSYu40aYSc8tH98jtSuqWMjfGq8CT9o1Dw==", + "version": "2.25.4", + "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.25.4.tgz", + "integrity": "sha512-LTWExtHp4g3VcLLCUMyeeyTXEAZawSQngmJ3/2Z93ysxpeu2/NS7lGG/ERGCQb2snbqmXK8dkZmfg44Tn4Qebw==", "dependencies": { + "@types/fs-extra": "^9.0.11", "@types/prompts": "^2.0.13", "chalk": "^4.1.1", "cli-table": "^0.3.6", @@ -1432,6 +1433,14 @@ "form-data": "*" } }, + "node_modules/@types/fs-extra": { + "version": "9.0.12", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.12.tgz", + "integrity": "sha512-I+bsBr67CurCGnSenZZ7v94gd3tc3+Aj2taxMT4yu4ABLuOgOjeFxX3dokG24ztSRg5tnT00sL8BszO7gSMoIw==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/graceful-fs": { "version": "4.1.5", "dev": true, @@ -11501,6 +11510,7 @@ }, "node_modules/querystring": { "version": "0.2.0", + "dev": true, "engines": { "node": ">=0.4.x" } @@ -13455,6 +13465,7 @@ }, "node_modules/url": { "version": "0.11.0", + "dev": true, "license": "MIT", "dependencies": { "punycode": "1.3.2", @@ -13468,6 +13479,7 @@ }, "node_modules/url/node_modules/punycode": { "version": "1.3.2", + "dev": true, "license": "MIT" }, "node_modules/use": { @@ -14779,10 +14791,11 @@ } }, "@sasjs/utils": { - "version": "2.25.1", - "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.25.1.tgz", - "integrity": "sha512-lwkPE+QsB81b8/1M8X2zLdhpuiA8pIjgOwJH57zhcsliuDnNs4uijSYu40aYSc8tH98jtSuqWMjfGq8CT9o1Dw==", + "version": "2.25.4", + "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.25.4.tgz", + "integrity": "sha512-LTWExtHp4g3VcLLCUMyeeyTXEAZawSQngmJ3/2Z93ysxpeu2/NS7lGG/ERGCQb2snbqmXK8dkZmfg44Tn4Qebw==", "requires": { + "@types/fs-extra": "^9.0.11", "@types/prompts": "^2.0.13", "chalk": "^4.1.1", "cli-table": "^0.3.6", @@ -14970,6 +14983,14 @@ "form-data": "*" } }, + "@types/fs-extra": { + "version": "9.0.12", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.12.tgz", + "integrity": "sha512-I+bsBr67CurCGnSenZZ7v94gd3tc3+Aj2taxMT4yu4ABLuOgOjeFxX3dokG24ztSRg5tnT00sL8BszO7gSMoIw==", + "requires": { + "@types/node": "*" + } + }, "@types/graceful-fs": { "version": "4.1.5", "dev": true, @@ -21835,7 +21856,8 @@ "dev": true }, "querystring": { - "version": "0.2.0" + "version": "0.2.0", + "dev": true }, "querystring-es3": { "version": "0.2.1", @@ -23117,13 +23139,15 @@ }, "url": { "version": "0.11.0", + "dev": true, "requires": { "punycode": "1.3.2", "querystring": "0.2.0" }, "dependencies": { "punycode": { - "version": "1.3.2" + "version": "1.3.2", + "dev": true } } }, diff --git a/package.json b/package.json index e190315..756785b 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ }, "main": "index.js", "dependencies": { - "@sasjs/utils": "^2.25.1", + "@sasjs/utils": "^2.25.4", "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", "form-data": "^4.0.0",