From 04ccbf68437a107a35f998bebb39e3b2256ebf60 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Wed, 7 Jul 2021 10:02:14 +0100 Subject: [PATCH 01/39] feat(log): write logs to file when polling for job status --- src/SASViyaApiClient.ts | 541 +++------------------------------- src/api/viya/executeScript.ts | 303 +++++++++++++++++++ src/api/viya/pollJobState.ts | 178 +++++++++++ src/api/viya/uploadTables.ts | 35 +++ src/auth/tokens.ts | 122 ++++++++ src/request/RequestClient.ts | 5 + src/types/PollOptions.ts | 6 +- 7 files changed, 689 insertions(+), 501 deletions(-) create mode 100644 src/api/viya/executeScript.ts create mode 100644 src/api/viya/pollJobState.ts create mode 100644 src/api/viya/uploadTables.ts create mode 100644 src/auth/tokens.ts diff --git a/src/SASViyaApiClient.ts b/src/SASViyaApiClient.ts index 068e780..3c1b324 100644 --- a/src/SASViyaApiClient.ts +++ b/src/SASViyaApiClient.ts @@ -1,10 +1,4 @@ -import { - convertToCSV, - isRelativePath, - isUri, - isUrl, - fetchLogByChunks -} from './utils' +import { isRelativePath, isUri, isUrl } from './utils' import * as NodeFormData from 'form-data' import { Job, @@ -17,28 +11,18 @@ import { JobDefinition, PollOptions } from './types' -import { - ComputeJobExecutionError, - JobExecutionError, - NotFoundError -} from './types/errors' -import { formatDataForRequest } from './utils/formatDataForRequest' +import { JobExecutionError } from './types/errors' import { SessionManager } from './SessionManager' import { ContextManager } from './ContextManager' -import { - timestampToYYYYMMDDHHMMSS, - isAccessTokenExpiring, - isRefreshTokenExpiring, - Logger, - LogLevel, - SasAuthResponse, - MacroVar, - AuthConfig -} from '@sasjs/utils' +import { SasAuthResponse, MacroVar, AuthConfig } from '@sasjs/utils' import { isAuthorizeFormRequired } from './auth/isAuthorizeFormRequired' import { RequestClient } from './request/RequestClient' import { prefixMessage } from '@sasjs/utils/error' import * as mime from 'mime' +import { pollJobState } from './api/viya/pollJobState' +import { getAccessToken, getTokens, refreshTokens } from './auth/tokens' +import { uploadTables } from './api/viya/uploadTables' +import { executeScript } from './api/viya/executeScript' /** * A client for interfacing with the SAS Viya REST API. @@ -174,13 +158,6 @@ export class SASViyaApiClient { throw new Error(`Execution context ${contextName} not found.`) } - const createSessionRequest = { - method: 'POST', - headers: { - Authorization: `Bearer ${accessToken}`, - 'Content-Type': 'application/json' - } - } const { result: createdSession } = await this.requestClient.post( `/compute/contexts/${executionContext.id}/sessions`, {}, @@ -295,249 +272,22 @@ export class SASViyaApiClient { printPid = false, variables?: MacroVar ): Promise { - let access_token = (authConfig || {}).access_token - if (authConfig) { - ;({ access_token } = await this.getTokens(authConfig)) - } - - const logger = process.logger || console - - try { - let executionSessionId: string - - const session = await this.sessionManager - .getSession(access_token) - .catch((err) => { - throw prefixMessage(err, 'Error while getting session. ') - }) - - executionSessionId = session!.id - - if (printPid) { - const { result: jobIdVariable } = await this.sessionManager - .getVariable(executionSessionId, 'SYSJOBID', access_token) - .catch((err) => { - throw prefixMessage(err, 'Error while getting session variable. ') - }) - - if (jobIdVariable && jobIdVariable.value) { - const relativeJobPath = this.rootFolderName - ? jobPath.split(this.rootFolderName).join('').replace(/^\//, '') - : jobPath - - const logger = new Logger(debug ? LogLevel.Debug : LogLevel.Info) - - logger.info( - `Triggered '${relativeJobPath}' with PID ${ - jobIdVariable.value - } at ${timestampToYYYYMMDDHHMMSS()}` - ) - } - } - - const jobArguments: { [key: string]: any } = { - _contextName: contextName, - _OMITJSONLISTING: true, - _OMITJSONLOG: true, - _OMITSESSIONRESULTS: true, - _OMITTEXTLISTING: true, - _OMITTEXTLOG: true - } - - if (debug) { - jobArguments['_OMITTEXTLOG'] = false - jobArguments['_OMITSESSIONRESULTS'] = false - } - - let fileName - - if (isRelativePath(jobPath)) { - fileName = `exec-${ - jobPath.includes('/') ? jobPath.split('/')[1] : jobPath - }` - } else { - const jobPathParts = jobPath.split('/') - fileName = jobPathParts.pop() - } - - let jobVariables: any = { - SYS_JES_JOB_URI: '', - _program: isRelativePath(jobPath) - ? this.rootFolderName + '/' + jobPath - : jobPath - } - - if (variables) jobVariables = { ...jobVariables, ...variables } - - if (debug) jobVariables = { ...jobVariables, _DEBUG: 131 } - - let files: any[] = [] - - if (data) { - if (JSON.stringify(data).includes(';')) { - files = await this.uploadTables(data, access_token).catch((err) => { - throw prefixMessage(err, 'Error while uploading tables. ') - }) - - jobVariables['_webin_file_count'] = files.length - - files.forEach((fileInfo, index) => { - jobVariables[ - `_webin_fileuri${index + 1}` - ] = `/files/files/${fileInfo.file.id}` - jobVariables[`_webin_name${index + 1}`] = fileInfo.tableName - }) - } else { - jobVariables = { ...jobVariables, ...formatDataForRequest(data) } - } - } - - // Execute job in session - const jobRequestBody = { - name: fileName, - description: 'Powered by SASjs', - code: linesOfCode, - variables: jobVariables, - arguments: jobArguments - } - - const { result: postedJob, etag } = await this.requestClient - .post( - `/compute/sessions/${executionSessionId}/jobs`, - jobRequestBody, - access_token - ) - .catch((err) => { - throw prefixMessage(err, 'Error while posting job. ') - }) - - if (!waitForResult) return session - - if (debug) { - logger.info(`Job has been submitted for '${fileName}'.`) - logger.info( - `You can monitor the job progress at '${this.serverUrl}${ - postedJob.links.find((l: any) => l.rel === 'state')!.href - }'.` - ) - } - - const jobStatus = await this.pollJobState( - postedJob, - etag, - authConfig, - pollOptions - ).catch(async (err) => { - const error = err?.response?.data - const result = /err=[0-9]*,/.exec(error) - - const errorCode = '5113' - if (result?.[0]?.slice(4, -1) === errorCode) { - const sessionLogUrl = - postedJob.links.find((l: any) => l.rel === 'up')!.href + '/log' - const logCount = 1000000 - err.log = await fetchLogByChunks( - this.requestClient, - access_token!, - sessionLogUrl, - logCount - ) - } - throw prefixMessage(err, 'Error while polling job status. ') - }) - - if (authConfig) { - ;({ access_token } = await this.getTokens(authConfig)) - } - - const { result: currentJob } = await this.requestClient - .get( - `/compute/sessions/${executionSessionId}/jobs/${postedJob.id}`, - access_token - ) - .catch((err) => { - throw prefixMessage(err, 'Error while getting job. ') - }) - - let jobResult - let log = '' - - const logLink = currentJob.links.find((l) => l.rel === 'log') - - if (debug && logLink) { - const logUrl = `${logLink.href}/content` - const logCount = currentJob.logStatistics?.lineCount ?? 1000000 - log = await fetchLogByChunks( - this.requestClient, - access_token!, - logUrl, - logCount - ) - } - - if (jobStatus === 'failed' || jobStatus === 'error') { - return Promise.reject(new ComputeJobExecutionError(currentJob, log)) - } - - let resultLink - - if (expectWebout) { - resultLink = `/compute/sessions/${executionSessionId}/filerefs/_webout/content` - } else { - return { job: currentJob, log } - } - - if (resultLink) { - jobResult = await this.requestClient - .get(resultLink, access_token, 'text/plain') - .catch(async (e) => { - if (e instanceof NotFoundError) { - if (logLink) { - const logUrl = `${logLink.href}/content` - const logCount = currentJob.logStatistics?.lineCount ?? 1000000 - log = await fetchLogByChunks( - this.requestClient, - access_token!, - logUrl, - logCount - ) - - return Promise.reject({ - status: 500, - log - }) - } - } - - return { - result: JSON.stringify(e) - } - }) - } - - await this.sessionManager - .clearSession(executionSessionId, access_token) - .catch((err) => { - throw prefixMessage(err, 'Error while clearing session. ') - }) - - return { result: jobResult?.result, log } - } catch (e) { - if (e && e.status === 404) { - return this.executeScript( - jobPath, - linesOfCode, - contextName, - authConfig, - data, - debug, - false, - true - ) - } else { - throw prefixMessage(e, 'Error while executing script. ') - } - } + return executeScript( + this.requestClient, + this.sessionManager, + this.rootFolderName, + jobPath, + linesOfCode, + contextName, + authConfig, + data, + debug, + expectWebout, + waitForResult, + pollOptions, + printPid, + variables + ) } /** @@ -772,37 +522,7 @@ export class SASViyaApiClient { clientSecret: string, authCode: string ): Promise { - const url = this.serverUrl + '/SASLogon/oauth/token' - let token - if (typeof Buffer === 'undefined') { - token = btoa(clientId + ':' + clientSecret) - } else { - token = Buffer.from(clientId + ':' + clientSecret).toString('base64') - } - const headers = { - Authorization: 'Basic ' + token - } - - let formData - if (typeof FormData === 'undefined') { - formData = new NodeFormData() - } else { - formData = new FormData() - } - formData.append('grant_type', 'authorization_code') - formData.append('code', authCode) - - const authResponse = await this.requestClient - .post( - url, - formData, - undefined, - 'multipart/form-data; boundary=' + (formData as any)._boundary, - headers - ) - .then((res) => res.result as SasAuthResponse) - - return authResponse + return getAccessToken(this.requestClient, clientId, clientSecret, authCode) } /** @@ -816,39 +536,12 @@ export class SASViyaApiClient { clientSecret: string, refreshToken: string ) { - const url = this.serverUrl + '/SASLogon/oauth/token' - let token - if (typeof Buffer === 'undefined') { - token = btoa(clientId + ':' + clientSecret) - } else { - token = Buffer.from(clientId + ':' + clientSecret).toString('base64') - } - const headers = { - Authorization: 'Basic ' + token - } - - let formData - if (typeof FormData === 'undefined') { - formData = new NodeFormData() - formData.append('grant_type', 'refresh_token') - formData.append('refresh_token', refreshToken) - } else { - formData = new FormData() - formData.append('grant_type', 'refresh_token') - formData.append('refresh_token', refreshToken) - } - - const authResponse = await this.requestClient - .post( - url, - formData, - undefined, - 'multipart/form-data; boundary=' + (formData as any)._boundary, - headers - ) - .then((res) => res.result) - - return authResponse + return refreshTokens( + this.requestClient, + clientId, + clientSecret, + refreshToken + ) } /** @@ -895,7 +588,7 @@ export class SASViyaApiClient { ) { let access_token = (authConfig || {}).access_token if (authConfig) { - ;({ access_token } = await this.getTokens(authConfig)) + ;({ access_token } = await getTokens(this.requestClient, authConfig)) } if (isRelativePath(sasJob) && !this.rootFolderName) { @@ -991,7 +684,7 @@ export class SASViyaApiClient { ) { let access_token = (authConfig || {}).access_token if (authConfig) { - ;({ access_token } = await this.getTokens(authConfig)) + ;({ access_token } = await getTokens(this.requestClient, authConfig)) } if (isRelativePath(sasJob) && !this.rootFolderName) { throw new Error( @@ -1140,157 +833,24 @@ export class SASViyaApiClient { this.folderMap.set(path, itemsAtRoot) } - // REFACTOR: set default value for 'pollOptions' attribute private async pollJobState( - postedJob: any, + postedJob: Job, etag: string | null, authConfig?: AuthConfig, pollOptions?: PollOptions ) { - const logger = process.logger || console - - let POLL_INTERVAL = 300 - let MAX_POLL_COUNT = 1000 - let MAX_ERROR_COUNT = 5 - let access_token = (authConfig || {}).access_token - if (authConfig) { - ;({ access_token } = await this.getTokens(authConfig)) - } - - if (pollOptions) { - POLL_INTERVAL = pollOptions.POLL_INTERVAL || POLL_INTERVAL - MAX_POLL_COUNT = pollOptions.MAX_POLL_COUNT || MAX_POLL_COUNT - } - - let postedJobState = '' - let pollCount = 0 - let errorCount = 0 - const headers: any = { - 'Content-Type': 'application/json', - 'If-None-Match': etag - } - if (access_token) { - headers.Authorization = `Bearer ${access_token}` - } - const stateLink = postedJob.links.find((l: any) => l.rel === 'state') - if (!stateLink) { - Promise.reject(`Job state link was not found.`) - } - - const { result: state } = await this.requestClient - .get( - `${this.serverUrl}${stateLink.href}?_action=wait&wait=300`, - access_token, - 'text/plain', - {}, - this.debug - ) - .catch((err) => { - console.error( - `Error fetching job state from ${this.serverUrl}${stateLink.href}. Starting poll, assuming job to be running.`, - err - ) - return { result: 'unavailable' } - }) - - const currentState = state.trim() - if (currentState === 'completed') { - return Promise.resolve(currentState) - } - - return new Promise(async (resolve, _) => { - let printedState = '' - - const interval = setInterval(async () => { - if ( - postedJobState === 'running' || - postedJobState === '' || - postedJobState === 'pending' || - postedJobState === 'unavailable' - ) { - if (authConfig) { - ;({ access_token } = await this.getTokens(authConfig)) - } - - if (stateLink) { - const { result: jobState } = await this.requestClient - .get( - `${this.serverUrl}${stateLink.href}?_action=wait&wait=300`, - access_token, - 'text/plain', - {}, - this.debug - ) - .catch((err) => { - errorCount++ - if ( - pollCount >= MAX_POLL_COUNT || - errorCount >= MAX_ERROR_COUNT - ) { - throw prefixMessage( - err, - 'Error while getting job state after interval. ' - ) - } - console.error( - `Error fetching job state from ${this.serverUrl}${stateLink.href}. Resuming poll, assuming job to be running.`, - err - ) - return { result: 'unavailable' } - }) - - postedJobState = jobState.trim() - if (postedJobState != 'unavailable' && errorCount > 0) { - errorCount = 0 - } - - if (this.debug && printedState !== postedJobState) { - logger.info('Polling job status...') - logger.info(`Current job state: ${postedJobState}`) - - printedState = postedJobState - } - - pollCount++ - - if (pollCount >= MAX_POLL_COUNT) { - resolve(postedJobState) - } - } - } else { - clearInterval(interval) - resolve(postedJobState) - } - }, POLL_INTERVAL) - }) + return pollJobState( + this.requestClient, + postedJob, + this.debug, + etag, + authConfig, + pollOptions + ) } private async uploadTables(data: any, accessToken?: string) { - const uploadedFiles = [] - const headers: any = { - 'Content-Type': 'application/json' - } - if (accessToken) { - headers.Authorization = `Bearer ${accessToken}` - } - - for (const tableName in data) { - const csv = convertToCSV(data[tableName]) - if (csv === 'ERROR: LARGE STRING LENGTH') { - throw new Error( - 'The max length of a string value in SASjs is 32765 characters.' - ) - } - - const uploadResponse = await this.requestClient - .uploadFile(`${this.serverUrl}/files/files#rawUpload`, csv, accessToken) - .catch((err) => { - throw prefixMessage(err, 'Error while uploading file. ') - }) - - uploadedFiles.push({ tableName, file: uploadResponse.result }) - } - return uploadedFiles + return uploadTables(this.requestClient, data, accessToken) } private async getFolderDetails( @@ -1493,21 +1053,4 @@ export class SASViyaApiClient { return movedFolder } - - private async getTokens(authConfig: AuthConfig): Promise { - const logger = process.logger || console - let { access_token, refresh_token, client, secret } = authConfig - if ( - isAccessTokenExpiring(access_token) || - isRefreshTokenExpiring(refresh_token) - ) { - logger.info('Refreshing access and refresh tokens.') - ;({ access_token, refresh_token } = await this.refreshTokens( - client, - secret, - refresh_token - )) - } - return { access_token, refresh_token, client, secret } - } } diff --git a/src/api/viya/executeScript.ts b/src/api/viya/executeScript.ts new file mode 100644 index 0000000..0ea25c6 --- /dev/null +++ b/src/api/viya/executeScript.ts @@ -0,0 +1,303 @@ +import { + AuthConfig, + MacroVar, + Logger, + LogLevel, + timestampToYYYYMMDDHHMMSS +} from '@sasjs/utils' +import { prefixMessage } from '@sasjs/utils/error' +import { + PollOptions, + Job, + ComputeJobExecutionError, + NotFoundError +} from '../..' +import { getTokens } from '../../auth/tokens' +import { RequestClient } from '../../request/RequestClient' +import { SessionManager } from '../../SessionManager' +import { isRelativePath, fetchLogByChunks } from '../../utils' +import { formatDataForRequest } from '../../utils/formatDataForRequest' +import { pollJobState } from './pollJobState' +import { uploadTables } from './uploadTables' + +/** + * Executes code on the current SAS Viya server. + * @param jobPath - the path to the file being submitted for execution. + * @param linesOfCode - an array of code lines to execute. + * @param contextName - the context to execute the code in. + * @param authConfig - an object containing an access token, refresh token, client ID and secret. + * @param data - execution data. + * @param debug - when set to true, the log will be returned. + * @param expectWebout - when set to true, the automatic _webout fileref will be checked for content, and that content returned. This fileref is used when the Job contains a SASjs web request (as opposed to executing arbitrary SAS code). + * @param waitForResult - when set to true, function will return the session + * @param pollOptions - an object that represents poll interval(milliseconds) and maximum amount of attempts. Object example: { MAX_POLL_COUNT: 24 * 60 * 60, POLL_INTERVAL: 1000 }. + * @param printPid - a boolean that indicates whether the function should print (PID) of the started job. + * @param variables - an object that represents macro variables. + */ +export async function executeScript( + requestClient: RequestClient, + sessionManager: SessionManager, + rootFolderName: string, + jobPath: string, + linesOfCode: string[], + contextName: string, + authConfig?: AuthConfig, + data = null, + debug: boolean = false, + expectWebout = false, + waitForResult = true, + pollOptions?: PollOptions, + printPid = false, + variables?: MacroVar +): Promise { + let access_token = (authConfig || {}).access_token + if (authConfig) { + ;({ access_token } = await getTokens(requestClient, authConfig)) + } + + const logger = process.logger || console + + try { + let executionSessionId: string + + const session = await sessionManager + .getSession(access_token) + .catch((err) => { + throw prefixMessage(err, 'Error while getting session. ') + }) + + executionSessionId = session!.id + + if (printPid) { + const { result: jobIdVariable } = await sessionManager + .getVariable(executionSessionId, 'SYSJOBID', access_token) + .catch((err) => { + throw prefixMessage(err, 'Error while getting session variable. ') + }) + + if (jobIdVariable && jobIdVariable.value) { + const relativeJobPath = rootFolderName + ? jobPath.split(rootFolderName).join('').replace(/^\//, '') + : jobPath + + const logger = new Logger(debug ? LogLevel.Debug : LogLevel.Info) + + logger.info( + `Triggered '${relativeJobPath}' with PID ${ + jobIdVariable.value + } at ${timestampToYYYYMMDDHHMMSS()}` + ) + } + } + + const jobArguments: { [key: string]: any } = { + _contextName: contextName, + _OMITJSONLISTING: true, + _OMITJSONLOG: true, + _OMITSESSIONRESULTS: true, + _OMITTEXTLISTING: true, + _OMITTEXTLOG: true + } + + if (debug) { + jobArguments['_OMITTEXTLOG'] = false + jobArguments['_OMITSESSIONRESULTS'] = false + } + + let fileName + + if (isRelativePath(jobPath)) { + fileName = `exec-${ + jobPath.includes('/') ? jobPath.split('/')[1] : jobPath + }` + } else { + const jobPathParts = jobPath.split('/') + fileName = jobPathParts.pop() + } + + let jobVariables: any = { + SYS_JES_JOB_URI: '', + _program: isRelativePath(jobPath) + ? rootFolderName + '/' + jobPath + : jobPath + } + + if (variables) jobVariables = { ...jobVariables, ...variables } + + if (debug) jobVariables = { ...jobVariables, _DEBUG: 131 } + + let files: any[] = [] + + if (data) { + if (JSON.stringify(data).includes(';')) { + files = await uploadTables(requestClient, data, access_token).catch( + (err) => { + throw prefixMessage(err, 'Error while uploading tables. ') + } + ) + + jobVariables['_webin_file_count'] = files.length + + files.forEach((fileInfo, index) => { + jobVariables[ + `_webin_fileuri${index + 1}` + ] = `/files/files/${fileInfo.file.id}` + jobVariables[`_webin_name${index + 1}`] = fileInfo.tableName + }) + } else { + jobVariables = { ...jobVariables, ...formatDataForRequest(data) } + } + } + + // Execute job in session + const jobRequestBody = { + name: fileName, + description: 'Powered by SASjs', + code: linesOfCode, + variables: jobVariables, + arguments: jobArguments + } + + const { result: postedJob, etag } = await requestClient + .post( + `/compute/sessions/${executionSessionId}/jobs`, + jobRequestBody, + access_token + ) + .catch((err) => { + throw prefixMessage(err, 'Error while posting job. ') + }) + + if (!waitForResult) return session + + if (debug) { + logger.info(`Job has been submitted for '${fileName}'.`) + logger.info( + `You can monitor the job progress at '${requestClient.getBaseUrl()}${ + postedJob.links.find((l: any) => l.rel === 'state')!.href + }'.` + ) + } + + const jobStatus = await pollJobState( + requestClient, + postedJob, + debug, + etag, + authConfig, + pollOptions + ).catch(async (err) => { + const error = err?.response?.data + const result = /err=[0-9]*,/.exec(error) + + const errorCode = '5113' + if (result?.[0]?.slice(4, -1) === errorCode) { + const sessionLogUrl = + postedJob.links.find((l: any) => l.rel === 'up')!.href + '/log' + const logCount = 1000000 + err.log = await fetchLogByChunks( + requestClient, + access_token!, + sessionLogUrl, + logCount + ) + } + throw prefixMessage(err, 'Error while polling job status. ') + }) + + if (authConfig) { + ;({ access_token } = await getTokens(requestClient, authConfig)) + } + + const { result: currentJob } = await requestClient + .get( + `/compute/sessions/${executionSessionId}/jobs/${postedJob.id}`, + access_token + ) + .catch((err) => { + throw prefixMessage(err, 'Error while getting job. ') + }) + + let jobResult + let log = '' + + const logLink = currentJob.links.find((l) => l.rel === 'log') + + if (debug && logLink) { + const logUrl = `${logLink.href}/content` + const logCount = currentJob.logStatistics?.lineCount ?? 1000000 + log = await fetchLogByChunks( + requestClient, + access_token!, + logUrl, + logCount + ) + } + + if (jobStatus === 'failed' || jobStatus === 'error') { + return Promise.reject(new ComputeJobExecutionError(currentJob, log)) + } + + let resultLink + + if (expectWebout) { + resultLink = `/compute/sessions/${executionSessionId}/filerefs/_webout/content` + } else { + return { job: currentJob, log } + } + + if (resultLink) { + jobResult = await requestClient + .get(resultLink, access_token, 'text/plain') + .catch(async (e) => { + if (e instanceof NotFoundError) { + if (logLink) { + const logUrl = `${logLink.href}/content` + const logCount = currentJob.logStatistics?.lineCount ?? 1000000 + log = await fetchLogByChunks( + requestClient, + access_token!, + logUrl, + logCount + ) + + return Promise.reject({ + status: 500, + log + }) + } + } + + return { + result: JSON.stringify(e) + } + }) + } + + await sessionManager + .clearSession(executionSessionId, access_token) + .catch((err) => { + throw prefixMessage(err, 'Error while clearing session. ') + }) + + return { result: jobResult?.result, log } + } catch (e) { + if (e && e.status === 404) { + return executeScript( + requestClient, + sessionManager, + rootFolderName, + jobPath, + linesOfCode, + contextName, + authConfig, + data, + debug, + false, + true + ) + } else { + throw prefixMessage(e, 'Error while executing script. ') + } + } +} diff --git a/src/api/viya/pollJobState.ts b/src/api/viya/pollJobState.ts new file mode 100644 index 0000000..1bd85f8 --- /dev/null +++ b/src/api/viya/pollJobState.ts @@ -0,0 +1,178 @@ +import { AuthConfig, createFile, generateTimestamp } from '@sasjs/utils' +import { prefixMessage } from '@sasjs/utils/error' +import { Job, PollOptions } from '../..' +import { getTokens } from '../../auth/tokens' +import { RequestClient } from '../../request/RequestClient' +import { fetchLogByChunks } from '../../utils' + +export async function pollJobState( + requestClient: RequestClient, + postedJob: Job, + debug: boolean, + etag: string | null, + authConfig?: AuthConfig, + pollOptions?: PollOptions +) { + const logger = process.logger || console + + let POLL_INTERVAL = 300 + let MAX_POLL_COUNT = 1000 + let MAX_ERROR_COUNT = 5 + let access_token = (authConfig || {}).access_token + if (authConfig) { + ;({ access_token } = await getTokens(requestClient, authConfig)) + } + + if (pollOptions) { + POLL_INTERVAL = pollOptions.pollInterval || POLL_INTERVAL + MAX_POLL_COUNT = pollOptions.maxPollCount || MAX_POLL_COUNT + } + + let postedJobState = '' + let pollCount = 0 + let errorCount = 0 + const headers: any = { + 'Content-Type': 'application/json', + 'If-None-Match': etag + } + if (access_token) { + headers.Authorization = `Bearer ${access_token}` + } + const stateLink = postedJob.links.find((l: any) => l.rel === 'state') + if (!stateLink) { + return Promise.reject(`Job state link was not found.`) + } + + const { result: state } = await requestClient + .get( + `${stateLink.href}?_action=wait&wait=300`, + access_token, + 'text/plain', + {}, + debug + ) + .catch((err) => { + logger.error( + `Error fetching job state from ${stateLink.href}. Starting poll, assuming job to be running.`, + err + ) + return { result: 'unavailable' } + }) + + const currentState = state.trim() + if (currentState === 'completed') { + return Promise.resolve(currentState) + } + + return new Promise(async (resolve, _) => { + let printedState = '' + + const interval = setInterval(async () => { + if ( + postedJobState === 'running' || + postedJobState === '' || + postedJobState === 'pending' || + postedJobState === 'unavailable' + ) { + if (authConfig) { + ;({ access_token } = await getTokens(requestClient, authConfig)) + } + + if (stateLink) { + const { result: jobState } = await requestClient + .get( + `${stateLink.href}?_action=wait&wait=300`, + access_token, + 'text/plain', + {}, + debug + ) + .catch((err) => { + errorCount++ + if ( + pollCount >= MAX_POLL_COUNT || + errorCount >= MAX_ERROR_COUNT + ) { + throw prefixMessage( + err, + 'Error while getting job state after interval. ' + ) + } + logger.error( + `Error fetching job state from ${stateLink.href}. Resuming poll, assuming job to be running.`, + err + ) + return { result: 'unavailable' } + }) + + postedJobState = jobState.trim() + if (postedJobState != 'unavailable' && errorCount > 0) { + errorCount = 0 + } + + if (debug && printedState !== postedJobState) { + logger.info('Polling job status...') + logger.info(`Current job state: ${postedJobState}`) + + printedState = postedJobState + } + + pollCount++ + + await saveLog( + postedJob, + requestClient, + pollOptions?.streamLog || false, + pollOptions?.logFilePath, + access_token + ) + + if (pollCount >= MAX_POLL_COUNT) { + resolve(postedJobState) + } + } + } else { + clearInterval(interval) + resolve(postedJobState) + } + }, POLL_INTERVAL) + }) +} + +async function saveLog( + job: Job, + requestClient: RequestClient, + shouldSaveLog: boolean, + logFilePath?: string, + accessToken?: string +) { + if (!shouldSaveLog) { + return + } + + if (!accessToken) { + throw new Error( + `Logs for job ${job.id} cannot be fetched without a valid access token.` + ) + } + + const logger = process.logger || console + const logFileName = `${job.name || 'job'}-${generateTimestamp()}.log` + const logPath = `${logFilePath || process.cwd()}/${logFileName}` + const jobLogUrl = job.links.find((l) => l.rel === 'log') + + if (!jobLogUrl) { + throw new Error(`Log URL for job ${job.id} was not found.`) + } + + const logCount = job.logStatistics?.lineCount ?? 1000000 + const log = await fetchLogByChunks( + requestClient, + accessToken, + `${jobLogUrl.href}/content`, + logCount + ) + + logger.info(`Writing logs to ${logPath}`) + await createFile(logPath, log) +} diff --git a/src/api/viya/uploadTables.ts b/src/api/viya/uploadTables.ts new file mode 100644 index 0000000..13bf7b8 --- /dev/null +++ b/src/api/viya/uploadTables.ts @@ -0,0 +1,35 @@ +import { prefixMessage } from '@sasjs/utils/error' +import { RequestClient } from '../../request/RequestClient' +import { convertToCSV } from '../../utils/convertToCsv' + +export async function uploadTables( + requestClient: RequestClient, + data: any, + accessToken?: string +) { + const uploadedFiles = [] + const headers: any = { + 'Content-Type': 'application/json' + } + if (accessToken) { + headers.Authorization = `Bearer ${accessToken}` + } + + for (const tableName in data) { + const csv = convertToCSV(data[tableName]) + if (csv === 'ERROR: LARGE STRING LENGTH') { + throw new Error( + 'The max length of a string value in SASjs is 32765 characters.' + ) + } + + const uploadResponse = await requestClient + .uploadFile(`/files/files#rawUpload`, csv, accessToken) + .catch((err) => { + throw prefixMessage(err, 'Error while uploading file. ') + }) + + uploadedFiles.push({ tableName, file: uploadResponse.result }) + } + return uploadedFiles +} diff --git a/src/auth/tokens.ts b/src/auth/tokens.ts new file mode 100644 index 0000000..bc940e2 --- /dev/null +++ b/src/auth/tokens.ts @@ -0,0 +1,122 @@ +import { + AuthConfig, + isAccessTokenExpiring, + isRefreshTokenExpiring, + SasAuthResponse +} from '@sasjs/utils' +import * as NodeFormData from 'form-data' +import { RequestClient } from '../request/RequestClient' + +/** + * Exchanges the auth code for an access token for the given client. + * @param requestClient - the pre-configured HTTP request client + * @param clientId - the client ID to authenticate with. + * @param clientSecret - the client secret to authenticate with. + * @param authCode - the auth code received from the server. + */ +export async function getAccessToken( + requestClient: RequestClient, + clientId: string, + clientSecret: string, + authCode: string +): Promise { + const url = '/SASLogon/oauth/token' + let token + if (typeof Buffer === 'undefined') { + token = btoa(clientId + ':' + clientSecret) + } else { + token = Buffer.from(clientId + ':' + clientSecret).toString('base64') + } + const headers = { + Authorization: 'Basic ' + token + } + + let formData + if (typeof FormData === 'undefined') { + formData = new NodeFormData() + } else { + formData = new FormData() + } + formData.append('grant_type', 'authorization_code') + formData.append('code', authCode) + + const authResponse = await requestClient + .post( + url, + formData, + undefined, + 'multipart/form-data; boundary=' + (formData as any)._boundary, + headers + ) + .then((res) => res.result as SasAuthResponse) + + return authResponse +} + +/** + * Returns the auth configuration, refreshing the tokens if necessary. + * @param requestClient - the pre-configured HTTP request client + * @param authConfig - an object containing a client ID, secret, access token and refresh token + */ +export async function getTokens( + requestClient: RequestClient, + authConfig: AuthConfig +): Promise { + const logger = process.logger || console + let { access_token, refresh_token, client, secret } = authConfig + if ( + isAccessTokenExpiring(access_token) || + isRefreshTokenExpiring(refresh_token) + ) { + logger.info('Refreshing access and refresh tokens.') + ;({ access_token, refresh_token } = await refreshTokens( + requestClient, + client, + secret, + refresh_token + )) + } + return { access_token, refresh_token, client, secret } +} + +/** + * Exchanges the refresh token for an access token for the given client. + * @param requestClient - the pre-configured HTTP request client + * @param clientId - the client ID to authenticate with. + * @param clientSecret - the client secret to authenticate with. + * @param authCode - the refresh token received from the server. + */ +export async function refreshTokens( + requestClient: RequestClient, + clientId: string, + clientSecret: string, + refreshToken: string +) { + const url = '/SASLogon/oauth/token' + let token + if (typeof Buffer === 'undefined') { + token = btoa(clientId + ':' + clientSecret) + } else { + token = Buffer.from(clientId + ':' + clientSecret).toString('base64') + } + const headers = { + Authorization: 'Basic ' + token + } + + const formData = + typeof FormData === 'undefined' ? new NodeFormData() : new FormData() + formData.append('grant_type', 'refresh_token') + formData.append('refresh_token', refreshToken) + + const authResponse = await requestClient + .post( + url, + formData, + undefined, + 'multipart/form-data; boundary=' + (formData as any)._boundary, + headers + ) + .then((res) => res.result) + + return authResponse +} diff --git a/src/request/RequestClient.ts b/src/request/RequestClient.ts index eb6aac2..2701986 100644 --- a/src/request/RequestClient.ts +++ b/src/request/RequestClient.ts @@ -43,6 +43,7 @@ export interface HttpClient { getCsrfToken(type: 'general' | 'file'): CsrfToken | undefined clearCsrfTokens(): void + getBaseUrl(): string } export class RequestClient implements HttpClient { @@ -78,6 +79,10 @@ export class RequestClient implements HttpClient { this.fileUploadCsrfToken = { headerName: '', value: '' } } + public getBaseUrl() { + return this.httpClient.defaults.baseURL || '' + } + public async get( url: string, accessToken: string | undefined, diff --git a/src/types/PollOptions.ts b/src/types/PollOptions.ts index 0472f0c..6daa1b7 100644 --- a/src/types/PollOptions.ts +++ b/src/types/PollOptions.ts @@ -1,4 +1,6 @@ export interface PollOptions { - MAX_POLL_COUNT?: number - POLL_INTERVAL?: number + maxPollCount: number + pollInterval: number + streamLog: boolean + logFilePath?: string } From 13be2f9c701ba2ce05f80a43ff57c4c252a4291b Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Fri, 9 Jul 2021 09:17:26 +0100 Subject: [PATCH 02/39] chore(*): remove unused dependencies and variables, fix imports --- package-lock.json | 46 ++++++++++++++++++++--- package.json | 3 +- src/SASViyaApiClient.ts | 12 ------ src/SASjs.ts | 8 +++- src/api/viya/executeScript.ts | 69 +++++++++++++++-------------------- src/api/viya/pollJobState.ts | 39 +++++++++++--------- tsconfig.json | 2 +- 7 files changed, 100 insertions(+), 79 deletions(-) diff --git a/package-lock.json b/package-lock.json index 059a16c..b59f89d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,8 @@ "url": "^0.11.0" }, "devDependencies": { + "@types/axios": "^0.14.0", + "@types/form-data": "^2.5.0", "@types/jest": "^26.0.23", "@types/mime": "^2.0.3", "@types/tough-cookie": "^4.0.0", @@ -24,7 +26,6 @@ "dotenv": "^10.0.0", "jest": "^27.0.6", "jest-extended": "^0.11.5", - "mime": "^2.5.2", "node-polyfill-webpack-plugin": "^1.1.4", "path": "^0.12.7", "process": "^0.11.10", @@ -1435,6 +1436,16 @@ "node": ">= 6" } }, + "node_modules/@types/axios": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@types/axios/-/axios-0.14.0.tgz", + "integrity": "sha1-7CMA++fX3d1+udOr+HmZlkyvzkY=", + "deprecated": "This is a stub types definition for axios (https://github.com/mzabriskie/axios). axios provides its own type definitions, so you don't need @types/axios installed!", + "dev": true, + "dependencies": { + "axios": "*" + } + }, "node_modules/@types/babel__core": { "version": "7.1.14", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.14.tgz", @@ -1502,6 +1513,16 @@ "integrity": "sha512-LfZwXoGUDo0C3me81HXgkBg5CTQYb6xzEl+fNmbO4JdRiSKQ8A0GD1OBBvKAIsbCUgoyAty7m99GqqMQe784ew==", "dev": true }, + "node_modules/@types/form-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.5.0.tgz", + "integrity": "sha512-23/wYiuckYYtFpL+4RPWiWmRQH2BjFuqCUi2+N3amB1a1Drv+i/byTrGvlLwRVLFNAZbwpbQ7JvTK+VCAPMbcg==", + "deprecated": "This is a stub types definition. form-data provides its own type definitions, so you do not need this installed.", + "dev": true, + "dependencies": { + "form-data": "*" + } + }, "node_modules/@types/graceful-fs": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz", @@ -11155,11 +11176,6 @@ "safer-buffer": "^2.0.2", "tweetnacl": "~0.14.0" }, - "bin": { - "sshpk-conv": "bin/sshpk-conv", - "sshpk-sign": "bin/sshpk-sign", - "sshpk-verify": "bin/sshpk-verify" - }, "engines": { "node": ">=0.10.0" } @@ -15848,6 +15864,15 @@ "integrity": "sha512-RbzJvlNzmRq5c3O09UipeuXno4tA1FE6ikOjxZK0tuxVv3412l64l5t1W5pj4+rJq9vpkm/kwiR07aZXnsKPxw==", "dev": true }, + "@types/axios": { + "version": "0.14.0", + "resolved": "https://registry.npmjs.org/@types/axios/-/axios-0.14.0.tgz", + "integrity": "sha1-7CMA++fX3d1+udOr+HmZlkyvzkY=", + "dev": true, + "requires": { + "axios": "*" + } + }, "@types/babel__core": { "version": "7.1.14", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.1.14.tgz", @@ -15915,6 +15940,15 @@ "integrity": "sha512-LfZwXoGUDo0C3me81HXgkBg5CTQYb6xzEl+fNmbO4JdRiSKQ8A0GD1OBBvKAIsbCUgoyAty7m99GqqMQe784ew==", "dev": true }, + "@types/form-data": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@types/form-data/-/form-data-2.5.0.tgz", + "integrity": "sha512-23/wYiuckYYtFpL+4RPWiWmRQH2BjFuqCUi2+N3amB1a1Drv+i/byTrGvlLwRVLFNAZbwpbQ7JvTK+VCAPMbcg==", + "dev": true, + "requires": { + "form-data": "*" + } + }, "@types/graceful-fs": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz", diff --git a/package.json b/package.json index 65fd231..471ca36 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,8 @@ }, "license": "ISC", "devDependencies": { + "@types/axios": "^0.14.0", + "@types/form-data": "^2.5.0", "@types/jest": "^26.0.23", "@types/mime": "^2.0.3", "@types/tough-cookie": "^4.0.0", @@ -46,7 +48,6 @@ "dotenv": "^10.0.0", "jest": "^27.0.6", "jest-extended": "^0.11.5", - "mime": "^2.5.2", "node-polyfill-webpack-plugin": "^1.1.4", "path": "^0.12.7", "process": "^0.11.10", diff --git a/src/SASViyaApiClient.ts b/src/SASViyaApiClient.ts index 8d33de0..78d92b5 100644 --- a/src/SASViyaApiClient.ts +++ b/src/SASViyaApiClient.ts @@ -18,7 +18,6 @@ import { SasAuthResponse, MacroVar, AuthConfig } from '@sasjs/utils/types' import { isAuthorizeFormRequired } from './auth/isAuthorizeFormRequired' import { RequestClient } from './request/RequestClient' import { prefixMessage } from '@sasjs/utils/error' -import * as mime from 'mime' import { pollJobState } from './api/viya/pollJobState' import { getAccessToken, getTokens, refreshTokens } from './auth/tokens' import { uploadTables } from './api/viya/uploadTables' @@ -334,9 +333,6 @@ export class SASViyaApiClient { const formData = new NodeFormData() formData.append('file', contentBuffer, fileName) - const mimeType = - mime.getType(fileName.match(/\.[0-9a-z]+$/i)?.[0] || '') ?? 'text/plain' - return ( await this.requestClient.post( `/files/files?parentFolderUri=${parentFolderUri}&typeDefName=file#rawUpload`, @@ -939,14 +935,6 @@ export class SASViyaApiClient { ? sourceFolder : await this.getFolderUri(sourceFolder, accessToken) - const requestInfo = { - method: 'GET', - headers: { - 'Content-Type': 'application/json', - Authorization: 'Bearer ' + accessToken - } - } - const { result: members } = await this.requestClient.get<{ items: any[] }>( `${this.serverUrl}${sourceFolderUri}/members?limit=${limit}`, accessToken diff --git a/src/SASjs.ts b/src/SASjs.ts index d60ad27..b91e034 100644 --- a/src/SASjs.ts +++ b/src/SASjs.ts @@ -4,7 +4,12 @@ import { SASViyaApiClient } from './SASViyaApiClient' import { SAS9ApiClient } from './SAS9ApiClient' import { FileUploader } from './FileUploader' import { AuthManager } from './auth' -import { ServerType, MacroVar, AuthConfig } from '@sasjs/utils/types' +import { + ServerType, + MacroVar, + AuthConfig, + ExtraResponseAttributes +} from '@sasjs/utils/types' import { RequestClient } from './request/RequestClient' import { JobExecutor, @@ -14,7 +19,6 @@ import { Sas9JobExecutor } from './job-execution' import { ErrorResponse } from './types/errors' -import { ExtraResponseAttributes } from '@sasjs/utils/types' const defaultConfig: SASjsConfig = { serverUrl: '', diff --git a/src/api/viya/executeScript.ts b/src/api/viya/executeScript.ts index 0ea25c6..0abe466 100644 --- a/src/api/viya/executeScript.ts +++ b/src/api/viya/executeScript.ts @@ -1,10 +1,5 @@ -import { - AuthConfig, - MacroVar, - Logger, - LogLevel, - timestampToYYYYMMDDHHMMSS -} from '@sasjs/utils' +import { timestampToYYYYMMDDHHMMSS } from '@sasjs/utils/time' +import { AuthConfig, MacroVar } from '@sasjs/utils/types' import { prefixMessage } from '@sasjs/utils/error' import { PollOptions, @@ -42,7 +37,7 @@ export async function executeScript( linesOfCode: string[], contextName: string, authConfig?: AuthConfig, - data = null, + data: any = null, debug: boolean = false, expectWebout = false, waitForResult = true, @@ -80,7 +75,7 @@ export async function executeScript( ? jobPath.split(rootFolderName).join('').replace(/^\//, '') : jobPath - const logger = new Logger(debug ? LogLevel.Debug : LogLevel.Info) + const logger = process.logger || console logger.info( `Triggered '${relativeJobPath}' with PID ${ @@ -235,44 +230,40 @@ export async function executeScript( } if (jobStatus === 'failed' || jobStatus === 'error') { - return Promise.reject(new ComputeJobExecutionError(currentJob, log)) + throw new ComputeJobExecutionError(currentJob, log) } - let resultLink - - if (expectWebout) { - resultLink = `/compute/sessions/${executionSessionId}/filerefs/_webout/content` - } else { + if (!expectWebout) { return { job: currentJob, log } } - if (resultLink) { - jobResult = await requestClient - .get(resultLink, access_token, 'text/plain') - .catch(async (e) => { - if (e instanceof NotFoundError) { - if (logLink) { - const logUrl = `${logLink.href}/content` - const logCount = currentJob.logStatistics?.lineCount ?? 1000000 - log = await fetchLogByChunks( - requestClient, - access_token!, - logUrl, - logCount - ) + const resultLink = `/compute/sessions/${executionSessionId}/filerefs/_webout/content` - return Promise.reject({ - status: 500, - log - }) - } - } + jobResult = await requestClient + .get(resultLink, access_token, 'text/plain') + .catch(async (e) => { + if (e instanceof NotFoundError) { + if (logLink) { + const logUrl = `${logLink.href}/content` + const logCount = currentJob.logStatistics?.lineCount ?? 1000000 + log = await fetchLogByChunks( + requestClient, + access_token!, + logUrl, + logCount + ) - return { - result: JSON.stringify(e) + return Promise.reject({ + status: 500, + log + }) } - }) - } + } + + return { + result: JSON.stringify(e) + } + }) await sessionManager .clearSession(executionSessionId, access_token) diff --git a/src/api/viya/pollJobState.ts b/src/api/viya/pollJobState.ts index 1bd85f8..63796ce 100644 --- a/src/api/viya/pollJobState.ts +++ b/src/api/viya/pollJobState.ts @@ -1,5 +1,7 @@ -import { AuthConfig, createFile, generateTimestamp } from '@sasjs/utils' +import { AuthConfig } from '@sasjs/utils' import { prefixMessage } from '@sasjs/utils/error' +import { generateTimestamp } from '@sasjs/utils/time' +import { createFile } from '@sasjs/utils/file' import { Job, PollOptions } from '../..' import { getTokens } from '../../auth/tokens' import { RequestClient } from '../../request/RequestClient' @@ -15,17 +17,23 @@ export async function pollJobState( ) { const logger = process.logger || console - let POLL_INTERVAL = 300 - let MAX_POLL_COUNT = 1000 - let MAX_ERROR_COUNT = 5 + let pollInterval = 300 + let maxPollCount = 1000 + let maxErrorCount = 5 let access_token = (authConfig || {}).access_token + + const logFileName = `${postedJob.name || 'job'}-${generateTimestamp()}.log` + const logFilePath = `${ + pollOptions?.logFilePath || process.cwd() + }/${logFileName}` + if (authConfig) { ;({ access_token } = await getTokens(requestClient, authConfig)) } if (pollOptions) { - POLL_INTERVAL = pollOptions.pollInterval || POLL_INTERVAL - MAX_POLL_COUNT = pollOptions.maxPollCount || MAX_POLL_COUNT + pollInterval = pollOptions.pollInterval || pollInterval + maxPollCount = pollOptions.maxPollCount || maxPollCount } let postedJobState = '' @@ -89,10 +97,7 @@ export async function pollJobState( ) .catch((err) => { errorCount++ - if ( - pollCount >= MAX_POLL_COUNT || - errorCount >= MAX_ERROR_COUNT - ) { + if (pollCount >= maxPollCount || errorCount >= maxErrorCount) { throw prefixMessage( err, 'Error while getting job state after interval. ' @@ -123,11 +128,11 @@ export async function pollJobState( postedJob, requestClient, pollOptions?.streamLog || false, - pollOptions?.logFilePath, + logFilePath, access_token ) - if (pollCount >= MAX_POLL_COUNT) { + if (pollCount >= maxPollCount) { resolve(postedJobState) } } @@ -135,7 +140,7 @@ export async function pollJobState( clearInterval(interval) resolve(postedJobState) } - }, POLL_INTERVAL) + }, pollInterval) }) } @@ -143,7 +148,7 @@ async function saveLog( job: Job, requestClient: RequestClient, shouldSaveLog: boolean, - logFilePath?: string, + logFilePath: string, accessToken?: string ) { if (!shouldSaveLog) { @@ -157,8 +162,6 @@ async function saveLog( } const logger = process.logger || console - const logFileName = `${job.name || 'job'}-${generateTimestamp()}.log` - const logPath = `${logFilePath || process.cwd()}/${logFileName}` const jobLogUrl = job.links.find((l) => l.rel === 'log') if (!jobLogUrl) { @@ -173,6 +176,6 @@ async function saveLog( logCount ) - logger.info(`Writing logs to ${logPath}`) - await createFile(logPath, log) + logger.info(`Writing logs to ${logFilePath}`) + await createFile(logFilePath, log) } diff --git a/tsconfig.json b/tsconfig.json index e3dc154..36161db 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -9,5 +9,5 @@ "sourceMap": true }, "include": ["src"], - "exclude": ["node_modules", "**/*.spec.ts"] + "exclude": ["node_modules"] } From 0114a80e382c6d41d760360e31fc0951656ea3a2 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Fri, 9 Jul 2021 09:17:49 +0100 Subject: [PATCH 03/39] chore(execute): add tests for executeScript --- src/api/viya/spec/executeScript.spec.ts | 678 ++++++++++++++++++++++++ src/api/viya/spec/mockResponses.ts | 56 ++ 2 files changed, 734 insertions(+) create mode 100644 src/api/viya/spec/executeScript.spec.ts create mode 100644 src/api/viya/spec/mockResponses.ts diff --git a/src/api/viya/spec/executeScript.spec.ts b/src/api/viya/spec/executeScript.spec.ts new file mode 100644 index 0000000..03af686 --- /dev/null +++ b/src/api/viya/spec/executeScript.spec.ts @@ -0,0 +1,678 @@ +import { RequestClient } from '../../../request/RequestClient' +import { SessionManager } from '../../../SessionManager' +import { executeScript } from '../executeScript' +import { mockSession, mockAuthConfig, mockJob } from './mockResponses' +import * as pollJobStateModule from '../pollJobState' +import * as uploadTablesModule from '../uploadTables' +import * as tokensModule from '../../../auth/tokens' +import * as formatDataModule from '../../../utils/formatDataForRequest' +import * as fetchLogsModule from '../../../utils/fetchLogByChunks' +import { PollOptions } from '../../../types' +import { ComputeJobExecutionError, NotFoundError } from '../../../types/errors' +import { Logger, LogLevel } from '@sasjs/utils' + +const sessionManager = new (>SessionManager)() +const requestClient = new (>RequestClient)() +const defaultPollOptions: PollOptions = { + maxPollCount: 100, + pollInterval: 500, + streamLog: false +} + +describe('executeScript', () => { + beforeEach(() => { + ;(process as any).logger = new Logger(LogLevel.Off) + setupMocks() + }) + + it('should not try to get fresh tokens if an authConfig is not provided', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put hello'], + 'test context' + ) + + expect(tokensModule.getTokens).not.toHaveBeenCalled() + }) + + it('should try to get fresh tokens if an authConfig is provided', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put hello'], + 'test context', + mockAuthConfig + ) + + expect(tokensModule.getTokens).toHaveBeenCalledWith( + requestClient, + mockAuthConfig + ) + }) + + it('should get a session from the session manager before executing', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put hello'], + 'test context' + ) + + expect(sessionManager.getSession).toHaveBeenCalledWith(undefined) + }) + + it('should handle errors while getting a session', async () => { + jest + .spyOn(sessionManager, 'getSession') + .mockImplementation(() => Promise.reject('Test Error')) + + const error = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put hello'], + 'test context' + ).catch((e) => e) + + expect(error.includes('Error while getting session.')).toBeTruthy() + }) + + it('should fetch the PID when printPid is true', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put hello'], + 'test context', + mockAuthConfig, + null, + false, + false, + false, + defaultPollOptions, + true + ) + + expect(sessionManager.getVariable).toHaveBeenCalledWith( + mockSession.id, + 'SYSJOBID', + mockAuthConfig.access_token + ) + }) + + it('should handle errors while getting the job PID', async () => { + jest + .spyOn(sessionManager, 'getVariable') + .mockImplementation(() => Promise.reject('Test Error')) + + const error = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put hello'], + 'test context', + mockAuthConfig, + null, + false, + false, + false, + defaultPollOptions, + true + ).catch((e) => e) + + expect(error.includes('Error while getting session variable.')).toBeTruthy() + }) + + it('should use the file upload approach when data contains semicolons', async () => { + jest + .spyOn(uploadTablesModule, 'uploadTables') + .mockImplementation(() => + Promise.resolve([{ tableName: 'test', file: { id: 1 } }]) + ) + + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put hello'], + 'test context', + mockAuthConfig, + { foo: 'bar;' }, + false, + false, + false, + defaultPollOptions, + true + ) + + expect(uploadTablesModule.uploadTables).toHaveBeenCalledWith( + requestClient, + { foo: 'bar;' }, + mockAuthConfig.access_token + ) + }) + + it('should format data as CSV when it does not contain semicolons', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put hello'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + false, + false, + defaultPollOptions, + true + ) + + expect(formatDataModule.formatDataForRequest).toHaveBeenCalledWith({ + foo: 'bar' + }) + }) + + it('should submit a job for execution via the compute API', async () => { + jest + .spyOn(formatDataModule, 'formatDataForRequest') + .mockImplementation(() => ({ sasjs_tables: 'foo', sasjs0data: 'bar' })) + + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + false, + false, + defaultPollOptions, + true + ) + + expect(requestClient.post).toHaveBeenCalledWith( + `/compute/sessions/${mockSession.id}/jobs`, + { + name: 'exec-test', + description: 'Powered by SASjs', + code: ['%put "hello";'], + variables: { + SYS_JES_JOB_URI: '', + _program: 'test/test', + sasjs_tables: 'foo', + sasjs0data: 'bar' + }, + arguments: { + _contextName: 'test context', + _OMITJSONLISTING: true, + _OMITJSONLOG: true, + _OMITSESSIONRESULTS: true, + _OMITTEXTLISTING: true, + _OMITTEXTLOG: true + } + }, + mockAuthConfig.access_token + ) + }) + + it('should set the correct variables when debug is true', async () => { + jest + .spyOn(formatDataModule, 'formatDataForRequest') + .mockImplementation(() => ({ sasjs_tables: 'foo', sasjs0data: 'bar' })) + + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + true, + false, + false, + defaultPollOptions, + true + ) + + expect(requestClient.post).toHaveBeenCalledWith( + `/compute/sessions/${mockSession.id}/jobs`, + { + name: 'exec-test', + description: 'Powered by SASjs', + code: ['%put "hello";'], + variables: { + SYS_JES_JOB_URI: '', + _program: 'test/test', + sasjs_tables: 'foo', + sasjs0data: 'bar', + _DEBUG: 131 + }, + arguments: { + _contextName: 'test context', + _OMITJSONLISTING: true, + _OMITJSONLOG: true, + _OMITSESSIONRESULTS: false, + _OMITTEXTLISTING: true, + _OMITTEXTLOG: false + } + }, + mockAuthConfig.access_token + ) + }) + + it('should handle errors during job submission', async () => { + jest + .spyOn(requestClient, 'post') + .mockImplementation(() => Promise.reject('Test Error')) + + const error = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + true, + false, + false, + defaultPollOptions, + true + ).catch((e) => e) + + console.log(error) + expect(error.includes('Error while posting job')).toBeTruthy() + }) + + it('should immediately return the session when waitForResult is false', async () => { + const result = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + true, + false, + false, + defaultPollOptions, + true + ) + + expect(result).toEqual(mockSession) + }) + + it('should poll for job completion when waitForResult is true', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + false, + true, + defaultPollOptions, + true + ) + + expect(pollJobStateModule.pollJobState).toHaveBeenCalledWith( + requestClient, + mockJob, + false, + '', + mockAuthConfig, + defaultPollOptions + ) + }) + + it('should handle general errors when polling for job status', async () => { + jest + .spyOn(pollJobStateModule, 'pollJobState') + .mockImplementation(() => Promise.reject('Poll Error')) + + const error = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + false, + true, + defaultPollOptions, + true + ).catch((e) => e) + + expect(error.includes('Error while polling job status.')).toBeTruthy() + }) + + it('should fetch the log and append it to the error in case of a 5113 error code', async () => { + jest + .spyOn(pollJobStateModule, 'pollJobState') + .mockImplementation(() => + Promise.reject({ response: { data: 'err=5113,' } }) + ) + + const error = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + false, + true, + defaultPollOptions, + true + ).catch((e) => e) + + expect(fetchLogsModule.fetchLogByChunks).toHaveBeenCalledWith( + requestClient, + mockAuthConfig.access_token, + mockJob.links.find((l) => l.rel === 'up')!.href + '/log', + 1000000 + ) + expect(error.log).toEqual('Test Log') + }) + + it('should fetch the logs for the job if debug is true and a log URL is available', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + true, + false, + true, + defaultPollOptions, + true + ) + + expect(fetchLogsModule.fetchLogByChunks).toHaveBeenCalledWith( + requestClient, + mockAuthConfig.access_token, + mockJob.links.find((l) => l.rel === 'log')!.href + '/content', + mockJob.logStatistics.lineCount + ) + }) + + it('should not fetch the logs for the job if debug is false', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + false, + true, + defaultPollOptions, + true + ) + + expect(fetchLogsModule.fetchLogByChunks).not.toHaveBeenCalled() + }) + + it('should throw a ComputeJobExecutionError if the job has failed', async () => { + jest + .spyOn(pollJobStateModule, 'pollJobState') + .mockImplementation(() => Promise.resolve('failed')) + + const error: ComputeJobExecutionError = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + true, + false, + true, + defaultPollOptions, + true + ).catch((e) => e) + + expect(fetchLogsModule.fetchLogByChunks).toHaveBeenCalledWith( + requestClient, + mockAuthConfig.access_token, + mockJob.links.find((l) => l.rel === 'log')!.href + '/content', + mockJob.logStatistics.lineCount + ) + + expect(error).toBeInstanceOf(ComputeJobExecutionError) + expect(error.log).toEqual('Test Log') + expect(error.job).toEqual(mockJob) + }) + + it('should throw a ComputeJobExecutionError if the job has errored out', async () => { + jest + .spyOn(pollJobStateModule, 'pollJobState') + .mockImplementation(() => Promise.resolve('error')) + + const error: ComputeJobExecutionError = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + true, + false, + true, + defaultPollOptions, + true + ).catch((e) => e) + + expect(fetchLogsModule.fetchLogByChunks).toHaveBeenCalledWith( + requestClient, + mockAuthConfig.access_token, + mockJob.links.find((l) => l.rel === 'log')!.href + '/content', + mockJob.logStatistics.lineCount + ) + + expect(error).toBeInstanceOf(ComputeJobExecutionError) + expect(error.log).toEqual('Test Log') + expect(error.job).toEqual(mockJob) + }) + + it('should fetch the result if expectWebout is true', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + true, + true, + defaultPollOptions, + true + ) + + expect(requestClient.get).toHaveBeenCalledWith( + `/compute/sessions/${mockSession.id}/filerefs/_webout/content`, + mockAuthConfig.access_token, + 'text/plain' + ) + }) + + it('should fetch the logs if the webout file was not found', async () => { + jest.spyOn(requestClient, 'get').mockImplementation((url, ...rest) => { + if (url.includes('_webout')) { + return Promise.reject(new NotFoundError(url)) + } + return Promise.resolve({ result: mockJob, etag: '' }) + }) + + const error = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + true, + true, + defaultPollOptions, + true + ).catch((e) => e) + + expect(requestClient.get).toHaveBeenCalledWith( + `/compute/sessions/${mockSession.id}/filerefs/_webout/content`, + mockAuthConfig.access_token, + 'text/plain' + ) + + expect(fetchLogsModule.fetchLogByChunks).toHaveBeenCalledWith( + requestClient, + mockAuthConfig.access_token, + mockJob.links.find((l) => l.rel === 'log')!.href + '/content', + mockJob.logStatistics.lineCount + ) + + expect(error.status).toEqual(500) + expect(error.log).toEqual('Test Log') + }) + + it('should clear the session after execution is complete', async () => { + await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + true, + true, + defaultPollOptions, + true + ) + + expect(sessionManager.clearSession).toHaveBeenCalledWith( + mockSession.id, + mockAuthConfig.access_token + ) + }) + + it('should handle errors while clearing a session', async () => { + jest + .spyOn(sessionManager, 'clearSession') + .mockImplementation(() => Promise.reject('Clear Session Error')) + + const error = await executeScript( + requestClient, + sessionManager, + 'test', + 'test', + ['%put "hello";'], + 'test context', + mockAuthConfig, + { foo: 'bar' }, + false, + true, + true, + defaultPollOptions, + true + ).catch((e) => { + console.log(e) + return e + }) + + expect(error.includes('Error while clearing session.')).toBeTruthy() + }) +}) + +const setupMocks = () => { + jest.restoreAllMocks() + jest.mock('../../../request/RequestClient') + jest.mock('../../../SessionManager') + jest.mock('../../../auth/tokens') + jest.mock('../pollJobState') + jest.mock('../uploadTables') + jest.mock('../../../utils/formatDataForRequest') + jest.mock('../../../utils/fetchLogByChunks') + + jest + .spyOn(requestClient, 'post') + .mockImplementation(() => Promise.resolve({ result: mockJob, etag: '' })) + jest + .spyOn(requestClient, 'get') + .mockImplementation(() => Promise.resolve({ result: mockJob, etag: '' })) + jest + .spyOn(requestClient, 'delete') + .mockImplementation(() => Promise.resolve({ result: {}, etag: '' })) + jest + .spyOn(tokensModule, 'getTokens') + .mockImplementation(() => Promise.resolve(mockAuthConfig)) + jest + .spyOn(pollJobStateModule, 'pollJobState') + .mockImplementation(() => Promise.resolve('completed')) + jest + .spyOn(sessionManager, 'getVariable') + .mockImplementation(() => + Promise.resolve({ result: { value: 'test' }, etag: 'test' }) + ) + jest + .spyOn(sessionManager, 'getSession') + .mockImplementation(() => Promise.resolve(mockSession)) + jest + .spyOn(sessionManager, 'clearSession') + .mockImplementation(() => Promise.resolve()) + jest + .spyOn(formatDataModule, 'formatDataForRequest') + .mockImplementation(() => ({ sasjs_tables: 'test', sasjs0data: 'test' })) + jest + .spyOn(fetchLogsModule, 'fetchLogByChunks') + .mockImplementation(() => Promise.resolve('Test Log')) +} diff --git a/src/api/viya/spec/mockResponses.ts b/src/api/viya/spec/mockResponses.ts new file mode 100644 index 0000000..e85028c --- /dev/null +++ b/src/api/viya/spec/mockResponses.ts @@ -0,0 +1,56 @@ +import { AuthConfig } from '@sasjs/utils/types' +import { Job, Session } from '../../../types' + +export const mockSession: Session = { + id: 's35510n', + state: 'idle', + links: [], + attributes: { + sessionInactiveTimeout: 1 + }, + creationTimeStamp: new Date().valueOf().toString() +} + +export const mockJob: Job = { + id: 'j0b', + name: 'test job', + uri: '/j0b', + createdBy: 'test user', + results: { + '_webout.json': 'test' + }, + logStatistics: { + lineCount: 100, + modifiedTimeStamp: new Date().valueOf().toString() + }, + links: [ + { + rel: 'log', + href: '/log', + method: 'GET', + type: 'log', + uri: 'log' + }, + { + rel: 'state', + href: '/state', + method: 'GET', + type: 'state', + uri: 'state' + }, + { + rel: 'up', + href: '/job', + method: 'GET', + type: 'up', + uri: 'job' + } + ] +} + +export const mockAuthConfig: AuthConfig = { + client: 'cl13nt', + secret: '53cr3t', + access_token: 'acc355', + refresh_token: 'r3fr35h' +} From 1c90f4f455bea77d410e93372d269e6a72bab00c Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Fri, 9 Jul 2021 09:29:57 +0100 Subject: [PATCH 04/39] chore(*): remove log --- src/api/viya/spec/executeScript.spec.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/api/viya/spec/executeScript.spec.ts b/src/api/viya/spec/executeScript.spec.ts index 03af686..bca8fdb 100644 --- a/src/api/viya/spec/executeScript.spec.ts +++ b/src/api/viya/spec/executeScript.spec.ts @@ -624,10 +624,7 @@ describe('executeScript', () => { true, defaultPollOptions, true - ).catch((e) => { - console.log(e) - return e - }) + ).catch((e) => e) expect(error.includes('Error while clearing session.')).toBeTruthy() }) From f57c7b8f7d097e0d36e16956b46d84223a1cfee1 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Mon, 12 Jul 2021 20:30:42 +0100 Subject: [PATCH 05/39] chore(deps): up utils version --- package-lock.json | 14 +++++++------- package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index b59f89d..970d08e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,7 @@ "name": "@sasjs/adapter", "license": "ISC", "dependencies": { - "@sasjs/utils": "^2.23.2", + "@sasjs/utils": "^2.24.0", "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", "form-data": "^4.0.0", @@ -1272,9 +1272,9 @@ } }, "node_modules/@sasjs/utils": { - "version": "2.23.2", - "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.23.2.tgz", - "integrity": "sha512-PGHrqLgi/QajseksqD/2CSL+b45tLLQQUZrBW/PpHzvx2qcwXxfrWvnSo6v3UwUigxgyu+xkPK5AIlEJ81Tndw==", + "version": "2.24.0", + "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.24.0.tgz", + "integrity": "sha512-tWCqY8u/1So7mcUOgwzaePdpJaCQ60ihNlvTNHSzlJP5WIR2LKraoyinMfocWrinf/FwrR6qNFuefKZ3d5M8LA==", "dependencies": { "@types/prompts": "^2.0.13", "chalk": "^4.1.1", @@ -15720,9 +15720,9 @@ } }, "@sasjs/utils": { - "version": "2.23.2", - "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.23.2.tgz", - "integrity": "sha512-PGHrqLgi/QajseksqD/2CSL+b45tLLQQUZrBW/PpHzvx2qcwXxfrWvnSo6v3UwUigxgyu+xkPK5AIlEJ81Tndw==", + "version": "2.24.0", + "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.24.0.tgz", + "integrity": "sha512-tWCqY8u/1So7mcUOgwzaePdpJaCQ60ihNlvTNHSzlJP5WIR2LKraoyinMfocWrinf/FwrR6qNFuefKZ3d5M8LA==", "requires": { "@types/prompts": "^2.0.13", "chalk": "^4.1.1", diff --git a/package.json b/package.json index 471ca36..5e217df 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ }, "main": "index.js", "dependencies": { - "@sasjs/utils": "^2.23.2", + "@sasjs/utils": "^2.24.0", "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", "form-data": "^4.0.0", From 123b9fb5356ee2147c66040fbf1bde98a9cc5f65 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Mon, 12 Jul 2021 20:31:17 +0100 Subject: [PATCH 06/39] chore(refactor): split up and add tests for core functionality --- src/SASViyaApiClient.ts | 4 +- src/api/viya/executeScript.ts | 2 +- src/api/viya/pollJobState.ts | 56 +---- src/api/viya/saveLog.ts | 40 ++++ src/api/viya/spec/executeScript.spec.ts | 20 +- src/api/viya/spec/pollJobState.spec.ts | 266 ++++++++++++++++++++++++ src/api/viya/spec/saveLog.spec.ts | 72 +++++++ src/auth/getAccessToken.ts | 49 +++++ src/auth/getTokens.ts | 40 ++++ src/auth/refreshTokens.ts | 49 +++++ src/auth/spec/getTokens.spec.ts | 79 +++++++ src/auth/spec/mockResponses.ts | 22 ++ src/auth/spec/refreshTokens.spec.ts | 75 +++++++ src/auth/tokens.ts | 122 ----------- 14 files changed, 717 insertions(+), 179 deletions(-) create mode 100644 src/api/viya/saveLog.ts create mode 100644 src/api/viya/spec/pollJobState.spec.ts create mode 100644 src/api/viya/spec/saveLog.spec.ts create mode 100644 src/auth/getAccessToken.ts create mode 100644 src/auth/getTokens.ts create mode 100644 src/auth/refreshTokens.ts create mode 100644 src/auth/spec/getTokens.spec.ts create mode 100644 src/auth/spec/refreshTokens.spec.ts delete mode 100644 src/auth/tokens.ts diff --git a/src/SASViyaApiClient.ts b/src/SASViyaApiClient.ts index 78d92b5..ee439f2 100644 --- a/src/SASViyaApiClient.ts +++ b/src/SASViyaApiClient.ts @@ -19,9 +19,11 @@ import { isAuthorizeFormRequired } from './auth/isAuthorizeFormRequired' import { RequestClient } from './request/RequestClient' import { prefixMessage } from '@sasjs/utils/error' import { pollJobState } from './api/viya/pollJobState' -import { getAccessToken, getTokens, refreshTokens } from './auth/tokens' +import { getTokens } from './auth/getTokens' import { uploadTables } from './api/viya/uploadTables' import { executeScript } from './api/viya/executeScript' +import { getAccessToken } from './auth/getAccessToken' +import { refreshTokens } from './auth/refreshTokens' /** * A client for interfacing with the SAS Viya REST API. diff --git a/src/api/viya/executeScript.ts b/src/api/viya/executeScript.ts index 0abe466..c75a3e5 100644 --- a/src/api/viya/executeScript.ts +++ b/src/api/viya/executeScript.ts @@ -7,7 +7,7 @@ import { ComputeJobExecutionError, NotFoundError } from '../..' -import { getTokens } from '../../auth/tokens' +import { getTokens } from '../../auth/getTokens' import { RequestClient } from '../../request/RequestClient' import { SessionManager } from '../../SessionManager' import { isRelativePath, fetchLogByChunks } from '../../utils' diff --git a/src/api/viya/pollJobState.ts b/src/api/viya/pollJobState.ts index 63796ce..8d95c42 100644 --- a/src/api/viya/pollJobState.ts +++ b/src/api/viya/pollJobState.ts @@ -1,11 +1,10 @@ -import { AuthConfig } from '@sasjs/utils' +import { AuthConfig } from '@sasjs/utils/types' import { prefixMessage } from '@sasjs/utils/error' import { generateTimestamp } from '@sasjs/utils/time' -import { createFile } from '@sasjs/utils/file' import { Job, PollOptions } from '../..' -import { getTokens } from '../../auth/tokens' +import { getTokens } from '../../auth/getTokens' import { RequestClient } from '../../request/RequestClient' -import { fetchLogByChunks } from '../../utils' +import { saveLog } from './saveLog' export async function pollJobState( requestClient: RequestClient, @@ -48,7 +47,7 @@ export async function pollJobState( } const stateLink = postedJob.links.find((l: any) => l.rel === 'state') if (!stateLink) { - return Promise.reject(`Job state link was not found.`) + throw new Error(`Job state link was not found.`) } const { result: state } = await requestClient @@ -72,7 +71,7 @@ export async function pollJobState( return Promise.resolve(currentState) } - return new Promise(async (resolve, _) => { + return new Promise(async (resolve, reject) => { let printedState = '' const interval = setInterval(async () => { @@ -98,9 +97,12 @@ export async function pollJobState( .catch((err) => { errorCount++ if (pollCount >= maxPollCount || errorCount >= maxErrorCount) { - throw prefixMessage( - err, - 'Error while getting job state after interval. ' + clearInterval(interval) + reject( + prefixMessage( + err, + 'Error while getting job state after interval. ' + ) ) } logger.error( @@ -143,39 +145,3 @@ export async function pollJobState( }, pollInterval) }) } - -async function saveLog( - job: Job, - requestClient: RequestClient, - shouldSaveLog: boolean, - logFilePath: string, - accessToken?: string -) { - if (!shouldSaveLog) { - return - } - - if (!accessToken) { - throw new Error( - `Logs for job ${job.id} cannot be fetched without a valid access token.` - ) - } - - const logger = process.logger || console - const jobLogUrl = job.links.find((l) => l.rel === 'log') - - if (!jobLogUrl) { - throw new Error(`Log URL for job ${job.id} was not found.`) - } - - const logCount = job.logStatistics?.lineCount ?? 1000000 - const log = await fetchLogByChunks( - requestClient, - accessToken, - `${jobLogUrl.href}/content`, - logCount - ) - - logger.info(`Writing logs to ${logFilePath}`) - await createFile(logFilePath, log) -} diff --git a/src/api/viya/saveLog.ts b/src/api/viya/saveLog.ts new file mode 100644 index 0000000..9200930 --- /dev/null +++ b/src/api/viya/saveLog.ts @@ -0,0 +1,40 @@ +import { createFile } from '@sasjs/utils/file' +import { Job } from '../..' +import { RequestClient } from '../../request/RequestClient' +import { fetchLogByChunks } from '../../utils' + +export async function saveLog( + job: Job, + requestClient: RequestClient, + shouldSaveLog: boolean, + logFilePath: string, + accessToken?: string +) { + if (!shouldSaveLog) { + return + } + + if (!accessToken) { + throw new Error( + `Logs for job ${job.id} cannot be fetched without a valid access token.` + ) + } + + const logger = process.logger || console + const jobLogUrl = job.links.find((l) => l.rel === 'log') + + if (!jobLogUrl) { + throw new Error(`Log URL for job ${job.id} was not found.`) + } + + const logCount = job.logStatistics?.lineCount ?? 1000000 + const log = await fetchLogByChunks( + requestClient, + accessToken, + `${jobLogUrl.href}/content`, + logCount + ) + + logger.info(`Writing logs to ${logFilePath}`) + await createFile(logFilePath, log) +} diff --git a/src/api/viya/spec/executeScript.spec.ts b/src/api/viya/spec/executeScript.spec.ts index bca8fdb..314c70b 100644 --- a/src/api/viya/spec/executeScript.spec.ts +++ b/src/api/viya/spec/executeScript.spec.ts @@ -4,7 +4,7 @@ import { executeScript } from '../executeScript' import { mockSession, mockAuthConfig, mockJob } from './mockResponses' import * as pollJobStateModule from '../pollJobState' import * as uploadTablesModule from '../uploadTables' -import * as tokensModule from '../../../auth/tokens' +import * as getTokensModule from '../../../auth/getTokens' import * as formatDataModule from '../../../utils/formatDataForRequest' import * as fetchLogsModule from '../../../utils/fetchLogByChunks' import { PollOptions } from '../../../types' @@ -35,7 +35,7 @@ describe('executeScript', () => { 'test context' ) - expect(tokensModule.getTokens).not.toHaveBeenCalled() + expect(getTokensModule.getTokens).not.toHaveBeenCalled() }) it('should try to get fresh tokens if an authConfig is provided', async () => { @@ -49,7 +49,7 @@ describe('executeScript', () => { mockAuthConfig ) - expect(tokensModule.getTokens).toHaveBeenCalledWith( + expect(getTokensModule.getTokens).toHaveBeenCalledWith( requestClient, mockAuthConfig ) @@ -82,7 +82,7 @@ describe('executeScript', () => { 'test context' ).catch((e) => e) - expect(error.includes('Error while getting session.')).toBeTruthy() + expect(error).toContain('Error while getting session.') }) it('should fetch the PID when printPid is true', async () => { @@ -130,7 +130,7 @@ describe('executeScript', () => { true ).catch((e) => e) - expect(error.includes('Error while getting session variable.')).toBeTruthy() + expect(error).toContain('Error while getting session variable.') }) it('should use the file upload approach when data contains semicolons', async () => { @@ -300,7 +300,7 @@ describe('executeScript', () => { ).catch((e) => e) console.log(error) - expect(error.includes('Error while posting job')).toBeTruthy() + expect(error).toContain('Error while posting job') }) it('should immediately return the session when waitForResult is false', async () => { @@ -371,7 +371,7 @@ describe('executeScript', () => { true ).catch((e) => e) - expect(error.includes('Error while polling job status.')).toBeTruthy() + expect(error).toContain('Error while polling job status.') }) it('should fetch the log and append it to the error in case of a 5113 error code', async () => { @@ -626,7 +626,7 @@ describe('executeScript', () => { true ).catch((e) => e) - expect(error.includes('Error while clearing session.')).toBeTruthy() + expect(error).toContain('Error while clearing session.') }) }) @@ -634,7 +634,7 @@ const setupMocks = () => { jest.restoreAllMocks() jest.mock('../../../request/RequestClient') jest.mock('../../../SessionManager') - jest.mock('../../../auth/tokens') + jest.mock('../../../auth/getTokens') jest.mock('../pollJobState') jest.mock('../uploadTables') jest.mock('../../../utils/formatDataForRequest') @@ -650,7 +650,7 @@ const setupMocks = () => { .spyOn(requestClient, 'delete') .mockImplementation(() => Promise.resolve({ result: {}, etag: '' })) jest - .spyOn(tokensModule, 'getTokens') + .spyOn(getTokensModule, 'getTokens') .mockImplementation(() => Promise.resolve(mockAuthConfig)) jest .spyOn(pollJobStateModule, 'pollJobState') diff --git a/src/api/viya/spec/pollJobState.spec.ts b/src/api/viya/spec/pollJobState.spec.ts new file mode 100644 index 0000000..23ac560 --- /dev/null +++ b/src/api/viya/spec/pollJobState.spec.ts @@ -0,0 +1,266 @@ +import { RequestClient } from '../../../request/RequestClient' +import { mockAuthConfig, mockJob } from './mockResponses' +import { pollJobState } from '../pollJobState' +import * as getTokensModule from '../../../auth/getTokens' +import * as saveLogModule from '../saveLog' +import { PollOptions } from '../../../types' +import { Logger, LogLevel } from '@sasjs/utils' + +const requestClient = new (>RequestClient)() +const defaultPollOptions: PollOptions = { + maxPollCount: 100, + pollInterval: 500, + streamLog: false +} + +describe('pollJobState', () => { + beforeEach(() => { + ;(process as any).logger = new Logger(LogLevel.Off) + setupMocks() + }) + + it('should get valid tokens if the authConfig has been provided', async () => { + await pollJobState( + requestClient, + mockJob, + false, + 'test', + mockAuthConfig, + defaultPollOptions + ) + + expect(getTokensModule.getTokens).toHaveBeenCalledWith( + requestClient, + mockAuthConfig + ) + }) + + it('should not attempt to get tokens if the authConfig has not been provided', async () => { + await pollJobState( + requestClient, + mockJob, + false, + 'test', + undefined, + defaultPollOptions + ) + + expect(getTokensModule.getTokens).not.toHaveBeenCalled() + }) + + it('should throw an error if the job does not have a state link', async () => { + const error = await pollJobState( + requestClient, + { ...mockJob, links: mockJob.links.filter((l) => l.rel !== 'state') }, + false, + 'test', + undefined, + defaultPollOptions + ).catch((e) => e) + + expect((error as Error).message).toContain('Job state link was not found.') + }) + + it('should attempt to refresh tokens before each poll', async () => { + jest + .spyOn(requestClient, 'get') + .mockImplementationOnce(() => + Promise.resolve({ result: 'pending', etag: '' }) + ) + .mockImplementationOnce(() => + Promise.resolve({ result: 'running', etag: '' }) + ) + .mockImplementation(() => + Promise.resolve({ result: 'completed', etag: '' }) + ) + + await pollJobState( + requestClient, + mockJob, + false, + 'test', + mockAuthConfig, + defaultPollOptions + ) + + expect(getTokensModule.getTokens).toHaveBeenCalledTimes(3) + }) + + it('should attempt to fetch and save the log after each poll', async () => { + jest + .spyOn(requestClient, 'get') + .mockImplementationOnce(() => + Promise.resolve({ result: 'pending', etag: '' }) + ) + .mockImplementationOnce(() => + Promise.resolve({ result: 'running', etag: '' }) + ) + .mockImplementation(() => + Promise.resolve({ result: 'completed', etag: '' }) + ) + + await pollJobState( + requestClient, + mockJob, + false, + 'test', + mockAuthConfig, + defaultPollOptions + ) + + expect(saveLogModule.saveLog).toHaveBeenCalledTimes(2) + }) + + it('should return the current status when the max poll count is reached', async () => { + jest + .spyOn(requestClient, 'get') + .mockImplementationOnce(() => + Promise.resolve({ result: 'pending', etag: '' }) + ) + .mockImplementationOnce(() => + Promise.resolve({ result: 'running', etag: '' }) + ) + + const state = await pollJobState( + requestClient, + mockJob, + false, + 'test', + mockAuthConfig, + { + ...defaultPollOptions, + maxPollCount: 1 + } + ) + + expect(state).toEqual('running') + }) + + it('should continue polling until the job completes or errors', async () => { + jest + .spyOn(requestClient, 'get') + .mockImplementationOnce(() => + Promise.resolve({ result: 'pending', etag: '' }) + ) + .mockImplementationOnce(() => + Promise.resolve({ result: 'running', etag: '' }) + ) + .mockImplementation(() => + Promise.resolve({ result: 'completed', etag: '' }) + ) + + const state = await pollJobState( + requestClient, + mockJob, + false, + 'test', + undefined, + defaultPollOptions + ) + + expect(requestClient.get).toHaveBeenCalledTimes(4) + expect(state).toEqual('completed') + }) + + it('should print the state to the console when debug is on', async () => { + jest.spyOn((process as any).logger, 'info') + jest + .spyOn(requestClient, 'get') + .mockImplementationOnce(() => + Promise.resolve({ result: 'pending', etag: '' }) + ) + .mockImplementationOnce(() => + Promise.resolve({ result: 'running', etag: '' }) + ) + .mockImplementation(() => + Promise.resolve({ result: 'completed', etag: '' }) + ) + + await pollJobState( + requestClient, + mockJob, + true, + 'test', + undefined, + defaultPollOptions + ) + + expect((process as any).logger.info).toHaveBeenCalledTimes(4) + expect((process as any).logger.info).toHaveBeenNthCalledWith( + 1, + 'Polling job status...' + ) + expect((process as any).logger.info).toHaveBeenNthCalledWith( + 2, + 'Current job state: running' + ) + expect((process as any).logger.info).toHaveBeenNthCalledWith( + 3, + 'Polling job status...' + ) + expect((process as any).logger.info).toHaveBeenNthCalledWith( + 4, + 'Current job state: completed' + ) + }) + + it('should continue polling when there is a single error in between', async () => { + jest + .spyOn(requestClient, 'get') + .mockImplementationOnce(() => + Promise.resolve({ result: 'pending', etag: '' }) + ) + .mockImplementationOnce(() => Promise.reject('Status Error')) + .mockImplementationOnce(() => + Promise.resolve({ result: 'completed', etag: '' }) + ) + + const state = await pollJobState( + requestClient, + mockJob, + false, + 'test', + undefined, + defaultPollOptions + ) + + expect(requestClient.get).toHaveBeenCalledTimes(3) + expect(state).toEqual('completed') + }) + + it('should throw an error when the error count exceeds the set value of 5', async () => { + jest + .spyOn(requestClient, 'get') + .mockImplementation(() => Promise.reject('Status Error')) + + const error = await pollJobState( + requestClient, + mockJob, + false, + 'test', + undefined, + defaultPollOptions + ).catch((e) => e) + + expect(error).toContain('Error while getting job state after interval.') + }) +}) + +const setupMocks = () => { + jest.restoreAllMocks() + jest.mock('../../../request/RequestClient') + jest.mock('../../../auth/getTokens') + jest.mock('../saveLog') + + jest + .spyOn(requestClient, 'get') + .mockImplementation(() => + Promise.resolve({ result: 'completed', etag: '' }) + ) + jest + .spyOn(getTokensModule, 'getTokens') + .mockImplementation(() => Promise.resolve(mockAuthConfig)) + jest + .spyOn(saveLogModule, 'saveLog') + .mockImplementation(() => Promise.resolve()) +} diff --git a/src/api/viya/spec/saveLog.spec.ts b/src/api/viya/spec/saveLog.spec.ts new file mode 100644 index 0000000..c4b8b9d --- /dev/null +++ b/src/api/viya/spec/saveLog.spec.ts @@ -0,0 +1,72 @@ +import { Logger, LogLevel } from '@sasjs/utils' +import * as fileModule from '@sasjs/utils/file' +import { RequestClient } from '../../../request/RequestClient' +import * as fetchLogsModule from '../../../utils/fetchLogByChunks' +import { saveLog } from '../saveLog' +import { mockJob } from './mockResponses' + +const requestClient = new (>RequestClient)() + +describe('saveLog', () => { + beforeEach(() => { + ;(process as any).logger = new Logger(LogLevel.Off) + setupMocks() + }) + + it('should return immediately if shouldSaveLog is false', async () => { + await saveLog(mockJob, requestClient, false, '/test', 't0k3n') + + expect(fetchLogsModule.fetchLogByChunks).not.toHaveBeenCalled() + expect(fileModule.createFile).not.toHaveBeenCalled() + }) + + it('should throw an error when a valid access token is not provided', async () => { + const error = await saveLog(mockJob, requestClient, true, '/test').catch( + (e) => e + ) + + expect(error.message).toContain( + `Logs for job ${mockJob.id} cannot be fetched without a valid access token.` + ) + }) + + it('should throw an error when the log URL is not available', async () => { + const error = await saveLog( + { ...mockJob, links: mockJob.links.filter((l) => l.rel !== 'log') }, + requestClient, + true, + '/test', + 't0k3n' + ).catch((e) => e) + + expect(error.message).toContain( + `Log URL for job ${mockJob.id} was not found.` + ) + }) + + it('should fetch and save logs to the given path', async () => { + await saveLog(mockJob, requestClient, true, '/test', 't0k3n') + + expect(fetchLogsModule.fetchLogByChunks).toHaveBeenCalledWith( + requestClient, + 't0k3n', + '/log/content', + 100 + ) + expect(fileModule.createFile).toHaveBeenCalledWith('/test', 'Test Log') + }) +}) + +const setupMocks = () => { + jest.restoreAllMocks() + jest.mock('../../../request/RequestClient') + jest.mock('../../../utils/fetchLogByChunks') + jest.mock('@sasjs/utils') + + jest + .spyOn(fetchLogsModule, 'fetchLogByChunks') + .mockImplementation(() => Promise.resolve('Test Log')) + jest + .spyOn(fileModule, 'createFile') + .mockImplementation(() => Promise.resolve()) +} diff --git a/src/auth/getAccessToken.ts b/src/auth/getAccessToken.ts new file mode 100644 index 0000000..0b11340 --- /dev/null +++ b/src/auth/getAccessToken.ts @@ -0,0 +1,49 @@ +import { SasAuthResponse } from '@sasjs/utils' +import * as NodeFormData from 'form-data' +import { RequestClient } from '../request/RequestClient' + +/** + * Exchanges the auth code for an access token for the given client. + * @param requestClient - the pre-configured HTTP request client + * @param clientId - the client ID to authenticate with. + * @param clientSecret - the client secret to authenticate with. + * @param authCode - the auth code received from the server. + */ +export async function getAccessToken( + requestClient: RequestClient, + clientId: string, + clientSecret: string, + authCode: string +): Promise { + const url = '/SASLogon/oauth/token' + let token + if (typeof Buffer === 'undefined') { + token = btoa(clientId + ':' + clientSecret) + } else { + token = Buffer.from(clientId + ':' + clientSecret).toString('base64') + } + const headers = { + Authorization: 'Basic ' + token + } + + let formData + if (typeof FormData === 'undefined') { + formData = new NodeFormData() + } else { + formData = new FormData() + } + formData.append('grant_type', 'authorization_code') + formData.append('code', authCode) + + const authResponse = await requestClient + .post( + url, + formData, + undefined, + 'multipart/form-data; boundary=' + (formData as any)._boundary, + headers + ) + .then((res) => res.result as SasAuthResponse) + + return authResponse +} diff --git a/src/auth/getTokens.ts b/src/auth/getTokens.ts new file mode 100644 index 0000000..031c6a3 --- /dev/null +++ b/src/auth/getTokens.ts @@ -0,0 +1,40 @@ +import { + AuthConfig, + isAccessTokenExpiring, + isRefreshTokenExpiring, + hasTokenExpired +} from '@sasjs/utils' +import { RequestClient } from '../request/RequestClient' +import { refreshTokens } from './refreshTokens' + +/** + * Returns the auth configuration, refreshing the tokens if necessary. + * @param requestClient - the pre-configured HTTP request client + * @param authConfig - an object containing a client ID, secret, access token and refresh token + */ +export async function getTokens( + requestClient: RequestClient, + authConfig: AuthConfig +): Promise { + const logger = process.logger || console + let { access_token, refresh_token, client, secret } = authConfig + if ( + isAccessTokenExpiring(access_token) || + isRefreshTokenExpiring(refresh_token) + ) { + if (hasTokenExpired(refresh_token)) { + const error = + 'Unable to obtain new access token. Your refresh token has expired.' + logger.error(error) + throw new Error(error) + } + logger.info('Refreshing access and refresh tokens.') + ;({ access_token, refresh_token } = await refreshTokens( + requestClient, + client, + secret, + refresh_token + )) + } + return { access_token, refresh_token, client, secret } +} diff --git a/src/auth/refreshTokens.ts b/src/auth/refreshTokens.ts new file mode 100644 index 0000000..5871d63 --- /dev/null +++ b/src/auth/refreshTokens.ts @@ -0,0 +1,49 @@ +import { SasAuthResponse } from '@sasjs/utils/types' +import { prefixMessage } from '@sasjs/utils/error' +import * as NodeFormData from 'form-data' +import { RequestClient } from '../request/RequestClient' + +/** + * Exchanges the refresh token for an access token for the given client. + * @param requestClient - the pre-configured HTTP request client + * @param clientId - the client ID to authenticate with. + * @param clientSecret - the client secret to authenticate with. + * @param authCode - the refresh token received from the server. + */ +export async function refreshTokens( + requestClient: RequestClient, + clientId: string, + clientSecret: string, + refreshToken: string +) { + const url = '/SASLogon/oauth/token' + let token + token = + typeof Buffer === 'undefined' + ? btoa(clientId + ':' + clientSecret) + : Buffer.from(clientId + ':' + clientSecret).toString('base64') + + const headers = { + Authorization: 'Basic ' + token + } + + const formData = + typeof FormData === 'undefined' ? new NodeFormData() : new FormData() + formData.append('grant_type', 'refresh_token') + formData.append('refresh_token', refreshToken) + + const authResponse = await requestClient + .post( + url, + formData, + undefined, + 'multipart/form-data; boundary=' + (formData as any)._boundary, + headers + ) + .then((res) => res.result) + .catch((err) => { + throw prefixMessage(err, 'Error while refreshing tokens') + }) + + return authResponse +} diff --git a/src/auth/spec/getTokens.spec.ts b/src/auth/spec/getTokens.spec.ts new file mode 100644 index 0000000..de4397c --- /dev/null +++ b/src/auth/spec/getTokens.spec.ts @@ -0,0 +1,79 @@ +import { AuthConfig } from '@sasjs/utils' +import * as refreshTokensModule from '../refreshTokens' +import { generateToken, mockAuthResponse } from './mockResponses' +import { getTokens } from '../getTokens' +import { RequestClient } from '../../request/RequestClient' + +const requestClient = new (>RequestClient)() + +describe('getTokens', () => { + it('should attempt to refresh tokens if the access token is expiring', async () => { + setupMocks() + const access_token = generateToken(30) + const refresh_token = generateToken(86400000) + const authConfig: AuthConfig = { + access_token, + refresh_token, + client: 'cl13nt', + secret: 's3cr3t' + } + + await getTokens(requestClient, authConfig) + + expect(refreshTokensModule.refreshTokens).toHaveBeenCalledWith( + requestClient, + authConfig.client, + authConfig.secret, + authConfig.refresh_token + ) + }) + + it('should attempt to refresh tokens if the refresh token is expiring', async () => { + setupMocks() + const access_token = generateToken(86400000) + const refresh_token = generateToken(30) + const authConfig: AuthConfig = { + access_token, + refresh_token, + client: 'cl13nt', + secret: 's3cr3t' + } + + await getTokens(requestClient, authConfig) + + expect(refreshTokensModule.refreshTokens).toHaveBeenCalledWith( + requestClient, + authConfig.client, + authConfig.secret, + authConfig.refresh_token + ) + }) + + it('should throw an error if the refresh token has already expired', async () => { + setupMocks() + const access_token = generateToken(86400000) + const refresh_token = generateToken(-36000) + const authConfig: AuthConfig = { + access_token, + refresh_token, + client: 'cl13nt', + secret: 's3cr3t' + } + const expectedError = + 'Unable to obtain new access token. Your refresh token has expired.' + + const error = await getTokens(requestClient, authConfig).catch((e) => e) + + expect(error.message).toEqual(expectedError) + }) +}) + +const setupMocks = () => { + jest.restoreAllMocks() + jest.mock('../../request/RequestClient') + jest.mock('../refreshTokens') + + jest + .spyOn(refreshTokensModule, 'refreshTokens') + .mockImplementation(() => Promise.resolve(mockAuthResponse)) +} diff --git a/src/auth/spec/mockResponses.ts b/src/auth/spec/mockResponses.ts index 4ffcfb2..e15391a 100644 --- a/src/auth/spec/mockResponses.ts +++ b/src/auth/spec/mockResponses.ts @@ -1,2 +1,24 @@ +import { SasAuthResponse } from '@sasjs/utils/types' + export const mockLoginAuthoriseRequiredResponse = `
` export const mockLoginSuccessResponse = `You have signed in` + +export const mockAuthResponse: SasAuthResponse = { + access_token: 'acc355', + refresh_token: 'r3fr35h', + id_token: 'id', + token_type: 'bearer', + expires_in: new Date().valueOf(), + scope: 'default', + jti: 'test' +} + +export const generateToken = (timeToLiveSeconds: number): string => { + const exp = + new Date(new Date().getTime() + timeToLiveSeconds * 1000).getTime() / 1000 + const header = 'eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9' + const payload = Buffer.from(JSON.stringify({ exp })).toString('base64') + const signature = '4-iaDojEVl0pJQMjrbM1EzUIfAZgsbK_kgnVyVxFSVo' + const token = `${header}.${payload}.${signature}` + return token +} diff --git a/src/auth/spec/refreshTokens.spec.ts b/src/auth/spec/refreshTokens.spec.ts new file mode 100644 index 0000000..8f88f1d --- /dev/null +++ b/src/auth/spec/refreshTokens.spec.ts @@ -0,0 +1,75 @@ +import { AuthConfig } from '@sasjs/utils' +import * as NodeFormData from 'form-data' +import { generateToken, mockAuthResponse } from './mockResponses' +import { RequestClient } from '../../request/RequestClient' +import { refreshTokens } from '../refreshTokens' + +const requestClient = new (>RequestClient)() + +describe('refreshTokens', () => { + it('should attempt to refresh tokens', async () => { + setupMocks() + const access_token = generateToken(30) + const refresh_token = generateToken(30) + const authConfig: AuthConfig = { + access_token, + refresh_token, + client: 'cl13nt', + secret: 's3cr3t' + } + jest + .spyOn(requestClient, 'post') + .mockImplementation(() => + Promise.resolve({ result: mockAuthResponse, etag: '' }) + ) + const token = Buffer.from( + authConfig.client + ':' + authConfig.secret + ).toString('base64') + + await refreshTokens( + requestClient, + authConfig.client, + authConfig.secret, + authConfig.refresh_token + ) + + expect(requestClient.post).toHaveBeenCalledWith( + '/SASLogon/oauth/token', + expect.any(NodeFormData), + undefined, + expect.stringContaining('multipart/form-data; boundary='), + { + Authorization: 'Basic ' + token + } + ) + }) + + it('should handle errors while refreshing tokens', async () => { + setupMocks() + const access_token = generateToken(30) + const refresh_token = generateToken(30) + const authConfig: AuthConfig = { + access_token, + refresh_token, + client: 'cl13nt', + secret: 's3cr3t' + } + jest + .spyOn(requestClient, 'post') + .mockImplementation(() => Promise.reject('Token Error')) + + const error = await refreshTokens( + requestClient, + authConfig.client, + authConfig.secret, + authConfig.refresh_token + ).catch((e) => e) + + expect(error).toContain('Error while refreshing tokens') + }) +}) + +const setupMocks = () => { + jest.restoreAllMocks() + jest.mock('../../request/RequestClient') +} diff --git a/src/auth/tokens.ts b/src/auth/tokens.ts deleted file mode 100644 index bc940e2..0000000 --- a/src/auth/tokens.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { - AuthConfig, - isAccessTokenExpiring, - isRefreshTokenExpiring, - SasAuthResponse -} from '@sasjs/utils' -import * as NodeFormData from 'form-data' -import { RequestClient } from '../request/RequestClient' - -/** - * Exchanges the auth code for an access token for the given client. - * @param requestClient - the pre-configured HTTP request client - * @param clientId - the client ID to authenticate with. - * @param clientSecret - the client secret to authenticate with. - * @param authCode - the auth code received from the server. - */ -export async function getAccessToken( - requestClient: RequestClient, - clientId: string, - clientSecret: string, - authCode: string -): Promise { - const url = '/SASLogon/oauth/token' - let token - if (typeof Buffer === 'undefined') { - token = btoa(clientId + ':' + clientSecret) - } else { - token = Buffer.from(clientId + ':' + clientSecret).toString('base64') - } - const headers = { - Authorization: 'Basic ' + token - } - - let formData - if (typeof FormData === 'undefined') { - formData = new NodeFormData() - } else { - formData = new FormData() - } - formData.append('grant_type', 'authorization_code') - formData.append('code', authCode) - - const authResponse = await requestClient - .post( - url, - formData, - undefined, - 'multipart/form-data; boundary=' + (formData as any)._boundary, - headers - ) - .then((res) => res.result as SasAuthResponse) - - return authResponse -} - -/** - * Returns the auth configuration, refreshing the tokens if necessary. - * @param requestClient - the pre-configured HTTP request client - * @param authConfig - an object containing a client ID, secret, access token and refresh token - */ -export async function getTokens( - requestClient: RequestClient, - authConfig: AuthConfig -): Promise { - const logger = process.logger || console - let { access_token, refresh_token, client, secret } = authConfig - if ( - isAccessTokenExpiring(access_token) || - isRefreshTokenExpiring(refresh_token) - ) { - logger.info('Refreshing access and refresh tokens.') - ;({ access_token, refresh_token } = await refreshTokens( - requestClient, - client, - secret, - refresh_token - )) - } - return { access_token, refresh_token, client, secret } -} - -/** - * Exchanges the refresh token for an access token for the given client. - * @param requestClient - the pre-configured HTTP request client - * @param clientId - the client ID to authenticate with. - * @param clientSecret - the client secret to authenticate with. - * @param authCode - the refresh token received from the server. - */ -export async function refreshTokens( - requestClient: RequestClient, - clientId: string, - clientSecret: string, - refreshToken: string -) { - const url = '/SASLogon/oauth/token' - let token - if (typeof Buffer === 'undefined') { - token = btoa(clientId + ':' + clientSecret) - } else { - token = Buffer.from(clientId + ':' + clientSecret).toString('base64') - } - const headers = { - Authorization: 'Basic ' + token - } - - const formData = - typeof FormData === 'undefined' ? new NodeFormData() : new FormData() - formData.append('grant_type', 'refresh_token') - formData.append('refresh_token', refreshToken) - - const authResponse = await requestClient - .post( - url, - formData, - undefined, - 'multipart/form-data; boundary=' + (formData as any)._boundary, - headers - ) - .then((res) => res.result) - - return authResponse -} From a0fbe1a74088de10227ca1d9ea4c325718e804ba Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Mon, 12 Jul 2021 20:42:49 +0100 Subject: [PATCH 07/39] chore(ci): add coverage report action --- .github/workflows/coverage.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/coverage.yml diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 0000000..7fc938f --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,25 @@ +name: SASjs Adapter Coverage Report + +on: + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [12.x] + + steps: + - uses: actions/checkout@v2 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Install Dependencies + run: npm ci + - name: Generate coverage report + uses: artiomtr/jest-coverage-report-action@v2.0-rc.2 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} From 4257ec78aac6a829adcfe5e78ce7fb0898724ea5 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Mon, 12 Jul 2021 20:45:09 +0100 Subject: [PATCH 08/39] chore(ci): add coverage report to build workflow --- .github/workflows/build.yml | 4 ++++ .github/workflows/coverage.yml | 25 ------------------------- 2 files changed, 4 insertions(+), 25 deletions(-) delete mode 100644 .github/workflows/coverage.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 188d999..10d8ea1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,6 +27,10 @@ jobs: run: npm run lint - name: Run unit tests run: npm test + - name: Generate coverage report + uses: artiomtr/jest-coverage-report-action@v2.0-rc.2 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} - name: Build Package run: npm run package:lib env: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml deleted file mode 100644 index 7fc938f..0000000 --- a/.github/workflows/coverage.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: SASjs Adapter Coverage Report - -on: - pull_request: - -jobs: - build: - runs-on: ubuntu-latest - - strategy: - matrix: - node-version: [12.x] - - steps: - - uses: actions/checkout@v2 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - name: Install Dependencies - run: npm ci - - name: Generate coverage report - uses: artiomtr/jest-coverage-report-action@v2.0-rc.2 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} From b9f368193d5be7abb1c6c9897b3f7bae13587927 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Tue, 13 Jul 2021 08:12:15 +0100 Subject: [PATCH 09/39] chore(refactor): add more tests --- src/api/viya/spec/uploadTables.spec.ts | 67 +++++++++++++++++++++++ src/api/viya/uploadTables.ts | 6 --- src/auth/getAccessToken.ts | 6 ++- src/auth/spec/getAccessToken.spec.ts | 75 ++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 7 deletions(-) create mode 100644 src/api/viya/spec/uploadTables.spec.ts create mode 100644 src/auth/spec/getAccessToken.spec.ts diff --git a/src/api/viya/spec/uploadTables.spec.ts b/src/api/viya/spec/uploadTables.spec.ts new file mode 100644 index 0000000..529c6e2 --- /dev/null +++ b/src/api/viya/spec/uploadTables.spec.ts @@ -0,0 +1,67 @@ +import { RequestClient } from '../../../request/RequestClient' +import * as convertToCsvModule from '../../../utils/convertToCsv' +import { uploadTables } from '../uploadTables' + +const requestClient = new (>RequestClient)() + +describe('uploadTables', () => { + beforeEach(() => { + setupMocks() + }) + + it('should return a list of uploaded files', async () => { + const data = { foo: 'bar' } + + const files = await uploadTables(requestClient, data, 't0k3n') + + expect(files).toEqual([{ tableName: 'foo', file: 'test-file' }]) + expect(requestClient.uploadFile).toHaveBeenCalledTimes(1) + expect(requestClient.uploadFile).toHaveBeenCalledWith( + '/files/files#rawUpload', + 'Test CSV', + 't0k3n' + ) + }) + + it('should throw an error when the CSV exceeds the maximum length', async () => { + const data = { foo: 'bar' } + jest + .spyOn(convertToCsvModule, 'convertToCSV') + .mockImplementation(() => 'ERROR: LARGE STRING LENGTH') + + const error = await uploadTables(requestClient, data, 't0k3n').catch( + (e) => e + ) + + expect(requestClient.uploadFile).not.toHaveBeenCalled() + expect(error.message).toEqual( + 'The max length of a string value in SASjs is 32765 characters.' + ) + }) + + it('should throw an error when the file upload fails', async () => { + const data = { foo: 'bar' } + jest + .spyOn(requestClient, 'uploadFile') + .mockImplementation(() => Promise.reject('Upload Error')) + + const error = await uploadTables(requestClient, data, 't0k3n').catch( + (e) => e + ) + + expect(error).toContain('Error while uploading file.') + }) +}) + +const setupMocks = () => { + jest.restoreAllMocks() + jest.mock('../../../utils/convertToCsv') + jest + .spyOn(convertToCsvModule, 'convertToCSV') + .mockImplementation(() => 'Test CSV') + jest + .spyOn(requestClient, 'uploadFile') + .mockImplementation(() => + Promise.resolve({ result: 'test-file', etag: '' }) + ) +} diff --git a/src/api/viya/uploadTables.ts b/src/api/viya/uploadTables.ts index 13bf7b8..e7f5b66 100644 --- a/src/api/viya/uploadTables.ts +++ b/src/api/viya/uploadTables.ts @@ -8,12 +8,6 @@ export async function uploadTables( accessToken?: string ) { const uploadedFiles = [] - const headers: any = { - 'Content-Type': 'application/json' - } - if (accessToken) { - headers.Authorization = `Bearer ${accessToken}` - } for (const tableName in data) { const csv = convertToCSV(data[tableName]) diff --git a/src/auth/getAccessToken.ts b/src/auth/getAccessToken.ts index 0b11340..d51833e 100644 --- a/src/auth/getAccessToken.ts +++ b/src/auth/getAccessToken.ts @@ -1,4 +1,5 @@ -import { SasAuthResponse } from '@sasjs/utils' +import { SasAuthResponse } from '@sasjs/utils/types' +import { prefixMessage } from '@sasjs/utils/error' import * as NodeFormData from 'form-data' import { RequestClient } from '../request/RequestClient' @@ -44,6 +45,9 @@ export async function getAccessToken( headers ) .then((res) => res.result as SasAuthResponse) + .catch((err) => { + throw prefixMessage(err, 'Error while getting access token') + }) return authResponse } diff --git a/src/auth/spec/getAccessToken.spec.ts b/src/auth/spec/getAccessToken.spec.ts new file mode 100644 index 0000000..e4fa00f --- /dev/null +++ b/src/auth/spec/getAccessToken.spec.ts @@ -0,0 +1,75 @@ +import { AuthConfig } from '@sasjs/utils' +import * as NodeFormData from 'form-data' +import { generateToken, mockAuthResponse } from './mockResponses' +import { RequestClient } from '../../request/RequestClient' +import { getAccessToken } from '../getAccessToken' + +const requestClient = new (>RequestClient)() + +describe('getAccessToken', () => { + it('should attempt to refresh tokens', async () => { + setupMocks() + const access_token = generateToken(30) + const refresh_token = generateToken(30) + const authConfig: AuthConfig = { + access_token, + refresh_token, + client: 'cl13nt', + secret: 's3cr3t' + } + jest + .spyOn(requestClient, 'post') + .mockImplementation(() => + Promise.resolve({ result: mockAuthResponse, etag: '' }) + ) + const token = Buffer.from( + authConfig.client + ':' + authConfig.secret + ).toString('base64') + + await getAccessToken( + requestClient, + authConfig.client, + authConfig.secret, + authConfig.refresh_token + ) + + expect(requestClient.post).toHaveBeenCalledWith( + '/SASLogon/oauth/token', + expect.any(NodeFormData), + undefined, + expect.stringContaining('multipart/form-data; boundary='), + { + Authorization: 'Basic ' + token + } + ) + }) + + it('should handle errors while refreshing tokens', async () => { + setupMocks() + const access_token = generateToken(30) + const refresh_token = generateToken(30) + const authConfig: AuthConfig = { + access_token, + refresh_token, + client: 'cl13nt', + secret: 's3cr3t' + } + jest + .spyOn(requestClient, 'post') + .mockImplementation(() => Promise.reject('Token Error')) + + const error = await getAccessToken( + requestClient, + authConfig.client, + authConfig.secret, + authConfig.refresh_token + ).catch((e) => e) + + expect(error).toContain('Error while getting access token') + }) +}) + +const setupMocks = () => { + jest.restoreAllMocks() + jest.mock('../../request/RequestClient') +} From db578564ba2096ba13c3dff9c4165ea8d5b5c1ec Mon Sep 17 00:00:00 2001 From: Saad Jutt Date: Tue, 13 Jul 2021 17:11:49 +0500 Subject: [PATCH 10/39] fix: removed url package --- package-lock.json | 20 ++++++++++---------- package.json | 3 +-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index 059a16c..2432dd9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,8 +12,7 @@ "axios-cookiejar-support": "^1.0.1", "form-data": "^4.0.0", "https": "^1.0.0", - "tough-cookie": "^4.0.0", - "url": "^0.11.0" + "tough-cookie": "^4.0.0" }, "devDependencies": { "@types/jest": "^26.0.23", @@ -11155,11 +11154,6 @@ "safer-buffer": "^2.0.2", "tweetnacl": "~0.14.0" }, - "bin": { - "sshpk-conv": "bin/sshpk-conv", - "sshpk-sign": "bin/sshpk-sign", - "sshpk-verify": "bin/sshpk-verify" - }, "engines": { "node": ">=0.10.0" } @@ -12113,6 +12107,7 @@ "version": "0.2.0", "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "dev": true, "engines": { "node": ">=0.4.x" } @@ -14171,6 +14166,7 @@ "version": "0.11.0", "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz", "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=", + "dev": true, "dependencies": { "punycode": "1.3.2", "querystring": "0.2.0" @@ -14185,7 +14181,8 @@ "node_modules/url/node_modules/punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=", + "dev": true }, "node_modules/use": { "version": "3.1.1", @@ -24130,7 +24127,8 @@ "querystring": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "dev": true }, "querystring-es3": { "version": "0.2.1", @@ -25781,6 +25779,7 @@ "version": "0.11.0", "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz", "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=", + "dev": true, "requires": { "punycode": "1.3.2", "querystring": "0.2.0" @@ -25789,7 +25788,8 @@ "punycode": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=", + "dev": true } } }, diff --git a/package.json b/package.json index 65fd231..64900d3 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,6 @@ "axios-cookiejar-support": "^1.0.1", "form-data": "^4.0.0", "https": "^1.0.0", - "tough-cookie": "^4.0.0", - "url": "^0.11.0" + "tough-cookie": "^4.0.0" } } From d4725d2e54bb0f885ff74da4abb79e15e06f63d4 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Wed, 14 Jul 2021 07:50:25 +0100 Subject: [PATCH 11/39] chore(refactor): change property name in PollOptions --- src/api/viya/pollJobState.ts | 2 +- src/types/PollOptions.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/viya/pollJobState.ts b/src/api/viya/pollJobState.ts index 8d95c42..5e74dda 100644 --- a/src/api/viya/pollJobState.ts +++ b/src/api/viya/pollJobState.ts @@ -23,7 +23,7 @@ export async function pollJobState( const logFileName = `${postedJob.name || 'job'}-${generateTimestamp()}.log` const logFilePath = `${ - pollOptions?.logFilePath || process.cwd() + pollOptions?.logFolderPath || process.cwd() }/${logFileName}` if (authConfig) { diff --git a/src/types/PollOptions.ts b/src/types/PollOptions.ts index 6daa1b7..82daf9e 100644 --- a/src/types/PollOptions.ts +++ b/src/types/PollOptions.ts @@ -2,5 +2,5 @@ export interface PollOptions { maxPollCount: number pollInterval: number streamLog: boolean - logFilePath?: string + logFolderPath?: string } From 1ff3937d117c763bef435b850d3fa20dddd91371 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Wed, 14 Jul 2021 08:03:54 +0100 Subject: [PATCH 12/39] chore(deps): update dependencies --- package-lock.json | 440 +++++++++++++++++++++++----------------------- package.json | 10 +- 2 files changed, 224 insertions(+), 226 deletions(-) diff --git a/package-lock.json b/package-lock.json index 970d08e..08531a8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,9 +18,9 @@ "devDependencies": { "@types/axios": "^0.14.0", "@types/form-data": "^2.5.0", - "@types/jest": "^26.0.23", + "@types/jest": "^26.0.24", "@types/mime": "^2.0.3", - "@types/tough-cookie": "^4.0.0", + "@types/tough-cookie": "^4.0.1", "copyfiles": "^2.4.1", "cp": "^0.2.0", "dotenv": "^10.0.0", @@ -36,11 +36,11 @@ "ts-loader": "^9.2.2", "tslint": "^6.1.3", "tslint-config-prettier": "^1.18.0", - "typedoc": "^0.21.2", + "typedoc": "^0.21.4", "typedoc-neo-theme": "^1.1.1", "typedoc-plugin-external-module-name": "^4.0.6", - "typescript": "^4.3.4", - "webpack": "^5.41.1", + "typescript": "^4.3.5", + "webpack": "^5.44.0", "webpack-cli": "^4.7.2" } }, @@ -1508,9 +1508,9 @@ } }, "node_modules/@types/estree": { - "version": "0.0.48", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.48.tgz", - "integrity": "sha512-LfZwXoGUDo0C3me81HXgkBg5CTQYb6xzEl+fNmbO4JdRiSKQ8A0GD1OBBvKAIsbCUgoyAty7m99GqqMQe784ew==", + "version": "0.0.50", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.50.tgz", + "integrity": "sha512-C6N5s2ZFtuZRj54k2/zyRhNDjJwwcViAM3Nbm8zjBpbqAdZ00mr0CFxvSKeO8Y/e03WVFLpQMdHYVfUd6SB+Hw==", "dev": true }, "node_modules/@types/form-data": { @@ -1557,9 +1557,9 @@ } }, "node_modules/@types/jest": { - "version": "26.0.23", - "resolved": "https://registry.npmjs.org/@types/jest/-/jest-26.0.23.tgz", - "integrity": "sha512-ZHLmWMJ9jJ9PTiT58juykZpL7KjwJywFN3Rr2pTSkyQfydf/rk22yS7W8p5DaVUMQ2BQC7oYiU3FjbTM/mYrOA==", + "version": "26.0.24", + "resolved": "https://registry.npmjs.org/@types/jest/-/jest-26.0.24.tgz", + "integrity": "sha512-E/X5Vib8BWqZNRlDxj9vYXhsDwPYbPINqKF9BsnSoon4RQ0D9moEuLD8txgyypFLH7J4+Lho9Nr/c8H0Fi+17w==", "dev": true, "dependencies": { "jest-diff": "^26.0.0", @@ -1628,9 +1628,9 @@ "dev": true }, "node_modules/@types/tough-cookie": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.0.tgz", - "integrity": "sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.1.tgz", + "integrity": "sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==", "dev": true }, "node_modules/@types/yargs": { @@ -1649,148 +1649,148 @@ "dev": true }, "node_modules/@webassemblyjs/ast": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.0.tgz", - "integrity": "sha512-kX2W49LWsbthrmIRMbQZuQDhGtjyqXfEmmHyEi4XWnSZtPmxY0+3anPIzsnRb45VH/J55zlOfWvZuY47aJZTJg==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.1.tgz", + "integrity": "sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw==", "dev": true, "dependencies": { - "@webassemblyjs/helper-numbers": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0" + "@webassemblyjs/helper-numbers": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1" } }, "node_modules/@webassemblyjs/floating-point-hex-parser": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.0.tgz", - "integrity": "sha512-Q/aVYs/VnPDVYvsCBL/gSgwmfjeCb4LW8+TMrO3cSzJImgv8lxxEPM2JA5jMrivE7LSz3V+PFqtMbls3m1exDA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.1.tgz", + "integrity": "sha512-iGRfyc5Bq+NnNuX8b5hwBrRjzf0ocrJPI6GWFodBFzmFnyvrQ83SHKhmilCU/8Jv67i4GJZBMhEzltxzcNagtQ==", "dev": true }, "node_modules/@webassemblyjs/helper-api-error": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.0.tgz", - "integrity": "sha512-baT/va95eXiXb2QflSx95QGT5ClzWpGaa8L7JnJbgzoYeaA27FCvuBXU758l+KXWRndEmUXjP0Q5fibhavIn8w==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.1.tgz", + "integrity": "sha512-RlhS8CBCXfRUR/cwo2ho9bkheSXG0+NwooXcc3PAILALf2QLdFyj7KGsKRbVc95hZnhnERon4kW/D3SZpp6Tcg==", "dev": true }, "node_modules/@webassemblyjs/helper-buffer": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.0.tgz", - "integrity": "sha512-u9HPBEl4DS+vA8qLQdEQ6N/eJQ7gT7aNvMIo8AAWvAl/xMrcOSiI2M0MAnMCy3jIFke7bEee/JwdX1nUpCtdyA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.1.tgz", + "integrity": "sha512-gwikF65aDNeeXa8JxXa2BAk+REjSyhrNC9ZwdT0f8jc4dQQeDQ7G4m0f2QCLPJiMTTO6wfDmRmj/pW0PsUvIcA==", "dev": true }, "node_modules/@webassemblyjs/helper-numbers": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.0.tgz", - "integrity": "sha512-DhRQKelIj01s5IgdsOJMKLppI+4zpmcMQ3XboFPLwCpSNH6Hqo1ritgHgD0nqHeSYqofA6aBN/NmXuGjM1jEfQ==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.1.tgz", + "integrity": "sha512-vDkbxiB8zfnPdNK9Rajcey5C0w+QJugEglN0of+kmO8l7lDb77AnlKYQF7aarZuCrv+l0UvqL+68gSDr3k9LPQ==", "dev": true, "dependencies": { - "@webassemblyjs/floating-point-hex-parser": "1.11.0", - "@webassemblyjs/helper-api-error": "1.11.0", + "@webassemblyjs/floating-point-hex-parser": "1.11.1", + "@webassemblyjs/helper-api-error": "1.11.1", "@xtuc/long": "4.2.2" } }, "node_modules/@webassemblyjs/helper-wasm-bytecode": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.0.tgz", - "integrity": "sha512-MbmhvxXExm542tWREgSFnOVo07fDpsBJg3sIl6fSp9xuu75eGz5lz31q7wTLffwL3Za7XNRCMZy210+tnsUSEA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.1.tgz", + "integrity": "sha512-PvpoOGiJwXeTrSf/qfudJhwlvDQxFgelbMqtq52WWiXC6Xgg1IREdngmPN3bs4RoO83PnL/nFrxucXj1+BX62Q==", "dev": true }, "node_modules/@webassemblyjs/helper-wasm-section": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.0.tgz", - "integrity": "sha512-3Eb88hcbfY/FCukrg6i3EH8H2UsD7x8Vy47iVJrP967A9JGqgBVL9aH71SETPx1JrGsOUVLo0c7vMCN22ytJew==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.1.tgz", + "integrity": "sha512-10P9No29rYX1j7F3EVPX3JvGPQPae+AomuSTPiF9eBQeChHI6iqjMIwR9JmOJXwpnn/oVGDk7I5IlskuMwU/pg==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1" } }, "node_modules/@webassemblyjs/ieee754": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.0.tgz", - "integrity": "sha512-KXzOqpcYQwAfeQ6WbF6HXo+0udBNmw0iXDmEK5sFlmQdmND+tr773Ti8/5T/M6Tl/413ArSJErATd8In3B+WBA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.1.tgz", + "integrity": "sha512-hJ87QIPtAMKbFq6CGTkZYJivEwZDbQUgYd3qKSadTNOhVY7p+gfP6Sr0lLRVTaG1JjFj+r3YchoqRYxNH3M0GQ==", "dev": true, "dependencies": { "@xtuc/ieee754": "^1.2.0" } }, "node_modules/@webassemblyjs/leb128": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.0.tgz", - "integrity": "sha512-aqbsHa1mSQAbeeNcl38un6qVY++hh8OpCOzxhixSYgbRfNWcxJNJQwe2rezK9XEcssJbbWIkblaJRwGMS9zp+g==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.1.tgz", + "integrity": "sha512-BJ2P0hNZ0u+Th1YZXJpzW6miwqQUGcIHT1G/sf72gLVD9DZ5AdYTqPNbHZh6K1M5VmKvFXwGSWZADz+qBWxeRw==", "dev": true, "dependencies": { "@xtuc/long": "4.2.2" } }, "node_modules/@webassemblyjs/utf8": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.0.tgz", - "integrity": "sha512-A/lclGxH6SpSLSyFowMzO/+aDEPU4hvEiooCMXQPcQFPPJaYcPQNKGOCLUySJsYJ4trbpr+Fs08n4jelkVTGVw==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.1.tgz", + "integrity": "sha512-9kqcxAEdMhiwQkHpkNiorZzqpGrodQQ2IGrHHxCy+Ozng0ofyMA0lTqiLkVs1uzTRejX+/O0EOT7KxqVPuXosQ==", "dev": true }, "node_modules/@webassemblyjs/wasm-edit": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.0.tgz", - "integrity": "sha512-JHQ0damXy0G6J9ucyKVXO2j08JVJ2ntkdJlq1UTiUrIgfGMmA7Ik5VdC/L8hBK46kVJgujkBIoMtT8yVr+yVOQ==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.1.tgz", + "integrity": "sha512-g+RsupUC1aTHfR8CDgnsVRVZFJqdkFHpsHMfJuWQzWU3tvnLC07UqHICfP+4XyL2tnr1amvl1Sdp06TnYCmVkA==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/helper-wasm-section": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0", - "@webassemblyjs/wasm-opt": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0", - "@webassemblyjs/wast-printer": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/helper-wasm-section": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1", + "@webassemblyjs/wasm-opt": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1", + "@webassemblyjs/wast-printer": "1.11.1" } }, "node_modules/@webassemblyjs/wasm-gen": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.0.tgz", - "integrity": "sha512-BEUv1aj0WptCZ9kIS30th5ILASUnAPEvE3tVMTrItnZRT9tXCLW2LEXT8ezLw59rqPP9klh9LPmpU+WmRQmCPQ==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.1.tgz", + "integrity": "sha512-F7QqKXwwNlMmsulj6+O7r4mmtAlCWfO/0HdgOxSklZfQcDu0TpLiD1mRt/zF25Bk59FIjEuGAIyn5ei4yMfLhA==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/ieee754": "1.11.0", - "@webassemblyjs/leb128": "1.11.0", - "@webassemblyjs/utf8": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/ieee754": "1.11.1", + "@webassemblyjs/leb128": "1.11.1", + "@webassemblyjs/utf8": "1.11.1" } }, "node_modules/@webassemblyjs/wasm-opt": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.0.tgz", - "integrity": "sha512-tHUSP5F4ywyh3hZ0+fDQuWxKx3mJiPeFufg+9gwTpYp324mPCQgnuVKwzLTZVqj0duRDovnPaZqDwoyhIO8kYg==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz", + "integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1" } }, "node_modules/@webassemblyjs/wasm-parser": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.0.tgz", - "integrity": "sha512-6L285Sgu9gphrcpDXINvm0M9BskznnzJTE7gYkjDbxET28shDqp27wpruyx3C2S/dvEwiigBwLA1cz7lNUi0kw==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.1.tgz", + "integrity": "sha512-rrBujw+dJu32gYB7/Lup6UhdkPx9S9SnobZzRVL7VcBH9Bt9bCBLEuX/YXOOtBsOZ4NQrRykKhffRWHvigQvOA==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-api-error": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/ieee754": "1.11.0", - "@webassemblyjs/leb128": "1.11.0", - "@webassemblyjs/utf8": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-api-error": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/ieee754": "1.11.1", + "@webassemblyjs/leb128": "1.11.1", + "@webassemblyjs/utf8": "1.11.1" } }, "node_modules/@webassemblyjs/wast-printer": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.0.tgz", - "integrity": "sha512-Fg5OX46pRdTgB7rKIUojkh9vXaVN6sGYCnEiJN1GYkb0RPwShZXp6KTDqmoMdQPKhcroOXh3fEzmkWmCYaKYhQ==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.1.tgz", + "integrity": "sha512-IQboUWM4eKzWW+N/jij2sRatKMh99QEelo3Eb2q0qXkvPRISAj8Qxtmw5itwqK+TTkBuUIE45AxYPToqPtL5gg==", "dev": true, "dependencies": { - "@webassemblyjs/ast": "1.11.0", + "@webassemblyjs/ast": "1.11.1", "@xtuc/long": "4.2.2" } }, @@ -1834,9 +1834,9 @@ "dev": true }, "node_modules/acorn": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.3.0.tgz", - "integrity": "sha512-tqPKHZ5CaBJw0Xmy0ZZvLs1qTV+BNFSyvn77ASXkpBNfIRk8ev26fKrD9iLGwGA9zedPao52GSHzq8lyZG0NUw==", + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.4.1.tgz", + "integrity": "sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA==", "dev": true, "bin": { "acorn": "bin/acorn" @@ -3753,9 +3753,9 @@ } }, "node_modules/es-module-lexer": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.6.0.tgz", - "integrity": "sha512-f8kcHX1ArhllUtb/wVSyvygoKCznIjnxhLxy7TCvIiMdT7fL4ZDTIKaadMe6eLvOXg6Wk02UeoFgUoZ2EKZZUA==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.7.1.tgz", + "integrity": "sha512-MgtWFl5No+4S3TmhDmCz2ObFGm6lEpTnzbQi+Dd+pw4mlTIZTmM2iAs5gRlmx5zS9luzobCSBSI90JM/1/JgOw==", "dev": true }, "node_modules/es-to-primitive": { @@ -13879,14 +13879,13 @@ } }, "node_modules/typedoc": { - "version": "0.21.2", - "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.21.2.tgz", - "integrity": "sha512-SR1ByJB3USg+jxoxwzMRP07g/0f/cQUE5t7gOh1iTUyjTPyJohu9YSKRlK+MSXXqlhIq+m0jkEHEG5HoY7/Adg==", + "version": "0.21.4", + "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.21.4.tgz", + "integrity": "sha512-slZQhvD9U0d9KacktYAyuNMMOXJRFNHy+Gd8xY2Qrqq3eTTTv3frv3N4au/cFnab9t3T5WA0Orb6QUjMc+1bDA==", "dev": true, "dependencies": { "glob": "^7.1.7", "handlebars": "^4.7.7", - "lodash": "^4.17.21", "lunr": "^2.3.9", "marked": "^2.1.1", "minimatch": "^3.0.0", @@ -14030,9 +14029,9 @@ } }, "node_modules/typescript": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz", - "integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew==", + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.5.tgz", + "integrity": "sha512-DqQgihaQ9cUrskJo9kIyW/+g0Vxsk8cDtZ52a3NGh0YNTfpUSArXSohyUGnvbPazEPLu398C0UxmKSOrPumUzA==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -14342,21 +14341,21 @@ } }, "node_modules/webpack": { - "version": "5.41.1", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.41.1.tgz", - "integrity": "sha512-AJZIIsqJ/MVTmegEq9Tlw5mk5EHdGiJbDdz9qP15vmUH+oxI1FdWcL0E9EO8K/zKaRPWqEs7G/OPxq1P61u5Ug==", + "version": "5.44.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.44.0.tgz", + "integrity": "sha512-I1S1w4QLoKmH19pX6YhYN0NiSXaWY8Ou00oA+aMcr9IUGeF5azns+IKBkfoAAG9Bu5zOIzZt/mN35OffBya8AQ==", "dev": true, "dependencies": { "@types/eslint-scope": "^3.7.0", - "@types/estree": "^0.0.48", - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/wasm-edit": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0", - "acorn": "^8.2.1", + "@types/estree": "^0.0.50", + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/wasm-edit": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1", + "acorn": "^8.4.1", "browserslist": "^4.14.5", "chrome-trace-event": "^1.0.2", "enhanced-resolve": "^5.8.0", - "es-module-lexer": "^0.6.0", + "es-module-lexer": "^0.7.1", "eslint-scope": "5.1.1", "events": "^3.2.0", "glob-to-regexp": "^0.4.1", @@ -15935,9 +15934,9 @@ } }, "@types/estree": { - "version": "0.0.48", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.48.tgz", - "integrity": "sha512-LfZwXoGUDo0C3me81HXgkBg5CTQYb6xzEl+fNmbO4JdRiSKQ8A0GD1OBBvKAIsbCUgoyAty7m99GqqMQe784ew==", + "version": "0.0.50", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-0.0.50.tgz", + "integrity": "sha512-C6N5s2ZFtuZRj54k2/zyRhNDjJwwcViAM3Nbm8zjBpbqAdZ00mr0CFxvSKeO8Y/e03WVFLpQMdHYVfUd6SB+Hw==", "dev": true }, "@types/form-data": { @@ -15983,9 +15982,9 @@ } }, "@types/jest": { - "version": "26.0.23", - "resolved": "https://registry.npmjs.org/@types/jest/-/jest-26.0.23.tgz", - "integrity": "sha512-ZHLmWMJ9jJ9PTiT58juykZpL7KjwJywFN3Rr2pTSkyQfydf/rk22yS7W8p5DaVUMQ2BQC7oYiU3FjbTM/mYrOA==", + "version": "26.0.24", + "resolved": "https://registry.npmjs.org/@types/jest/-/jest-26.0.24.tgz", + "integrity": "sha512-E/X5Vib8BWqZNRlDxj9vYXhsDwPYbPINqKF9BsnSoon4RQ0D9moEuLD8txgyypFLH7J4+Lho9Nr/c8H0Fi+17w==", "dev": true, "requires": { "jest-diff": "^26.0.0", @@ -16054,9 +16053,9 @@ "dev": true }, "@types/tough-cookie": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.0.tgz", - "integrity": "sha512-I99sngh224D0M7XgW1s120zxCt3VYQ3IQsuw3P3jbq5GG4yc79+ZjyKznyOGIQrflfylLgcfekeZW/vk0yng6A==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@types/tough-cookie/-/tough-cookie-4.0.1.tgz", + "integrity": "sha512-Y0K95ThC3esLEYD6ZuqNek29lNX2EM1qxV8y2FTLUB0ff5wWrk7az+mLrnNFUnaXcgKye22+sFBRXOgpPILZNg==", "dev": true }, "@types/yargs": { @@ -16075,148 +16074,148 @@ "dev": true }, "@webassemblyjs/ast": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.0.tgz", - "integrity": "sha512-kX2W49LWsbthrmIRMbQZuQDhGtjyqXfEmmHyEi4XWnSZtPmxY0+3anPIzsnRb45VH/J55zlOfWvZuY47aJZTJg==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.11.1.tgz", + "integrity": "sha512-ukBh14qFLjxTQNTXocdyksN5QdM28S1CxHt2rdskFyL+xFV7VremuBLVbmCePj+URalXBENx/9Lm7lnhihtCSw==", "dev": true, "requires": { - "@webassemblyjs/helper-numbers": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0" + "@webassemblyjs/helper-numbers": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1" } }, "@webassemblyjs/floating-point-hex-parser": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.0.tgz", - "integrity": "sha512-Q/aVYs/VnPDVYvsCBL/gSgwmfjeCb4LW8+TMrO3cSzJImgv8lxxEPM2JA5jMrivE7LSz3V+PFqtMbls3m1exDA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.1.tgz", + "integrity": "sha512-iGRfyc5Bq+NnNuX8b5hwBrRjzf0ocrJPI6GWFodBFzmFnyvrQ83SHKhmilCU/8Jv67i4GJZBMhEzltxzcNagtQ==", "dev": true }, "@webassemblyjs/helper-api-error": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.0.tgz", - "integrity": "sha512-baT/va95eXiXb2QflSx95QGT5ClzWpGaa8L7JnJbgzoYeaA27FCvuBXU758l+KXWRndEmUXjP0Q5fibhavIn8w==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.1.tgz", + "integrity": "sha512-RlhS8CBCXfRUR/cwo2ho9bkheSXG0+NwooXcc3PAILALf2QLdFyj7KGsKRbVc95hZnhnERon4kW/D3SZpp6Tcg==", "dev": true }, "@webassemblyjs/helper-buffer": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.0.tgz", - "integrity": "sha512-u9HPBEl4DS+vA8qLQdEQ6N/eJQ7gT7aNvMIo8AAWvAl/xMrcOSiI2M0MAnMCy3jIFke7bEee/JwdX1nUpCtdyA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.11.1.tgz", + "integrity": "sha512-gwikF65aDNeeXa8JxXa2BAk+REjSyhrNC9ZwdT0f8jc4dQQeDQ7G4m0f2QCLPJiMTTO6wfDmRmj/pW0PsUvIcA==", "dev": true }, "@webassemblyjs/helper-numbers": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.0.tgz", - "integrity": "sha512-DhRQKelIj01s5IgdsOJMKLppI+4zpmcMQ3XboFPLwCpSNH6Hqo1ritgHgD0nqHeSYqofA6aBN/NmXuGjM1jEfQ==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.1.tgz", + "integrity": "sha512-vDkbxiB8zfnPdNK9Rajcey5C0w+QJugEglN0of+kmO8l7lDb77AnlKYQF7aarZuCrv+l0UvqL+68gSDr3k9LPQ==", "dev": true, "requires": { - "@webassemblyjs/floating-point-hex-parser": "1.11.0", - "@webassemblyjs/helper-api-error": "1.11.0", + "@webassemblyjs/floating-point-hex-parser": "1.11.1", + "@webassemblyjs/helper-api-error": "1.11.1", "@xtuc/long": "4.2.2" } }, "@webassemblyjs/helper-wasm-bytecode": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.0.tgz", - "integrity": "sha512-MbmhvxXExm542tWREgSFnOVo07fDpsBJg3sIl6fSp9xuu75eGz5lz31q7wTLffwL3Za7XNRCMZy210+tnsUSEA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.1.tgz", + "integrity": "sha512-PvpoOGiJwXeTrSf/qfudJhwlvDQxFgelbMqtq52WWiXC6Xgg1IREdngmPN3bs4RoO83PnL/nFrxucXj1+BX62Q==", "dev": true }, "@webassemblyjs/helper-wasm-section": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.0.tgz", - "integrity": "sha512-3Eb88hcbfY/FCukrg6i3EH8H2UsD7x8Vy47iVJrP967A9JGqgBVL9aH71SETPx1JrGsOUVLo0c7vMCN22ytJew==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.11.1.tgz", + "integrity": "sha512-10P9No29rYX1j7F3EVPX3JvGPQPae+AomuSTPiF9eBQeChHI6iqjMIwR9JmOJXwpnn/oVGDk7I5IlskuMwU/pg==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1" } }, "@webassemblyjs/ieee754": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.0.tgz", - "integrity": "sha512-KXzOqpcYQwAfeQ6WbF6HXo+0udBNmw0iXDmEK5sFlmQdmND+tr773Ti8/5T/M6Tl/413ArSJErATd8In3B+WBA==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.1.tgz", + "integrity": "sha512-hJ87QIPtAMKbFq6CGTkZYJivEwZDbQUgYd3qKSadTNOhVY7p+gfP6Sr0lLRVTaG1JjFj+r3YchoqRYxNH3M0GQ==", "dev": true, "requires": { "@xtuc/ieee754": "^1.2.0" } }, "@webassemblyjs/leb128": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.0.tgz", - "integrity": "sha512-aqbsHa1mSQAbeeNcl38un6qVY++hh8OpCOzxhixSYgbRfNWcxJNJQwe2rezK9XEcssJbbWIkblaJRwGMS9zp+g==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.1.tgz", + "integrity": "sha512-BJ2P0hNZ0u+Th1YZXJpzW6miwqQUGcIHT1G/sf72gLVD9DZ5AdYTqPNbHZh6K1M5VmKvFXwGSWZADz+qBWxeRw==", "dev": true, "requires": { "@xtuc/long": "4.2.2" } }, "@webassemblyjs/utf8": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.0.tgz", - "integrity": "sha512-A/lclGxH6SpSLSyFowMzO/+aDEPU4hvEiooCMXQPcQFPPJaYcPQNKGOCLUySJsYJ4trbpr+Fs08n4jelkVTGVw==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.1.tgz", + "integrity": "sha512-9kqcxAEdMhiwQkHpkNiorZzqpGrodQQ2IGrHHxCy+Ozng0ofyMA0lTqiLkVs1uzTRejX+/O0EOT7KxqVPuXosQ==", "dev": true }, "@webassemblyjs/wasm-edit": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.0.tgz", - "integrity": "sha512-JHQ0damXy0G6J9ucyKVXO2j08JVJ2ntkdJlq1UTiUrIgfGMmA7Ik5VdC/L8hBK46kVJgujkBIoMtT8yVr+yVOQ==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.11.1.tgz", + "integrity": "sha512-g+RsupUC1aTHfR8CDgnsVRVZFJqdkFHpsHMfJuWQzWU3tvnLC07UqHICfP+4XyL2tnr1amvl1Sdp06TnYCmVkA==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/helper-wasm-section": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0", - "@webassemblyjs/wasm-opt": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0", - "@webassemblyjs/wast-printer": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/helper-wasm-section": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1", + "@webassemblyjs/wasm-opt": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1", + "@webassemblyjs/wast-printer": "1.11.1" } }, "@webassemblyjs/wasm-gen": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.0.tgz", - "integrity": "sha512-BEUv1aj0WptCZ9kIS30th5ILASUnAPEvE3tVMTrItnZRT9tXCLW2LEXT8ezLw59rqPP9klh9LPmpU+WmRQmCPQ==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.11.1.tgz", + "integrity": "sha512-F7QqKXwwNlMmsulj6+O7r4mmtAlCWfO/0HdgOxSklZfQcDu0TpLiD1mRt/zF25Bk59FIjEuGAIyn5ei4yMfLhA==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/ieee754": "1.11.0", - "@webassemblyjs/leb128": "1.11.0", - "@webassemblyjs/utf8": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/ieee754": "1.11.1", + "@webassemblyjs/leb128": "1.11.1", + "@webassemblyjs/utf8": "1.11.1" } }, "@webassemblyjs/wasm-opt": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.0.tgz", - "integrity": "sha512-tHUSP5F4ywyh3hZ0+fDQuWxKx3mJiPeFufg+9gwTpYp324mPCQgnuVKwzLTZVqj0duRDovnPaZqDwoyhIO8kYg==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.11.1.tgz", + "integrity": "sha512-VqnkNqnZlU5EB64pp1l7hdm3hmQw7Vgqa0KF/KCNO9sIpI6Fk6brDEiX+iCOYrvMuBWDws0NkTOxYEb85XQHHw==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-buffer": "1.11.0", - "@webassemblyjs/wasm-gen": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-buffer": "1.11.1", + "@webassemblyjs/wasm-gen": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1" } }, "@webassemblyjs/wasm-parser": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.0.tgz", - "integrity": "sha512-6L285Sgu9gphrcpDXINvm0M9BskznnzJTE7gYkjDbxET28shDqp27wpruyx3C2S/dvEwiigBwLA1cz7lNUi0kw==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.11.1.tgz", + "integrity": "sha512-rrBujw+dJu32gYB7/Lup6UhdkPx9S9SnobZzRVL7VcBH9Bt9bCBLEuX/YXOOtBsOZ4NQrRykKhffRWHvigQvOA==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/helper-api-error": "1.11.0", - "@webassemblyjs/helper-wasm-bytecode": "1.11.0", - "@webassemblyjs/ieee754": "1.11.0", - "@webassemblyjs/leb128": "1.11.0", - "@webassemblyjs/utf8": "1.11.0" + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/helper-api-error": "1.11.1", + "@webassemblyjs/helper-wasm-bytecode": "1.11.1", + "@webassemblyjs/ieee754": "1.11.1", + "@webassemblyjs/leb128": "1.11.1", + "@webassemblyjs/utf8": "1.11.1" } }, "@webassemblyjs/wast-printer": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.0.tgz", - "integrity": "sha512-Fg5OX46pRdTgB7rKIUojkh9vXaVN6sGYCnEiJN1GYkb0RPwShZXp6KTDqmoMdQPKhcroOXh3fEzmkWmCYaKYhQ==", + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.11.1.tgz", + "integrity": "sha512-IQboUWM4eKzWW+N/jij2sRatKMh99QEelo3Eb2q0qXkvPRISAj8Qxtmw5itwqK+TTkBuUIE45AxYPToqPtL5gg==", "dev": true, "requires": { - "@webassemblyjs/ast": "1.11.0", + "@webassemblyjs/ast": "1.11.1", "@xtuc/long": "4.2.2" } }, @@ -16260,9 +16259,9 @@ "dev": true }, "acorn": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.3.0.tgz", - "integrity": "sha512-tqPKHZ5CaBJw0Xmy0ZZvLs1qTV+BNFSyvn77ASXkpBNfIRk8ev26fKrD9iLGwGA9zedPao52GSHzq8lyZG0NUw==", + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.4.1.tgz", + "integrity": "sha512-asabaBSkEKosYKMITunzX177CXxQ4Q8BSSzMTKD+FefUhipQC70gfW5SiUDhYQ3vk8G+81HqQk7Fv9OXwwn9KA==", "dev": true }, "acorn-globals": { @@ -17837,9 +17836,9 @@ } }, "es-module-lexer": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.6.0.tgz", - "integrity": "sha512-f8kcHX1ArhllUtb/wVSyvygoKCznIjnxhLxy7TCvIiMdT7fL4ZDTIKaadMe6eLvOXg6Wk02UeoFgUoZ2EKZZUA==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.7.1.tgz", + "integrity": "sha512-MgtWFl5No+4S3TmhDmCz2ObFGm6lEpTnzbQi+Dd+pw4mlTIZTmM2iAs5gRlmx5zS9luzobCSBSI90JM/1/JgOw==", "dev": true }, "es-to-primitive": { @@ -25588,14 +25587,13 @@ } }, "typedoc": { - "version": "0.21.2", - "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.21.2.tgz", - "integrity": "sha512-SR1ByJB3USg+jxoxwzMRP07g/0f/cQUE5t7gOh1iTUyjTPyJohu9YSKRlK+MSXXqlhIq+m0jkEHEG5HoY7/Adg==", + "version": "0.21.4", + "resolved": "https://registry.npmjs.org/typedoc/-/typedoc-0.21.4.tgz", + "integrity": "sha512-slZQhvD9U0d9KacktYAyuNMMOXJRFNHy+Gd8xY2Qrqq3eTTTv3frv3N4au/cFnab9t3T5WA0Orb6QUjMc+1bDA==", "dev": true, "requires": { "glob": "^7.1.7", "handlebars": "^4.7.7", - "lodash": "^4.17.21", "lunr": "^2.3.9", "marked": "^2.1.1", "minimatch": "^3.0.0", @@ -25694,9 +25692,9 @@ } }, "typescript": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.4.tgz", - "integrity": "sha512-uauPG7XZn9F/mo+7MrsRjyvbxFpzemRjKEZXS4AK83oP2KKOJPvb+9cO/gmnv8arWZvhnjVOXz7B49m1l0e9Ew==", + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.3.5.tgz", + "integrity": "sha512-DqQgihaQ9cUrskJo9kIyW/+g0Vxsk8cDtZ52a3NGh0YNTfpUSArXSohyUGnvbPazEPLu398C0UxmKSOrPumUzA==", "dev": true }, "uglify-js": { @@ -25958,21 +25956,21 @@ "dev": true }, "webpack": { - "version": "5.41.1", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.41.1.tgz", - "integrity": "sha512-AJZIIsqJ/MVTmegEq9Tlw5mk5EHdGiJbDdz9qP15vmUH+oxI1FdWcL0E9EO8K/zKaRPWqEs7G/OPxq1P61u5Ug==", + "version": "5.44.0", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.44.0.tgz", + "integrity": "sha512-I1S1w4QLoKmH19pX6YhYN0NiSXaWY8Ou00oA+aMcr9IUGeF5azns+IKBkfoAAG9Bu5zOIzZt/mN35OffBya8AQ==", "dev": true, "requires": { "@types/eslint-scope": "^3.7.0", - "@types/estree": "^0.0.48", - "@webassemblyjs/ast": "1.11.0", - "@webassemblyjs/wasm-edit": "1.11.0", - "@webassemblyjs/wasm-parser": "1.11.0", - "acorn": "^8.2.1", + "@types/estree": "^0.0.50", + "@webassemblyjs/ast": "1.11.1", + "@webassemblyjs/wasm-edit": "1.11.1", + "@webassemblyjs/wasm-parser": "1.11.1", + "acorn": "^8.4.1", "browserslist": "^4.14.5", "chrome-trace-event": "^1.0.2", "enhanced-resolve": "^5.8.0", - "es-module-lexer": "^0.6.0", + "es-module-lexer": "^0.7.1", "eslint-scope": "5.1.1", "events": "^3.2.0", "glob-to-regexp": "^0.4.1", diff --git a/package.json b/package.json index 5e217df..95e7deb 100644 --- a/package.json +++ b/package.json @@ -40,9 +40,9 @@ "devDependencies": { "@types/axios": "^0.14.0", "@types/form-data": "^2.5.0", - "@types/jest": "^26.0.23", + "@types/jest": "^26.0.24", "@types/mime": "^2.0.3", - "@types/tough-cookie": "^4.0.0", + "@types/tough-cookie": "^4.0.1", "copyfiles": "^2.4.1", "cp": "^0.2.0", "dotenv": "^10.0.0", @@ -58,11 +58,11 @@ "ts-loader": "^9.2.2", "tslint": "^6.1.3", "tslint-config-prettier": "^1.18.0", - "typedoc": "^0.21.2", + "typedoc": "^0.21.4", "typedoc-neo-theme": "^1.1.1", "typedoc-plugin-external-module-name": "^4.0.6", - "typescript": "^4.3.4", - "webpack": "^5.41.1", + "typescript": "^4.3.5", + "webpack": "^5.44.0", "webpack-cli": "^4.7.2" }, "main": "index.js", From 69999d8e8bdb6aa3a1014e99396cbe3f673384b3 Mon Sep 17 00:00:00 2001 From: sabhas Date: Sun, 18 Jul 2021 21:34:16 +0500 Subject: [PATCH 13/39] fix: update fileUpload method to override existing config --- src/SASjs.ts | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/SASjs.ts b/src/SASjs.ts index d60ad27..6e7df78 100644 --- a/src/SASjs.ts +++ b/src/SASjs.ts @@ -540,11 +540,22 @@ export default class SASjs { * Process). Is prepended at runtime with the value of `appLoc`. * @param files - array of files to be uploaded, including File object and file name. * @param params - request URL parameters. + * @param config - object to override existing config (optional) */ - public uploadFile(sasJob: string, files: UploadFile[], params: any) { - const fileUploader = - this.fileUploader || - new FileUploader(this.sasjsConfig, this.jobsPath, this.requestClient!) + public uploadFile( + sasJob: string, + files: UploadFile[], + params: any, + config?: any + ) { + const fileUploader = config + ? new FileUploader( + { ...this.sasjsConfig, ...config }, + this.jobsPath, + this.requestClient! + ) + : this.fileUploader || + new FileUploader(this.sasjsConfig, this.jobsPath, this.requestClient!) return fileUploader.uploadFile(sasJob, files, params) } From c69be8ffc3169816761151a0fa0ba3a2bb3c8023 Mon Sep 17 00:00:00 2001 From: sabhas Date: Sun, 18 Jul 2021 21:37:08 +0500 Subject: [PATCH 14/39] fix: move parseSasViyaDebugResponse method to utils folder --- src/job-execution/WebJobExecutor.ts | 26 +++++++++----------------- src/utils/index.ts | 1 + src/utils/parseViyaDebugResponse.ts | 29 +++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 17 deletions(-) create mode 100644 src/utils/parseViyaDebugResponse.ts diff --git a/src/job-execution/WebJobExecutor.ts b/src/job-execution/WebJobExecutor.ts index 30a6e2f..91a0fc8 100644 --- a/src/job-execution/WebJobExecutor.ts +++ b/src/job-execution/WebJobExecutor.ts @@ -8,7 +8,11 @@ import { generateFileUploadForm } from '../file/generateFileUploadForm' import { generateTableUploadForm } from '../file/generateTableUploadForm' import { RequestClient } from '../request/RequestClient' import { SASViyaApiClient } from '../SASViyaApiClient' -import { isRelativePath, isValidJson } from '../utils' +import { + isRelativePath, + isValidJson, + parseSasViyaDebugResponse +} from '../utils' import { BaseJobExecutor } from './JobExecutor' import { parseWeboutResponse } from '../utils/parseWeboutResponse' @@ -95,8 +99,10 @@ export class WebJobExecutor extends BaseJobExecutor { this.requestClient!.post(apiUrl, formData, undefined) .then(async (res) => { if (this.serverType === ServerType.SasViya && config.debug) { - const jsonResponse = await this.parseSasViyaDebugResponse( - res.result as string + const jsonResponse = await parseSasViyaDebugResponse( + res.result as string, + this.requestClient, + this.serverUrl ) this.appendRequest(res, sasJob, config.debug) resolve(jsonResponse) @@ -151,20 +157,6 @@ export class WebJobExecutor extends BaseJobExecutor { return requestPromise } - private parseSasViyaDebugResponse = async (response: string) => { - const iframeStart = response.split( - '')[0] : null - if (!jsonUrl) { - throw new Error('Unable to find webout file URL.') - } - - return this.requestClient - .get(this.serverUrl + jsonUrl, undefined) - .then((res) => res.result) - } - private async getJobUri(sasJob: string) { if (!this.sasViyaApiClient) return '' let uri = '' diff --git a/src/utils/index.ts b/src/utils/index.ts index 3f6ec1d..9cc5244 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -13,3 +13,4 @@ export * from './splitChunks' export * from './parseWeboutResponse' export * from './fetchLogByChunks' export * from './isValidJson' +export * from './parseViyaDebugResponse' diff --git a/src/utils/parseViyaDebugResponse.ts b/src/utils/parseViyaDebugResponse.ts new file mode 100644 index 0000000..3137995 --- /dev/null +++ b/src/utils/parseViyaDebugResponse.ts @@ -0,0 +1,29 @@ +import { RequestClient } from '../request/RequestClient' + +/** + * When querying a Viya job using the Web approach (as opposed to using the APIs) with _DEBUG enabled, + * the first response contains the log with the content in an iframe. Therefore when debug is enabled, + * and the serverType is VIYA, and useComputeApi is null (WEB), we call this function to extract the + * (_webout) content from the iframe. + * @param response - first response from viya job + * @param requestClient + * @param serverUrl + * @returns + */ +export const parseSasViyaDebugResponse = async ( + response: string, + requestClient: RequestClient, + serverUrl: string +) => { + const iframeStart = response.split( + '')[0] : null + if (!jsonUrl) { + throw new Error('Unable to find webout file URL.') + } + + return requestClient + .get(serverUrl + jsonUrl, undefined) + .then((res) => res.result) +} From 5098342dfed942cb658b502347d2635f9a2c8839 Mon Sep 17 00:00:00 2001 From: sabhas Date: Sun, 18 Jul 2021 21:39:57 +0500 Subject: [PATCH 15/39] fix: retrieve content from the iframe in first response when viya Web approach used with debug enabled --- src/FileUploader.ts | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/FileUploader.ts b/src/FileUploader.ts index 148f534..922571a 100644 --- a/src/FileUploader.ts +++ b/src/FileUploader.ts @@ -1,4 +1,4 @@ -import { isUrl } from './utils' +import { isUrl, isValidJson, parseSasViyaDebugResponse } from './utils' import { UploadFile } from './types/UploadFile' import { ErrorResponse, LoginRequiredError } from './types/errors' import { RequestClient } from './request/RequestClient' @@ -63,13 +63,28 @@ export class FileUploader { return this.requestClient .post(uploadUrl, formData, undefined, 'application/json', headers) - .then((res) => { - let result + .then(async (res) => { + // for web approach on Viya + if ( + this.sasjsConfig.debug && + (this.sasjsConfig.useComputeApi === null || + this.sasjsConfig.useComputeApi === undefined) && + this.sasjsConfig.serverType === ServerType.SasViya + ) { + const jsonResponse = await parseSasViyaDebugResponse( + res.result as string, + this.requestClient, + this.sasjsConfig.serverUrl + ) + return typeof jsonResponse === 'string' + ? isValidJson(jsonResponse) + : jsonResponse + } - result = - typeof res.result === 'string' ? JSON.parse(res.result) : res.result + return typeof res.result === 'string' + ? isValidJson(res.result) + : res.result - return result //TODO: append to SASjs requests }) .catch((err: Error) => { From 5347aeba09e42195d40a34effdce9f5022da947e Mon Sep 17 00:00:00 2001 From: sabhas Date: Sun, 18 Jul 2021 23:24:22 +0500 Subject: [PATCH 16/39] fix: replace isValidJson with getValidJson --- src/FileUploader.ts | 6 +++--- src/job-execution/WebJobExecutor.ts | 6 +++--- src/request/RequestClient.ts | 4 ++-- src/test/utils/isValidJson.spec.ts | 8 ++++---- src/utils/{isValidJson.ts => getValidJson.ts} | 2 +- src/utils/index.ts | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) rename src/utils/{isValidJson.ts => getValidJson.ts} (81%) diff --git a/src/FileUploader.ts b/src/FileUploader.ts index 922571a..7001bcb 100644 --- a/src/FileUploader.ts +++ b/src/FileUploader.ts @@ -1,4 +1,4 @@ -import { isUrl, isValidJson, parseSasViyaDebugResponse } from './utils' +import { isUrl, getValidJson, parseSasViyaDebugResponse } from './utils' import { UploadFile } from './types/UploadFile' import { ErrorResponse, LoginRequiredError } from './types/errors' import { RequestClient } from './request/RequestClient' @@ -77,12 +77,12 @@ export class FileUploader { this.sasjsConfig.serverUrl ) return typeof jsonResponse === 'string' - ? isValidJson(jsonResponse) + ? getValidJson(jsonResponse) : jsonResponse } return typeof res.result === 'string' - ? isValidJson(res.result) + ? getValidJson(res.result) : res.result //TODO: append to SASjs requests diff --git a/src/job-execution/WebJobExecutor.ts b/src/job-execution/WebJobExecutor.ts index 91a0fc8..98063fa 100644 --- a/src/job-execution/WebJobExecutor.ts +++ b/src/job-execution/WebJobExecutor.ts @@ -10,7 +10,7 @@ import { RequestClient } from '../request/RequestClient' import { SASViyaApiClient } from '../SASViyaApiClient' import { isRelativePath, - isValidJson, + getValidJson, parseSasViyaDebugResponse } from '../utils' import { BaseJobExecutor } from './JobExecutor' @@ -115,11 +115,11 @@ export class WebJobExecutor extends BaseJobExecutor { ) } - isValidJson(jsonResponse) + getValidJson(jsonResponse) this.appendRequest(res, sasJob, config.debug) resolve(res.result) } - isValidJson(res.result as string) + getValidJson(res.result as string) this.appendRequest(res, sasJob, config.debug) resolve(res.result) }) diff --git a/src/request/RequestClient.ts b/src/request/RequestClient.ts index 5bf7414..d1946f7 100644 --- a/src/request/RequestClient.ts +++ b/src/request/RequestClient.ts @@ -11,7 +11,7 @@ import { import { parseWeboutResponse } from '../utils/parseWeboutResponse' import { prefixMessage } from '@sasjs/utils/error' import { SAS9AuthError } from '../types/errors/SAS9AuthError' -import { isValidJson } from '../utils' +import { getValidJson } from '../utils' export interface HttpClient { get( @@ -429,7 +429,7 @@ export class RequestClient implements HttpClient { throw new Error('Valid JSON could not be extracted from response.') } - const jsonResponse = isValidJson(weboutResponse) + const jsonResponse = getValidJson(weboutResponse) parsedResponse = jsonResponse } catch { parsedResponse = response.data diff --git a/src/test/utils/isValidJson.spec.ts b/src/test/utils/isValidJson.spec.ts index b7af712..1901ba1 100644 --- a/src/test/utils/isValidJson.spec.ts +++ b/src/test/utils/isValidJson.spec.ts @@ -1,4 +1,4 @@ -import { isValidJson } from '../../utils' +import { getValidJson } from '../../utils' describe('jsonValidator', () => { it('should not throw an error with an valid json', () => { @@ -6,7 +6,7 @@ describe('jsonValidator', () => { test: 'test' } - expect(isValidJson(json)).toBe(json) + expect(getValidJson(json)).toBe(json) }) it('should not throw an error with an valid json string', () => { @@ -14,7 +14,7 @@ describe('jsonValidator', () => { test: 'test' } - expect(isValidJson(JSON.stringify(json))).toStrictEqual(json) + expect(getValidJson(JSON.stringify(json))).toStrictEqual(json) }) it('should throw an error with an invalid json', () => { @@ -22,7 +22,7 @@ describe('jsonValidator', () => { expect(() => { try { - isValidJson(json) + getValidJson(json) } catch (err) { throw new Error() } diff --git a/src/utils/isValidJson.ts b/src/utils/getValidJson.ts similarity index 81% rename from src/utils/isValidJson.ts rename to src/utils/getValidJson.ts index 253440f..738f679 100644 --- a/src/utils/isValidJson.ts +++ b/src/utils/getValidJson.ts @@ -2,7 +2,7 @@ * Checks if string is in valid JSON format else throw error. * @param str - string to check. */ -export const isValidJson = (str: string | object) => { +export const getValidJson = (str: string | object) => { try { if (typeof str === 'object') return str diff --git a/src/utils/index.ts b/src/utils/index.ts index 9cc5244..9f70293 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -12,5 +12,5 @@ export * from './serialize' export * from './splitChunks' export * from './parseWeboutResponse' export * from './fetchLogByChunks' -export * from './isValidJson' +export * from './getValidJson' export * from './parseViyaDebugResponse' From 4a61fb8f7fc665e0145ad86a6979d1a316784fc5 Mon Sep 17 00:00:00 2001 From: sabhas Date: Mon, 19 Jul 2021 13:00:06 +0500 Subject: [PATCH 17/39] chore: update variable name from config to ovverrideSasjsConfig --- src/SASjs.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/SASjs.ts b/src/SASjs.ts index 6e7df78..ea54489 100644 --- a/src/SASjs.ts +++ b/src/SASjs.ts @@ -540,17 +540,17 @@ export default class SASjs { * Process). Is prepended at runtime with the value of `appLoc`. * @param files - array of files to be uploaded, including File object and file name. * @param params - request URL parameters. - * @param config - object to override existing config (optional) + * @param overrideSasjsConfig - object to override existing config (optional) */ public uploadFile( sasJob: string, files: UploadFile[], params: any, - config?: any + overrideSasjsConfig?: any ) { - const fileUploader = config + const fileUploader = overrideSasjsConfig ? new FileUploader( - { ...this.sasjsConfig, ...config }, + { ...this.sasjsConfig, ...overrideSasjsConfig }, this.jobsPath, this.requestClient! ) From 85e5ade93aed4ef1d857fd68d472f64a470ec2c5 Mon Sep 17 00:00:00 2001 From: sabhas Date: Mon, 19 Jul 2021 13:01:18 +0500 Subject: [PATCH 18/39] fix: handle the case when array is passed in getValidJson method --- src/test/utils/getValidJson.spec.ts | 41 +++++++++++++++++++++++++++++ src/test/utils/isValidJson.spec.ts | 31 ---------------------- src/utils/getValidJson.ts | 5 +++- 3 files changed, 45 insertions(+), 32 deletions(-) create mode 100644 src/test/utils/getValidJson.spec.ts delete mode 100644 src/test/utils/isValidJson.spec.ts diff --git a/src/test/utils/getValidJson.spec.ts b/src/test/utils/getValidJson.spec.ts new file mode 100644 index 0000000..e7fbf66 --- /dev/null +++ b/src/test/utils/getValidJson.spec.ts @@ -0,0 +1,41 @@ +import { getValidJson } from '../../utils' + +describe('jsonValidator', () => { + it('should not throw an error with a valid json', () => { + const json = { + test: 'test' + } + + expect(getValidJson(json)).toBe(json) + }) + + it('should not throw an error with a valid json string', () => { + const json = { + test: 'test' + } + + expect(getValidJson(JSON.stringify(json))).toStrictEqual(json) + }) + + it('should throw an error with an invalid json', () => { + const json = `{\"test\":\"test\"\"test2\":\"test\"}` + let errorThrown = false + try { + getValidJson(json) + } catch (error) { + errorThrown = true + } + expect(errorThrown).toBe(true) + }) + + it('should throw an error when an array is passed', () => { + const array = ['hello', 'world'] + let errorThrown = false + try { + getValidJson(array) + } catch (error) { + errorThrown = true + } + expect(errorThrown).toBe(true) + }) +}) diff --git a/src/test/utils/isValidJson.spec.ts b/src/test/utils/isValidJson.spec.ts deleted file mode 100644 index 1901ba1..0000000 --- a/src/test/utils/isValidJson.spec.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { getValidJson } from '../../utils' - -describe('jsonValidator', () => { - it('should not throw an error with an valid json', () => { - const json = { - test: 'test' - } - - expect(getValidJson(json)).toBe(json) - }) - - it('should not throw an error with an valid json string', () => { - const json = { - test: 'test' - } - - expect(getValidJson(JSON.stringify(json))).toStrictEqual(json) - }) - - it('should throw an error with an invalid json', () => { - const json = `{\"test\":\"test\"\"test2\":\"test\"}` - - expect(() => { - try { - getValidJson(json) - } catch (err) { - throw new Error() - } - }).toThrowError - }) -}) diff --git a/src/utils/getValidJson.ts b/src/utils/getValidJson.ts index 738f679..0313157 100644 --- a/src/utils/getValidJson.ts +++ b/src/utils/getValidJson.ts @@ -1,9 +1,12 @@ /** - * Checks if string is in valid JSON format else throw error. + * if string passed then parse the string to json else if throw error for all other types unless it is not a valid json object. * @param str - string to check. */ export const getValidJson = (str: string | object) => { try { + if (Array.isArray(str)) { + throw new Error('Can not parse array object to json.') + } if (typeof str === 'object') return str return JSON.parse(str) From 5c8d311ae887daa71f3735fa6202bee117640159 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Tue, 20 Jul 2021 09:25:39 +0100 Subject: [PATCH 19/39] chore(streamlog): optimise polling mechanism --- package-lock.json | 188 ++++++++++++++- package.json | 2 +- src/SASViyaApiClient.ts | 16 +- src/api/viya/executeScript.ts | 1 - src/api/viya/pollJobState.ts | 307 ++++++++++++++++-------- src/api/viya/saveLog.ts | 28 ++- src/api/viya/spec/executeScript.spec.ts | 9 +- src/api/viya/spec/mockResponses.ts | 17 ++ src/api/viya/spec/pollJobState.spec.ts | 193 +++++++++------ src/api/viya/spec/saveLog.spec.ts | 40 ++- src/api/viya/writeStream.ts | 15 ++ src/types/errors/JobStatePollError.ts | 11 + src/types/errors/index.ts | 1 + src/utils/fetchLogByChunks.ts | 26 +- 14 files changed, 624 insertions(+), 230 deletions(-) create mode 100644 src/api/viya/writeStream.ts create mode 100644 src/types/errors/JobStatePollError.ts diff --git a/package-lock.json b/package-lock.json index 928b41e..b62d148 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,7 @@ "name": "@sasjs/adapter", "license": "ISC", "dependencies": { - "@sasjs/utils": "^2.24.0", + "@sasjs/utils": "^2.25.1", "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", "form-data": "^4.0.0", @@ -1188,8 +1188,9 @@ } }, "node_modules/@sasjs/utils": { - "version": "2.24.0", - "license": "ISC", + "version": "2.25.1", + "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.25.1.tgz", + "integrity": "sha512-lwkPE+QsB81b8/1M8X2zLdhpuiA8pIjgOwJH57zhcsliuDnNs4uijSYu40aYSc8tH98jtSuqWMjfGq8CT9o1Dw==", "dependencies": { "@types/prompts": "^2.0.13", "chalk": "^4.1.1", @@ -7932,7 +7933,182 @@ "treeverse", "validate-npm-package-name", "which", - "write-file-atomic" + "write-file-atomic", + "@npmcli/disparity-colors", + "@npmcli/git", + "@npmcli/installed-package-contents", + "@npmcli/map-workspaces", + "@npmcli/metavuln-calculator", + "@npmcli/move-file", + "@npmcli/name-from-folder", + "@npmcli/node-gyp", + "@npmcli/promise-spawn", + "@tootallnate/once", + "agent-base", + "agentkeepalive", + "aggregate-error", + "ajv", + "ansi-regex", + "ansi-styles", + "aproba", + "are-we-there-yet", + "asap", + "asn1", + "assert-plus", + "asynckit", + "aws-sign2", + "aws4", + "balanced-match", + "bcrypt-pbkdf", + "bin-links", + "binary-extensions", + "brace-expansion", + "builtins", + "caseless", + "cidr-regex", + "clean-stack", + "clone", + "cmd-shim", + "code-point-at", + "color-convert", + "color-name", + "colors", + "combined-stream", + "common-ancestor-path", + "concat-map", + "console-control-strings", + "core-util-is", + "dashdash", + "debug", + "debuglog", + "defaults", + "delayed-stream", + "delegates", + "depd", + "dezalgo", + "diff", + "ecc-jsbn", + "emoji-regex", + "encoding", + "env-paths", + "err-code", + "extend", + "extsprintf", + "fast-deep-equal", + "fast-json-stable-stringify", + "forever-agent", + "fs-minipass", + "fs.realpath", + "function-bind", + "gauge", + "getpass", + "har-schema", + "har-validator", + "has", + "has-flag", + "has-unicode", + "http-cache-semantics", + "http-proxy-agent", + "http-signature", + "https-proxy-agent", + "humanize-ms", + "iconv-lite", + "ignore-walk", + "imurmurhash", + "indent-string", + "infer-owner", + "inflight", + "inherits", + "ip", + "ip-regex", + "is-core-module", + "is-fullwidth-code-point", + "is-lambda", + "is-typedarray", + "isarray", + "isexe", + "isstream", + "jsbn", + "json-schema", + "json-schema-traverse", + "json-stringify-nice", + "json-stringify-safe", + "jsonparse", + "jsprim", + "just-diff", + "just-diff-apply", + "lru-cache", + "mime-db", + "mime-types", + "minimatch", + "minipass-collect", + "minipass-fetch", + "minipass-flush", + "minipass-json-stream", + "minipass-sized", + "minizlib", + "mute-stream", + "negotiator", + "normalize-package-data", + "npm-bundled", + "npm-install-checks", + "npm-normalize-package-bin", + "npm-packlist", + "number-is-nan", + "oauth-sign", + "object-assign", + "once", + "p-map", + "path-is-absolute", + "path-parse", + "performance-now", + "proc-log", + "process-nextick-args", + "promise-all-reject-late", + "promise-call-limit", + "promise-inflight", + "promise-retry", + "promzard", + "psl", + "punycode", + "qs", + "read-cmd-shim", + "readable-stream", + "request", + "resolve", + "retry", + "safe-buffer", + "safer-buffer", + "set-blocking", + "signal-exit", + "smart-buffer", + "socks", + "socks-proxy-agent", + "spdx-correct", + "spdx-exceptions", + "spdx-expression-parse", + "spdx-license-ids", + "sshpk", + "string_decoder", + "string-width", + "stringify-package", + "strip-ansi", + "supports-color", + "tunnel-agent", + "tweetnacl", + "typedarray-to-buffer", + "unique-filename", + "unique-slug", + "uri-js", + "util-deprecate", + "uuid", + "validate-npm-package-license", + "verror", + "walk-up-path", + "wcwidth", + "wide-align", + "wrappy", + "yallist" ], "dev": true, "license": "Artistic-2.0", @@ -14604,7 +14780,9 @@ } }, "@sasjs/utils": { - "version": "2.24.0", + "version": "2.25.1", + "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.25.1.tgz", + "integrity": "sha512-lwkPE+QsB81b8/1M8X2zLdhpuiA8pIjgOwJH57zhcsliuDnNs4uijSYu40aYSc8tH98jtSuqWMjfGq8CT9o1Dw==", "requires": { "@types/prompts": "^2.0.13", "chalk": "^4.1.1", diff --git a/package.json b/package.json index 95e7deb..332b564 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ }, "main": "index.js", "dependencies": { - "@sasjs/utils": "^2.24.0", + "@sasjs/utils": "^2.25.1", "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", "form-data": "^4.0.0", diff --git a/src/SASViyaApiClient.ts b/src/SASViyaApiClient.ts index ee439f2..e68082c 100644 --- a/src/SASViyaApiClient.ts +++ b/src/SASViyaApiClient.ts @@ -754,18 +754,16 @@ export class SASViyaApiClient { jobDefinition, arguments: jobArguments } - const { result: postedJob, etag } = await this.requestClient.post( + const { result: postedJob } = await this.requestClient.post( `${this.serverUrl}/jobExecution/jobs?_action=wait`, postJobRequestBody, access_token ) - const jobStatus = await this.pollJobState( - postedJob, - etag, - authConfig - ).catch((err) => { - throw prefixMessage(err, 'Error while polling job status. ') - }) + const jobStatus = await this.pollJobState(postedJob, authConfig).catch( + (err) => { + throw prefixMessage(err, 'Error while polling job status. ') + } + ) const { result: currentJob } = await this.requestClient.get( `${this.serverUrl}/jobExecution/jobs/${postedJob.id}`, access_token @@ -833,7 +831,6 @@ export class SASViyaApiClient { private async pollJobState( postedJob: Job, - etag: string | null, authConfig?: AuthConfig, pollOptions?: PollOptions ) { @@ -841,7 +838,6 @@ export class SASViyaApiClient { this.requestClient, postedJob, this.debug, - etag, authConfig, pollOptions ) diff --git a/src/api/viya/executeScript.ts b/src/api/viya/executeScript.ts index c75a3e5..e54143f 100644 --- a/src/api/viya/executeScript.ts +++ b/src/api/viya/executeScript.ts @@ -178,7 +178,6 @@ export async function executeScript( requestClient, postedJob, debug, - etag, authConfig, pollOptions ).catch(async (err) => { diff --git a/src/api/viya/pollJobState.ts b/src/api/viya/pollJobState.ts index 5e74dda..b777382 100644 --- a/src/api/viya/pollJobState.ts +++ b/src/api/viya/pollJobState.ts @@ -1,16 +1,19 @@ import { AuthConfig } from '@sasjs/utils/types' -import { prefixMessage } from '@sasjs/utils/error' -import { generateTimestamp } from '@sasjs/utils/time' import { Job, PollOptions } from '../..' import { getTokens } from '../../auth/getTokens' import { RequestClient } from '../../request/RequestClient' +import { JobStatePollError } from '../../types/errors' +import { generateTimestamp } from '@sasjs/utils/time' import { saveLog } from './saveLog' +import { createWriteStream } from '@sasjs/utils/file' +import { WriteStream } from 'fs' +import { Link } from '../../types' +import { prefixMessage } from '@sasjs/utils/error' export async function pollJobState( requestClient: RequestClient, postedJob: Job, debug: boolean, - etag: string | null, authConfig?: AuthConfig, pollOptions?: PollOptions ) { @@ -18,130 +21,226 @@ export async function pollJobState( let pollInterval = 300 let maxPollCount = 1000 - let maxErrorCount = 5 - let access_token = (authConfig || {}).access_token - - const logFileName = `${postedJob.name || 'job'}-${generateTimestamp()}.log` - const logFilePath = `${ - pollOptions?.logFolderPath || process.cwd() - }/${logFileName}` - - if (authConfig) { - ;({ access_token } = await getTokens(requestClient, authConfig)) - } if (pollOptions) { pollInterval = pollOptions.pollInterval || pollInterval maxPollCount = pollOptions.maxPollCount || maxPollCount } - let postedJobState = '' - let pollCount = 0 - let errorCount = 0 - const headers: any = { - 'Content-Type': 'application/json', - 'If-None-Match': etag - } - if (access_token) { - headers.Authorization = `Bearer ${access_token}` - } const stateLink = postedJob.links.find((l: any) => l.rel === 'state') if (!stateLink) { throw new Error(`Job state link was not found.`) } - const { result: state } = await requestClient - .get( - `${stateLink.href}?_action=wait&wait=300`, - access_token, - 'text/plain', - {}, - debug + let currentState = await getJobState( + requestClient, + postedJob, + '', + debug, + authConfig + ).catch((err) => { + logger.error( + `Error fetching job state from ${stateLink.href}. Starting poll, assuming job to be running.`, + err ) - .catch((err) => { - logger.error( - `Error fetching job state from ${stateLink.href}. Starting poll, assuming job to be running.`, - err - ) - return { result: 'unavailable' } - }) + return 'unavailable' + }) + + let pollCount = 0 - const currentState = state.trim() if (currentState === 'completed') { return Promise.resolve(currentState) } - return new Promise(async (resolve, reject) => { - let printedState = '' + let logFileStream + if (pollOptions?.streamLog) { + const logFileName = `${postedJob.name || 'job'}-${generateTimestamp()}.log` + const logFilePath = `${ + pollOptions?.logFolderPath || process.cwd() + }/${logFileName}` - const interval = setInterval(async () => { - if ( - postedJobState === 'running' || - postedJobState === '' || - postedJobState === 'pending' || - postedJobState === 'unavailable' - ) { - if (authConfig) { - ;({ access_token } = await getTokens(requestClient, authConfig)) - } + logFileStream = await createWriteStream(logFilePath) + } - if (stateLink) { - const { result: jobState } = await requestClient - .get( - `${stateLink.href}?_action=wait&wait=300`, - access_token, - 'text/plain', - {}, - debug - ) - .catch((err) => { - errorCount++ - if (pollCount >= maxPollCount || errorCount >= maxErrorCount) { - clearInterval(interval) - reject( - prefixMessage( - err, - 'Error while getting job state after interval. ' - ) - ) - } - logger.error( - `Error fetching job state from ${stateLink.href}. Resuming poll, assuming job to be running.`, - err - ) - return { result: 'unavailable' } - }) + let result = await doPoll( + requestClient, + postedJob, + currentState, + debug, + pollCount, + authConfig, + pollOptions, + logFileStream + ) - postedJobState = jobState.trim() - if (postedJobState != 'unavailable' && errorCount > 0) { - errorCount = 0 - } + currentState = result.state + pollCount = result.pollCount - if (debug && printedState !== postedJobState) { - logger.info('Polling job status...') - logger.info(`Current job state: ${postedJobState}`) + if (!needsRetry(currentState) || pollCount >= maxPollCount) { + return currentState + } - printedState = postedJobState - } + // If we get to this point, this is a long-running job that needs longer polling. + // We will resume polling with a bigger interval of 1 minute + let longJobPollOptions: PollOptions = { + maxPollCount: 24 * 60, + pollInterval: 60000, + streamLog: false + } + if (pollOptions) { + longJobPollOptions.streamLog = pollOptions.streamLog + longJobPollOptions.logFolderPath = pollOptions.logFolderPath + } - pollCount++ + result = await doPoll( + requestClient, + postedJob, + currentState, + debug, + pollCount, + authConfig, + longJobPollOptions, + logFileStream + ) - await saveLog( - postedJob, - requestClient, - pollOptions?.streamLog || false, - logFilePath, - access_token - ) + currentState = result.state + pollCount = result.pollCount - if (pollCount >= maxPollCount) { - resolve(postedJobState) - } - } - } else { - clearInterval(interval) - resolve(postedJobState) - } - }, pollInterval) - }) + if (logFileStream) { + logFileStream.end() + } + + return currentState } + +const getJobState = async ( + requestClient: RequestClient, + job: Job, + currentState: string, + debug: boolean, + authConfig?: AuthConfig +) => { + const stateLink = job.links.find((l: any) => l.rel === 'state') + if (!stateLink) { + throw new Error(`Job state link was not found.`) + } + + if (needsRetry(currentState)) { + let tokens + if (authConfig) { + tokens = await getTokens(requestClient, authConfig) + } + + const { result: jobState } = await requestClient + .get( + `${stateLink.href}?_action=wait&wait=300`, + tokens?.access_token, + 'text/plain', + {}, + debug + ) + .catch((err) => { + throw new JobStatePollError(job.id, err) + }) + + return jobState.trim() + } else { + return currentState + } +} + +const needsRetry = (state: string) => + state === 'running' || + state === '' || + state === 'pending' || + state === 'unavailable' + +const doPoll = async ( + requestClient: RequestClient, + postedJob: Job, + currentState: string, + debug: boolean, + pollCount: number, + authConfig?: AuthConfig, + pollOptions?: PollOptions, + logStream?: WriteStream +): Promise<{ state: string; pollCount: number }> => { + let pollInterval = 300 + let maxPollCount = 1000 + let maxErrorCount = 5 + let errorCount = 0 + let state = currentState + let printedState = '' + let startLogLine = 0 + + const logger = process.logger || console + + if (pollOptions) { + pollInterval = pollOptions.pollInterval || pollInterval + maxPollCount = pollOptions.maxPollCount || maxPollCount + } + + const stateLink = postedJob.links.find((l: Link) => l.rel === 'state') + if (!stateLink) { + throw new Error(`Job state link was not found.`) + } + + while (needsRetry(state) && pollCount <= 100 && pollCount <= maxPollCount) { + state = await getJobState( + requestClient, + postedJob, + state, + debug, + authConfig + ).catch((err) => { + errorCount++ + if (pollCount >= maxPollCount || errorCount >= maxErrorCount) { + throw err + } + logger.error( + `Error fetching job state from ${stateLink.href}. Resuming poll, assuming job to be running.`, + err + ) + return 'unavailable' + }) + + pollCount++ + + const jobUrl = postedJob.links.find((l: Link) => l.rel === 'self') + const { result: job } = await requestClient.get( + jobUrl!.href, + authConfig?.access_token + ) + + const endLogLine = job.logStatistics?.lineCount ?? 1000000 + + await saveLog( + postedJob, + requestClient, + pollOptions?.streamLog || false, + startLogLine, + endLogLine, + logStream, + authConfig?.access_token + ) + + startLogLine += job.logStatistics.lineCount + + if (debug && printedState !== state) { + logger.info('Polling job status...') + logger.info(`Current job state: ${state}`) + + printedState = state + } + + if (state != 'unavailable' && errorCount > 0) { + errorCount = 0 + } + + await delay(pollInterval) + } + + return { state, pollCount } +} + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) diff --git a/src/api/viya/saveLog.ts b/src/api/viya/saveLog.ts index 9200930..94fbbfd 100644 --- a/src/api/viya/saveLog.ts +++ b/src/api/viya/saveLog.ts @@ -1,15 +1,19 @@ -import { createFile } from '@sasjs/utils/file' import { Job } from '../..' import { RequestClient } from '../../request/RequestClient' -import { fetchLogByChunks } from '../../utils' +import { fetchLog } from '../../utils' +import { WriteStream } from 'fs' +import { writeStream } from './writeStream' export async function saveLog( job: Job, requestClient: RequestClient, shouldSaveLog: boolean, - logFilePath: string, + startLine: number, + endLine: number, + logFileStream?: WriteStream, accessToken?: string ) { + console.log('startLine: ', startLine, ' endLine: ', endLine) if (!shouldSaveLog) { return } @@ -20,6 +24,12 @@ export async function saveLog( ) } + if (!logFileStream) { + throw new Error( + `Logs for job ${job.id} cannot be written without a valid write stream.` + ) + } + const logger = process.logger || console const jobLogUrl = job.links.find((l) => l.rel === 'log') @@ -27,14 +37,14 @@ export async function saveLog( throw new Error(`Log URL for job ${job.id} was not found.`) } - const logCount = job.logStatistics?.lineCount ?? 1000000 - const log = await fetchLogByChunks( + const log = await fetchLog( requestClient, accessToken, `${jobLogUrl.href}/content`, - logCount - ) + startLine, + endLine + ).catch((e) => console.log(e)) - logger.info(`Writing logs to ${logFilePath}`) - await createFile(logFilePath, log) + logger.info(`Writing logs to ${logFileStream.path}`) + await writeStream(logFileStream, log || '').catch((e) => console.log(e)) } diff --git a/src/api/viya/spec/executeScript.spec.ts b/src/api/viya/spec/executeScript.spec.ts index 314c70b..5e2fb11 100644 --- a/src/api/viya/spec/executeScript.spec.ts +++ b/src/api/viya/spec/executeScript.spec.ts @@ -344,7 +344,6 @@ describe('executeScript', () => { requestClient, mockJob, false, - '', mockAuthConfig, defaultPollOptions ) @@ -546,7 +545,7 @@ describe('executeScript', () => { if (url.includes('_webout')) { return Promise.reject(new NotFoundError(url)) } - return Promise.resolve({ result: mockJob, etag: '' }) + return Promise.resolve({ result: mockJob, etag: '', status: 200 }) }) const error = await executeScript( @@ -645,7 +644,9 @@ const setupMocks = () => { .mockImplementation(() => Promise.resolve({ result: mockJob, etag: '' })) jest .spyOn(requestClient, 'get') - .mockImplementation(() => Promise.resolve({ result: mockJob, etag: '' })) + .mockImplementation(() => + Promise.resolve({ result: mockJob, etag: '', status: 200 }) + ) jest .spyOn(requestClient, 'delete') .mockImplementation(() => Promise.resolve({ result: {}, etag: '' })) @@ -658,7 +659,7 @@ const setupMocks = () => { jest .spyOn(sessionManager, 'getVariable') .mockImplementation(() => - Promise.resolve({ result: { value: 'test' }, etag: 'test' }) + Promise.resolve({ result: { value: 'test' }, etag: 'test', status: 200 }) ) jest .spyOn(sessionManager, 'getSession') diff --git a/src/api/viya/spec/mockResponses.ts b/src/api/viya/spec/mockResponses.ts index e85028c..22580f7 100644 --- a/src/api/viya/spec/mockResponses.ts +++ b/src/api/viya/spec/mockResponses.ts @@ -31,6 +31,13 @@ export const mockJob: Job = { type: 'log', uri: 'log' }, + { + rel: 'self', + href: '/job', + method: 'GET', + type: 'job', + uri: 'job' + }, { rel: 'state', href: '/state', @@ -54,3 +61,13 @@ export const mockAuthConfig: AuthConfig = { access_token: 'acc355', refresh_token: 'r3fr35h' } + +export class MockStream { + _write(chunk: string, _: any, next: Function) { + next() + } + + reset() {} + + destroy() {} +} diff --git a/src/api/viya/spec/pollJobState.spec.ts b/src/api/viya/spec/pollJobState.spec.ts index 23ac560..dcbd2b9 100644 --- a/src/api/viya/spec/pollJobState.spec.ts +++ b/src/api/viya/spec/pollJobState.spec.ts @@ -1,10 +1,11 @@ +import * as fs from 'fs' +import { Logger, LogLevel } from '@sasjs/utils' import { RequestClient } from '../../../request/RequestClient' import { mockAuthConfig, mockJob } from './mockResponses' import { pollJobState } from '../pollJobState' import * as getTokensModule from '../../../auth/getTokens' import * as saveLogModule from '../saveLog' import { PollOptions } from '../../../types' -import { Logger, LogLevel } from '@sasjs/utils' const requestClient = new (>RequestClient)() const defaultPollOptions: PollOptions = { @@ -24,7 +25,6 @@ describe('pollJobState', () => { requestClient, mockJob, false, - 'test', mockAuthConfig, defaultPollOptions ) @@ -40,7 +40,6 @@ describe('pollJobState', () => { requestClient, mockJob, false, - 'test', undefined, defaultPollOptions ) @@ -53,7 +52,6 @@ describe('pollJobState', () => { requestClient, { ...mockJob, links: mockJob.links.filter((l) => l.rel !== 'state') }, false, - 'test', undefined, defaultPollOptions ).catch((e) => e) @@ -62,23 +60,12 @@ describe('pollJobState', () => { }) it('should attempt to refresh tokens before each poll', async () => { - jest - .spyOn(requestClient, 'get') - .mockImplementationOnce(() => - Promise.resolve({ result: 'pending', etag: '' }) - ) - .mockImplementationOnce(() => - Promise.resolve({ result: 'running', etag: '' }) - ) - .mockImplementation(() => - Promise.resolve({ result: 'completed', etag: '' }) - ) + mockSimplePoll() await pollJobState( requestClient, mockJob, false, - 'test', mockAuthConfig, defaultPollOptions ) @@ -87,23 +74,12 @@ describe('pollJobState', () => { }) it('should attempt to fetch and save the log after each poll', async () => { - jest - .spyOn(requestClient, 'get') - .mockImplementationOnce(() => - Promise.resolve({ result: 'pending', etag: '' }) - ) - .mockImplementationOnce(() => - Promise.resolve({ result: 'running', etag: '' }) - ) - .mockImplementation(() => - Promise.resolve({ result: 'completed', etag: '' }) - ) + mockSimplePoll() await pollJobState( requestClient, mockJob, false, - 'test', mockAuthConfig, defaultPollOptions ) @@ -112,20 +88,12 @@ describe('pollJobState', () => { }) it('should return the current status when the max poll count is reached', async () => { - jest - .spyOn(requestClient, 'get') - .mockImplementationOnce(() => - Promise.resolve({ result: 'pending', etag: '' }) - ) - .mockImplementationOnce(() => - Promise.resolve({ result: 'running', etag: '' }) - ) + mockRunningPoll() const state = await pollJobState( requestClient, mockJob, false, - 'test', mockAuthConfig, { ...defaultPollOptions, @@ -136,51 +104,47 @@ describe('pollJobState', () => { expect(state).toEqual('running') }) - it('should continue polling until the job completes or errors', async () => { - jest - .spyOn(requestClient, 'get') - .mockImplementationOnce(() => - Promise.resolve({ result: 'pending', etag: '' }) - ) - .mockImplementationOnce(() => - Promise.resolve({ result: 'running', etag: '' }) - ) - .mockImplementation(() => - Promise.resolve({ result: 'completed', etag: '' }) - ) + it('should poll with a larger interval for longer running jobs', async () => { + mockLongPoll() + + const state = await pollJobState( + requestClient, + mockJob, + false, + mockAuthConfig, + { + ...defaultPollOptions, + maxPollCount: 200, + pollInterval: 10 + } + ) + + expect(state).toEqual('completed') + }, 200000) + + it('should continue polling until the job completes or errors', async () => { + mockSimplePoll(1) const state = await pollJobState( requestClient, mockJob, false, - 'test', undefined, defaultPollOptions ) - expect(requestClient.get).toHaveBeenCalledTimes(4) + expect(requestClient.get).toHaveBeenCalledTimes(3) expect(state).toEqual('completed') }) it('should print the state to the console when debug is on', async () => { jest.spyOn((process as any).logger, 'info') - jest - .spyOn(requestClient, 'get') - .mockImplementationOnce(() => - Promise.resolve({ result: 'pending', etag: '' }) - ) - .mockImplementationOnce(() => - Promise.resolve({ result: 'running', etag: '' }) - ) - .mockImplementation(() => - Promise.resolve({ result: 'completed', etag: '' }) - ) + mockSimplePoll() await pollJobState( requestClient, mockJob, true, - 'test', undefined, defaultPollOptions ) @@ -205,21 +169,12 @@ describe('pollJobState', () => { }) it('should continue polling when there is a single error in between', async () => { - jest - .spyOn(requestClient, 'get') - .mockImplementationOnce(() => - Promise.resolve({ result: 'pending', etag: '' }) - ) - .mockImplementationOnce(() => Promise.reject('Status Error')) - .mockImplementationOnce(() => - Promise.resolve({ result: 'completed', etag: '' }) - ) + mockPollWithSingleError() const state = await pollJobState( requestClient, mockJob, false, - 'test', undefined, defaultPollOptions ) @@ -229,20 +184,19 @@ describe('pollJobState', () => { }) it('should throw an error when the error count exceeds the set value of 5', async () => { - jest - .spyOn(requestClient, 'get') - .mockImplementation(() => Promise.reject('Status Error')) + mockErroredPoll() const error = await pollJobState( requestClient, mockJob, false, - 'test', undefined, defaultPollOptions ).catch((e) => e) - expect(error).toContain('Error while getting job state after interval.') + expect(error.message).toEqual( + 'Error while polling job state for job j0b: Status Error' + ) }) }) @@ -251,11 +205,12 @@ const setupMocks = () => { jest.mock('../../../request/RequestClient') jest.mock('../../../auth/getTokens') jest.mock('../saveLog') + jest.mock('fs') jest .spyOn(requestClient, 'get') .mockImplementation(() => - Promise.resolve({ result: 'completed', etag: '' }) + Promise.resolve({ result: 'completed', etag: '', status: 200 }) ) jest .spyOn(getTokensModule, 'getTokens') @@ -263,4 +218,84 @@ const setupMocks = () => { jest .spyOn(saveLogModule, 'saveLog') .mockImplementation(() => Promise.resolve()) + jest + .spyOn(fs, 'createWriteStream') + .mockImplementation(() => ({} as unknown as fs.WriteStream)) +} + +const mockSimplePoll = (runningCount = 2) => { + let count = 0 + jest.spyOn(requestClient, 'get').mockImplementation((url) => { + count++ + if (url.includes('job')) { + return Promise.resolve({ result: mockJob, etag: '', status: 200 }) + } + return Promise.resolve({ + result: + count === 0 + ? 'pending' + : count <= runningCount + ? 'running' + : 'completed', + etag: '', + status: 200 + }) + }) +} + +const mockRunningPoll = () => { + let count = 0 + jest.spyOn(requestClient, 'get').mockImplementation((url) => { + count++ + if (url.includes('job')) { + return Promise.resolve({ result: mockJob, etag: '', status: 200 }) + } + return Promise.resolve({ + result: count === 0 ? 'pending' : 'running', + etag: '', + status: 200 + }) + }) +} + +const mockLongPoll = () => { + let count = 0 + jest.spyOn(requestClient, 'get').mockImplementation((url) => { + count++ + if (url.includes('job')) { + return Promise.resolve({ result: mockJob, etag: '', status: 200 }) + } + return Promise.resolve({ + result: count <= 101 ? 'running' : 'completed', + etag: '', + status: 200 + }) + }) +} + +const mockPollWithSingleError = () => { + let count = 0 + jest.spyOn(requestClient, 'get').mockImplementation((url) => { + count++ + if (url.includes('job')) { + return Promise.resolve({ result: mockJob, etag: '', status: 200 }) + } + if (count === 1) { + return Promise.reject('Status Error') + } + return Promise.resolve({ + result: count === 0 ? 'pending' : 'completed', + etag: '', + status: 200 + }) + }) +} + +const mockErroredPoll = () => { + jest.spyOn(requestClient, 'get').mockImplementation((url) => { + if (url.includes('job')) { + return Promise.resolve({ result: mockJob, etag: '', status: 200 }) + } + return Promise.reject('Status Error') + }) } diff --git a/src/api/viya/spec/saveLog.spec.ts b/src/api/viya/spec/saveLog.spec.ts index c4b8b9d..4d35c6f 100644 --- a/src/api/viya/spec/saveLog.spec.ts +++ b/src/api/viya/spec/saveLog.spec.ts @@ -1,11 +1,13 @@ import { Logger, LogLevel } from '@sasjs/utils' -import * as fileModule from '@sasjs/utils/file' import { RequestClient } from '../../../request/RequestClient' import * as fetchLogsModule from '../../../utils/fetchLogByChunks' +import * as writeStreamModule from '../writeStream' import { saveLog } from '../saveLog' import { mockJob } from './mockResponses' +import { WriteStream } from 'fs' const requestClient = new (>RequestClient)() +const stream = {} as unknown as WriteStream describe('saveLog', () => { beforeEach(() => { @@ -14,16 +16,21 @@ describe('saveLog', () => { }) it('should return immediately if shouldSaveLog is false', async () => { - await saveLog(mockJob, requestClient, false, '/test', 't0k3n') + await saveLog(mockJob, requestClient, false, 0, 100, stream, 't0k3n') - expect(fetchLogsModule.fetchLogByChunks).not.toHaveBeenCalled() - expect(fileModule.createFile).not.toHaveBeenCalled() + expect(fetchLogsModule.fetchLog).not.toHaveBeenCalled() + expect(writeStreamModule.writeStream).not.toHaveBeenCalled() }) it('should throw an error when a valid access token is not provided', async () => { - const error = await saveLog(mockJob, requestClient, true, '/test').catch( - (e) => e - ) + const error = await saveLog( + mockJob, + requestClient, + true, + 0, + 100, + stream + ).catch((e) => e) expect(error.message).toContain( `Logs for job ${mockJob.id} cannot be fetched without a valid access token.` @@ -35,7 +42,9 @@ describe('saveLog', () => { { ...mockJob, links: mockJob.links.filter((l) => l.rel !== 'log') }, requestClient, true, - '/test', + 0, + 100, + stream, 't0k3n' ).catch((e) => e) @@ -45,15 +54,19 @@ describe('saveLog', () => { }) it('should fetch and save logs to the given path', async () => { - await saveLog(mockJob, requestClient, true, '/test', 't0k3n') + await saveLog(mockJob, requestClient, true, 0, 100, stream, 't0k3n') - expect(fetchLogsModule.fetchLogByChunks).toHaveBeenCalledWith( + expect(fetchLogsModule.fetchLog).toHaveBeenCalledWith( requestClient, 't0k3n', '/log/content', + 0, 100 ) - expect(fileModule.createFile).toHaveBeenCalledWith('/test', 'Test Log') + expect(writeStreamModule.writeStream).toHaveBeenCalledWith( + stream, + 'Test Log' + ) }) }) @@ -62,11 +75,12 @@ const setupMocks = () => { jest.mock('../../../request/RequestClient') jest.mock('../../../utils/fetchLogByChunks') jest.mock('@sasjs/utils') + jest.mock('../writeStream') jest - .spyOn(fetchLogsModule, 'fetchLogByChunks') + .spyOn(fetchLogsModule, 'fetchLog') .mockImplementation(() => Promise.resolve('Test Log')) jest - .spyOn(fileModule, 'createFile') + .spyOn(writeStreamModule, 'writeStream') .mockImplementation(() => Promise.resolve()) } diff --git a/src/api/viya/writeStream.ts b/src/api/viya/writeStream.ts new file mode 100644 index 0000000..dc09885 --- /dev/null +++ b/src/api/viya/writeStream.ts @@ -0,0 +1,15 @@ +import { WriteStream } from 'fs' + +export const writeStream = async ( + stream: WriteStream, + content: string +): Promise => { + return new Promise((resolve, reject) => { + stream.write(content + '\n\nnext chunk\n\n', (e) => { + if (e) { + return reject(e) + } + return resolve() + }) + }) +} diff --git a/src/types/errors/JobStatePollError.ts b/src/types/errors/JobStatePollError.ts new file mode 100644 index 0000000..2584b1c --- /dev/null +++ b/src/types/errors/JobStatePollError.ts @@ -0,0 +1,11 @@ +export class JobStatePollError extends Error { + constructor(id: string, public originalError: Error) { + super( + `Error while polling job state for job ${id}: ${ + originalError.message || originalError + }` + ) + this.name = 'JobStatePollError' + Object.setPrototypeOf(this, JobStatePollError.prototype) + } +} diff --git a/src/types/errors/index.ts b/src/types/errors/index.ts index 72b63cc..f8595d4 100644 --- a/src/types/errors/index.ts +++ b/src/types/errors/index.ts @@ -2,6 +2,7 @@ export * from './AuthorizeError' export * from './ComputeJobExecutionError' export * from './InternalServerError' export * from './JobExecutionError' +export * from './JobStatePollError' export * from './LoginRequiredError' export * from './NotFoundError' export * from './ErrorResponse' diff --git a/src/utils/fetchLogByChunks.ts b/src/utils/fetchLogByChunks.ts index a149c4c..22e888a 100644 --- a/src/utils/fetchLogByChunks.ts +++ b/src/utils/fetchLogByChunks.ts @@ -14,18 +14,36 @@ export const fetchLogByChunks = async ( accessToken: string, logUrl: string, logCount: number +): Promise => { + return await fetchLog(requestClient, accessToken, logUrl, 0, logCount) +} + +/** + * Fetches a section of the log file delineated by start and end lines + * @param {object} requestClient - client object of Request Client. + * @param {string} accessToken - an access token for an authorized user. + * @param {string} logUrl - url of the log file. + * @param {number} start - the line at which to start fetching the log. + * @param {number} end - the line at which to stop fetching the log. + * @returns an string containing log lines. + */ +export const fetchLog = async ( + requestClient: RequestClient, + accessToken: string, + logUrl: string, + start: number, + end: number ): Promise => { const logger = process.logger || console let log: string = '' - const loglimit = logCount < 10000 ? logCount : 10000 - let start = 0 + const loglimit = end < 10000 ? end : 10000 do { logger.info( `Fetching logs from line no: ${start + 1} to ${ start + loglimit - } of ${logCount}.` + } of ${end}.` ) const logChunkJson = await requestClient! .get(`${logUrl}?start=${start}&limit=${loglimit}`, accessToken) @@ -40,6 +58,6 @@ export const fetchLogByChunks = async ( log += logChunk start += loglimit - } while (start < logCount) + } while (start < end) return log } From cfa0c8b9aff3b99639ac6075a392a9d2bed84308 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Wed, 21 Jul 2021 08:12:34 +0100 Subject: [PATCH 20/39] chore(refactor): only fetch job if streaming logs, fix tests, add JSDoc comments --- src/api/viya/pollJobState.ts | 34 +++++++++++++------------- src/api/viya/saveLog.ts | 17 ++++++++----- src/api/viya/spec/pollJobState.spec.ts | 28 +++++++++++++++------ src/api/viya/spec/saveLog.spec.ts | 21 +++------------- src/api/viya/uploadTables.ts | 8 ++++++ 5 files changed, 60 insertions(+), 48 deletions(-) diff --git a/src/api/viya/pollJobState.ts b/src/api/viya/pollJobState.ts index b777382..6d66244 100644 --- a/src/api/viya/pollJobState.ts +++ b/src/api/viya/pollJobState.ts @@ -8,7 +8,6 @@ import { saveLog } from './saveLog' import { createWriteStream } from '@sasjs/utils/file' import { WriteStream } from 'fs' import { Link } from '../../types' -import { prefixMessage } from '@sasjs/utils/error' export async function pollJobState( requestClient: RequestClient, @@ -206,25 +205,26 @@ const doPoll = async ( pollCount++ - const jobUrl = postedJob.links.find((l: Link) => l.rel === 'self') - const { result: job } = await requestClient.get( - jobUrl!.href, - authConfig?.access_token - ) + if (pollOptions?.streamLog) { + const jobUrl = postedJob.links.find((l: Link) => l.rel === 'self') + const { result: job } = await requestClient.get( + jobUrl!.href, + authConfig?.access_token + ) - const endLogLine = job.logStatistics?.lineCount ?? 1000000 + const endLogLine = job.logStatistics?.lineCount ?? 1000000 - await saveLog( - postedJob, - requestClient, - pollOptions?.streamLog || false, - startLogLine, - endLogLine, - logStream, - authConfig?.access_token - ) + await saveLog( + postedJob, + requestClient, + startLogLine, + endLogLine, + logStream, + authConfig?.access_token + ) - startLogLine += job.logStatistics.lineCount + startLogLine += endLogLine + } if (debug && printedState !== state) { logger.info('Polling job status...') diff --git a/src/api/viya/saveLog.ts b/src/api/viya/saveLog.ts index 94fbbfd..a659acf 100644 --- a/src/api/viya/saveLog.ts +++ b/src/api/viya/saveLog.ts @@ -4,20 +4,25 @@ import { fetchLog } from '../../utils' import { WriteStream } from 'fs' import { writeStream } from './writeStream' +/** + * Appends logs to a supplied write stream. + * This is useful for getting quick feedback on longer running jobs. + * @param job - the job to fetch logs for + * @param requestClient - the pre-configured HTTP request client + * @param startLine - the line at which to start fetching the log + * @param endLine - the line at which to stop fetching the log + * @param logFileStream - the write stream to which the log is appended + * @accessToken - an optional access token for authentication/authorization + * The access token is not required when fetching logs from the browser. + */ export async function saveLog( job: Job, requestClient: RequestClient, - shouldSaveLog: boolean, startLine: number, endLine: number, logFileStream?: WriteStream, accessToken?: string ) { - console.log('startLine: ', startLine, ' endLine: ', endLine) - if (!shouldSaveLog) { - return - } - if (!accessToken) { throw new Error( `Logs for job ${job.id} cannot be fetched without a valid access token.` diff --git a/src/api/viya/spec/pollJobState.spec.ts b/src/api/viya/spec/pollJobState.spec.ts index dcbd2b9..a415b98 100644 --- a/src/api/viya/spec/pollJobState.spec.ts +++ b/src/api/viya/spec/pollJobState.spec.ts @@ -1,11 +1,12 @@ -import * as fs from 'fs' import { Logger, LogLevel } from '@sasjs/utils' +import * as fileModule from '@sasjs/utils/file' import { RequestClient } from '../../../request/RequestClient' import { mockAuthConfig, mockJob } from './mockResponses' import { pollJobState } from '../pollJobState' import * as getTokensModule from '../../../auth/getTokens' import * as saveLogModule from '../saveLog' import { PollOptions } from '../../../types' +import { WriteStream } from 'fs' const requestClient = new (>RequestClient)() const defaultPollOptions: PollOptions = { @@ -73,7 +74,18 @@ describe('pollJobState', () => { expect(getTokensModule.getTokens).toHaveBeenCalledTimes(3) }) - it('should attempt to fetch and save the log after each poll', async () => { + it('should attempt to fetch and save the log after each poll when streamLog is true', async () => { + mockSimplePoll() + + await pollJobState(requestClient, mockJob, false, mockAuthConfig, { + ...defaultPollOptions, + streamLog: true + }) + + expect(saveLogModule.saveLog).toHaveBeenCalledTimes(2) + }) + + it('should not attempt to fetch and save the log after each poll when streamLog is false', async () => { mockSimplePoll() await pollJobState( @@ -84,7 +96,7 @@ describe('pollJobState', () => { defaultPollOptions ) - expect(saveLogModule.saveLog).toHaveBeenCalledTimes(2) + expect(saveLogModule.saveLog).not.toHaveBeenCalled() }) it('should return the current status when the max poll count is reached', async () => { @@ -133,7 +145,7 @@ describe('pollJobState', () => { defaultPollOptions ) - expect(requestClient.get).toHaveBeenCalledTimes(3) + expect(requestClient.get).toHaveBeenCalledTimes(2) expect(state).toEqual('completed') }) @@ -179,7 +191,7 @@ describe('pollJobState', () => { defaultPollOptions ) - expect(requestClient.get).toHaveBeenCalledTimes(3) + expect(requestClient.get).toHaveBeenCalledTimes(2) expect(state).toEqual('completed') }) @@ -205,7 +217,7 @@ const setupMocks = () => { jest.mock('../../../request/RequestClient') jest.mock('../../../auth/getTokens') jest.mock('../saveLog') - jest.mock('fs') + jest.mock('@sasjs/utils/file') jest .spyOn(requestClient, 'get') @@ -219,8 +231,8 @@ const setupMocks = () => { .spyOn(saveLogModule, 'saveLog') .mockImplementation(() => Promise.resolve()) jest - .spyOn(fs, 'createWriteStream') - .mockImplementation(() => ({} as unknown as fs.WriteStream)) + .spyOn(fileModule, 'createWriteStream') + .mockImplementation(() => Promise.resolve({} as unknown as WriteStream)) } const mockSimplePoll = (runningCount = 2) => { diff --git a/src/api/viya/spec/saveLog.spec.ts b/src/api/viya/spec/saveLog.spec.ts index 4d35c6f..a6c662b 100644 --- a/src/api/viya/spec/saveLog.spec.ts +++ b/src/api/viya/spec/saveLog.spec.ts @@ -15,22 +15,10 @@ describe('saveLog', () => { setupMocks() }) - it('should return immediately if shouldSaveLog is false', async () => { - await saveLog(mockJob, requestClient, false, 0, 100, stream, 't0k3n') - - expect(fetchLogsModule.fetchLog).not.toHaveBeenCalled() - expect(writeStreamModule.writeStream).not.toHaveBeenCalled() - }) - it('should throw an error when a valid access token is not provided', async () => { - const error = await saveLog( - mockJob, - requestClient, - true, - 0, - 100, - stream - ).catch((e) => e) + const error = await saveLog(mockJob, requestClient, 0, 100, stream).catch( + (e) => e + ) expect(error.message).toContain( `Logs for job ${mockJob.id} cannot be fetched without a valid access token.` @@ -41,7 +29,6 @@ describe('saveLog', () => { const error = await saveLog( { ...mockJob, links: mockJob.links.filter((l) => l.rel !== 'log') }, requestClient, - true, 0, 100, stream, @@ -54,7 +41,7 @@ describe('saveLog', () => { }) it('should fetch and save logs to the given path', async () => { - await saveLog(mockJob, requestClient, true, 0, 100, stream, 't0k3n') + await saveLog(mockJob, requestClient, 0, 100, stream, 't0k3n') expect(fetchLogsModule.fetchLog).toHaveBeenCalledWith( requestClient, diff --git a/src/api/viya/uploadTables.ts b/src/api/viya/uploadTables.ts index e7f5b66..b9e4402 100644 --- a/src/api/viya/uploadTables.ts +++ b/src/api/viya/uploadTables.ts @@ -2,6 +2,14 @@ import { prefixMessage } from '@sasjs/utils/error' import { RequestClient } from '../../request/RequestClient' import { convertToCSV } from '../../utils/convertToCsv' +/** + * Uploads tables to SAS as specially formatted CSVs. + * This is more compact than JSON, and easier to read within SAS. + * @param requestClient - the pre-configured HTTP request client + * @param data - the JSON representation of the data to be uploaded + * @param accessToken - an optional access token for authentication/authorization + * The access token is not required when uploading tables from the browser. + */ export async function uploadTables( requestClient: RequestClient, data: any, From 7bd2e31f3b5bca20be35e8464efd42ec0452438e Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Wed, 21 Jul 2021 08:13:45 +0100 Subject: [PATCH 21/39] chore(cleanup): remove console logs --- src/api/viya/saveLog.ts | 4 ++-- src/api/viya/spec/executeScript.spec.ts | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/api/viya/saveLog.ts b/src/api/viya/saveLog.ts index a659acf..3bbd498 100644 --- a/src/api/viya/saveLog.ts +++ b/src/api/viya/saveLog.ts @@ -48,8 +48,8 @@ export async function saveLog( `${jobLogUrl.href}/content`, startLine, endLine - ).catch((e) => console.log(e)) + ) logger.info(`Writing logs to ${logFileStream.path}`) - await writeStream(logFileStream, log || '').catch((e) => console.log(e)) + await writeStream(logFileStream, log || '') } diff --git a/src/api/viya/spec/executeScript.spec.ts b/src/api/viya/spec/executeScript.spec.ts index 5e2fb11..9eb80c4 100644 --- a/src/api/viya/spec/executeScript.spec.ts +++ b/src/api/viya/spec/executeScript.spec.ts @@ -299,7 +299,6 @@ describe('executeScript', () => { true ).catch((e) => e) - console.log(error) expect(error).toContain('Error while posting job') }) From c8e029cff49ace43d5566bc61a3a2dd774a0ea88 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Wed, 21 Jul 2021 08:37:45 +0100 Subject: [PATCH 22/39] chore(deps): bump utils --- package-lock.json | 42 +++++++++++++++++++++++++++++++++--------- package.json | 2 +- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8f8364f..e241b7f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,7 @@ "name": "@sasjs/adapter", "license": "ISC", "dependencies": { - "@sasjs/utils": "^2.25.1", + "@sasjs/utils": "^2.25.4", "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", "form-data": "^4.0.0", @@ -1187,10 +1187,11 @@ } }, "node_modules/@sasjs/utils": { - "version": "2.25.1", - "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.25.1.tgz", - "integrity": "sha512-lwkPE+QsB81b8/1M8X2zLdhpuiA8pIjgOwJH57zhcsliuDnNs4uijSYu40aYSc8tH98jtSuqWMjfGq8CT9o1Dw==", + "version": "2.25.4", + "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.25.4.tgz", + "integrity": "sha512-LTWExtHp4g3VcLLCUMyeeyTXEAZawSQngmJ3/2Z93ysxpeu2/NS7lGG/ERGCQb2snbqmXK8dkZmfg44Tn4Qebw==", "dependencies": { + "@types/fs-extra": "^9.0.11", "@types/prompts": "^2.0.13", "chalk": "^4.1.1", "cli-table": "^0.3.6", @@ -1432,6 +1433,14 @@ "form-data": "*" } }, + "node_modules/@types/fs-extra": { + "version": "9.0.12", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.12.tgz", + "integrity": "sha512-I+bsBr67CurCGnSenZZ7v94gd3tc3+Aj2taxMT4yu4ABLuOgOjeFxX3dokG24ztSRg5tnT00sL8BszO7gSMoIw==", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/graceful-fs": { "version": "4.1.5", "dev": true, @@ -11501,6 +11510,7 @@ }, "node_modules/querystring": { "version": "0.2.0", + "dev": true, "engines": { "node": ">=0.4.x" } @@ -13455,6 +13465,7 @@ }, "node_modules/url": { "version": "0.11.0", + "dev": true, "license": "MIT", "dependencies": { "punycode": "1.3.2", @@ -13468,6 +13479,7 @@ }, "node_modules/url/node_modules/punycode": { "version": "1.3.2", + "dev": true, "license": "MIT" }, "node_modules/use": { @@ -14779,10 +14791,11 @@ } }, "@sasjs/utils": { - "version": "2.25.1", - "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.25.1.tgz", - "integrity": "sha512-lwkPE+QsB81b8/1M8X2zLdhpuiA8pIjgOwJH57zhcsliuDnNs4uijSYu40aYSc8tH98jtSuqWMjfGq8CT9o1Dw==", + "version": "2.25.4", + "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.25.4.tgz", + "integrity": "sha512-LTWExtHp4g3VcLLCUMyeeyTXEAZawSQngmJ3/2Z93ysxpeu2/NS7lGG/ERGCQb2snbqmXK8dkZmfg44Tn4Qebw==", "requires": { + "@types/fs-extra": "^9.0.11", "@types/prompts": "^2.0.13", "chalk": "^4.1.1", "cli-table": "^0.3.6", @@ -14970,6 +14983,14 @@ "form-data": "*" } }, + "@types/fs-extra": { + "version": "9.0.12", + "resolved": "https://registry.npmjs.org/@types/fs-extra/-/fs-extra-9.0.12.tgz", + "integrity": "sha512-I+bsBr67CurCGnSenZZ7v94gd3tc3+Aj2taxMT4yu4ABLuOgOjeFxX3dokG24ztSRg5tnT00sL8BszO7gSMoIw==", + "requires": { + "@types/node": "*" + } + }, "@types/graceful-fs": { "version": "4.1.5", "dev": true, @@ -21835,7 +21856,8 @@ "dev": true }, "querystring": { - "version": "0.2.0" + "version": "0.2.0", + "dev": true }, "querystring-es3": { "version": "0.2.1", @@ -23117,13 +23139,15 @@ }, "url": { "version": "0.11.0", + "dev": true, "requires": { "punycode": "1.3.2", "querystring": "0.2.0" }, "dependencies": { "punycode": { - "version": "1.3.2" + "version": "1.3.2", + "dev": true } } }, diff --git a/package.json b/package.json index e190315..756785b 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ }, "main": "index.js", "dependencies": { - "@sasjs/utils": "^2.25.1", + "@sasjs/utils": "^2.25.4", "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", "form-data": "^4.0.0", From 33280d7a5b26353cdd4b4d5a8c0009a14a4fb630 Mon Sep 17 00:00:00 2001 From: "allcontributors[bot]" <46447321+allcontributors[bot]@users.noreply.github.com> Date: Wed, 21 Jul 2021 15:53:38 +0000 Subject: [PATCH 23/39] docs: update README.md [skip ci] --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index dfbb8d6..7081e58 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ # @sasjs/adapter + +[![All Contributors](https://img.shields.io/badge/all_contributors-1-orange.svg?style=flat-square)](#contributors-) + [![npm package][npm-image]][npm-url] [![Github Workflow][githubworkflow-image]][githubworkflow-url] @@ -234,3 +237,23 @@ If you are a SAS 9 or SAS Viya customer you can also request a copy of [Data Con If you find this library useful, help us grow our star graph! ![](https://starchart.cc/sasjs/adapter.svg) + +## Contributors ✨ + +Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): + + + + + + + + +

Mihajlo Medjedovic
💻
+ + + + + + +This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! \ No newline at end of file From 6d573d38972b9a3644b7d01aaf3b620adf10df86 Mon Sep 17 00:00:00 2001 From: "allcontributors[bot]" <46447321+allcontributors[bot]@users.noreply.github.com> Date: Wed, 21 Jul 2021 15:53:39 +0000 Subject: [PATCH 24/39] docs: create .all-contributorsrc [skip ci] --- .all-contributorsrc | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .all-contributorsrc diff --git a/.all-contributorsrc b/.all-contributorsrc new file mode 100644 index 0000000..d10e7c8 --- /dev/null +++ b/.all-contributorsrc @@ -0,0 +1,24 @@ +{ + "files": [ + "README.md" + ], + "imageSize": 100, + "commit": false, + "contributors": [ + { + "login": "medjedovicm", + "name": "Mihajlo Medjedovic", + "avatar_url": "https://avatars.githubusercontent.com/u/18329105?v=4", + "profile": "https://github.com/medjedovicm", + "contributions": [ + "code" + ] + } + ], + "contributorsPerLine": 7, + "projectName": "adapter", + "projectOwner": "sasjs", + "repoType": "github", + "repoHost": "https://github.com", + "skipCi": true +} From 3fb0d863e9aa2a43d7f98d24ed29bb4067d33c59 Mon Sep 17 00:00:00 2001 From: Allan Bowe <4420615+allanbowe@users.noreply.github.com> Date: Wed, 21 Jul 2021 18:55:01 +0300 Subject: [PATCH 25/39] Update README.md --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 7081e58..b78ac60 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,4 @@ # @sasjs/adapter - -[![All Contributors](https://img.shields.io/badge/all_contributors-1-orange.svg?style=flat-square)](#contributors-) - [![npm package][npm-image]][npm-url] [![Github Workflow][githubworkflow-image]][githubworkflow-url] @@ -256,4 +253,4 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d -This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! \ No newline at end of file +This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! From 0bb42c5e3c0cdf909e3c9a325d6dc0020e75a823 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Thu, 22 Jul 2021 09:25:55 +0100 Subject: [PATCH 26/39] fix(streamlog): use filepath if provided --- package-lock.json | 55 +++++++++----------------- package.json | 2 +- src/api/viya/pollJobState.ts | 20 +++++++--- src/api/viya/spec/pollJobState.spec.ts | 30 ++++++++++++++ 4 files changed, 63 insertions(+), 44 deletions(-) diff --git a/package-lock.json b/package-lock.json index e241b7f..cb95543 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,7 @@ "name": "@sasjs/adapter", "license": "ISC", "dependencies": { - "@sasjs/utils": "^2.25.4", + "@sasjs/utils": "^2.27.1", "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", "form-data": "^4.0.0", @@ -1187,9 +1187,9 @@ } }, "node_modules/@sasjs/utils": { - "version": "2.25.4", - "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.25.4.tgz", - "integrity": "sha512-LTWExtHp4g3VcLLCUMyeeyTXEAZawSQngmJ3/2Z93ysxpeu2/NS7lGG/ERGCQb2snbqmXK8dkZmfg44Tn4Qebw==", + "version": "2.27.1", + "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.27.1.tgz", + "integrity": "sha512-CYTQwEj89cc7H3tGiQQcyDkZYaWRc1HZJpOF8o2RHYS37fIAOy0SyyJdq6mcQ74Nb1u5AmFXPFIvnRCMEcTYeQ==", "dependencies": { "@types/fs-extra": "^9.0.11", "@types/prompts": "^2.0.13", @@ -1199,7 +1199,11 @@ "fs-extra": "^10.0.0", "jwt-decode": "^3.1.2", "prompts": "^2.4.1", + "rimraf": "^3.0.2", "valid-url": "^1.0.9" + }, + "engines": { + "node": ">=15" } }, "node_modules/@semantic-release/commit-analyzer": { @@ -2157,7 +2161,6 @@ }, "node_modules/balanced-match": { "version": "1.0.2", - "dev": true, "license": "MIT" }, "node_modules/base": { @@ -2259,7 +2262,6 @@ }, "node_modules/brace-expansion": { "version": "1.1.11", - "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -2741,7 +2743,6 @@ }, "node_modules/concat-map": { "version": "0.0.1", - "dev": true, "license": "MIT" }, "node_modules/consola": { @@ -4113,7 +4114,6 @@ }, "node_modules/fs.realpath": { "version": "1.0.0", - "dev": true, "license": "ISC" }, "node_modules/fsevents": { @@ -4243,7 +4243,6 @@ }, "node_modules/glob": { "version": "7.1.7", - "dev": true, "license": "ISC", "dependencies": { "fs.realpath": "^1.0.0", @@ -4657,7 +4656,6 @@ }, "node_modules/inflight": { "version": "1.0.6", - "dev": true, "license": "ISC", "dependencies": { "once": "^1.3.0", @@ -4666,7 +4664,6 @@ }, "node_modules/inherits": { "version": "2.0.4", - "dev": true, "license": "ISC" }, "node_modules/ini": { @@ -7610,7 +7607,6 @@ }, "node_modules/minimatch": { "version": "3.0.4", - "dev": true, "license": "ISC", "dependencies": { "brace-expansion": "^1.1.7" @@ -11007,7 +11003,6 @@ }, "node_modules/once": { "version": "1.4.0", - "dev": true, "license": "ISC", "dependencies": { "wrappy": "1" @@ -11242,7 +11237,6 @@ }, "node_modules/path-is-absolute": { "version": "1.0.1", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -11776,7 +11770,6 @@ }, "node_modules/rimraf": { "version": "3.0.2", - "dev": true, "license": "ISC", "dependencies": { "glob": "^7.1.3" @@ -13853,7 +13846,6 @@ }, "node_modules/wrappy": { "version": "1.0.2", - "dev": true, "license": "ISC" }, "node_modules/write-file-atomic": { @@ -14791,9 +14783,9 @@ } }, "@sasjs/utils": { - "version": "2.25.4", - "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.25.4.tgz", - "integrity": "sha512-LTWExtHp4g3VcLLCUMyeeyTXEAZawSQngmJ3/2Z93ysxpeu2/NS7lGG/ERGCQb2snbqmXK8dkZmfg44Tn4Qebw==", + "version": "2.27.1", + "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.27.1.tgz", + "integrity": "sha512-CYTQwEj89cc7H3tGiQQcyDkZYaWRc1HZJpOF8o2RHYS37fIAOy0SyyJdq6mcQ74Nb1u5AmFXPFIvnRCMEcTYeQ==", "requires": { "@types/fs-extra": "^9.0.11", "@types/prompts": "^2.0.13", @@ -14803,6 +14795,7 @@ "fs-extra": "^10.0.0", "jwt-decode": "^3.1.2", "prompts": "^2.4.1", + "rimraf": "^3.0.2", "valid-url": "^1.0.9" } }, @@ -15504,8 +15497,7 @@ } }, "balanced-match": { - "version": "1.0.2", - "dev": true + "version": "1.0.2" }, "base": { "version": "0.11.2", @@ -15570,7 +15562,6 @@ }, "brace-expansion": { "version": "1.1.11", - "dev": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -15892,8 +15883,7 @@ "dev": true }, "concat-map": { - "version": "0.0.1", - "dev": true + "version": "0.0.1" }, "consola": { "version": "2.15.3" @@ -16846,8 +16836,7 @@ } }, "fs.realpath": { - "version": "1.0.0", - "dev": true + "version": "1.0.0" }, "fsevents": { "version": "2.3.2", @@ -16938,7 +16927,6 @@ }, "glob": { "version": "7.1.7", - "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -17186,15 +17174,13 @@ }, "inflight": { "version": "1.0.6", - "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" } }, "inherits": { - "version": "2.0.4", - "dev": true + "version": "2.0.4" }, "ini": { "version": "1.3.8", @@ -19219,7 +19205,6 @@ }, "minimatch": { "version": "3.0.4", - "dev": true, "requires": { "brace-expansion": "^1.1.7" } @@ -21534,7 +21519,6 @@ }, "once": { "version": "1.4.0", - "dev": true, "requires": { "wrappy": "1" } @@ -21688,8 +21672,7 @@ "dev": true }, "path-is-absolute": { - "version": "1.0.1", - "dev": true + "version": "1.0.1" }, "path-key": { "version": "3.1.1", @@ -22020,7 +22003,6 @@ }, "rimraf": { "version": "3.0.2", - "dev": true, "requires": { "glob": "^7.1.3" } @@ -23403,8 +23385,7 @@ } }, "wrappy": { - "version": "1.0.2", - "dev": true + "version": "1.0.2" }, "write-file-atomic": { "version": "3.0.3", diff --git a/package.json b/package.json index 756785b..640e244 100644 --- a/package.json +++ b/package.json @@ -67,7 +67,7 @@ }, "main": "index.js", "dependencies": { - "@sasjs/utils": "^2.25.4", + "@sasjs/utils": "^2.27.1", "axios": "^0.21.1", "axios-cookiejar-support": "^1.0.1", "form-data": "^4.0.0", diff --git a/src/api/viya/pollJobState.ts b/src/api/viya/pollJobState.ts index 6d66244..3f033a5 100644 --- a/src/api/viya/pollJobState.ts +++ b/src/api/viya/pollJobState.ts @@ -5,7 +5,7 @@ import { RequestClient } from '../../request/RequestClient' import { JobStatePollError } from '../../types/errors' import { generateTimestamp } from '@sasjs/utils/time' import { saveLog } from './saveLog' -import { createWriteStream } from '@sasjs/utils/file' +import { createWriteStream, isFolder } from '@sasjs/utils/file' import { WriteStream } from 'fs' import { Link } from '../../types' @@ -53,12 +53,20 @@ export async function pollJobState( let logFileStream if (pollOptions?.streamLog) { - const logFileName = `${postedJob.name || 'job'}-${generateTimestamp()}.log` - const logFilePath = `${ - pollOptions?.logFolderPath || process.cwd() - }/${logFileName}` + const logPath = pollOptions?.logFolderPath || process.cwd() + const isFolderPath = await isFolder(logPath) + if (isFolderPath) { + const logFileName = `${ + postedJob.name || 'job' + }-${generateTimestamp()}.log` + const logFilePath = `${ + pollOptions?.logFolderPath || process.cwd() + }/${logFileName}` - logFileStream = await createWriteStream(logFilePath) + logFileStream = await createWriteStream(logFilePath) + } else { + logFileStream = await createWriteStream(logPath) + } } let result = await doPoll( diff --git a/src/api/viya/spec/pollJobState.spec.ts b/src/api/viya/spec/pollJobState.spec.ts index a415b98..7c82d06 100644 --- a/src/api/viya/spec/pollJobState.spec.ts +++ b/src/api/viya/spec/pollJobState.spec.ts @@ -1,4 +1,5 @@ import { Logger, LogLevel } from '@sasjs/utils' +import * as path from 'path' import * as fileModule from '@sasjs/utils/file' import { RequestClient } from '../../../request/RequestClient' import { mockAuthConfig, mockJob } from './mockResponses' @@ -85,6 +86,35 @@ describe('pollJobState', () => { expect(saveLogModule.saveLog).toHaveBeenCalledTimes(2) }) + it('should use the given log path if it points to a file', async () => { + mockSimplePoll() + + await pollJobState(requestClient, mockJob, false, mockAuthConfig, { + ...defaultPollOptions, + streamLog: true, + logFolderPath: path.join(__dirname, 'test.log') + }) + + expect(fileModule.createWriteStream).toHaveBeenCalledWith( + path.join(__dirname, 'test.log') + ) + }) + + it('should generate a log file path with a timestamp if it points to a folder', async () => { + mockSimplePoll() + + await pollJobState(requestClient, mockJob, false, mockAuthConfig, { + ...defaultPollOptions, + streamLog: true, + logFolderPath: path.join(__dirname) + }) + + expect(fileModule.createWriteStream).not.toHaveBeenCalledWith(__dirname) + expect(fileModule.createWriteStream).toHaveBeenCalledWith( + expect.stringContaining(__dirname + '/20') + ) + }) + it('should not attempt to fetch and save the log after each poll when streamLog is false', async () => { mockSimplePoll() From e3f189eed489fbbfb2fa8b5e5e5b18308a083d7c Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Thu, 22 Jul 2021 09:31:32 +0100 Subject: [PATCH 27/39] chore(test): fix test --- src/api/viya/spec/pollJobState.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/viya/spec/pollJobState.spec.ts b/src/api/viya/spec/pollJobState.spec.ts index 7c82d06..7855f57 100644 --- a/src/api/viya/spec/pollJobState.spec.ts +++ b/src/api/viya/spec/pollJobState.spec.ts @@ -111,7 +111,7 @@ describe('pollJobState', () => { expect(fileModule.createWriteStream).not.toHaveBeenCalledWith(__dirname) expect(fileModule.createWriteStream).toHaveBeenCalledWith( - expect.stringContaining(__dirname + '/20') + expect.stringContaining(__dirname + '/test job-20') ) }) From 405eea1d6cab8468263a77ba0066ea4222dc6b42 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Thu, 22 Jul 2021 09:41:30 +0100 Subject: [PATCH 28/39] chore(infra): set minimum node version to 15 --- .github/workflows/build.yml | 2 +- package-lock.json | 3 +++ package.json | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 10d8ea1..2aee220 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: - node-version: [12.x] + node-version: [15.x] steps: - uses: actions/checkout@v2 diff --git a/package-lock.json b/package-lock.json index cb95543..63e1e4f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -41,6 +41,9 @@ "typescript": "^4.3.5", "webpack": "^5.44.0", "webpack-cli": "^4.7.2" + }, + "engines": { + "node": ">=15" } }, "node_modules/@babel/code-frame": { diff --git a/package.json b/package.json index 640e244..f11698d 100644 --- a/package.json +++ b/package.json @@ -73,5 +73,8 @@ "form-data": "^4.0.0", "https": "^1.0.0", "tough-cookie": "^4.0.0" + }, + "engines": { + "node": ">=15" } } From 9600fa25123159a3f222e2c651fb02e40cceba12 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Thu, 22 Jul 2021 11:31:10 +0100 Subject: [PATCH 29/39] fix(poll): add default poll options --- src/api/viya/pollJobState.ts | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/api/viya/pollJobState.ts b/src/api/viya/pollJobState.ts index 3f033a5..c69546e 100644 --- a/src/api/viya/pollJobState.ts +++ b/src/api/viya/pollJobState.ts @@ -21,11 +21,14 @@ export async function pollJobState( let pollInterval = 300 let maxPollCount = 1000 - if (pollOptions) { - pollInterval = pollOptions.pollInterval || pollInterval - maxPollCount = pollOptions.maxPollCount || maxPollCount + const defaultPollOptions: PollOptions = { + maxPollCount, + pollInterval, + streamLog: false } + pollOptions = { ...defaultPollOptions, ...(pollOptions || {}) } + const stateLink = postedJob.links.find((l: any) => l.rel === 'state') if (!stateLink) { throw new Error(`Job state link was not found.`) @@ -52,7 +55,7 @@ export async function pollJobState( } let logFileStream - if (pollOptions?.streamLog) { + if (pollOptions.streamLog) { const logPath = pollOptions?.logFolderPath || process.cwd() const isFolderPath = await isFolder(logPath) if (isFolderPath) { @@ -69,6 +72,7 @@ export async function pollJobState( } } + // Poll up to the first 100 times with the specified poll interval let result = await doPoll( requestClient, postedJob, @@ -76,14 +80,18 @@ export async function pollJobState( debug, pollCount, authConfig, - pollOptions, + { + ...pollOptions, + maxPollCount: + pollOptions.maxPollCount <= 100 ? pollOptions.maxPollCount : 100 + }, logFileStream ) currentState = result.state pollCount = result.pollCount - if (!needsRetry(currentState) || pollCount >= maxPollCount) { + if (!needsRetry(currentState) || pollCount >= pollOptions.maxPollCount) { return currentState } @@ -192,7 +200,7 @@ const doPoll = async ( throw new Error(`Job state link was not found.`) } - while (needsRetry(state) && pollCount <= 100 && pollCount <= maxPollCount) { + while (needsRetry(state) && pollCount <= maxPollCount) { state = await getJobState( requestClient, postedJob, From c2ff28c3234136919e45b3bd0257c8aca7dbc133 Mon Sep 17 00:00:00 2001 From: Allan Bowe <4420615+allanbowe@users.noreply.github.com> Date: Fri, 23 Jul 2021 13:04:38 +0300 Subject: [PATCH 30/39] Update PULL_REQUEST_TEMPLATE.md --- PULL_REQUEST_TEMPLATE.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/PULL_REQUEST_TEMPLATE.md b/PULL_REQUEST_TEMPLATE.md index 744ed15..03f5c8a 100644 --- a/PULL_REQUEST_TEMPLATE.md +++ b/PULL_REQUEST_TEMPLATE.md @@ -12,9 +12,9 @@ What code changes have been made to achieve the intent. ## Checks -No PR (that involves a non-trivial code change) should be merged, unless all four of the items below are confirmed! If an urgent fix is needed - use a tar file. +No PR (that involves a non-trivial code change) should be merged, unless all items below are confirmed! If an urgent fix is needed - use a tar file. + -- [ ] Code is formatted correctly (`npm run lint:fix`). -- [ ] All unit tests are passing (`npm test`). - [ ] All `sasjs-cli` unit tests are passing (`npm test`). - [ ] All `sasjs-tests` are passing (instructions available [here](https://github.com/sasjs/adapter/blob/master/sasjs-tests/README.md)). +- [ ] [Data Controller](https://datacontroller.io) builds and is functional on both SAS 9 and Viya From 2a9526d056dc1e3074c0fa05fdc6cde1728854d9 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Fri, 23 Jul 2021 22:23:05 +0100 Subject: [PATCH 31/39] fix(node): add util to check if running in node --- src/utils/isNode.ts | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 src/utils/isNode.ts diff --git a/src/utils/isNode.ts b/src/utils/isNode.ts new file mode 100644 index 0000000..4a47e8f --- /dev/null +++ b/src/utils/isNode.ts @@ -0,0 +1,4 @@ +export const isNode = () => + typeof process !== 'undefined' && + process.versions != null && + process.versions.node != null From 0a6c5a0ec414c4728fe930b85df588e2f2e6de2d Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Fri, 23 Jul 2021 22:24:04 +0100 Subject: [PATCH 32/39] fix(fs): replace fs imports with locally defined WriteStream interface --- src/api/viya/saveLog.ts | 2 +- src/api/viya/writeStream.ts | 4 ++-- src/types/WriteStream.ts | 4 ++++ src/types/index.ts | 1 + src/utils/index.ts | 1 + 5 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 src/types/WriteStream.ts diff --git a/src/api/viya/saveLog.ts b/src/api/viya/saveLog.ts index 3bbd498..2b5ec08 100644 --- a/src/api/viya/saveLog.ts +++ b/src/api/viya/saveLog.ts @@ -1,7 +1,7 @@ import { Job } from '../..' import { RequestClient } from '../../request/RequestClient' import { fetchLog } from '../../utils' -import { WriteStream } from 'fs' +import { WriteStream } from '../../types' import { writeStream } from './writeStream' /** diff --git a/src/api/viya/writeStream.ts b/src/api/viya/writeStream.ts index dc09885..0baaaa0 100644 --- a/src/api/viya/writeStream.ts +++ b/src/api/viya/writeStream.ts @@ -1,11 +1,11 @@ -import { WriteStream } from 'fs' +import { WriteStream } from '../../types' export const writeStream = async ( stream: WriteStream, content: string ): Promise => { return new Promise((resolve, reject) => { - stream.write(content + '\n\nnext chunk\n\n', (e) => { + stream.write(content + '\n', (e) => { if (e) { return reject(e) } diff --git a/src/types/WriteStream.ts b/src/types/WriteStream.ts new file mode 100644 index 0000000..83a1d13 --- /dev/null +++ b/src/types/WriteStream.ts @@ -0,0 +1,4 @@ +export interface WriteStream { + write: (content: string, callback: (err?: Error) => any) => void + path: string +} diff --git a/src/types/index.ts b/src/types/index.ts index 313aef2..2303619 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -11,3 +11,4 @@ export * from './SASjsRequest' export * from './Session' export * from './UploadFile' export * from './PollOptions' +export * from './WriteStream' diff --git a/src/utils/index.ts b/src/utils/index.ts index 9f70293..2a05d63 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,6 +1,7 @@ export * from './asyncForEach' export * from './compareTimestamps' export * from './convertToCsv' +export * from './isNode' export * from './isRelativePath' export * from './isUri' export * from './isUrl' From 15d5f9ec915c75ef438876a449b72f0368857574 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Fri, 23 Jul 2021 22:24:21 +0100 Subject: [PATCH 33/39] chore(paths): fix import paths --- src/auth/getTokens.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/auth/getTokens.ts b/src/auth/getTokens.ts index 031c6a3..fe7779d 100644 --- a/src/auth/getTokens.ts +++ b/src/auth/getTokens.ts @@ -1,9 +1,9 @@ import { - AuthConfig, isAccessTokenExpiring, isRefreshTokenExpiring, hasTokenExpired -} from '@sasjs/utils' +} from '@sasjs/utils/auth' +import { AuthConfig } from '@sasjs/utils/types' import { RequestClient } from '../request/RequestClient' import { refreshTokens } from './refreshTokens' From 281a145beffab01ed16f0a679b853aaf04daa2a9 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Fri, 23 Jul 2021 22:24:41 +0100 Subject: [PATCH 34/39] fix(node): only create and write file stream if running in node --- src/api/viya/getFileStream.ts | 16 +++++++++++++ src/api/viya/pollJobState.ts | 44 +++++++++++++---------------------- 2 files changed, 32 insertions(+), 28 deletions(-) create mode 100644 src/api/viya/getFileStream.ts diff --git a/src/api/viya/getFileStream.ts b/src/api/viya/getFileStream.ts new file mode 100644 index 0000000..620ddc0 --- /dev/null +++ b/src/api/viya/getFileStream.ts @@ -0,0 +1,16 @@ +import { isFolder } from '@sasjs/utils/file' +import { generateTimestamp } from '@sasjs/utils/time' +import { Job } from '../../types' + +export const getFileStream = async (job: Job, filePath?: string) => { + const { createWriteStream } = require('@sasjs/utils/file') + const logPath = filePath || process.cwd() + const isFolderPath = await isFolder(logPath) + if (isFolderPath) { + const logFileName = `${job.name || 'job'}-${generateTimestamp()}.log` + const logFilePath = `${filePath || process.cwd()}/${logFileName}` + return await createWriteStream(logFilePath) + } else { + return await createWriteStream(logPath) + } +} diff --git a/src/api/viya/pollJobState.ts b/src/api/viya/pollJobState.ts index c69546e..c4b05d0 100644 --- a/src/api/viya/pollJobState.ts +++ b/src/api/viya/pollJobState.ts @@ -3,11 +3,8 @@ import { Job, PollOptions } from '../..' import { getTokens } from '../../auth/getTokens' import { RequestClient } from '../../request/RequestClient' import { JobStatePollError } from '../../types/errors' -import { generateTimestamp } from '@sasjs/utils/time' -import { saveLog } from './saveLog' -import { createWriteStream, isFolder } from '@sasjs/utils/file' -import { WriteStream } from 'fs' -import { Link } from '../../types' +import { Link, WriteStream } from '../../types' +import { isNode } from '../../utils' export async function pollJobState( requestClient: RequestClient, @@ -55,21 +52,9 @@ export async function pollJobState( } let logFileStream - if (pollOptions.streamLog) { - const logPath = pollOptions?.logFolderPath || process.cwd() - const isFolderPath = await isFolder(logPath) - if (isFolderPath) { - const logFileName = `${ - postedJob.name || 'job' - }-${generateTimestamp()}.log` - const logFilePath = `${ - pollOptions?.logFolderPath || process.cwd() - }/${logFileName}` - - logFileStream = await createWriteStream(logFilePath) - } else { - logFileStream = await createWriteStream(logPath) - } + if (pollOptions.streamLog && isNode()) { + const { getFileStream } = require('./getFileStream') + logFileStream = await getFileStream(postedJob, pollOptions.logFolderPath) } // Poll up to the first 100 times with the specified poll interval @@ -230,14 +215,17 @@ const doPoll = async ( const endLogLine = job.logStatistics?.lineCount ?? 1000000 - await saveLog( - postedJob, - requestClient, - startLogLine, - endLogLine, - logStream, - authConfig?.access_token - ) + const { saveLog } = isNode() ? require('./saveLog') : { saveLog: null } + if (saveLog) { + await saveLog( + postedJob, + requestClient, + startLogLine, + endLogLine, + logStream, + authConfig?.access_token + ) + } startLogLine += endLogLine } From 7cf681bea3836c4b02ef055c037cbf55feac5c51 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Fri, 23 Jul 2021 22:24:48 +0100 Subject: [PATCH 35/39] chore(tests): fix tests --- src/api/viya/spec/getFileStream.spec.ts | 41 +++++++++++++++++++++++++ src/api/viya/spec/pollJobState.spec.ts | 39 ++++++++++++----------- src/api/viya/spec/saveLog.spec.ts | 2 +- src/api/viya/spec/writeStream.spec.ts | 25 +++++++++++++++ 4 files changed, 88 insertions(+), 19 deletions(-) create mode 100644 src/api/viya/spec/getFileStream.spec.ts create mode 100644 src/api/viya/spec/writeStream.spec.ts diff --git a/src/api/viya/spec/getFileStream.spec.ts b/src/api/viya/spec/getFileStream.spec.ts new file mode 100644 index 0000000..0722e37 --- /dev/null +++ b/src/api/viya/spec/getFileStream.spec.ts @@ -0,0 +1,41 @@ +import { Logger, LogLevel } from '@sasjs/utils/logger' +import * as path from 'path' +import * as fileModule from '@sasjs/utils/file' +import { getFileStream } from '../getFileStream' +import { mockJob } from './mockResponses' +import { WriteStream } from '../../../types' + +describe('getFileStream', () => { + beforeEach(() => { + ;(process as any).logger = new Logger(LogLevel.Off) + setupMocks() + }) + it('should use the given log path if it points to a file', async () => { + const { createWriteStream } = require('@sasjs/utils/file') + + await getFileStream(mockJob, path.join(__dirname, 'test.log')) + + expect(createWriteStream).toHaveBeenCalledWith( + path.join(__dirname, 'test.log') + ) + }) + + it('should generate a log file path with a timestamp if it points to a folder', async () => { + const { createWriteStream } = require('@sasjs/utils/file') + + await getFileStream(mockJob, __dirname) + + expect(createWriteStream).not.toHaveBeenCalledWith(__dirname) + expect(createWriteStream).toHaveBeenCalledWith( + expect.stringContaining(__dirname + '/test job-20') + ) + }) +}) + +const setupMocks = () => { + jest.restoreAllMocks() + jest.mock('@sasjs/utils/file/file') + jest + .spyOn(fileModule, 'createWriteStream') + .mockImplementation(() => Promise.resolve({} as unknown as WriteStream)) +} diff --git a/src/api/viya/spec/pollJobState.spec.ts b/src/api/viya/spec/pollJobState.spec.ts index 7855f57..aba468c 100644 --- a/src/api/viya/spec/pollJobState.spec.ts +++ b/src/api/viya/spec/pollJobState.spec.ts @@ -1,11 +1,11 @@ import { Logger, LogLevel } from '@sasjs/utils' -import * as path from 'path' -import * as fileModule from '@sasjs/utils/file' import { RequestClient } from '../../../request/RequestClient' import { mockAuthConfig, mockJob } from './mockResponses' import { pollJobState } from '../pollJobState' import * as getTokensModule from '../../../auth/getTokens' import * as saveLogModule from '../saveLog' +import * as getFileStreamModule from '../getFileStream' +import * as isNodeModule from '../../../utils/isNode' import { PollOptions } from '../../../types' import { WriteStream } from 'fs' @@ -77,42 +77,43 @@ describe('pollJobState', () => { it('should attempt to fetch and save the log after each poll when streamLog is true', async () => { mockSimplePoll() + const { saveLog } = require('../saveLog') await pollJobState(requestClient, mockJob, false, mockAuthConfig, { ...defaultPollOptions, streamLog: true }) - expect(saveLogModule.saveLog).toHaveBeenCalledTimes(2) + expect(saveLog).toHaveBeenCalledTimes(2) }) - it('should use the given log path if it points to a file', async () => { + it('should create a write stream in Node.js environment when streamLog is true', async () => { mockSimplePoll() + const { getFileStream } = require('../getFileStream') + const { saveLog } = require('../saveLog') await pollJobState(requestClient, mockJob, false, mockAuthConfig, { ...defaultPollOptions, - streamLog: true, - logFolderPath: path.join(__dirname, 'test.log') + streamLog: true }) - expect(fileModule.createWriteStream).toHaveBeenCalledWith( - path.join(__dirname, 'test.log') - ) + expect(getFileStream).toHaveBeenCalled() + expect(saveLog).toHaveBeenCalledTimes(2) }) - it('should generate a log file path with a timestamp if it points to a folder', async () => { + it('should not create a write stream in a non-Node.js environment', async () => { mockSimplePoll() + jest.spyOn(isNodeModule, 'isNode').mockImplementation(() => false) + const { saveLog } = require('../saveLog') + const { getFileStream } = require('../getFileStream') await pollJobState(requestClient, mockJob, false, mockAuthConfig, { ...defaultPollOptions, - streamLog: true, - logFolderPath: path.join(__dirname) + streamLog: true }) - expect(fileModule.createWriteStream).not.toHaveBeenCalledWith(__dirname) - expect(fileModule.createWriteStream).toHaveBeenCalledWith( - expect.stringContaining(__dirname + '/test job-20') - ) + expect(getFileStream).not.toHaveBeenCalled() + expect(saveLog).not.toHaveBeenCalled() }) it('should not attempt to fetch and save the log after each poll when streamLog is false', async () => { @@ -247,7 +248,8 @@ const setupMocks = () => { jest.mock('../../../request/RequestClient') jest.mock('../../../auth/getTokens') jest.mock('../saveLog') - jest.mock('@sasjs/utils/file') + jest.mock('../getFileStream') + jest.mock('../../../utils/isNode') jest .spyOn(requestClient, 'get') @@ -261,8 +263,9 @@ const setupMocks = () => { .spyOn(saveLogModule, 'saveLog') .mockImplementation(() => Promise.resolve()) jest - .spyOn(fileModule, 'createWriteStream') + .spyOn(getFileStreamModule, 'getFileStream') .mockImplementation(() => Promise.resolve({} as unknown as WriteStream)) + jest.spyOn(isNodeModule, 'isNode').mockImplementation(() => true) } const mockSimplePoll = (runningCount = 2) => { diff --git a/src/api/viya/spec/saveLog.spec.ts b/src/api/viya/spec/saveLog.spec.ts index a6c662b..261438e 100644 --- a/src/api/viya/spec/saveLog.spec.ts +++ b/src/api/viya/spec/saveLog.spec.ts @@ -4,7 +4,7 @@ import * as fetchLogsModule from '../../../utils/fetchLogByChunks' import * as writeStreamModule from '../writeStream' import { saveLog } from '../saveLog' import { mockJob } from './mockResponses' -import { WriteStream } from 'fs' +import { WriteStream } from '../../../types' const requestClient = new (>RequestClient)() const stream = {} as unknown as WriteStream diff --git a/src/api/viya/spec/writeStream.spec.ts b/src/api/viya/spec/writeStream.spec.ts new file mode 100644 index 0000000..358c82a --- /dev/null +++ b/src/api/viya/spec/writeStream.spec.ts @@ -0,0 +1,25 @@ +import { WriteStream } from '../../../types' +import { writeStream } from '../writeStream' +import 'jest-extended' + +describe('writeStream', () => { + const stream: WriteStream = { + write: jest.fn(), + path: 'test' + } + + it('should resolve when the stream is written successfully', async () => { + expect(writeStream(stream, 'test')).toResolve() + + expect(stream.write).toHaveBeenCalledWith('test\n', expect.anything()) + }) + + it('should reject when the write errors out', async () => { + jest + .spyOn(stream, 'write') + .mockImplementation((_, callback) => callback(new Error('Test Error'))) + const error = await writeStream(stream, 'test').catch((e) => e) + + expect(error.message).toEqual('Test Error') + }) +}) From 87e2edbd6cfa6da61ad10c2ff607e894e4240ea1 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Sat, 24 Jul 2021 00:12:11 +0100 Subject: [PATCH 36/39] chore(test): fix long poll count --- src/api/viya/spec/pollJobState.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/viya/spec/pollJobState.spec.ts b/src/api/viya/spec/pollJobState.spec.ts index aba468c..74f39e1 100644 --- a/src/api/viya/spec/pollJobState.spec.ts +++ b/src/api/viya/spec/pollJobState.spec.ts @@ -311,7 +311,7 @@ const mockLongPoll = () => { return Promise.resolve({ result: mockJob, etag: '', status: 200 }) } return Promise.resolve({ - result: count <= 101 ? 'running' : 'completed', + result: count <= 102 ? 'running' : 'completed', etag: '', status: 200 }) From 626fc2e15f0a51cc7e8bed05e0f0b46683ef52e6 Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Sat, 24 Jul 2021 09:53:39 +0100 Subject: [PATCH 37/39] fix(path): make log file path platform-agnostic --- src/api/viya/getFileStream.ts | 3 ++- src/api/viya/spec/getFileStream.spec.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/api/viya/getFileStream.ts b/src/api/viya/getFileStream.ts index 620ddc0..c647f3e 100644 --- a/src/api/viya/getFileStream.ts +++ b/src/api/viya/getFileStream.ts @@ -8,7 +8,8 @@ export const getFileStream = async (job: Job, filePath?: string) => { const isFolderPath = await isFolder(logPath) if (isFolderPath) { const logFileName = `${job.name || 'job'}-${generateTimestamp()}.log` - const logFilePath = `${filePath || process.cwd()}/${logFileName}` + const path = require('path') + const logFilePath = path.join(filePath || process.cwd(), logFileName) return await createWriteStream(logFilePath) } else { return await createWriteStream(logPath) diff --git a/src/api/viya/spec/getFileStream.spec.ts b/src/api/viya/spec/getFileStream.spec.ts index 0722e37..a05b5cb 100644 --- a/src/api/viya/spec/getFileStream.spec.ts +++ b/src/api/viya/spec/getFileStream.spec.ts @@ -27,7 +27,7 @@ describe('getFileStream', () => { expect(createWriteStream).not.toHaveBeenCalledWith(__dirname) expect(createWriteStream).toHaveBeenCalledWith( - expect.stringContaining(__dirname + '/test job-20') + expect.stringContaining(path.join(__dirname, '/test job-20')) ) }) }) From eac9da22bfbdce332d95e6ada0943a01123877cc Mon Sep 17 00:00:00 2001 From: Krishna Acondy Date: Sat, 24 Jul 2021 10:27:31 +0100 Subject: [PATCH 38/39] chore(test): fix assertion --- src/api/viya/spec/getFileStream.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/api/viya/spec/getFileStream.spec.ts b/src/api/viya/spec/getFileStream.spec.ts index a05b5cb..9ab766b 100644 --- a/src/api/viya/spec/getFileStream.spec.ts +++ b/src/api/viya/spec/getFileStream.spec.ts @@ -27,7 +27,7 @@ describe('getFileStream', () => { expect(createWriteStream).not.toHaveBeenCalledWith(__dirname) expect(createWriteStream).toHaveBeenCalledWith( - expect.stringContaining(path.join(__dirname, '/test job-20')) + expect.stringContaining(path.join(__dirname, 'test job-20')) ) }) }) From dfbe2d8f9449a12f31d0787afae7e33f8726c1c4 Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Sat, 24 Jul 2021 21:31:51 +0300 Subject: [PATCH 39/39] chore: contributors --- .all-contributorsrc | 102 ++++++++++++++++++++++++++++++++++++++++++++ README.md | 31 +++++++++++++- 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 .all-contributorsrc diff --git a/.all-contributorsrc b/.all-contributorsrc new file mode 100644 index 0000000..a0325db --- /dev/null +++ b/.all-contributorsrc @@ -0,0 +1,102 @@ +{ + "projectName": "adapter", + "projectOwner": "sasjs", + "repoType": "github", + "repoHost": "https://github.com", + "files": [ + "README.md" + ], + "imageSize": 100, + "commit": false, + "commitConvention": "angular", + "contributors": [ + { + "login": "krishna-acondy", + "name": "Krishna Acondy", + "avatar_url": "https://avatars.githubusercontent.com/u/2980428?v=4", + "profile": "https://krishna-acondy.io/", + "contributions": [ + "code", + "infra", + "blog", + "content", + "ideas", + "video" + ] + }, + { + "login": "YuryShkoda", + "name": "Yury Shkoda", + "avatar_url": "https://avatars.githubusercontent.com/u/25773492?v=4", + "profile": "https://www.erudicat.com/", + "contributions": [ + "code", + "infra", + "ideas", + "test", + "video" + ] + }, + { + "login": "medjedovicm", + "name": "Mihajlo Medjedovic", + "avatar_url": "https://avatars.githubusercontent.com/u/18329105?v=4", + "profile": "https://github.com/medjedovicm", + "contributions": [ + "code", + "infra", + "test", + "review" + ] + }, + { + "login": "allanbowe", + "name": "Allan Bowe", + "avatar_url": "https://avatars.githubusercontent.com/u/4420615?v=4", + "profile": "https://github.com/allanbowe", + "contributions": [ + "code", + "review", + "test", + "mentoring", + "maintenance" + ] + }, + { + "login": "saadjutt01", + "name": "Muhammad Saad ", + "avatar_url": "https://avatars.githubusercontent.com/u/8914650?v=4", + "profile": "https://github.com/saadjutt01", + "contributions": [ + "code", + "review", + "test", + "mentoring", + "infra" + ] + }, + { + "login": "sabhas", + "name": "Sabir Hassan", + "avatar_url": "https://avatars.githubusercontent.com/u/82647447?v=4", + "profile": "https://github.com/sabhas", + "contributions": [ + "code", + "review", + "test", + "ideas" + ] + }, + { + "login": "VladislavParhomchik", + "name": "VladislavParhomchik", + "avatar_url": "https://avatars.githubusercontent.com/u/83717836?v=4", + "profile": "https://github.com/VladislavParhomchik", + "contributions": [ + "test", + "review" + ] + } + ], + "contributorsPerLine": 7 +} diff --git a/README.md b/README.md index 5665baf..9dc32b4 100644 --- a/README.md +++ b/README.md @@ -172,7 +172,7 @@ Configuration on the client side involves passing an object on startup, which ca * `serverType` - either `SAS9` or `SASVIYA`. * `serverUrl` - the location (including http protocol and port) of the SAS Server. Can be omitted, eg if serving directly from the SAS Web Server, or in streaming mode. * `debug` - if `true` then SAS Logs and extra debug information is returned. -* `useComputeApi` - Only relevant when the serverType is `SASVIYA`. If `true` the [Compute API](#using-the-compute-api) is used. If `false` the [JES API](#using-the-jes-api) is used. If `null` or `undefined` the [Web](#using-jes-web-app) approach is used. +* `useComputeApi` - Only relevant when the serverType is `SASVIYA`. If `true` the [Compute API](#using-the-compute-api) is used. If `false` the [JES API](#using-the-jes-api) is used. If `null` or `undefined` the [Web](#using-jes-web-app) approach is used. * `contextName` - Compute context on which the requests will be called. If missing or not provided, defaults to `Job Execution Compute context`. The adapter supports a number of approaches for interfacing with Viya (`serverType` is `SASVIYA`). For maximum performance, be sure to [configure your compute context](https://sasjs.io/guide-viya/#shared-account-and-server-re-use) with `reuseServerProcesses` as `true` and a system account in `runServerAs`. This functionality is available since Viya 3.5. This configuration is supported when [creating contexts using the CLI](https://sasjs.io/sasjs-cli-context/#sasjs-context-create). @@ -230,3 +230,32 @@ If you are a SAS 9 or SAS Viya customer you can also request a copy of [Data Con If you find this library useful, help us grow our star graph! ![](https://starchart.cc/sasjs/adapter.svg) + +## Contributors ✨ + +[![All Contributors](https://img.shields.io/badge/all_contributors-7-orange.svg?style=flat-square)](#contributors-) + + +Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)): + + + + + + + + + + + + + + +

Krishna Acondy
💻 🚇 📝 🖋 🤔 📹
Yury Shkoda
💻 🚇 🤔 ⚠️ 📹
Mihajlo Medjedovic
💻 🚇 ⚠️ 👀
Allan Bowe
💻 👀 ⚠️ 🧑‍🏫 🚧
Muhammad Saad
💻 👀 ⚠️ 🧑‍🏫 🚇
Sabir Hassan
💻 👀 ⚠️ 🤔
VladislavParhomchik
⚠️ 👀
+ + + + + + +This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome!