diff --git a/src/SASViyaApiClient.ts b/src/SASViyaApiClient.ts index beaf51b..68f5cdd 100644 --- a/src/SASViyaApiClient.ts +++ b/src/SASViyaApiClient.ts @@ -1,4 +1,10 @@ -import { convertToCSV, isRelativePath, isUri, isUrl } from './utils' +import { + convertToCSV, + isRelativePath, + isUri, + isUrl, + fetchLogFileContentByChunks +} from './utils' import * as NodeFormData from 'form-data' import { Job, @@ -420,19 +426,19 @@ export class SASViyaApiClient { }) let jobResult - let log + let log = '' const logLink = currentJob.links.find((l) => l.rel === 'log') if (debug && logLink) { - log = await this.requestClient - .get(`${logLink.href}/content?limit=10000`, accessToken) - .then((res: any) => - res.result.items.map((i: any) => i.line).join('\n') - ) - .catch((err) => { - throw prefixMessage(err, 'Error while getting log. ') - }) + const logUrl = `${logLink.href}/content` + const logCount = currentJob.logStatistics?.lineCount ?? 1000000 + log = await fetchLogFileContentByChunks( + this.requestClient, + accessToken!, + logUrl, + logCount + ) } if (jobStatus === 'failed' || jobStatus === 'error') { @@ -453,14 +459,14 @@ export class SASViyaApiClient { .catch(async (e) => { if (e instanceof NotFoundError) { if (logLink) { - log = await this.requestClient - .get(`${logLink.href}/content?limit=10000`, accessToken) - .then((res: any) => - res.result.items.map((i: any) => i.line).join('\n') - ) - .catch((err) => { - throw prefixMessage(err, 'Error while getting log. ') - }) + const logUrl = `${logLink.href}/content` + const logCount = currentJob.logStatistics?.lineCount ?? 1000000 + log = await fetchLogFileContentByChunks( + this.requestClient, + accessToken!, + logUrl, + logCount + ) return Promise.reject({ status: 500, diff --git a/src/types/Job.ts b/src/types/Job.ts index 71b72dd..4b9e0ad 100644 --- a/src/types/Job.ts +++ b/src/types/Job.ts @@ -1,5 +1,6 @@ import { Link } from './Link' import { JobResult } from './JobResult' +import { LogStatistics } from './LogStatistics' export interface Job { id: string @@ -10,4 +11,5 @@ export interface Job { links: Link[] results: JobResult error?: any + logStatistics: LogStatistics } diff --git a/src/types/LogStatistics.ts b/src/types/LogStatistics.ts new file mode 100644 index 0000000..d98a516 --- /dev/null +++ b/src/types/LogStatistics.ts @@ -0,0 +1,4 @@ +export interface LogStatistics { + lineCount: number + modifiedTimeStamp: string +} diff --git a/src/utils/fetchLogFileContentByChunks.ts b/src/utils/fetchLogFileContentByChunks.ts new file mode 100644 index 0000000..742121d --- /dev/null +++ b/src/utils/fetchLogFileContentByChunks.ts @@ -0,0 +1,43 @@ +import { RequestClient } from '../request/RequestClient' +import { prefixMessage } from '@sasjs/utils/error' + +/** + * Fetches content of the log file + * @param {object} requestClient - client object of Request Client. + * @param {string} accessToken - an access token for an authorized user. + * @param {string} logUrl - url of the log file. + * @param {number} logCount- total number of log lines in file. + * @returns an string containing log lines. + */ +export const fetchLogFileContentByChunks = async ( + requestClient: RequestClient, + accessToken: string, + logUrl: string, + logCount: number +): Promise => { + let log: string = '' + + const loglimit = logCount < 10000 ? logCount : 10000 + let start = 0 + do { + console.log( + `Fetching logs from line no: ${start + 1} to ${ + start + loglimit + } of ${logCount}.` + ) + const logChunkJson = await requestClient! + .get(`${logUrl}?start=${start}&limit=${loglimit}`, accessToken) + .then((res: any) => res.result) + .catch((err) => { + throw prefixMessage(err, 'Error while getting log. ') + }) + + if (logChunkJson.items.length === 0) break + + const logChunk = logChunkJson.items.map((i: any) => i.line).join('\n') + log += logChunk + + start += loglimit + } while (start < logCount) + return log +} diff --git a/src/utils/index.ts b/src/utils/index.ts index f740b45..b19cc9f 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -11,3 +11,4 @@ export * from './parseSasViyaLog' export * from './serialize' export * from './splitChunks' export * from './parseWeboutResponse' +export * from './fetchLogFileContentByChunks'