1
0
mirror of https://github.com/sasjs/adapter.git synced 2026-01-03 18:50:05 +00:00

feat(jes-api): implement job execution via API to JES

This commit is contained in:
Krishna Acondy
2020-07-13 18:38:30 +01:00
parent c22b9066d8
commit 92504b0c16
4 changed files with 356 additions and 283 deletions

View File

@@ -6,6 +6,6 @@
"appLoc": "/Public/app",
"serverType": "SASVIYA",
"debug": false,
"contextName": null
"contextName": "SAS Job Execution compute context"
}
}

View File

@@ -6,7 +6,7 @@ import {
} from "./utils";
import * as NodeFormData from "form-data";
import * as path from "path";
import { Job, Session, Context, Folder } from "./types";
import { Job, Session, Context, Folder, CsrfToken } from "./types";
/**
* A client for interfacing with the SAS Viya REST API
@@ -32,7 +32,7 @@ export class SASViyaApiClient {
if (this.rootFolderMap.size) {
return this.rootFolderMap;
}
this.populateRootFolderMap();
return this.rootFolderMap;
}
@@ -276,12 +276,12 @@ export class SASViyaApiClient {
}
/**
* Creates a folder in the specified location. Either parentFolderPath or
* parentFolderUri must be provided.
* Creates a folder in the specified location. Either parentFolderPath or
* parentFolderUri must be provided.
* @param folderName - the name of the new folder.
* @param parentFolderPath - the full path to the parent folder. If not
* @param parentFolderPath - the full path to the parent folder. If not
* provided, the parentFolderUri must be provided.
* @param parentFolderUri - the URI (eg /folders/folders/UUID) of the parent
* @param parentFolderUri - the URI (eg /folders/folders/UUID) of the parent
* folder. If not provided, the parentFolderPath must be provided.
*/
public async createFolder(
@@ -296,17 +296,27 @@ export class SASViyaApiClient {
if (!parentFolderUri && parentFolderPath) {
parentFolderUri = await this.getFolderUri(parentFolderPath, accessToken);
if (!parentFolderUri){
if (!parentFolderUri) {
console.log(`Parent folder is not present: ${parentFolderPath}`);
const newParentFolderPath = parentFolderPath.substring(0, parentFolderPath.lastIndexOf("/"));
const newParentFolderPath = parentFolderPath.substring(
0,
parentFolderPath.lastIndexOf("/")
);
const newFolderName = `${parentFolderPath.split("/").pop()}`;
if (newParentFolderPath === ""){
if (newParentFolderPath === "") {
throw new Error("Root Folder should have been present on server");
}
console.log(`Creating Parent Folder:\n${newFolderName} in ${newParentFolderPath}`)
const parentFolder = await this.createFolder(newFolderName, newParentFolderPath, undefined, accessToken)
console.log(`Parent Folder "${newFolderName}" successfully created.`)
console.log(
`Creating Parent Folder:\n${newFolderName} in ${newParentFolderPath}`
);
const parentFolder = await this.createFolder(
newFolderName,
newParentFolderPath,
undefined,
accessToken
);
console.log(`Parent Folder "${newFolderName}" successfully created.`);
parentFolderUri = `/folders/folders/${parentFolder.id}`;
}
}
@@ -350,7 +360,9 @@ export class SASViyaApiClient {
accessToken?: string
) {
if (!parentFolderPath && !parentFolderUri) {
throw new Error('Either parentFolderPath or parentFolderUri must be provided');
throw new Error(
"Either parentFolderPath or parentFolderUri must be provided"
);
}
if (!parentFolderUri && parentFolderPath) {
@@ -365,12 +377,12 @@ export class SASViyaApiClient {
},
body: JSON.stringify({
name: jobName,
parameters:[
parameters: [
{
"name":"_addjesbeginendmacros",
"type":"CHARACTER",
"defaultValue":"false"
}
name: "_addjesbeginendmacros",
type: "CHARACTER",
defaultValue: "false",
},
],
type: "Compute",
code,
@@ -578,6 +590,7 @@ export class SASViyaApiClient {
let files: any[] = [];
if (data && Object.keys(data).length) {
files = await this.uploadTables(data, accessToken);
console.log("Uploaded table files: ", files);
}
const jobName = path.basename(sasJob);
const jobFolder = sasJob.replace(`/${jobName}`, "");
@@ -612,15 +625,15 @@ export class SASViyaApiClient {
};
if (debug) {
jobArguments["_omittextlog"] = "false";
jobArguments["_omitsessionresults"] = "false";
jobArguments["_debug"] = 131;
jobArguments["_OMITTEXTLOG"] = "false";
jobArguments["_OMITSESSIONRESULTS"] = "false";
jobArguments["_DEBUG"] = 131;
}
files.forEach((fileInfo, index) => {
jobArguments[
`_webin_fileuri${index + 1}`
] = `/files/files/${fileInfo.id}`;
] = `/files/files/${fileInfo.file.id}`;
jobArguments[`_webin_name${index + 1}`] = fileInfo.tableName;
});
@@ -643,16 +656,29 @@ export class SASViyaApiClient {
`${this.serverUrl}/jobExecution/jobs/${postedJob.id}`,
{ headers }
);
const resultLink = currentJob.results["_webout.json"];
if (resultLink) {
const result = await this.request<any>(
`${this.serverUrl}${resultLink}/content`,
{ headers }
);
return result;
}
return postedJob;
let result, log;
if (jobStatus === "failed") {
return Promise.reject(currentJob.error);
}
const resultLink = currentJob.results["_webout.json"];
const logLink = currentJob.links.find((l) => l.rel === "log");
if (resultLink) {
result = await this.request<any>(
`${this.serverUrl}${resultLink}/content`,
{ headers },
"text"
);
}
if (debug && logLink) {
log = await this.request<any>(
`${this.serverUrl}${logLink.href}/content`,
{
headers,
}
).then((res: any) => res.items.map((i: any) => i.line).join("\n"));
}
return { result, log };
} else {
throw new Error(
`The job ${sasJob} was not found at the location ${this.rootFolderName}`
@@ -673,7 +699,7 @@ export class SASViyaApiClient {
`${this.serverUrl}${url}`,
requestInfo
);
if (!folder){
if (!folder) {
throw new Error("Cannot populate RootFolderMap unless rootFolder exists");
}
const members = await this.request<{ items: any[] }>(
@@ -734,6 +760,8 @@ export class SASViyaApiClient {
accessToken?: string,
silent = false
) {
const MAX_POLL_COUNT = 1000;
const POLL_INTERVAL = 300;
let postedJobState = "";
let pollCount = 0;
const headers: any = {
@@ -767,7 +795,7 @@ export class SASViyaApiClient {
console.log(`Current state: ${postedJobState}\n`);
}
pollCount++;
if (pollCount >= 100) {
if (pollCount >= MAX_POLL_COUNT) {
resolve(postedJobState);
}
}
@@ -775,7 +803,7 @@ export class SASViyaApiClient {
clearInterval(interval);
resolve(postedJobState);
}
}, 100);
}, POLL_INTERVAL);
});
}
@@ -814,21 +842,24 @@ export class SASViyaApiClient {
private async getFolderUri(folderPath: string, accessToken?: string) {
const url = "/folders/folders/@item?path=" + folderPath;
const requestInfo: any = {
method: "GET",
};
if (accessToken) {
requestInfo.headers = { Authorization: `Bearer ${accessToken}` };
}
const folder = await this.request<Folder>(
`${this.serverUrl}${url}`,
requestInfo
);
if (!folder)
return undefined;
return `/folders/folders/${folder.id}`;
const requestInfo: any = {
method: "GET",
};
if (accessToken) {
requestInfo.headers = { Authorization: `Bearer ${accessToken}` };
}
const folder = await this.request<Folder>(
`${this.serverUrl}${url}`,
requestInfo
);
if (!folder) return undefined;
return `/folders/folders/${folder.id}`;
}
setCsrfToken = (csrfToken: CsrfToken) => {
this.csrfToken = csrfToken;
};
private async request<T>(
url: string,
options: RequestInit,
@@ -840,11 +871,6 @@ export class SASViyaApiClient {
[this.csrfToken.headerName]: this.csrfToken.value,
};
}
return await makeRequest<T>(
url,
options,
(csrfToken) => (this.csrfToken = csrfToken),
contentType
);
return await makeRequest<T>(url, options, this.setCsrfToken, contentType);
}
}

View File

@@ -386,257 +386,244 @@ export default class SASjs {
loginRequiredCallback?: any,
accessToken?: string
) {
const sasjsWaitingRequest: SASjsWaitingRequest = {
requestPromise: {
promise: null,
resolve: null,
reject: null,
},
SASjob: sasJob,
data,
params,
};
if (
this.sasjsConfig.serverType === ServerType.SASViya &&
this.sasjsConfig.contextName
) {
return await this.executeViaJesApi(
sasJob,
data,
params,
loginRequiredCallback,
accessToken
);
} else {
const sasjsWaitingRequest: SASjsWaitingRequest = {
requestPromise: {
promise: null,
resolve: null,
reject: null,
},
SASjob: sasJob,
data,
params,
};
const program = this.sasjsConfig.appLoc
? this.sasjsConfig.appLoc.replace(/\/?$/, "/") +
sasJob.replace(/^\//, "")
: sasJob;
const jobUri =
this.sasjsConfig.serverType === "SASVIYA"
? await this.getJobUri(sasJob)
: "";
const apiUrl = `${this.sasjsConfig.serverUrl}${this.jobsPath}/?${
jobUri.length > 0
? "__program=" + program + "&_job=" + jobUri
: "_program=" + program
}`;
// if (
// this.sasjsConfig.serverType === ServerType.SASViya &&
// this.sasjsConfig.contextName
// ) {
// sasjsWaitingRequest.requestPromise.promise = new Promise(
// async (resolve, reject) => {
// const session = await this.checkSession();
const inputParams = params ? params : {};
const requestParams = {
...inputParams,
...this.getRequestParams(),
};
// if (!session.isLoggedIn) {
// if (loginRequiredCallback) loginRequiredCallback(true);
// logInRequired = true;
// sasjsWaitingRequest.requestPromise.resolve = resolve;
// sasjsWaitingRequest.requestPromise.reject = reject;
// this.sasjsWaitingRequests.push(sasjsWaitingRequest);
// } else {
// resolve(
// await this.sasViyaApiClient?.executeJob(
// sasJob,
// this.sasjsConfig.contextName,
// this.sasjsConfig.debug,
// data,
// accessToken
// )
// );
// }
// }
// );
// return sasjsWaitingRequest.requestPromise.promise;
// } else {
const program = this.sasjsConfig.appLoc
? this.sasjsConfig.appLoc.replace(/\/?$/, "/") + sasJob.replace(/^\//, "")
: sasJob;
const jobUri =
this.sasjsConfig.serverType === "SASVIYA"
? await this.getJobUri(sasJob)
: "";
const apiUrl = `${this.sasjsConfig.serverUrl}${this.jobsPath}/?${
jobUri.length > 0
? "__program=" + program + "&_job=" + jobUri
: "_program=" + program
}`;
const formData = new FormData();
const inputParams = params ? params : {};
const requestParams = {
...inputParams,
...this.getRequestParams(),
};
let isError = false;
let errorMsg = "";
const formData = new FormData();
if (data) {
console.log("Input data", data);
const stringifiedData = JSON.stringify(data);
if (
this.sasjsConfig.serverType === ServerType.SAS9 ||
stringifiedData.length > 500000 ||
stringifiedData.includes(";")
) {
// file upload approach
for (const tableName in data) {
console.log("TableName: ", tableName);
if (isError) {
return;
}
const name = tableName;
const csv = convertToCSV(data[tableName]);
console.log("Converted CSV", csv);
if (csv === "ERROR: LARGE STRING LENGTH") {
console.log("String too long");
isError = true;
errorMsg =
"The max length of a string value in SASjs is 32765 characters.";
}
let isError = false;
let errorMsg = "";
if (data) {
console.log("Input data", data);
const stringifiedData = JSON.stringify(data);
if (
this.sasjsConfig.serverType === ServerType.SAS9 ||
stringifiedData.length > 500000 ||
stringifiedData.includes(";")
) {
// file upload approach
for (const tableName in data) {
console.log("TableName: ", tableName);
if (isError) {
return;
}
const name = tableName;
const csv = convertToCSV(data[tableName]);
console.log("Converted CSV", csv);
if (csv === "ERROR: LARGE STRING LENGTH") {
console.log("String too long");
isError = true;
errorMsg =
"The max length of a string value in SASjs is 32765 characters.";
}
const file = new Blob([csv], { type: "application/csv" });
console.log("File", file);
formData.append(name, file, `${name}.csv`);
}
} else {
// param based approach
const sasjsTables = [];
let tableCounter = 0;
for (const tableName in data) {
if (isError) {
return;
}
tableCounter++;
sasjsTables.push(tableName);
const csv = convertToCSV(data[tableName]);
if (csv === "ERROR: LARGE STRING LENGTH") {
isError = true;
errorMsg =
"The max length of a string value in SASjs is 32765 characters.";
}
// if csv has length more then 16k, send in chunks
if (csv.length > 16000) {
const csvChunks = splitChunks(csv);
// append chunks to form data with same key
csvChunks.map((chunk) => {
formData.append(`sasjs${tableCounter}data`, chunk);
const file = new Blob([csv], {
type: "application/csv",
});
} else {
requestParams[`sasjs${tableCounter}data`] = csv;
console.log("File", file);
formData.append(name, file, `${name}.csv`);
}
} else {
// param based approach
const sasjsTables = [];
let tableCounter = 0;
for (const tableName in data) {
if (isError) {
return;
}
tableCounter++;
sasjsTables.push(tableName);
const csv = convertToCSV(data[tableName]);
if (csv === "ERROR: LARGE STRING LENGTH") {
isError = true;
errorMsg =
"The max length of a string value in SASjs is 32765 characters.";
}
// if csv has length more then 16k, send in chunks
if (csv.length > 16000) {
const csvChunks = splitChunks(csv);
// append chunks to form data with same key
csvChunks.map((chunk) => {
formData.append(`sasjs${tableCounter}data`, chunk);
});
} else {
requestParams[`sasjs${tableCounter}data`] = csv;
}
}
requestParams["sasjs_tables"] = sasjsTables.join(" ");
}
requestParams["sasjs_tables"] = sasjsTables.join(" ");
}
}
for (const key in requestParams) {
if (requestParams.hasOwnProperty(key)) {
formData.append(key, requestParams[key]);
for (const key in requestParams) {
if (requestParams.hasOwnProperty(key)) {
formData.append(key, requestParams[key]);
}
}
}
console.log("Form data", formData);
console.log("Form data", formData);
let isRedirected = false;
let isRedirected = false;
sasjsWaitingRequest.requestPromise.promise = new Promise(
(resolve, reject) => {
if (isError) {
reject({ MESSAGE: errorMsg });
}
const headers: any = {};
if (this._csrfHeader && this._csrf) {
headers[this._csrfHeader] = this._csrf;
}
fetch(apiUrl, {
method: "POST",
body: formData,
referrerPolicy: "same-origin",
headers,
})
.then(async (response) => {
if (!response.ok) {
if (response.status === 403) {
const tokenHeader = response.headers.get("X-CSRF-HEADER");
sasjsWaitingRequest.requestPromise.promise = new Promise(
(resolve, reject) => {
if (isError) {
reject({ MESSAGE: errorMsg });
}
const headers: any = {};
if (this._csrfHeader && this._csrf) {
headers[this._csrfHeader] = this._csrf;
}
fetch(apiUrl, {
method: "POST",
body: formData,
referrerPolicy: "same-origin",
headers,
})
.then(async (response) => {
if (!response.ok) {
if (response.status === 403) {
const tokenHeader = response.headers.get("X-CSRF-HEADER");
if (tokenHeader) {
const token = response.headers.get(tokenHeader);
this._csrfHeader = tokenHeader;
this._csrf = token;
if (tokenHeader) {
const token = response.headers.get(tokenHeader);
this._csrfHeader = tokenHeader;
this._csrf = token;
}
}
}
}
if (
response.redirected &&
this.sasjsConfig.serverType === ServerType.SAS9
) {
isRedirected = true;
}
if (
response.redirected &&
this.sasjsConfig.serverType === ServerType.SAS9
) {
isRedirected = true;
}
return response.text();
})
.then((responseText) => {
if (
(needsRetry(responseText) || isRedirected) &&
!isLogInRequired(responseText)
) {
if (this.retryCount < requestRetryLimit) {
this.retryCount++;
this.request(sasJob, data, params).then(
(res: any) => resolve(res),
(err: any) => reject(err)
);
return response.text();
})
.then((responseText) => {
if (
(needsRetry(responseText) || isRedirected) &&
!isLogInRequired(responseText)
) {
if (this.retryCount < requestRetryLimit) {
this.retryCount++;
this.request(sasJob, data, params).then(
(res: any) => resolve(res),
(err: any) => reject(err)
);
} else {
this.retryCount = 0;
reject(responseText);
}
} else {
this.retryCount = 0;
reject(responseText);
}
} else {
this.retryCount = 0;
this.parseLogFromResponse(responseText, program);
this.parseLogFromResponse(responseText, program);
if (isLogInRequired(responseText)) {
if (loginRequiredCallback) loginRequiredCallback(true);
sasjsWaitingRequest.requestPromise.resolve = resolve;
sasjsWaitingRequest.requestPromise.reject = reject;
this.sasjsWaitingRequests.push(sasjsWaitingRequest);
} else {
if (
this.sasjsConfig.serverType === ServerType.SAS9 &&
this.sasjsConfig.debug
) {
this.updateUsername(responseText);
const jsonResponseText = this.parseSAS9Response(responseText);
if (jsonResponseText !== "") {
resolve(JSON.parse(jsonResponseText));
} else {
reject({
MESSAGE: this.parseSAS9ErrorResponse(responseText),
});
}
} else if (
this.sasjsConfig.serverType === ServerType.SASViya &&
this.sasjsConfig.debug
) {
try {
this.parseSASVIYADebugResponse(responseText).then(
(resText: any) => {
this.updateUsername(resText);
try {
resolve(JSON.parse(resText));
} catch (e) {
reject({ MESSAGE: resText });
}
},
(err: any) => {
reject({ MESSAGE: err });
}
);
} catch (e) {
reject({ MESSAGE: responseText });
}
if (isLogInRequired(responseText)) {
if (loginRequiredCallback) loginRequiredCallback(true);
sasjsWaitingRequest.requestPromise.resolve = resolve;
sasjsWaitingRequest.requestPromise.reject = reject;
this.sasjsWaitingRequests.push(sasjsWaitingRequest);
} else {
this.updateUsername(responseText);
try {
const parsedJson = JSON.parse(responseText);
resolve(parsedJson);
} catch (e) {
reject({ MESSAGE: responseText });
if (
this.sasjsConfig.serverType === ServerType.SAS9 &&
this.sasjsConfig.debug
) {
this.updateUsername(responseText);
const jsonResponseText = this.parseSAS9Response(
responseText
);
if (jsonResponseText !== "") {
resolve(JSON.parse(jsonResponseText));
} else {
reject({
MESSAGE: this.parseSAS9ErrorResponse(responseText),
});
}
} else if (
this.sasjsConfig.serverType === ServerType.SASViya &&
this.sasjsConfig.debug
) {
try {
this.parseSASVIYADebugResponse(responseText).then(
(resText: any) => {
this.updateUsername(resText);
try {
resolve(JSON.parse(resText));
} catch (e) {
reject({ MESSAGE: resText });
}
},
(err: any) => {
reject({ MESSAGE: err });
}
);
} catch (e) {
reject({ MESSAGE: responseText });
}
} else {
this.updateUsername(responseText);
try {
const parsedJson = JSON.parse(responseText);
resolve(parsedJson);
} catch (e) {
reject({ MESSAGE: responseText });
}
}
}
}
}
})
.catch((e: Error) => {
reject(e);
});
}
);
})
.catch((e: Error) => {
reject(e);
});
}
);
return sasjsWaitingRequest.requestPromise.promise;
// }
return sasjsWaitingRequest.requestPromise.promise;
}
}
/**
@@ -695,6 +682,59 @@ export default class SASjs {
);
}
private async executeViaJesApi(
sasJob: string,
data: any,
params?: any,
loginRequiredCallback?: any,
accessToken?: string
) {
const sasjsWaitingRequest: SASjsWaitingRequest = {
requestPromise: {
promise: null,
resolve: null,
reject: null,
},
SASjob: sasJob,
data,
params,
};
sasjsWaitingRequest.requestPromise.promise = new Promise(
async (resolve, reject) => {
const session = await this.checkSession();
if (!session.isLoggedIn) {
if (loginRequiredCallback) loginRequiredCallback(true);
sasjsWaitingRequest.requestPromise.resolve = resolve;
sasjsWaitingRequest.requestPromise.reject = reject;
this.sasjsWaitingRequests.push(sasjsWaitingRequest);
} else {
resolve(
await this.sasViyaApiClient
?.executeJob(
sasJob,
this.sasjsConfig.contextName,
this.sasjsConfig.debug,
data,
accessToken
)
.then((response) => {
if (!this.sasjsConfig.debug) {
this.appendSasjsRequest(null, sasJob, null);
} else {
this.appendSasjsRequest(response, sasJob, null);
}
return JSON.parse(response.result);
})
.catch((e) => reject({ MESSAGE: e.message }))
);
}
}
);
return sasjsWaitingRequest.requestPromise.promise;
}
private async resendWaitingRequests() {
for (const sasjsWaitingRequest of this.sasjsWaitingRequests) {
this.request(
@@ -856,14 +896,20 @@ export default class SASjs {
let generatedCode = "";
let sasWork = null;
if (response) {
sourceCode = parseSourceCode(response);
generatedCode = parseGeneratedCode(response);
sasWork = await this.parseSasWork(response);
if (response && response.result && response.log) {
sourceCode = parseSourceCode(response.log);
generatedCode = parseGeneratedCode(response.log);
sasWork = JSON.parse(response.result).WORK;
} else {
if (response) {
sourceCode = parseSourceCode(response);
generatedCode = parseGeneratedCode(response);
sasWork = await this.parseSasWork(response);
}
}
this.sasjsRequests.push({
logFile: response,
logFile: (response && response.log) || response,
serviceLink: program,
timestamp: new Date(),
sourceCode,

View File

@@ -8,4 +8,5 @@ export interface Job {
createdBy: string;
links: Link[];
results: JobResult;
error?: any;
}