1
0
mirror of https://github.com/sasjs/adapter.git synced 2026-01-20 10:30:06 +00:00

feat(jes-api): implement job execution via API to JES

This commit is contained in:
Krishna Acondy
2020-07-13 18:38:30 +01:00
parent c22b9066d8
commit 92504b0c16
4 changed files with 356 additions and 283 deletions

View File

@@ -6,6 +6,6 @@
"appLoc": "/Public/app", "appLoc": "/Public/app",
"serverType": "SASVIYA", "serverType": "SASVIYA",
"debug": false, "debug": false,
"contextName": null "contextName": "SAS Job Execution compute context"
} }
} }

View File

@@ -6,7 +6,7 @@ import {
} from "./utils"; } from "./utils";
import * as NodeFormData from "form-data"; import * as NodeFormData from "form-data";
import * as path from "path"; import * as path from "path";
import { Job, Session, Context, Folder } from "./types"; import { Job, Session, Context, Folder, CsrfToken } from "./types";
/** /**
* A client for interfacing with the SAS Viya REST API * A client for interfacing with the SAS Viya REST API
@@ -296,17 +296,27 @@ export class SASViyaApiClient {
if (!parentFolderUri && parentFolderPath) { if (!parentFolderUri && parentFolderPath) {
parentFolderUri = await this.getFolderUri(parentFolderPath, accessToken); parentFolderUri = await this.getFolderUri(parentFolderPath, accessToken);
if (!parentFolderUri){ if (!parentFolderUri) {
console.log(`Parent folder is not present: ${parentFolderPath}`); console.log(`Parent folder is not present: ${parentFolderPath}`);
const newParentFolderPath = parentFolderPath.substring(0, parentFolderPath.lastIndexOf("/")); const newParentFolderPath = parentFolderPath.substring(
0,
parentFolderPath.lastIndexOf("/")
);
const newFolderName = `${parentFolderPath.split("/").pop()}`; const newFolderName = `${parentFolderPath.split("/").pop()}`;
if (newParentFolderPath === ""){ if (newParentFolderPath === "") {
throw new Error("Root Folder should have been present on server"); throw new Error("Root Folder should have been present on server");
} }
console.log(`Creating Parent Folder:\n${newFolderName} in ${newParentFolderPath}`) console.log(
const parentFolder = await this.createFolder(newFolderName, newParentFolderPath, undefined, accessToken) `Creating Parent Folder:\n${newFolderName} in ${newParentFolderPath}`
console.log(`Parent Folder "${newFolderName}" successfully created.`) );
const parentFolder = await this.createFolder(
newFolderName,
newParentFolderPath,
undefined,
accessToken
);
console.log(`Parent Folder "${newFolderName}" successfully created.`);
parentFolderUri = `/folders/folders/${parentFolder.id}`; parentFolderUri = `/folders/folders/${parentFolder.id}`;
} }
} }
@@ -350,7 +360,9 @@ export class SASViyaApiClient {
accessToken?: string accessToken?: string
) { ) {
if (!parentFolderPath && !parentFolderUri) { if (!parentFolderPath && !parentFolderUri) {
throw new Error('Either parentFolderPath or parentFolderUri must be provided'); throw new Error(
"Either parentFolderPath or parentFolderUri must be provided"
);
} }
if (!parentFolderUri && parentFolderPath) { if (!parentFolderUri && parentFolderPath) {
@@ -365,12 +377,12 @@ export class SASViyaApiClient {
}, },
body: JSON.stringify({ body: JSON.stringify({
name: jobName, name: jobName,
parameters:[ parameters: [
{ {
"name":"_addjesbeginendmacros", name: "_addjesbeginendmacros",
"type":"CHARACTER", type: "CHARACTER",
"defaultValue":"false" defaultValue: "false",
} },
], ],
type: "Compute", type: "Compute",
code, code,
@@ -578,6 +590,7 @@ export class SASViyaApiClient {
let files: any[] = []; let files: any[] = [];
if (data && Object.keys(data).length) { if (data && Object.keys(data).length) {
files = await this.uploadTables(data, accessToken); files = await this.uploadTables(data, accessToken);
console.log("Uploaded table files: ", files);
} }
const jobName = path.basename(sasJob); const jobName = path.basename(sasJob);
const jobFolder = sasJob.replace(`/${jobName}`, ""); const jobFolder = sasJob.replace(`/${jobName}`, "");
@@ -612,15 +625,15 @@ export class SASViyaApiClient {
}; };
if (debug) { if (debug) {
jobArguments["_omittextlog"] = "false"; jobArguments["_OMITTEXTLOG"] = "false";
jobArguments["_omitsessionresults"] = "false"; jobArguments["_OMITSESSIONRESULTS"] = "false";
jobArguments["_debug"] = 131; jobArguments["_DEBUG"] = 131;
} }
files.forEach((fileInfo, index) => { files.forEach((fileInfo, index) => {
jobArguments[ jobArguments[
`_webin_fileuri${index + 1}` `_webin_fileuri${index + 1}`
] = `/files/files/${fileInfo.id}`; ] = `/files/files/${fileInfo.file.id}`;
jobArguments[`_webin_name${index + 1}`] = fileInfo.tableName; jobArguments[`_webin_name${index + 1}`] = fileInfo.tableName;
}); });
@@ -643,16 +656,29 @@ export class SASViyaApiClient {
`${this.serverUrl}/jobExecution/jobs/${postedJob.id}`, `${this.serverUrl}/jobExecution/jobs/${postedJob.id}`,
{ headers } { headers }
); );
const resultLink = currentJob.results["_webout.json"];
if (resultLink) {
const result = await this.request<any>(
`${this.serverUrl}${resultLink}/content`,
{ headers }
);
return result;
}
return postedJob; let result, log;
if (jobStatus === "failed") {
return Promise.reject(currentJob.error);
}
const resultLink = currentJob.results["_webout.json"];
const logLink = currentJob.links.find((l) => l.rel === "log");
if (resultLink) {
result = await this.request<any>(
`${this.serverUrl}${resultLink}/content`,
{ headers },
"text"
);
}
if (debug && logLink) {
log = await this.request<any>(
`${this.serverUrl}${logLink.href}/content`,
{
headers,
}
).then((res: any) => res.items.map((i: any) => i.line).join("\n"));
}
return { result, log };
} else { } else {
throw new Error( throw new Error(
`The job ${sasJob} was not found at the location ${this.rootFolderName}` `The job ${sasJob} was not found at the location ${this.rootFolderName}`
@@ -673,7 +699,7 @@ export class SASViyaApiClient {
`${this.serverUrl}${url}`, `${this.serverUrl}${url}`,
requestInfo requestInfo
); );
if (!folder){ if (!folder) {
throw new Error("Cannot populate RootFolderMap unless rootFolder exists"); throw new Error("Cannot populate RootFolderMap unless rootFolder exists");
} }
const members = await this.request<{ items: any[] }>( const members = await this.request<{ items: any[] }>(
@@ -734,6 +760,8 @@ export class SASViyaApiClient {
accessToken?: string, accessToken?: string,
silent = false silent = false
) { ) {
const MAX_POLL_COUNT = 1000;
const POLL_INTERVAL = 300;
let postedJobState = ""; let postedJobState = "";
let pollCount = 0; let pollCount = 0;
const headers: any = { const headers: any = {
@@ -767,7 +795,7 @@ export class SASViyaApiClient {
console.log(`Current state: ${postedJobState}\n`); console.log(`Current state: ${postedJobState}\n`);
} }
pollCount++; pollCount++;
if (pollCount >= 100) { if (pollCount >= MAX_POLL_COUNT) {
resolve(postedJobState); resolve(postedJobState);
} }
} }
@@ -775,7 +803,7 @@ export class SASViyaApiClient {
clearInterval(interval); clearInterval(interval);
resolve(postedJobState); resolve(postedJobState);
} }
}, 100); }, POLL_INTERVAL);
}); });
} }
@@ -814,21 +842,24 @@ export class SASViyaApiClient {
private async getFolderUri(folderPath: string, accessToken?: string) { private async getFolderUri(folderPath: string, accessToken?: string) {
const url = "/folders/folders/@item?path=" + folderPath; const url = "/folders/folders/@item?path=" + folderPath;
const requestInfo: any = { const requestInfo: any = {
method: "GET", method: "GET",
}; };
if (accessToken) { if (accessToken) {
requestInfo.headers = { Authorization: `Bearer ${accessToken}` }; requestInfo.headers = { Authorization: `Bearer ${accessToken}` };
} }
const folder = await this.request<Folder>( const folder = await this.request<Folder>(
`${this.serverUrl}${url}`, `${this.serverUrl}${url}`,
requestInfo requestInfo
); );
if (!folder) if (!folder) return undefined;
return undefined; return `/folders/folders/${folder.id}`;
return `/folders/folders/${folder.id}`;
} }
setCsrfToken = (csrfToken: CsrfToken) => {
this.csrfToken = csrfToken;
};
private async request<T>( private async request<T>(
url: string, url: string,
options: RequestInit, options: RequestInit,
@@ -840,11 +871,6 @@ export class SASViyaApiClient {
[this.csrfToken.headerName]: this.csrfToken.value, [this.csrfToken.headerName]: this.csrfToken.value,
}; };
} }
return await makeRequest<T>( return await makeRequest<T>(url, options, this.setCsrfToken, contentType);
url,
options,
(csrfToken) => (this.csrfToken = csrfToken),
contentType
);
} }
} }

View File

@@ -386,257 +386,244 @@ export default class SASjs {
loginRequiredCallback?: any, loginRequiredCallback?: any,
accessToken?: string accessToken?: string
) { ) {
const sasjsWaitingRequest: SASjsWaitingRequest = { if (
requestPromise: { this.sasjsConfig.serverType === ServerType.SASViya &&
promise: null, this.sasjsConfig.contextName
resolve: null, ) {
reject: null, return await this.executeViaJesApi(
}, sasJob,
SASjob: sasJob, data,
data, params,
params, loginRequiredCallback,
}; accessToken
);
} else {
const sasjsWaitingRequest: SASjsWaitingRequest = {
requestPromise: {
promise: null,
resolve: null,
reject: null,
},
SASjob: sasJob,
data,
params,
};
const program = this.sasjsConfig.appLoc
? this.sasjsConfig.appLoc.replace(/\/?$/, "/") +
sasJob.replace(/^\//, "")
: sasJob;
const jobUri =
this.sasjsConfig.serverType === "SASVIYA"
? await this.getJobUri(sasJob)
: "";
const apiUrl = `${this.sasjsConfig.serverUrl}${this.jobsPath}/?${
jobUri.length > 0
? "__program=" + program + "&_job=" + jobUri
: "_program=" + program
}`;
// if ( const inputParams = params ? params : {};
// this.sasjsConfig.serverType === ServerType.SASViya && const requestParams = {
// this.sasjsConfig.contextName ...inputParams,
// ) { ...this.getRequestParams(),
// sasjsWaitingRequest.requestPromise.promise = new Promise( };
// async (resolve, reject) => {
// const session = await this.checkSession();
// if (!session.isLoggedIn) { const formData = new FormData();
// if (loginRequiredCallback) loginRequiredCallback(true);
// logInRequired = true;
// sasjsWaitingRequest.requestPromise.resolve = resolve;
// sasjsWaitingRequest.requestPromise.reject = reject;
// this.sasjsWaitingRequests.push(sasjsWaitingRequest);
// } else {
// resolve(
// await this.sasViyaApiClient?.executeJob(
// sasJob,
// this.sasjsConfig.contextName,
// this.sasjsConfig.debug,
// data,
// accessToken
// )
// );
// }
// }
// );
// return sasjsWaitingRequest.requestPromise.promise;
// } else {
const program = this.sasjsConfig.appLoc
? this.sasjsConfig.appLoc.replace(/\/?$/, "/") + sasJob.replace(/^\//, "")
: sasJob;
const jobUri =
this.sasjsConfig.serverType === "SASVIYA"
? await this.getJobUri(sasJob)
: "";
const apiUrl = `${this.sasjsConfig.serverUrl}${this.jobsPath}/?${
jobUri.length > 0
? "__program=" + program + "&_job=" + jobUri
: "_program=" + program
}`;
const inputParams = params ? params : {}; let isError = false;
const requestParams = { let errorMsg = "";
...inputParams,
...this.getRequestParams(),
};
const formData = new FormData(); if (data) {
console.log("Input data", data);
const stringifiedData = JSON.stringify(data);
if (
this.sasjsConfig.serverType === ServerType.SAS9 ||
stringifiedData.length > 500000 ||
stringifiedData.includes(";")
) {
// file upload approach
for (const tableName in data) {
console.log("TableName: ", tableName);
if (isError) {
return;
}
const name = tableName;
const csv = convertToCSV(data[tableName]);
console.log("Converted CSV", csv);
if (csv === "ERROR: LARGE STRING LENGTH") {
console.log("String too long");
isError = true;
errorMsg =
"The max length of a string value in SASjs is 32765 characters.";
}
let isError = false; const file = new Blob([csv], {
let errorMsg = ""; type: "application/csv",
if (data) {
console.log("Input data", data);
const stringifiedData = JSON.stringify(data);
if (
this.sasjsConfig.serverType === ServerType.SAS9 ||
stringifiedData.length > 500000 ||
stringifiedData.includes(";")
) {
// file upload approach
for (const tableName in data) {
console.log("TableName: ", tableName);
if (isError) {
return;
}
const name = tableName;
const csv = convertToCSV(data[tableName]);
console.log("Converted CSV", csv);
if (csv === "ERROR: LARGE STRING LENGTH") {
console.log("String too long");
isError = true;
errorMsg =
"The max length of a string value in SASjs is 32765 characters.";
}
const file = new Blob([csv], { type: "application/csv" });
console.log("File", file);
formData.append(name, file, `${name}.csv`);
}
} else {
// param based approach
const sasjsTables = [];
let tableCounter = 0;
for (const tableName in data) {
if (isError) {
return;
}
tableCounter++;
sasjsTables.push(tableName);
const csv = convertToCSV(data[tableName]);
if (csv === "ERROR: LARGE STRING LENGTH") {
isError = true;
errorMsg =
"The max length of a string value in SASjs is 32765 characters.";
}
// if csv has length more then 16k, send in chunks
if (csv.length > 16000) {
const csvChunks = splitChunks(csv);
// append chunks to form data with same key
csvChunks.map((chunk) => {
formData.append(`sasjs${tableCounter}data`, chunk);
}); });
} else { console.log("File", file);
requestParams[`sasjs${tableCounter}data`] = csv;
formData.append(name, file, `${name}.csv`);
} }
} else {
// param based approach
const sasjsTables = [];
let tableCounter = 0;
for (const tableName in data) {
if (isError) {
return;
}
tableCounter++;
sasjsTables.push(tableName);
const csv = convertToCSV(data[tableName]);
if (csv === "ERROR: LARGE STRING LENGTH") {
isError = true;
errorMsg =
"The max length of a string value in SASjs is 32765 characters.";
}
// if csv has length more then 16k, send in chunks
if (csv.length > 16000) {
const csvChunks = splitChunks(csv);
// append chunks to form data with same key
csvChunks.map((chunk) => {
formData.append(`sasjs${tableCounter}data`, chunk);
});
} else {
requestParams[`sasjs${tableCounter}data`] = csv;
}
}
requestParams["sasjs_tables"] = sasjsTables.join(" ");
} }
requestParams["sasjs_tables"] = sasjsTables.join(" ");
} }
}
for (const key in requestParams) { for (const key in requestParams) {
if (requestParams.hasOwnProperty(key)) { if (requestParams.hasOwnProperty(key)) {
formData.append(key, requestParams[key]); formData.append(key, requestParams[key]);
}
} }
}
console.log("Form data", formData); console.log("Form data", formData);
let isRedirected = false; let isRedirected = false;
sasjsWaitingRequest.requestPromise.promise = new Promise( sasjsWaitingRequest.requestPromise.promise = new Promise(
(resolve, reject) => { (resolve, reject) => {
if (isError) { if (isError) {
reject({ MESSAGE: errorMsg }); reject({ MESSAGE: errorMsg });
} }
const headers: any = {}; const headers: any = {};
if (this._csrfHeader && this._csrf) { if (this._csrfHeader && this._csrf) {
headers[this._csrfHeader] = this._csrf; headers[this._csrfHeader] = this._csrf;
} }
fetch(apiUrl, { fetch(apiUrl, {
method: "POST", method: "POST",
body: formData, body: formData,
referrerPolicy: "same-origin", referrerPolicy: "same-origin",
headers, headers,
}) })
.then(async (response) => { .then(async (response) => {
if (!response.ok) { if (!response.ok) {
if (response.status === 403) { if (response.status === 403) {
const tokenHeader = response.headers.get("X-CSRF-HEADER"); const tokenHeader = response.headers.get("X-CSRF-HEADER");
if (tokenHeader) { if (tokenHeader) {
const token = response.headers.get(tokenHeader); const token = response.headers.get(tokenHeader);
this._csrfHeader = tokenHeader; this._csrfHeader = tokenHeader;
this._csrf = token; this._csrf = token;
}
} }
} }
}
if ( if (
response.redirected && response.redirected &&
this.sasjsConfig.serverType === ServerType.SAS9 this.sasjsConfig.serverType === ServerType.SAS9
) { ) {
isRedirected = true; isRedirected = true;
} }
return response.text(); return response.text();
}) })
.then((responseText) => { .then((responseText) => {
if ( if (
(needsRetry(responseText) || isRedirected) && (needsRetry(responseText) || isRedirected) &&
!isLogInRequired(responseText) !isLogInRequired(responseText)
) { ) {
if (this.retryCount < requestRetryLimit) { if (this.retryCount < requestRetryLimit) {
this.retryCount++; this.retryCount++;
this.request(sasJob, data, params).then( this.request(sasJob, data, params).then(
(res: any) => resolve(res), (res: any) => resolve(res),
(err: any) => reject(err) (err: any) => reject(err)
); );
} else {
this.retryCount = 0;
reject(responseText);
}
} else { } else {
this.retryCount = 0; this.retryCount = 0;
reject(responseText); this.parseLogFromResponse(responseText, program);
}
} else {
this.retryCount = 0;
this.parseLogFromResponse(responseText, program);
if (isLogInRequired(responseText)) { if (isLogInRequired(responseText)) {
if (loginRequiredCallback) loginRequiredCallback(true); if (loginRequiredCallback) loginRequiredCallback(true);
sasjsWaitingRequest.requestPromise.resolve = resolve; sasjsWaitingRequest.requestPromise.resolve = resolve;
sasjsWaitingRequest.requestPromise.reject = reject; sasjsWaitingRequest.requestPromise.reject = reject;
this.sasjsWaitingRequests.push(sasjsWaitingRequest); this.sasjsWaitingRequests.push(sasjsWaitingRequest);
} else {
if (
this.sasjsConfig.serverType === ServerType.SAS9 &&
this.sasjsConfig.debug
) {
this.updateUsername(responseText);
const jsonResponseText = this.parseSAS9Response(responseText);
if (jsonResponseText !== "") {
resolve(JSON.parse(jsonResponseText));
} else {
reject({
MESSAGE: this.parseSAS9ErrorResponse(responseText),
});
}
} else if (
this.sasjsConfig.serverType === ServerType.SASViya &&
this.sasjsConfig.debug
) {
try {
this.parseSASVIYADebugResponse(responseText).then(
(resText: any) => {
this.updateUsername(resText);
try {
resolve(JSON.parse(resText));
} catch (e) {
reject({ MESSAGE: resText });
}
},
(err: any) => {
reject({ MESSAGE: err });
}
);
} catch (e) {
reject({ MESSAGE: responseText });
}
} else { } else {
this.updateUsername(responseText); if (
try { this.sasjsConfig.serverType === ServerType.SAS9 &&
const parsedJson = JSON.parse(responseText); this.sasjsConfig.debug
resolve(parsedJson); ) {
} catch (e) { this.updateUsername(responseText);
reject({ MESSAGE: responseText }); const jsonResponseText = this.parseSAS9Response(
responseText
);
if (jsonResponseText !== "") {
resolve(JSON.parse(jsonResponseText));
} else {
reject({
MESSAGE: this.parseSAS9ErrorResponse(responseText),
});
}
} else if (
this.sasjsConfig.serverType === ServerType.SASViya &&
this.sasjsConfig.debug
) {
try {
this.parseSASVIYADebugResponse(responseText).then(
(resText: any) => {
this.updateUsername(resText);
try {
resolve(JSON.parse(resText));
} catch (e) {
reject({ MESSAGE: resText });
}
},
(err: any) => {
reject({ MESSAGE: err });
}
);
} catch (e) {
reject({ MESSAGE: responseText });
}
} else {
this.updateUsername(responseText);
try {
const parsedJson = JSON.parse(responseText);
resolve(parsedJson);
} catch (e) {
reject({ MESSAGE: responseText });
}
} }
} }
} }
} })
}) .catch((e: Error) => {
.catch((e: Error) => { reject(e);
reject(e); });
}); }
} );
);
return sasjsWaitingRequest.requestPromise.promise; return sasjsWaitingRequest.requestPromise.promise;
// } }
} }
/** /**
@@ -695,6 +682,59 @@ export default class SASjs {
); );
} }
private async executeViaJesApi(
sasJob: string,
data: any,
params?: any,
loginRequiredCallback?: any,
accessToken?: string
) {
const sasjsWaitingRequest: SASjsWaitingRequest = {
requestPromise: {
promise: null,
resolve: null,
reject: null,
},
SASjob: sasJob,
data,
params,
};
sasjsWaitingRequest.requestPromise.promise = new Promise(
async (resolve, reject) => {
const session = await this.checkSession();
if (!session.isLoggedIn) {
if (loginRequiredCallback) loginRequiredCallback(true);
sasjsWaitingRequest.requestPromise.resolve = resolve;
sasjsWaitingRequest.requestPromise.reject = reject;
this.sasjsWaitingRequests.push(sasjsWaitingRequest);
} else {
resolve(
await this.sasViyaApiClient
?.executeJob(
sasJob,
this.sasjsConfig.contextName,
this.sasjsConfig.debug,
data,
accessToken
)
.then((response) => {
if (!this.sasjsConfig.debug) {
this.appendSasjsRequest(null, sasJob, null);
} else {
this.appendSasjsRequest(response, sasJob, null);
}
return JSON.parse(response.result);
})
.catch((e) => reject({ MESSAGE: e.message }))
);
}
}
);
return sasjsWaitingRequest.requestPromise.promise;
}
private async resendWaitingRequests() { private async resendWaitingRequests() {
for (const sasjsWaitingRequest of this.sasjsWaitingRequests) { for (const sasjsWaitingRequest of this.sasjsWaitingRequests) {
this.request( this.request(
@@ -856,14 +896,20 @@ export default class SASjs {
let generatedCode = ""; let generatedCode = "";
let sasWork = null; let sasWork = null;
if (response) { if (response && response.result && response.log) {
sourceCode = parseSourceCode(response); sourceCode = parseSourceCode(response.log);
generatedCode = parseGeneratedCode(response); generatedCode = parseGeneratedCode(response.log);
sasWork = await this.parseSasWork(response); sasWork = JSON.parse(response.result).WORK;
} else {
if (response) {
sourceCode = parseSourceCode(response);
generatedCode = parseGeneratedCode(response);
sasWork = await this.parseSasWork(response);
}
} }
this.sasjsRequests.push({ this.sasjsRequests.push({
logFile: response, logFile: (response && response.log) || response,
serviceLink: program, serviceLink: program,
timestamp: new Date(), timestamp: new Date(),
sourceCode, sourceCode,

View File

@@ -8,4 +8,5 @@ export interface Job {
createdBy: string; createdBy: string;
links: Link[]; links: Link[];
results: JobResult; results: JobResult;
error?: any;
} }