From b06993ab9ea24b28d9e553763187387685aaa666 Mon Sep 17 00:00:00 2001 From: Sabir Hassan Date: Tue, 16 Aug 2022 15:51:37 +0500 Subject: [PATCH 1/7] feat: implement the logic for running python stored programs --- README.md | 17 +++-- api/.env.example | 3 +- api/src/controllers/internal/Session.ts | 46 ++++++++++++- .../internal/createPythonProgram.ts | 65 +++++++++++++++++++ api/src/controllers/internal/index.ts | 1 + .../controllers/internal/processProgram.ts | 43 +++++++++++- api/src/types/system/process.d.ts | 2 + api/src/utils/getDesktopFields.ts | 32 ++++++++- api/src/utils/getRunTimeAndFilePath.ts | 2 +- api/src/utils/setProcessVariables.ts | 4 +- api/src/utils/upload.ts | 31 ++++++++- api/src/utils/verifyEnvVariables.ts | 9 ++- 12 files changed, 235 insertions(+), 20 deletions(-) create mode 100644 api/src/controllers/internal/createPythonProgram.ts diff --git a/README.md b/README.md index 409ccc1..233f7b3 100644 --- a/README.md +++ b/README.md @@ -64,12 +64,22 @@ Example contents of a `.env` file: # Server mode is multi-user and suitable for intranet / internet use MODE= +# A comma separated string that defines the available runTimes. +# Priority is given to the runtime that comes first in the string. +# Possible options at the moment are sas and js + +# options: [sas|js|py|sas,js|js,sas|py,sas|sas,py|py,js|js,py|sas,js,py|sas,py,js|js,sas,py|js,py,sas|py,sas,js|py,js,sas] default:sas +RUN_TIMES= + # Path to SAS executable (sas.exe / sas.sh) SAS_PATH=/path/to/sas/executable.exe # Path to Node.js executable NODE_PATH=~/.nvm/versions/node/v16.14.0/bin/node +# Path to Python executable +PYTHON_PATH=/usr/bin/python + # Path to working directory # This location is for SAS WORK, staged files, DRIVE, configuration etc SASJS_ROOT=./sasjs_root @@ -139,13 +149,6 @@ LOG_FORMAT_MORGAN= # This location is for server logs with classical UNIX logrotate behavior LOG_LOCATION=./sasjs_root/logs -# A comma separated string that defines the available runTimes. -# Priority is given to the runtime that comes first in the string. -# Possible options at the moment are sas and js - -# options: [sas,js|js,sas|sas|js] default:sas -RUN_TIMES= - ``` ## Persisting the Session diff --git a/api/.env.example b/api/.env.example index e585aaf..705e0d2 100644 --- a/api/.env.example +++ b/api/.env.example @@ -14,9 +14,10 @@ HELMET_COEP=[true|false] if omitted HELMET default will be used DB_CONNECT=mongodb+srv://:@/?retryWrites=true&w=majority -RUN_TIMES=[sas|js|sas,js|js,sas] default considered as sas +RUN_TIMES=[sas|js|py|sas,js|js,sas|py,sas|sas,py|py,js|js,py|sas,js,py|sas,py,js|js,sas,py|js,py,sas|py,sas,js|py,js,sas] default considered as sas SAS_PATH=/opt/sas/sas9/SASHome/SASFoundation/9.4/sas NODE_PATH=~/.nvm/versions/node/v16.14.0/bin/node +PYTHON_PATH=/usr/bin/python SASJS_ROOT=./sasjs_root diff --git a/api/src/controllers/internal/Session.ts b/api/src/controllers/internal/Session.ts index 2db668c..3cbf4d9 100644 --- a/api/src/controllers/internal/Session.ts +++ b/api/src/controllers/internal/Session.ts @@ -197,9 +197,41 @@ export class JSSessionController extends SessionController { } } +export class PythonSessionController extends SessionController { + protected async createSession(): Promise { + const sessionId = generateUniqueFileName(generateTimestamp()) + const sessionFolder = path.join(getSessionsFolder(), sessionId) + + const creationTimeStamp = sessionId.split('-').pop() as string + // death time of session is 15 mins from creation + const deathTimeStamp = ( + parseInt(creationTimeStamp) + + 15 * 60 * 1000 - + 1000 + ).toString() + + const session: Session = { + id: sessionId, + ready: true, + inUse: true, + consumed: false, + completed: false, + creationTimeStamp, + deathTimeStamp, + path: sessionFolder + } + + const headersPath = path.join(session.path, 'stpsrv_header.txt') + await createFile(headersPath, 'Content-type: application/json') + + this.sessions.push(session) + return session + } +} + export const getSessionController = ( runTime: RunTimeType -): SASSessionController | JSSessionController => { +): SASSessionController | JSSessionController | PythonSessionController => { if (runTime === RunTimeType.SAS) { return getSASSessionController() } @@ -208,6 +240,10 @@ export const getSessionController = ( return getJSSessionController() } + if (runTime === RunTimeType.PY) { + return getPythonSessionController() + } + throw new Error('No Runtime is configured') } @@ -227,6 +263,14 @@ const getJSSessionController = (): JSSessionController => { return process.jsSessionController } +const getPythonSessionController = (): PythonSessionController => { + if (process.pythonSessionController) return process.pythonSessionController + + process.pythonSessionController = new PythonSessionController() + + return process.pythonSessionController +} + const autoExecContent = ` data _null_; /* remove the dummy SYSIN */ diff --git a/api/src/controllers/internal/createPythonProgram.ts b/api/src/controllers/internal/createPythonProgram.ts new file mode 100644 index 0000000..497ab27 --- /dev/null +++ b/api/src/controllers/internal/createPythonProgram.ts @@ -0,0 +1,65 @@ +import { isWindows } from '@sasjs/utils' +import { PreProgramVars, Session } from '../../types' +import { generateFileUploadPythonCode } from '../../utils' +import { ExecutionVars } from './' + +export const createPythonProgram = async ( + program: string, + preProgramVariables: PreProgramVars, + vars: ExecutionVars, + session: Session, + weboutPath: string, + tokenFile: string, + otherArgs?: any +) => { + const varStatments = Object.keys(vars).reduce( + (computed: string, key: string) => `${computed}${key} = '${vars[key]}';\n`, + '' + ) + + const preProgramVarStatments = ` +_SASJS_SESSION_PATH = '${ + isWindows() ? session.path.replace(/\\/g, '\\\\') : session.path + }'; +_WEBOUT = '${isWindows() ? weboutPath.replace(/\\/g, '\\\\') : weboutPath}'; +_SASJS_TOKENFILE = '${ + isWindows() ? tokenFile.replace(/\\/g, '\\\\') : tokenFile + }'; +_SASJS_USERNAME = '${preProgramVariables?.username}'; +_SASJS_USERID = '${preProgramVariables?.userId}'; +_SASJS_DISPLAYNAME = '${preProgramVariables?.displayName}'; +_METAPERSON = _SASJS_DISPLAYNAME; +_METAUSER = _SASJS_USERNAME; +SASJSPROCESSMODE = 'Stored Program'; +` + + const requiredModules = `import os` + + program = ` +# runtime vars +${varStatments} + +# dynamic user-provided vars +${preProgramVarStatments} + +# change working directory to session folder +os.chdir(_SASJS_SESSION_PATH) + +# actual job code +${program} + +` + // if no files are uploaded filesNamesMap will be undefined + if (otherArgs?.filesNamesMap) { + const uploadJSCode = await generateFileUploadPythonCode( + otherArgs.filesNamesMap, + session.path + ) + + //If js code for the file is generated it will be appended to the top of jsCode + if (uploadJSCode.length > 0) { + program = `${uploadJSCode}\n` + program + } + } + return requiredModules + program +} diff --git a/api/src/controllers/internal/index.ts b/api/src/controllers/internal/index.ts index 50672d7..2a64677 100644 --- a/api/src/controllers/internal/index.ts +++ b/api/src/controllers/internal/index.ts @@ -4,4 +4,5 @@ export * from './Execution' export * from './FileUploadController' export * from './createSASProgram' export * from './createJSProgram' +export * from './createPythonProgram' export * from './processProgram' diff --git a/api/src/controllers/internal/processProgram.ts b/api/src/controllers/internal/processProgram.ts index c059797..9f541c7 100644 --- a/api/src/controllers/internal/processProgram.ts +++ b/api/src/controllers/internal/processProgram.ts @@ -5,7 +5,12 @@ import { once } from 'stream' import { createFile, moveFile } from '@sasjs/utils' import { PreProgramVars, Session } from '../../types' import { RunTimeType } from '../../utils' -import { ExecutionVars, createSASProgram, createJSProgram } from './' +import { + ExecutionVars, + createSASProgram, + createJSProgram, + createPythonProgram +} from './' export const processProgram = async ( program: string, @@ -47,6 +52,42 @@ export const processProgram = async ( // copy the code.js program to log and end write stream writeStream.end(program) + session.completed = true + console.log('session completed', session) + } catch (err: any) { + session.completed = true + session.crashed = err.toString() + console.log('session crashed', session.id, session.crashed) + } + } else if (runTime === RunTimeType.PY) { + program = await createPythonProgram( + program, + preProgramVariables, + vars, + session, + weboutPath, + tokenFile, + otherArgs + ) + + const codePath = path.join(session.path, 'code.py') + + try { + await createFile(codePath, program) + + // create a stream that will write to console outputs to log file + const writeStream = fs.createWriteStream(logPath) + + // waiting for the open event so that we can have underlying file descriptor + await once(writeStream, 'open') + + execFileSync(process.pythonLoc!, [codePath], { + stdio: ['ignore', writeStream, writeStream] + }) + + // copy the code.py program to log and end write stream + writeStream.end(program) + session.completed = true console.log('session completed', session) } catch (err: any) { diff --git a/api/src/types/system/process.d.ts b/api/src/types/system/process.d.ts index bf29e16..f635d46 100644 --- a/api/src/types/system/process.d.ts +++ b/api/src/types/system/process.d.ts @@ -2,10 +2,12 @@ declare namespace NodeJS { export interface Process { sasLoc?: string nodeLoc?: string + pythonLoc?: string driveLoc: string logsLoc: string sasSessionController?: import('../../controllers/internal').SASSessionController jsSessionController?: import('../../controllers/internal').JSSessionController + pythonSessionController?: import('../../controllers/internal').PythonSessionController appStreamConfig: import('../').AppStreamConfig logger: import('@sasjs/utils/logger').Logger runTimes: import('../../utils').RunTimeType[] diff --git a/api/src/utils/getDesktopFields.ts b/api/src/utils/getDesktopFields.ts index e1e0966..bbf2391 100644 --- a/api/src/utils/getDesktopFields.ts +++ b/api/src/utils/getDesktopFields.ts @@ -4,9 +4,9 @@ import { createFolder, fileExists, folderExists, isWindows } from '@sasjs/utils' import { RunTimeType } from './verifyEnvVariables' export const getDesktopFields = async () => { - const { SAS_PATH, NODE_PATH } = process.env + const { SAS_PATH, NODE_PATH, PYTHON_PATH } = process.env - let sasLoc, nodeLoc + let sasLoc, nodeLoc, pythonLoc if (process.runTimes.includes(RunTimeType.SAS)) { sasLoc = SAS_PATH ?? (await getSASLocation()) @@ -16,7 +16,11 @@ export const getDesktopFields = async () => { nodeLoc = NODE_PATH ?? (await getNodeLocation()) } - return { sasLoc, nodeLoc } + if (process.runTimes.includes(RunTimeType.JS)) { + pythonLoc = PYTHON_PATH ?? (await getPythonLocation()) + } + + return { sasLoc, nodeLoc, pythonLoc } } const getDriveLocation = async (): Promise => { @@ -91,3 +95,25 @@ const getNodeLocation = async (): Promise => { return targetName } + +const getPythonLocation = async (): Promise => { + const validator = async (filePath: string) => { + if (!filePath) return 'Path to Python executable is required.' + + if (!(await fileExists(filePath))) { + return 'No file found at provided path.' + } + + return true + } + + const defaultLocation = isWindows() ? 'C:\\Python' : '/usr/bin/python' + + const targetName = await getString( + 'Please enter full path to a Python executable: ', + validator, + defaultLocation + ) + + return targetName +} diff --git a/api/src/utils/getRunTimeAndFilePath.ts b/api/src/utils/getRunTimeAndFilePath.ts index 259b0a9..e27fb2e 100644 --- a/api/src/utils/getRunTimeAndFilePath.ts +++ b/api/src/utils/getRunTimeAndFilePath.ts @@ -5,7 +5,7 @@ import { RunTimeType } from '.' export const getRunTimeAndFilePath = async (programPath: string) => { const ext = path.extname(programPath) - // If programPath (_program) is provided with a ".sas" or ".js" extension + // If programPath (_program) is provided with a ".sas", ".js" or ".py" extension // we should use that extension to determine the appropriate runTime if (ext && Object.values(RunTimeType).includes(ext.slice(1) as RunTimeType)) { const runTime = ext.slice(1) diff --git a/api/src/utils/setProcessVariables.ts b/api/src/utils/setProcessVariables.ts index 9fc3616..07c03e2 100644 --- a/api/src/utils/setProcessVariables.ts +++ b/api/src/utils/setProcessVariables.ts @@ -28,11 +28,13 @@ export const setProcessVariables = async () => { if (MODE === ModeType.Server) { process.sasLoc = process.env.SAS_PATH process.nodeLoc = process.env.NODE_PATH + process.pythonLoc = process.env.PYTHON_PATH } else { - const { sasLoc, nodeLoc } = await getDesktopFields() + const { sasLoc, nodeLoc, pythonLoc } = await getDesktopFields() process.sasLoc = sasLoc process.nodeLoc = nodeLoc + process.pythonLoc = pythonLoc } const { SASJS_ROOT } = process.env diff --git a/api/src/utils/upload.ts b/api/src/utils/upload.ts index 8a64e62..648f961 100644 --- a/api/src/utils/upload.ts +++ b/api/src/utils/upload.ts @@ -126,9 +126,34 @@ export const generateFileUploadJSCode = async ( } }) - if (fileCount) { - uploadCode = `\nconst _WEBIN_FILE_COUNT = ${fileCount}` + uploadCode - } + uploadCode += `\nconst _WEBIN_FILE_COUNT = ${fileCount}` + + return uploadCode +} + +/** + * Generates the python code that references uploaded files in the concurrent request + * @param filesNamesMap object that maps hashed file names and original file names + * @param sessionFolder name of the folder that is created for the purpose of files in concurrent request + * @returns generated python code + */ +export const generateFileUploadPythonCode = async ( + filesNamesMap: FilenamesMap, + sessionFolder: string +) => { + let uploadCode = '' + let fileCount = 0 + + const sessionFolderList: string[] = await listFilesInFolder(sessionFolder) + sessionFolderList.forEach(async (fileName) => { + if (fileName.includes('req_file')) { + fileCount++ + uploadCode += `\n_WEBIN_FILENAME${fileCount} = '${filesNamesMap[fileName].originalName}'` + uploadCode += `\n_WEBIN_NAME${fileCount} = '${filesNamesMap[fileName].fieldName}'` + } + }) + + uploadCode += `\n_WEBIN_FILE_COUNT = ${fileCount}` return uploadCode } diff --git a/api/src/utils/verifyEnvVariables.ts b/api/src/utils/verifyEnvVariables.ts index b0a40b2..023cf05 100644 --- a/api/src/utils/verifyEnvVariables.ts +++ b/api/src/utils/verifyEnvVariables.ts @@ -28,7 +28,8 @@ export enum LOG_FORMAT_MORGANType { export enum RunTimeType { SAS = 'sas', - JS = 'js' + JS = 'js', + PY = 'py' } export enum ReturnCode { @@ -228,7 +229,7 @@ const verifyRUN_TIMES = (): string[] => { const verifyExecutablePaths = () => { const errors: string[] = [] - const { RUN_TIMES, SAS_PATH, NODE_PATH, MODE } = process.env + const { RUN_TIMES, SAS_PATH, NODE_PATH, PYTHON_PATH, MODE } = process.env if (MODE === ModeType.Server) { const runTimes = RUN_TIMES?.split(',') @@ -240,6 +241,10 @@ const verifyExecutablePaths = () => { if (runTimes?.includes(RunTimeType.JS) && !NODE_PATH) { errors.push(`- NODE_PATH is required for ${RunTimeType.JS} run time`) } + + if (runTimes?.includes(RunTimeType.PY) && !PYTHON_PATH) { + errors.push(`- PYTHON_PATH is required for ${RunTimeType.PY} run time`) + } } return errors From a9b25b8880c39b733c332a19bef4faaf5ff074e8 Mon Sep 17 00:00:00 2001 From: Sabir Hassan Date: Tue, 16 Aug 2022 22:39:15 +0500 Subject: [PATCH 2/7] chore: added specs for stp --- api/src/routes/api/spec/stp.spec.ts | 446 ++++++++++++++++++++-------- 1 file changed, 315 insertions(+), 131 deletions(-) diff --git a/api/src/routes/api/spec/stp.spec.ts b/api/src/routes/api/spec/stp.spec.ts index 10525c5..5c1e0be 100644 --- a/api/src/routes/api/spec/stp.spec.ts +++ b/api/src/routes/api/spec/stp.spec.ts @@ -22,7 +22,8 @@ import { import { createFile, generateTimestamp, deleteFolder } from '@sasjs/utils' import { SASSessionController, - JSSessionController + JSSessionController, + PythonSessionController } from '../../../controllers/internal' import * as ProcessProgramModule from '../../../controllers/internal/processProgram' import { Session } from '../../../types' @@ -39,14 +40,16 @@ const user = { const sampleSasProgram = '%put hello world!;' const sampleJsProgram = `console.log('hello world!/')` +const samplePyProgram = `print('hello world!/')` const filesFolder = getFilesFolder() +let app: Express +let accessToken: string + describe('stp', () => { - let app: Express let con: Mongoose let mongoServer: MongoMemoryServer - let accessToken: string const userController = new UserController() const permissionController = new PermissionController() @@ -99,23 +102,7 @@ describe('stp', () => { await createFile(sasProgramPath, sampleSasProgram) await createFile(jsProgramPath, sampleJsProgram) - await request(app) - .get(`/SASjsApi/stp/execute?_program=${programPath}`) - .auth(accessToken, { type: 'bearer' }) - .send() - .expect(200) - - expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith( - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - RunTimeType.JS, - expect.anything(), - undefined - ) + await makeRequestAndAssert(programPath, 200, RunTimeType.JS) }) it('should throw error when js program is not present but sas program exists', async () => { @@ -123,11 +110,45 @@ describe('stp', () => { const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) await createFile(sasProgramPath, sampleSasProgram) - await request(app) - .get(`/SASjsApi/stp/execute?_program=${programPath}`) - .auth(accessToken, { type: 'bearer' }) - .send() - .expect(400) + await makeRequestAndAssert(programPath, 400) + }) + }) + + describe('with runtime py', () => { + const testFilesFolder = `test-stp-${generateTimestamp()}` + + beforeAll(() => { + process.runTimes = [RunTimeType.PY] + }) + + beforeEach(() => { + jest.resetModules() // it clears the cache + setupMocks() + }) + + afterEach(async () => { + jest.resetAllMocks() + await deleteFolder(path.join(filesFolder, testFilesFolder)) + }) + + it('should execute python program when python, js and sas programs are present', async () => { + const programPath = path.join(testFilesFolder, 'program') + const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) + const jsProgramPath = path.join(filesFolder, `${programPath}.js`) + const pyProgramPath = path.join(filesFolder, `${programPath}.py`) + await createFile(sasProgramPath, sampleSasProgram) + await createFile(jsProgramPath, sampleJsProgram) + await createFile(pyProgramPath, samplePyProgram) + + await makeRequestAndAssert(programPath, 200, RunTimeType.PY) + }) + + it('should throw error when py program is not present but js or sas program exists', async () => { + const programPath = path.join(testFilesFolder, 'program') + const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) + await createFile(sasProgramPath, sampleSasProgram) + + await makeRequestAndAssert(programPath, 400) }) }) @@ -153,23 +174,7 @@ describe('stp', () => { await createFile(sasProgramPath, sampleSasProgram) await createFile(jsProgramPath, sampleJsProgram) - await request(app) - .get(`/SASjsApi/stp/execute?_program=${programPath}`) - .auth(accessToken, { type: 'bearer' }) - .send() - .expect(200) - - expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith( - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - RunTimeType.SAS, - expect.anything(), - undefined - ) + await makeRequestAndAssert(programPath, 200, RunTimeType.SAS) }) it('should throw error when sas program do not exit but js exists', async () => { @@ -177,11 +182,7 @@ describe('stp', () => { const jsProgramPath = path.join(filesFolder, `${programPath}.js`) await createFile(jsProgramPath, sampleJsProgram) - await request(app) - .get(`/SASjsApi/stp/execute?_program=${programPath}`) - .auth(accessToken, { type: 'bearer' }) - .send() - .expect(400) + await makeRequestAndAssert(programPath, 400) }) }) @@ -207,23 +208,7 @@ describe('stp', () => { await createFile(sasProgramPath, sampleSasProgram) await createFile(jsProgramPath, sampleJsProgram) - await request(app) - .get(`/SASjsApi/stp/execute?_program=${programPath}`) - .auth(accessToken, { type: 'bearer' }) - .send() - .expect(200) - - expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith( - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - RunTimeType.JS, - expect.anything(), - undefined - ) + await makeRequestAndAssert(programPath, 200, RunTimeType.JS) }) it('should execute sas program when js program is not present but sas program exists', async () => { @@ -231,33 +216,53 @@ describe('stp', () => { const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) await createFile(sasProgramPath, sampleSasProgram) - await request(app) - .get(`/SASjsApi/stp/execute?_program=${programPath}`) - .auth(accessToken, { type: 'bearer' }) - .send() - .expect(200) - - expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith( - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - RunTimeType.SAS, - expect.anything(), - undefined - ) + await makeRequestAndAssert(programPath, 200, RunTimeType.SAS) }) it('should throw error when both sas and js programs do not exist', async () => { const programPath = path.join(testFilesFolder, 'program') - await request(app) - .get(`/SASjsApi/stp/execute?_program=${programPath}`) - .auth(accessToken, { type: 'bearer' }) - .send() - .expect(400) + await makeRequestAndAssert(programPath, 400) + }) + }) + + describe('with runtime py and sas', () => { + beforeAll(() => { + process.runTimes = [RunTimeType.PY, RunTimeType.SAS] + }) + + beforeEach(() => { + jest.resetModules() // it clears the cache + setupMocks() + }) + + afterEach(async () => { + jest.resetAllMocks() + await deleteFolder(path.join(filesFolder, testFilesFolder)) + }) + + it('should execute python program when both python and sas program are present', async () => { + const programPath = path.join(testFilesFolder, 'program') + const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) + const pyProgramPath = path.join(filesFolder, `${programPath}.py`) + await createFile(sasProgramPath, sampleSasProgram) + await createFile(pyProgramPath, samplePyProgram) + + await makeRequestAndAssert(programPath, 200, RunTimeType.PY) + }) + + it('should execute sas program when python program is not present but sas program exists', async () => { + const programPath = path.join(testFilesFolder, 'program') + const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) + await createFile(sasProgramPath, sampleSasProgram) + + await makeRequestAndAssert(programPath, 200, RunTimeType.SAS) + }) + + it('should throw error when both sas and js programs do not exist', async () => { + const programPath = path.join(testFilesFolder, 'program') + + await makeRequestAndAssert(programPath, 400) }) }) @@ -283,23 +288,7 @@ describe('stp', () => { await createFile(sasProgramPath, sampleSasProgram) await createFile(jsProgramPath, sampleJsProgram) - await request(app) - .get(`/SASjsApi/stp/execute?_program=${programPath}`) - .auth(accessToken, { type: 'bearer' }) - .send() - .expect(200) - - expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith( - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - RunTimeType.SAS, - expect.anything(), - undefined - ) + await makeRequestAndAssert(programPath, 200, RunTimeType.SAS) }) it('should execute js program when sas program is not present but js program exists', async () => { @@ -307,46 +296,237 @@ describe('stp', () => { const jsProgramPath = path.join(filesFolder, `${programPath}.js`) await createFile(jsProgramPath, sampleJsProgram) - await request(app) - .get(`/SASjsApi/stp/execute?_program=${programPath}`) - .auth(accessToken, { type: 'bearer' }) - .send() - .expect(200) - - expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith( - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - expect.anything(), - RunTimeType.JS, - expect.anything(), - undefined - ) + await makeRequestAndAssert(programPath, 200, RunTimeType.JS) }) it('should throw error when both sas and js programs do not exist', async () => { const programPath = path.join(testFilesFolder, 'program') - await request(app) - .get(`/SASjsApi/stp/execute?_program=${programPath}`) - .auth(accessToken, { type: 'bearer' }) - .send() - .expect(400) + await makeRequestAndAssert(programPath, 400) + }) + }) + + describe('with runtime sas and py', () => { + beforeAll(() => { + process.runTimes = [RunTimeType.SAS, RunTimeType.PY] + }) + + beforeEach(() => { + jest.resetModules() // it clears the cache + setupMocks() + }) + + afterEach(async () => { + jest.resetAllMocks() + await deleteFolder(path.join(filesFolder, testFilesFolder)) + }) + + it('should execute sas program when both sas and python programs exist', async () => { + const programPath = path.join(testFilesFolder, 'program') + const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) + const pyProgramPath = path.join(filesFolder, `${programPath}.py`) + await createFile(sasProgramPath, sampleSasProgram) + await createFile(pyProgramPath, samplePyProgram) + + await makeRequestAndAssert(programPath, 200, RunTimeType.SAS) + }) + + it('should execute python program when sas program is not present but python program exists', async () => { + const programPath = path.join(testFilesFolder, 'program') + const pyProgramPath = path.join(filesFolder, `${programPath}.py`) + await createFile(pyProgramPath, samplePyProgram) + + await makeRequestAndAssert(programPath, 200, RunTimeType.PY) + }) + + it('should throw error when both sas and python programs do not exist', async () => { + const programPath = path.join(testFilesFolder, 'program') + + await makeRequestAndAssert(programPath, 400) + }) + }) + + describe('with runtime sas, js and py', () => { + beforeAll(() => { + process.runTimes = [RunTimeType.SAS, RunTimeType.JS, RunTimeType.PY] + }) + + beforeEach(() => { + jest.resetModules() // it clears the cache + setupMocks() + }) + + afterEach(async () => { + jest.resetAllMocks() + await deleteFolder(path.join(filesFolder, testFilesFolder)) + }) + + it('should execute sas program when it exists, no matter js and python programs exist or not', async () => { + const programPath = path.join(testFilesFolder, 'program') + const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) + const jsProgramPath = path.join(filesFolder, `${programPath}.js`) + const pyProgramPath = path.join(filesFolder, `${programPath}.py`) + await createFile(sasProgramPath, sampleSasProgram) + await createFile(jsProgramPath, sampleJsProgram) + await createFile(pyProgramPath, samplePyProgram) + + await makeRequestAndAssert(programPath, 200, RunTimeType.SAS) + }) + + it('should execute js program when sas program is absent but js and python programs are present', async () => { + const programPath = path.join(testFilesFolder, 'program') + const jsProgramPath = path.join(filesFolder, `${programPath}.js`) + const pyProgramPath = path.join(filesFolder, `${programPath}.py`) + await createFile(jsProgramPath, sampleJsProgram) + await createFile(pyProgramPath, samplePyProgram) + + await makeRequestAndAssert(programPath, 200, RunTimeType.JS) + }) + + it('should execute python program when both sas and js programs are not present', async () => { + const programPath = path.join(testFilesFolder, 'program') + const pyProgramPath = path.join(filesFolder, `${programPath}.py`) + await createFile(pyProgramPath, samplePyProgram) + + await makeRequestAndAssert(programPath, 200, RunTimeType.PY) + }) + + it('should throw error when no program exists', async () => { + const programPath = path.join(testFilesFolder, 'program') + + await makeRequestAndAssert(programPath, 400) + }) + }) + + describe('with runtime js, sas and py', () => { + beforeAll(() => { + process.runTimes = [RunTimeType.JS, RunTimeType.SAS, RunTimeType.PY] + }) + + beforeEach(() => { + jest.resetModules() // it clears the cache + setupMocks() + }) + + afterEach(async () => { + jest.resetAllMocks() + await deleteFolder(path.join(filesFolder, testFilesFolder)) + }) + + it('should execute js program when it exists, no matter sas and python programs exist or not', async () => { + const programPath = path.join(testFilesFolder, 'program') + const jsProgramPath = path.join(filesFolder, `${programPath}.js`) + const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) + const pyProgramPath = path.join(filesFolder, `${programPath}.py`) + await createFile(jsProgramPath, sampleJsProgram) + await createFile(sasProgramPath, sampleSasProgram) + await createFile(pyProgramPath, samplePyProgram) + + await makeRequestAndAssert(programPath, 200, RunTimeType.JS) + }) + + it('should execute sas program when js program is absent but sas and python programs are present', async () => { + const programPath = path.join(testFilesFolder, 'program') + const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) + const pyProgramPath = path.join(filesFolder, `${programPath}.py`) + await createFile(sasProgramPath, sampleSasProgram) + await createFile(pyProgramPath, samplePyProgram) + + await makeRequestAndAssert(programPath, 200, RunTimeType.SAS) + }) + + it('should execute python program when both sas and js programs are not present', async () => { + const programPath = path.join(testFilesFolder, 'program') + const pyProgramPath = path.join(filesFolder, `${programPath}.py`) + await createFile(pyProgramPath, samplePyProgram) + + await makeRequestAndAssert(programPath, 200, RunTimeType.PY) + }) + + it('should throw error when no program exists', async () => { + const programPath = path.join(testFilesFolder, 'program') + + await makeRequestAndAssert(programPath, 400) + }) + }) + + describe('with runtime py, sas and js', () => { + beforeAll(() => { + process.runTimes = [RunTimeType.PY, RunTimeType.SAS, RunTimeType.JS] + }) + + beforeEach(() => { + jest.resetModules() // it clears the cache + setupMocks() + }) + + afterEach(async () => { + jest.resetAllMocks() + await deleteFolder(path.join(filesFolder, testFilesFolder)) + }) + + it('should execute python program when it exists, no matter sas and js programs exist or not', async () => { + const programPath = path.join(testFilesFolder, 'program') + const pyProgramPath = path.join(filesFolder, `${programPath}.py`) + const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) + const jsProgramPath = path.join(filesFolder, `${programPath}.js`) + await createFile(pyProgramPath, samplePyProgram) + await createFile(jsProgramPath, sampleJsProgram) + await createFile(sasProgramPath, sampleSasProgram) + + await makeRequestAndAssert(programPath, 200, RunTimeType.PY) + }) + + it('should execute sas program when python program is absent but sas and js programs are present', async () => { + const programPath = path.join(testFilesFolder, 'program') + const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) + const jsProgramPath = path.join(filesFolder, `${programPath}.js`) + await createFile(sasProgramPath, sampleSasProgram) + await createFile(jsProgramPath, sampleJsProgram) + + await makeRequestAndAssert(programPath, 200, RunTimeType.SAS) + }) + + it('should execute js program when both sas and python programs are not present', async () => { + const programPath = path.join(testFilesFolder, 'program') + const jsProgramPath = path.join(filesFolder, `${programPath}.js`) + await createFile(jsProgramPath, sampleJsProgram) + + await makeRequestAndAssert(programPath, 200, RunTimeType.JS) + }) + + it('should throw error when no program exists', async () => { + const programPath = path.join(testFilesFolder, 'program') + await makeRequestAndAssert(programPath, 400) }) }) }) }) }) -const generateSaveTokenAndCreateUser = async ( - someUser: any -): Promise => { - const userController = new UserController() - const dbUser = await userController.createUser(someUser) +const makeRequestAndAssert = async ( + programPath: string, + expectedStatusCode: number, + expectedRuntime?: RunTimeType +) => { + await request(app) + .get(`/SASjsApi/stp/execute?_program=${programPath}`) + .auth(accessToken, { type: 'bearer' }) + .send() + .expect(expectedStatusCode) - return generateAndSaveToken(dbUser.id) + if (expectedRuntime) + expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expectedRuntime, + expect.anything(), + undefined + ) } const generateAndSaveToken = async (userId: number) => { @@ -367,6 +547,10 @@ const setupMocks = async () => { .spyOn(JSSessionController.prototype, 'getSession') .mockImplementation(mockedGetSession) + jest + .spyOn(PythonSessionController.prototype, 'getSession') + .mockImplementation(mockedGetSession) + jest .spyOn(ProcessProgramModule, 'processProgram') .mockImplementation(() => Promise.resolve()) From 4c110827961174377b16e28f7c2552bc3a284ba9 Mon Sep 17 00:00:00 2001 From: Sabir Hassan Date: Wed, 17 Aug 2022 21:24:30 +0500 Subject: [PATCH 3/7] chore: addressed comments --- README.md | 7 +- api/.env.example | 2 +- api/src/routes/api/spec/stp.spec.ts | 271 ++++++++++------------------ 3 files changed, 106 insertions(+), 174 deletions(-) diff --git a/README.md b/README.md index 233f7b3..f3555d2 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,12 @@ MODE= # Priority is given to the runtime that comes first in the string. # Possible options at the moment are sas and js -# options: [sas|js|py|sas,js|js,sas|py,sas|sas,py|py,js|js,py|sas,js,py|sas,py,js|js,sas,py|js,py,sas|py,sas,js|py,js,sas] default:sas +# This string sets the priority of the available analytic runtimes +# Valid runtimes are SAS (sas), JavaScript (js) and Python (py) +# For each option provided, there should be a corresponding path, +# eg SAS_PATH, NODE_PATH or PYTHON_PATH +# Priority is given to runtimes earlier in the string +# Example options: [sas,js,py | js,py | sas | sas,js] RUN_TIMES= # Path to SAS executable (sas.exe / sas.sh) diff --git a/api/.env.example b/api/.env.example index 705e0d2..a8d7980 100644 --- a/api/.env.example +++ b/api/.env.example @@ -14,7 +14,7 @@ HELMET_COEP=[true|false] if omitted HELMET default will be used DB_CONNECT=mongodb+srv://:@/?retryWrites=true&w=majority -RUN_TIMES=[sas|js|py|sas,js|js,sas|py,sas|sas,py|py,js|js,py|sas,js,py|sas,py,js|js,sas,py|js,py,sas|py,sas,js|py,js,sas] default considered as sas +RUN_TIMES=[sas,js,py | js,py | sas | sas,js] default considered as sas SAS_PATH=/opt/sas/sas9/SASHome/SASFoundation/9.4/sas NODE_PATH=~/.nvm/versions/node/v16.14.0/bin/node PYTHON_PATH=/usr/bin/python diff --git a/api/src/routes/api/spec/stp.spec.ts b/api/src/routes/api/spec/stp.spec.ts index 5c1e0be..9b05b98 100644 --- a/api/src/routes/api/spec/stp.spec.ts +++ b/api/src/routes/api/spec/stp.spec.ts @@ -43,6 +43,7 @@ const sampleJsProgram = `console.log('hello world!/')` const samplePyProgram = `print('hello world!/')` const filesFolder = getFilesFolder() +const testFilesFolder = `test-stp-${generateTimestamp()}` let app: Express let accessToken: string @@ -75,8 +76,6 @@ describe('stp', () => { }) describe('execute', () => { - const testFilesFolder = `test-stp-${generateTimestamp()}` - describe('get', () => { describe('with runtime js', () => { const testFilesFolder = `test-stp-${generateTimestamp()}` @@ -96,21 +95,15 @@ describe('stp', () => { }) it('should execute js program when both js and sas program are present', async () => { - const programPath = path.join(testFilesFolder, 'program') - const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) - const jsProgramPath = path.join(filesFolder, `${programPath}.js`) - await createFile(sasProgramPath, sampleSasProgram) - await createFile(jsProgramPath, sampleJsProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.JS) + await makeRequestAndAssert( + [RunTimeType.JS, RunTimeType.SAS], + 200, + RunTimeType.JS + ) }) it('should throw error when js program is not present but sas program exists', async () => { - const programPath = path.join(testFilesFolder, 'program') - const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) - await createFile(sasProgramPath, sampleSasProgram) - - await makeRequestAndAssert(programPath, 400) + await makeRequestAndAssert([], 400) }) }) @@ -132,23 +125,15 @@ describe('stp', () => { }) it('should execute python program when python, js and sas programs are present', async () => { - const programPath = path.join(testFilesFolder, 'program') - const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) - const jsProgramPath = path.join(filesFolder, `${programPath}.js`) - const pyProgramPath = path.join(filesFolder, `${programPath}.py`) - await createFile(sasProgramPath, sampleSasProgram) - await createFile(jsProgramPath, sampleJsProgram) - await createFile(pyProgramPath, samplePyProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.PY) + await makeRequestAndAssert( + [RunTimeType.PY, RunTimeType.SAS, RunTimeType.JS], + 200, + RunTimeType.PY + ) }) it('should throw error when py program is not present but js or sas program exists', async () => { - const programPath = path.join(testFilesFolder, 'program') - const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) - await createFile(sasProgramPath, sampleSasProgram) - - await makeRequestAndAssert(programPath, 400) + await makeRequestAndAssert([], 400) }) }) @@ -168,21 +153,11 @@ describe('stp', () => { }) it('should execute sas program when both sas and js programs are present', async () => { - const programPath = path.join(testFilesFolder, 'program') - const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) - const jsProgramPath = path.join(filesFolder, `${programPath}.js`) - await createFile(sasProgramPath, sampleSasProgram) - await createFile(jsProgramPath, sampleJsProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.SAS) + await makeRequestAndAssert([RunTimeType.SAS], 200, RunTimeType.SAS) }) it('should throw error when sas program do not exit but js exists', async () => { - const programPath = path.join(testFilesFolder, 'program') - const jsProgramPath = path.join(filesFolder, `${programPath}.js`) - await createFile(jsProgramPath, sampleJsProgram) - - await makeRequestAndAssert(programPath, 400) + await makeRequestAndAssert([], 400) }) }) @@ -202,27 +177,19 @@ describe('stp', () => { }) it('should execute js program when both js and sas program are present', async () => { - const programPath = path.join(testFilesFolder, 'program') - const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) - const jsProgramPath = path.join(filesFolder, `${programPath}.js`) - await createFile(sasProgramPath, sampleSasProgram) - await createFile(jsProgramPath, sampleJsProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.JS) + await makeRequestAndAssert( + [RunTimeType.SAS, RunTimeType.JS], + 200, + RunTimeType.JS + ) }) it('should execute sas program when js program is not present but sas program exists', async () => { - const programPath = path.join(testFilesFolder, 'program') - const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) - await createFile(sasProgramPath, sampleSasProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.SAS) + await makeRequestAndAssert([RunTimeType.SAS], 200, RunTimeType.SAS) }) it('should throw error when both sas and js programs do not exist', async () => { - const programPath = path.join(testFilesFolder, 'program') - - await makeRequestAndAssert(programPath, 400) + await makeRequestAndAssert([], 400) }) }) @@ -242,27 +209,19 @@ describe('stp', () => { }) it('should execute python program when both python and sas program are present', async () => { - const programPath = path.join(testFilesFolder, 'program') - const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) - const pyProgramPath = path.join(filesFolder, `${programPath}.py`) - await createFile(sasProgramPath, sampleSasProgram) - await createFile(pyProgramPath, samplePyProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.PY) + await makeRequestAndAssert( + [RunTimeType.PY, RunTimeType.SAS], + 200, + RunTimeType.PY + ) }) it('should execute sas program when python program is not present but sas program exists', async () => { - const programPath = path.join(testFilesFolder, 'program') - const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) - await createFile(sasProgramPath, sampleSasProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.SAS) + await makeRequestAndAssert([RunTimeType.SAS], 200, RunTimeType.SAS) }) it('should throw error when both sas and js programs do not exist', async () => { - const programPath = path.join(testFilesFolder, 'program') - - await makeRequestAndAssert(programPath, 400) + await makeRequestAndAssert([], 400) }) }) @@ -282,27 +241,19 @@ describe('stp', () => { }) it('should execute sas program when both sas and js programs exist', async () => { - const programPath = path.join(testFilesFolder, 'program') - const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) - const jsProgramPath = path.join(filesFolder, `${programPath}.js`) - await createFile(sasProgramPath, sampleSasProgram) - await createFile(jsProgramPath, sampleJsProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.SAS) + await makeRequestAndAssert( + [RunTimeType.SAS, RunTimeType.JS], + 200, + RunTimeType.SAS + ) }) it('should execute js program when sas program is not present but js program exists', async () => { - const programPath = path.join(testFilesFolder, 'program') - const jsProgramPath = path.join(filesFolder, `${programPath}.js`) - await createFile(jsProgramPath, sampleJsProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.JS) + await makeRequestAndAssert([RunTimeType.JS], 200, RunTimeType.JS) }) it('should throw error when both sas and js programs do not exist', async () => { - const programPath = path.join(testFilesFolder, 'program') - - await makeRequestAndAssert(programPath, 400) + await makeRequestAndAssert([], 400) }) }) @@ -322,27 +273,19 @@ describe('stp', () => { }) it('should execute sas program when both sas and python programs exist', async () => { - const programPath = path.join(testFilesFolder, 'program') - const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) - const pyProgramPath = path.join(filesFolder, `${programPath}.py`) - await createFile(sasProgramPath, sampleSasProgram) - await createFile(pyProgramPath, samplePyProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.SAS) + await makeRequestAndAssert( + [RunTimeType.SAS, RunTimeType.PY], + 200, + RunTimeType.SAS + ) }) it('should execute python program when sas program is not present but python program exists', async () => { - const programPath = path.join(testFilesFolder, 'program') - const pyProgramPath = path.join(filesFolder, `${programPath}.py`) - await createFile(pyProgramPath, samplePyProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.PY) + await makeRequestAndAssert([RunTimeType.PY], 200, RunTimeType.PY) }) it('should throw error when both sas and python programs do not exist', async () => { - const programPath = path.join(testFilesFolder, 'program') - - await makeRequestAndAssert(programPath, 400) + await makeRequestAndAssert([], 400) }) }) @@ -362,39 +305,27 @@ describe('stp', () => { }) it('should execute sas program when it exists, no matter js and python programs exist or not', async () => { - const programPath = path.join(testFilesFolder, 'program') - const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) - const jsProgramPath = path.join(filesFolder, `${programPath}.js`) - const pyProgramPath = path.join(filesFolder, `${programPath}.py`) - await createFile(sasProgramPath, sampleSasProgram) - await createFile(jsProgramPath, sampleJsProgram) - await createFile(pyProgramPath, samplePyProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.SAS) + await makeRequestAndAssert( + [RunTimeType.SAS, RunTimeType.PY, RunTimeType.JS], + 200, + RunTimeType.SAS + ) }) it('should execute js program when sas program is absent but js and python programs are present', async () => { - const programPath = path.join(testFilesFolder, 'program') - const jsProgramPath = path.join(filesFolder, `${programPath}.js`) - const pyProgramPath = path.join(filesFolder, `${programPath}.py`) - await createFile(jsProgramPath, sampleJsProgram) - await createFile(pyProgramPath, samplePyProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.JS) + await makeRequestAndAssert( + [RunTimeType.JS, RunTimeType.PY], + 200, + RunTimeType.JS + ) }) it('should execute python program when both sas and js programs are not present', async () => { - const programPath = path.join(testFilesFolder, 'program') - const pyProgramPath = path.join(filesFolder, `${programPath}.py`) - await createFile(pyProgramPath, samplePyProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.PY) + await makeRequestAndAssert([RunTimeType.PY], 200, RunTimeType.PY) }) it('should throw error when no program exists', async () => { - const programPath = path.join(testFilesFolder, 'program') - - await makeRequestAndAssert(programPath, 400) + await makeRequestAndAssert([], 400) }) }) @@ -414,39 +345,27 @@ describe('stp', () => { }) it('should execute js program when it exists, no matter sas and python programs exist or not', async () => { - const programPath = path.join(testFilesFolder, 'program') - const jsProgramPath = path.join(filesFolder, `${programPath}.js`) - const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) - const pyProgramPath = path.join(filesFolder, `${programPath}.py`) - await createFile(jsProgramPath, sampleJsProgram) - await createFile(sasProgramPath, sampleSasProgram) - await createFile(pyProgramPath, samplePyProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.JS) + await makeRequestAndAssert( + [RunTimeType.JS, RunTimeType.SAS, RunTimeType.PY], + 200, + RunTimeType.JS + ) }) it('should execute sas program when js program is absent but sas and python programs are present', async () => { - const programPath = path.join(testFilesFolder, 'program') - const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) - const pyProgramPath = path.join(filesFolder, `${programPath}.py`) - await createFile(sasProgramPath, sampleSasProgram) - await createFile(pyProgramPath, samplePyProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.SAS) + await makeRequestAndAssert( + [RunTimeType.SAS, RunTimeType.PY], + 200, + RunTimeType.SAS + ) }) it('should execute python program when both sas and js programs are not present', async () => { - const programPath = path.join(testFilesFolder, 'program') - const pyProgramPath = path.join(filesFolder, `${programPath}.py`) - await createFile(pyProgramPath, samplePyProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.PY) + await makeRequestAndAssert([RunTimeType.PY], 200, RunTimeType.PY) }) it('should throw error when no program exists', async () => { - const programPath = path.join(testFilesFolder, 'program') - - await makeRequestAndAssert(programPath, 400) + await makeRequestAndAssert([], 400) }) }) @@ -466,38 +385,27 @@ describe('stp', () => { }) it('should execute python program when it exists, no matter sas and js programs exist or not', async () => { - const programPath = path.join(testFilesFolder, 'program') - const pyProgramPath = path.join(filesFolder, `${programPath}.py`) - const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) - const jsProgramPath = path.join(filesFolder, `${programPath}.js`) - await createFile(pyProgramPath, samplePyProgram) - await createFile(jsProgramPath, sampleJsProgram) - await createFile(sasProgramPath, sampleSasProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.PY) + await makeRequestAndAssert( + [RunTimeType.PY, RunTimeType.SAS, RunTimeType.JS], + 200, + RunTimeType.PY + ) }) it('should execute sas program when python program is absent but sas and js programs are present', async () => { - const programPath = path.join(testFilesFolder, 'program') - const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) - const jsProgramPath = path.join(filesFolder, `${programPath}.js`) - await createFile(sasProgramPath, sampleSasProgram) - await createFile(jsProgramPath, sampleJsProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.SAS) + await makeRequestAndAssert( + [RunTimeType.SAS, RunTimeType.JS], + 200, + RunTimeType.SAS + ) }) it('should execute js program when both sas and python programs are not present', async () => { - const programPath = path.join(testFilesFolder, 'program') - const jsProgramPath = path.join(filesFolder, `${programPath}.js`) - await createFile(jsProgramPath, sampleJsProgram) - - await makeRequestAndAssert(programPath, 200, RunTimeType.JS) + await makeRequestAndAssert([RunTimeType.JS], 200, RunTimeType.JS) }) it('should throw error when no program exists', async () => { - const programPath = path.join(testFilesFolder, 'program') - await makeRequestAndAssert(programPath, 400) + await makeRequestAndAssert([], 400) }) }) }) @@ -505,10 +413,29 @@ describe('stp', () => { }) const makeRequestAndAssert = async ( - programPath: string, + programTypes: RunTimeType[], expectedStatusCode: number, expectedRuntime?: RunTimeType ) => { + const programPath = path.join(testFilesFolder, 'program') + for (const programType of programTypes) { + if (programType === RunTimeType.JS) + await createFile( + path.join(filesFolder, `${programPath}.js`), + sampleJsProgram + ) + else if (programType === RunTimeType.PY) + await createFile( + path.join(filesFolder, `${programPath}.py`), + samplePyProgram + ) + else if (programType === RunTimeType.SAS) + await createFile( + path.join(filesFolder, `${programPath}.sas`), + sampleSasProgram + ) + } + await request(app) .get(`/SASjsApi/stp/execute?_program=${programPath}`) .auth(accessToken, { type: 'bearer' }) From 8780b800a34aa618631821e5d97e26e8b0f15806 Mon Sep 17 00:00:00 2001 From: Sabir Hassan Date: Thu, 18 Aug 2022 01:17:17 +0500 Subject: [PATCH 4/7] fix: update default content type for python and js runtimes --- api/src/controllers/internal/Session.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/controllers/internal/Session.ts b/api/src/controllers/internal/Session.ts index 3cbf4d9..56f1d39 100644 --- a/api/src/controllers/internal/Session.ts +++ b/api/src/controllers/internal/Session.ts @@ -190,7 +190,7 @@ export class JSSessionController extends SessionController { } const headersPath = path.join(session.path, 'stpsrv_header.txt') - await createFile(headersPath, 'Content-type: application/json') + await createFile(headersPath, 'Content-type: text/plain') this.sessions.push(session) return session @@ -222,7 +222,7 @@ export class PythonSessionController extends SessionController { } const headersPath = path.join(session.path, 'stpsrv_header.txt') - await createFile(headersPath, 'Content-type: application/json') + await createFile(headersPath, 'Content-type: text/plain') this.sessions.push(session) return session From 882bedd5d5da22de6ed45c03d0a261aadfb3a33c Mon Sep 17 00:00:00 2001 From: Sabir Hassan Date: Thu, 18 Aug 2022 01:19:47 +0500 Subject: [PATCH 5/7] fix: add a new variable _SASJS_WEBOUT_HEADERS to code.js and code.py --- api/src/controllers/internal/Execution.ts | 3 +++ api/src/controllers/internal/createJSProgram.ts | 16 +++++++++------- .../controllers/internal/createPythonProgram.ts | 2 ++ api/src/controllers/internal/processProgram.ts | 3 +++ api/src/routes/api/spec/stp.spec.ts | 1 + 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/api/src/controllers/internal/Execution.ts b/api/src/controllers/internal/Execution.ts index 1beb046..b5594a6 100644 --- a/api/src/controllers/internal/Execution.ts +++ b/api/src/controllers/internal/Execution.ts @@ -89,6 +89,8 @@ export class ExecutionController { tokenFile, preProgramVariables?.httpHeaders.join('\n') ?? '' ) + if (returnJson) + await createFile(headersPath, 'Content-type: application/json') await processProgram( program, @@ -96,6 +98,7 @@ export class ExecutionController { vars, session, weboutPath, + headersPath, tokenFile, runTime, logPath, diff --git a/api/src/controllers/internal/createJSProgram.ts b/api/src/controllers/internal/createJSProgram.ts index c669ac5..5738a82 100644 --- a/api/src/controllers/internal/createJSProgram.ts +++ b/api/src/controllers/internal/createJSProgram.ts @@ -9,6 +9,7 @@ export const createJSProgram = async ( vars: ExecutionVars, session: Session, weboutPath: string, + headersPath: string, tokenFile: string, otherArgs?: any ) => { @@ -23,15 +24,16 @@ let _webout = ''; const weboutPath = '${ isWindows() ? weboutPath.replace(/\\/g, '\\\\') : weboutPath }'; -const _sasjs_tokenfile = '${ +const _SASJS_TOKENFILE = '${ isWindows() ? tokenFile.replace(/\\/g, '\\\\') : tokenFile }'; -const _sasjs_username = '${preProgramVariables?.username}'; -const _sasjs_userid = '${preProgramVariables?.userId}'; -const _sasjs_displayname = '${preProgramVariables?.displayName}'; -const _metaperson = _sasjs_displayname; -const _metauser = _sasjs_username; -const sasjsprocessmode = 'Stored Program'; +const _SASJS_WEBOUT_HEADERS = '${headersPath}'; +const _SASJS_USERNAME = '${preProgramVariables?.username}'; +const _SASJS_USERID = '${preProgramVariables?.userId}'; +const _SASJS_DISPLAYNAME = '${preProgramVariables?.displayName}'; +const _METAPERSON = _SASJS_DISPLAYNAME; +const _METAUSER = _SASJS_USERNAME; +const SASJSPROCESSMODE = 'Stored Program'; ` const requiredModules = `const fs = require('fs')` diff --git a/api/src/controllers/internal/createPythonProgram.ts b/api/src/controllers/internal/createPythonProgram.ts index 497ab27..892ed14 100644 --- a/api/src/controllers/internal/createPythonProgram.ts +++ b/api/src/controllers/internal/createPythonProgram.ts @@ -9,6 +9,7 @@ export const createPythonProgram = async ( vars: ExecutionVars, session: Session, weboutPath: string, + headersPath: string, tokenFile: string, otherArgs?: any ) => { @@ -22,6 +23,7 @@ _SASJS_SESSION_PATH = '${ isWindows() ? session.path.replace(/\\/g, '\\\\') : session.path }'; _WEBOUT = '${isWindows() ? weboutPath.replace(/\\/g, '\\\\') : weboutPath}'; +_SASJS_WEBOUT_HEADERS = '${headersPath}'; _SASJS_TOKENFILE = '${ isWindows() ? tokenFile.replace(/\\/g, '\\\\') : tokenFile }'; diff --git a/api/src/controllers/internal/processProgram.ts b/api/src/controllers/internal/processProgram.ts index 9f541c7..db2db10 100644 --- a/api/src/controllers/internal/processProgram.ts +++ b/api/src/controllers/internal/processProgram.ts @@ -18,6 +18,7 @@ export const processProgram = async ( vars: ExecutionVars, session: Session, weboutPath: string, + headersPath: string, tokenFile: string, runTime: RunTimeType, logPath: string, @@ -30,6 +31,7 @@ export const processProgram = async ( vars, session, weboutPath, + headersPath, tokenFile, otherArgs ) @@ -66,6 +68,7 @@ export const processProgram = async ( vars, session, weboutPath, + headersPath, tokenFile, otherArgs ) diff --git a/api/src/routes/api/spec/stp.spec.ts b/api/src/routes/api/spec/stp.spec.ts index 9b05b98..7304a4f 100644 --- a/api/src/routes/api/spec/stp.spec.ts +++ b/api/src/routes/api/spec/stp.spec.ts @@ -450,6 +450,7 @@ const makeRequestAndAssert = async ( expect.anything(), expect.anything(), expect.anything(), + expect.anything(), expectedRuntime, expect.anything(), undefined From 02e88ae7280d020a753bc2c095a931c79ac392d1 Mon Sep 17 00:00:00 2001 From: Sabir Hassan Date: Thu, 18 Aug 2022 01:20:33 +0500 Subject: [PATCH 6/7] fix: update content for code.sas file --- api/src/controllers/internal/createSASProgram.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/api/src/controllers/internal/createSASProgram.ts b/api/src/controllers/internal/createSASProgram.ts index f589ef1..ee81593 100644 --- a/api/src/controllers/internal/createSASProgram.ts +++ b/api/src/controllers/internal/createSASProgram.ts @@ -23,10 +23,14 @@ export const createSASProgram = async ( %let _sasjs_displayname=${preProgramVariables?.displayName}; %let _sasjs_apiserverurl=${preProgramVariables?.serverUrl}; %let _sasjs_apipath=/SASjsApi/stp/execute; +%let _sasjs_webout_headers=%sysfunc(pathname(work))/../stpsrv_header.txt; %let _metaperson=&_sasjs_displayname; %let _metauser=&_sasjs_username; + +/* the below is here for compatibility and will be removed in a future release */ +%let sasjs_stpsrv_header_loc=&_sasjs_webout_headers; + %let sasjsprocessmode=Stored Program; -%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/../stpsrv_header.txt; %global SYSPROCESSMODE SYSTCPIPHOSTNAME SYSHOSTINFOLONG; %macro _sasjs_server_init(); From bd97363c1300f4d8a83bae09368ad330823a3b1a Mon Sep 17 00:00:00 2001 From: Sabir Hassan Date: Thu, 18 Aug 2022 01:39:03 +0500 Subject: [PATCH 7/7] chore: quick fixes --- api/src/controllers/internal/createJSProgram.ts | 9 +++++---- api/src/controllers/internal/createPythonProgram.ts | 9 +++++---- api/src/controllers/internal/createSASProgram.ts | 3 ++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/api/src/controllers/internal/createJSProgram.ts b/api/src/controllers/internal/createJSProgram.ts index 5738a82..2a7d594 100644 --- a/api/src/controllers/internal/createJSProgram.ts +++ b/api/src/controllers/internal/createJSProgram.ts @@ -57,14 +57,15 @@ if (_webout) { ` // if no files are uploaded filesNamesMap will be undefined if (otherArgs?.filesNamesMap) { - const uploadJSCode = await generateFileUploadJSCode( + const uploadJsCode = await generateFileUploadJSCode( otherArgs.filesNamesMap, session.path ) - //If js code for the file is generated it will be appended to the top of jsCode - if (uploadJSCode.length > 0) { - program = `${uploadJSCode}\n` + program + // If any files are uploaded, the program needs to be updated with some + // dynamically generated variables (pointers) for ease of ingestion + if (uploadJsCode.length > 0) { + program = `${uploadJsCode}\n` + program } } return requiredModules + program diff --git a/api/src/controllers/internal/createPythonProgram.ts b/api/src/controllers/internal/createPythonProgram.ts index 892ed14..d33231c 100644 --- a/api/src/controllers/internal/createPythonProgram.ts +++ b/api/src/controllers/internal/createPythonProgram.ts @@ -53,14 +53,15 @@ ${program} ` // if no files are uploaded filesNamesMap will be undefined if (otherArgs?.filesNamesMap) { - const uploadJSCode = await generateFileUploadPythonCode( + const uploadPythonCode = await generateFileUploadPythonCode( otherArgs.filesNamesMap, session.path ) - //If js code for the file is generated it will be appended to the top of jsCode - if (uploadJSCode.length > 0) { - program = `${uploadJSCode}\n` + program + // If any files are uploaded, the program needs to be updated with some + // dynamically generated variables (pointers) for ease of ingestion + if (uploadPythonCode.length > 0) { + program = `${uploadPythonCode}\n` + program } } return requiredModules + program diff --git a/api/src/controllers/internal/createSASProgram.ts b/api/src/controllers/internal/createSASProgram.ts index ee81593..b1745db 100644 --- a/api/src/controllers/internal/createSASProgram.ts +++ b/api/src/controllers/internal/createSASProgram.ts @@ -67,7 +67,8 @@ ${program}` session.path ) - //If sas code for the file is generated it will be appended to the top of sasCode + // If any files are uploaded, the program needs to be updated with some + // dynamically generated variables (pointers) for ease of ingestion if (uploadSasCode.length > 0) { program = `${uploadSasCode}` + program }