diff --git a/api/src/controllers/internal/Execution.ts b/api/src/controllers/internal/Execution.ts index 0391f7f..0bc80a7 100644 --- a/api/src/controllers/internal/Execution.ts +++ b/api/src/controllers/internal/Execution.ts @@ -19,7 +19,7 @@ import { getMacrosFolder, HTTPHeaders, isDebugOn, - SASJSRunTimes + RunTimeType } from '../../utils' export interface ExecutionVars { @@ -37,64 +37,56 @@ export interface ExecuteReturnJson { log?: string } -export class ExecutionController { - async executeFile( - programPath: string, - preProgramVariables: PreProgramVars, - vars: ExecutionVars, - otherArgs?: any, - returnJson?: boolean, - session?: Session - ) { - for (const runTime of process.runTimes) { - const codePath = - path - .join(getFilesFolder(), programPath) - .replace(new RegExp('/', 'g'), path.sep) + - '.' + - runTime - if (await fileExists(codePath)) { - const program = await readFile(codePath) +interface ExecuteFileParams { + programPath: string + preProgramVariables: PreProgramVars + vars: ExecutionVars + otherArgs?: any + returnJson?: boolean + session?: Session + runTime: RunTimeType +} - if (runTime === SASJSRunTimes.JS) { - return this.executeProgram( - program, - preProgramVariables, - vars, - otherArgs, - returnJson, - session, - runTime - ) - } else { - return this.executeProgram( - program, - preProgramVariables, - vars, - otherArgs, - returnJson, - session, - runTime - ) - } - } - } - throw `ExecutionController: The Stored Program at "${programPath}" does not exist, or you do not have permission to view it.` +interface ExecuteProgramParams extends Omit { + program: string +} + +export class ExecutionController { + async executeFile({ + programPath, + preProgramVariables, + vars, + otherArgs, + returnJson, + session, + runTime + }: ExecuteFileParams) { + const program = await readFile(programPath) + + return this.executeProgram({ + program, + preProgramVariables, + vars, + otherArgs, + returnJson, + session, + runTime + }) } - async executeProgram( - program: string, - preProgramVariables: PreProgramVars, - vars: ExecutionVars, - otherArgs?: any, - returnJson?: boolean, - sessionByFileUpload?: Session, - runTime: string = 'sas' - ): Promise { + async executeProgram({ + program, + preProgramVariables, + vars, + otherArgs, + returnJson, + session: sessionByFileUpload, + runTime + }: ExecuteProgramParams): Promise { const sessionController = - runTime === SASJSRunTimes.JS - ? getJSSessionController() - : getSASSessionController() + runTime === RunTimeType.SAS + ? getSASSessionController() + : getJSSessionController() const session = sessionByFileUpload ?? (await sessionController.getSession()) @@ -112,69 +104,17 @@ export class ExecutionController { preProgramVariables?.httpHeaders.join('\n') ?? '' ) - if (runTime === SASJSRunTimes.JS) { - program = await this.createJSProgram( - program, - preProgramVariables, - vars, - session, - weboutPath, - tokenFile, - otherArgs - ) - - const codePath = path.join(session.path, 'code.js') - - try { - await createFile(codePath, program) - - // create a stream that will write to console outputs to log file - const writeStream = fs.createWriteStream(logPath) - - // waiting for the open event so that we can have underlying file descriptor - await once(writeStream, 'open') - - execFileSync('node', [codePath], { - stdio: ['ignore', writeStream, writeStream] - }) - - // copy the code.js program to log and end write stream - writeStream.end(program) - - session.completed = true - console.log('session completed', session) - } catch (err: any) { - session.completed = true - session.crashed = err.toString() - console.log('session crashed', session.id, session.crashed) - } - } else { - program = await this.createSASProgram( - program, - preProgramVariables, - vars, - session, - weboutPath, - tokenFile, - otherArgs - ) - - const codePath = path.join(session.path, 'code.sas') - - // Creating this file in a RUNNING session will break out - // the autoexec loop and actually execute the program - // but - given it will take several milliseconds to create - // (which can mean SAS trying to run a partial program, or - // failing due to file lock) we first create the file THEN - // we rename it. - await createFile(codePath + '.bkp', program) - await moveFile(codePath + '.bkp', codePath) - - // we now need to poll the session status - while (!session.completed) { - await delay(50) - } - } + await processProgram( + program, + preProgramVariables, + vars, + session, + weboutPath, + tokenFile, + runTime, + logPath, + otherArgs + ) const log = (await fileExists(logPath)) ? await readFile(logPath) : '' const headersContent = (await fileExists(headersPath)) @@ -212,129 +152,6 @@ export class ExecutionController { } } - private async createSASProgram( - program: string, - preProgramVariables: PreProgramVars, - vars: ExecutionVars, - session: Session, - weboutPath: string, - tokenFile: string, - otherArgs?: any - ) { - const varStatments = Object.keys(vars).reduce( - (computed: string, key: string) => - `${computed}%let ${key}=${vars[key]};\n`, - '' - ) - - const preProgramVarStatments = ` -%let _sasjs_tokenfile=${tokenFile}; -%let _sasjs_username=${preProgramVariables?.username}; -%let _sasjs_userid=${preProgramVariables?.userId}; -%let _sasjs_displayname=${preProgramVariables?.displayName}; -%let _sasjs_apiserverurl=${preProgramVariables?.serverUrl}; -%let _sasjs_apipath=/SASjsApi/stp/execute; -%let _metaperson=&_sasjs_displayname; -%let _metauser=&_sasjs_username; -%let sasjsprocessmode=Stored Program; -%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/../stpsrv_header.txt; - -%global SYSPROCESSMODE SYSTCPIPHOSTNAME SYSHOSTINFOLONG; -%macro _sasjs_server_init(); - %if "&SYSPROCESSMODE"="" %then %let SYSPROCESSMODE=&sasjsprocessmode; - %if "&SYSTCPIPHOSTNAME"="" %then %let SYSTCPIPHOSTNAME=&_sasjs_apiserverurl; -%mend; -%_sasjs_server_init() -` - - program = ` -options insert=(SASAUTOS="${getMacrosFolder()}"); - -/* runtime vars */ -${varStatments} -filename _webout "${weboutPath}" mod; - -/* dynamic user-provided vars */ -${preProgramVarStatments} - -/* user autoexec starts */ -${otherArgs?.userAutoExec ?? ''} -/* user autoexec ends */ - -/* actual job code */ -${program}` - - // if no files are uploaded filesNamesMap will be undefined - if (otherArgs?.filesNamesMap) { - const uploadSasCode = await generateFileUploadSasCode( - otherArgs.filesNamesMap, - session.path - ) - - //If sas code for the file is generated it will be appended to the top of sasCode - if (uploadSasCode.length > 0) { - program = `${uploadSasCode}` + program - } - } - return program - } - - private async createJSProgram( - program: string, - preProgramVariables: PreProgramVars, - vars: ExecutionVars, - session: Session, - weboutPath: string, - tokenFile: string, - otherArgs?: any - ) { - const varStatments = Object.keys(vars).reduce( - (computed: string, key: string) => - `${computed}const ${key} = '${vars[key]}';\n`, - '' - ) - - const preProgramVarStatments = ` -const weboutPath = '${weboutPath}'; -const _sasjs_tokenfile = '${tokenFile}'; -const _sasjs_username = '${preProgramVariables?.username}'; -const _sasjs_userid = '${preProgramVariables?.userId}'; -const _sasjs_displayname = '${preProgramVariables?.displayName}'; -const _metaperson = _sasjs_displayname; -const _metauser = _sasjs_username; -const sasjsprocessmode = 'Stored Program'; -` - - const requiredModules = `const fs = require('fs-extra')` - - program = ` -/* runtime vars */ -${varStatments} - -/* dynamic user-provided vars */ -${preProgramVarStatments} - -/* actual job code */ -${program} - -/* write webout file*/ -fs.promises.writeFile(weboutPath, _webout) -` - // if no files are uploaded filesNamesMap will be undefined - if (otherArgs?.filesNamesMap) { - const uploadJSCode = await generateFileUploadJSCode( - otherArgs.filesNamesMap, - session.path - ) - - //If js code for the file is generated it will be appended to the top of jsCode - if (uploadJSCode.length > 0) { - program = `${uploadJSCode}\n` + program - } - } - return requiredModules + program - } - buildDirectoryTree() { const root: TreeNode = { name: 'files', @@ -374,3 +191,201 @@ fs.promises.writeFile(weboutPath, _webout) } const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +const createSASProgram = async ( + program: string, + preProgramVariables: PreProgramVars, + vars: ExecutionVars, + session: Session, + weboutPath: string, + tokenFile: string, + otherArgs?: any +) => { + const varStatments = Object.keys(vars).reduce( + (computed: string, key: string) => `${computed}%let ${key}=${vars[key]};\n`, + '' + ) + + const preProgramVarStatments = ` +%let _sasjs_tokenfile=${tokenFile}; +%let _sasjs_username=${preProgramVariables?.username}; +%let _sasjs_userid=${preProgramVariables?.userId}; +%let _sasjs_displayname=${preProgramVariables?.displayName}; +%let _sasjs_apiserverurl=${preProgramVariables?.serverUrl}; +%let _sasjs_apipath=/SASjsApi/stp/execute; +%let _metaperson=&_sasjs_displayname; +%let _metauser=&_sasjs_username; +%let sasjsprocessmode=Stored Program; +%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/../stpsrv_header.txt; + +%global SYSPROCESSMODE SYSTCPIPHOSTNAME SYSHOSTINFOLONG; +%macro _sasjs_server_init(); +%if "&SYSPROCESSMODE"="" %then %let SYSPROCESSMODE=&sasjsprocessmode; +%if "&SYSTCPIPHOSTNAME"="" %then %let SYSTCPIPHOSTNAME=&_sasjs_apiserverurl; +%mend; +%_sasjs_server_init() +` + + program = ` +options insert=(SASAUTOS="${getMacrosFolder()}"); + +/* runtime vars */ +${varStatments} +filename _webout "${weboutPath}" mod; + +/* dynamic user-provided vars */ +${preProgramVarStatments} + +/* user autoexec starts */ +${otherArgs?.userAutoExec ?? ''} +/* user autoexec ends */ + +/* actual job code */ +${program}` + + // if no files are uploaded filesNamesMap will be undefined + if (otherArgs?.filesNamesMap) { + const uploadSasCode = await generateFileUploadSasCode( + otherArgs.filesNamesMap, + session.path + ) + + //If sas code for the file is generated it will be appended to the top of sasCode + if (uploadSasCode.length > 0) { + program = `${uploadSasCode}` + program + } + } + return program +} + +const createJSProgram = async ( + program: string, + preProgramVariables: PreProgramVars, + vars: ExecutionVars, + session: Session, + weboutPath: string, + tokenFile: string, + otherArgs?: any +) => { + const varStatments = Object.keys(vars).reduce( + (computed: string, key: string) => + `${computed}const ${key} = '${vars[key]}';\n`, + '' + ) + + const preProgramVarStatments = ` +const weboutPath = '${weboutPath}'; +const _sasjs_tokenfile = '${tokenFile}'; +const _sasjs_username = '${preProgramVariables?.username}'; +const _sasjs_userid = '${preProgramVariables?.userId}'; +const _sasjs_displayname = '${preProgramVariables?.displayName}'; +const _metaperson = _sasjs_displayname; +const _metauser = _sasjs_username; +const sasjsprocessmode = 'Stored Program'; +` + + const requiredModules = `const fs = require('fs-extra')` + + program = ` +/* runtime vars */ +${varStatments} + +/* dynamic user-provided vars */ +${preProgramVarStatments} + +/* actual job code */ +${program} + +/* write webout file*/ +fs.promises.writeFile(weboutPath, _webout) +` + // if no files are uploaded filesNamesMap will be undefined + if (otherArgs?.filesNamesMap) { + const uploadJSCode = await generateFileUploadJSCode( + otherArgs.filesNamesMap, + session.path + ) + + //If js code for the file is generated it will be appended to the top of jsCode + if (uploadJSCode.length > 0) { + program = `${uploadJSCode}\n` + program + } + } + return requiredModules + program +} + +const processProgram = async ( + program: string, + preProgramVariables: PreProgramVars, + vars: ExecutionVars, + session: Session, + weboutPath: string, + tokenFile: string, + runTime: RunTimeType, + logPath: string, + otherArgs?: any +) => { + if (runTime === RunTimeType.JS) { + program = await createJSProgram( + program, + preProgramVariables, + vars, + session, + weboutPath, + tokenFile, + otherArgs + ) + + const codePath = path.join(session.path, 'code.js') + + try { + await createFile(codePath, program) + + // create a stream that will write to console outputs to log file + const writeStream = fs.createWriteStream(logPath) + + // waiting for the open event so that we can have underlying file descriptor + await once(writeStream, 'open') + + execFileSync('node', [codePath], { + stdio: ['ignore', writeStream, writeStream] + }) + + // copy the code.js program to log and end write stream + writeStream.end(program) + + session.completed = true + console.log('session completed', session) + } catch (err: any) { + session.completed = true + session.crashed = err.toString() + console.log('session crashed', session.id, session.crashed) + } + } else { + program = await createSASProgram( + program, + preProgramVariables, + vars, + session, + weboutPath, + tokenFile, + otherArgs + ) + + const codePath = path.join(session.path, 'code.sas') + + // Creating this file in a RUNNING session will break out + // the autoexec loop and actually execute the program + // but - given it will take several milliseconds to create + // (which can mean SAS trying to run a partial program, or + // failing due to file lock) we first create the file THEN + // we rename it. + await createFile(codePath + '.bkp', program) + await moveFile(codePath + '.bkp', codePath) + + // we now need to poll the session status + while (!session.completed) { + await delay(50) + } + } +} diff --git a/api/src/controllers/internal/FileUploadController.ts b/api/src/controllers/internal/FileUploadController.ts index 4e4b832..d267d43 100644 --- a/api/src/controllers/internal/FileUploadController.ts +++ b/api/src/controllers/internal/FileUploadController.ts @@ -1,9 +1,12 @@ -import path from 'path' import { Request, RequestHandler } from 'express' import multer from 'multer' -import { uuidv4, fileExists } from '@sasjs/utils' +import { uuidv4 } from '@sasjs/utils' import { getSASSessionController, getJSSessionController } from '.' -import { getFilesFolder, SASJSRunTimes } from '../../utils' +import { + executeProgramRawValidation, + getRunTimeAndFilePath, + RunTimeType +} from '../../utils' export class FileUploadController { private storage = multer.diskStorage({ @@ -22,31 +25,26 @@ export class FileUploadController { //It will intercept request and generate unique uuid to be used as a subfolder name //that will store the files uploaded public preUploadMiddleware: RequestHandler = async (req, res, next) => { - const programPath = req.query._program as string + const { error: errQ, value: query } = executeProgramRawValidation(req.query) + const { error: errB, value: body } = executeProgramRawValidation(req.body) - for (const runTime of process.runTimes) { - const codePath = - path - .join(getFilesFolder(), programPath) - .replace(new RegExp('/', 'g'), path.sep) + - '.' + - runTime + if (errQ && errB) return res.status(400).send(errB.details[0].message) - if (await fileExists(codePath)) { - let sessionController - if (runTime === SASJSRunTimes.JS) { - sessionController = getJSSessionController() - } else { - sessionController = getSASSessionController() - } - const session = await sessionController.getSession() - // marking consumed true, so that it's not available - // as readySession for any other request - session.consumed = true - req.sasjsSession = session - break - } - } + const programPath = (query?._program ?? body?._program) as string + + const { runTime } = await getRunTimeAndFilePath(programPath) + + const sessionController = + runTime === RunTimeType.SAS + ? getSASSessionController() + : getJSSessionController() + + const session = await sessionController.getSession() + // marking consumed true, so that it's not available + // as readySession for any other request + session.consumed = true + + req.sasjsSession = session next() } diff --git a/api/src/controllers/stp.ts b/api/src/controllers/stp.ts index d3b01bf..624762c 100644 --- a/api/src/controllers/stp.ts +++ b/api/src/controllers/stp.ts @@ -1,5 +1,4 @@ import express from 'express' -import path from 'path' import { Request, Security, @@ -19,12 +18,12 @@ import { } from './internal' import { getPreProgramVariables, - getFilesFolder, HTTPHeaders, isDebugOn, LogLine, makeFilesNamesMap, - parseLogToArray + parseLogToArray, + getRunTimeAndFilePath } from '../utils' import { MulterFile } from '../types/Upload' @@ -132,13 +131,16 @@ const executeReturnRaw = async ( ): Promise => { const query = req.query as ExecutionVars + const { codePath, runTime } = await getRunTimeAndFilePath(_program) + try { const { result, httpHeaders } = - (await new ExecutionController().executeFile( - _program, - getPreProgramVariables(req), - query - )) as ExecuteReturnRaw + (await new ExecutionController().executeFile({ + programPath: codePath, + preProgramVariables: getPreProgramVariables(req), + vars: query, + runTime + })) as ExecuteReturnRaw // Should over-ride response header for debug // on GET request to see entire log rendering on browser. @@ -167,20 +169,23 @@ const executeReturnJson = async ( req: express.Request, _program: string ): Promise => { + const { codePath, runTime } = await getRunTimeAndFilePath(_program) + const filesNamesMap = req.files?.length ? makeFilesNamesMap(req.files as MulterFile[]) : null try { const { webout, log, httpHeaders } = - (await new ExecutionController().executeFile( - _program, - getPreProgramVariables(req), - { ...req.query, ...req.body }, - { filesNamesMap: filesNamesMap }, - true, - req.sasjsSession - )) as ExecuteReturnJson + (await new ExecutionController().executeFile({ + programPath: codePath, + preProgramVariables: getPreProgramVariables(req), + vars: { ...req.query, ...req.body }, + otherArgs: { filesNamesMap: filesNamesMap }, + returnJson: true, + session: req.sasjsSession, + runTime + })) as ExecuteReturnJson let weboutRes: string | IRecordOfAny = webout if (httpHeaders['content-type']?.toLowerCase() === 'application/json') { diff --git a/api/src/routes/api/stp.ts b/api/src/routes/api/stp.ts index 5ea37db..858feb5 100644 --- a/api/src/routes/api/stp.ts +++ b/api/src/routes/api/stp.ts @@ -35,16 +35,17 @@ stpRouter.post( fileUploadController.preUploadMiddleware, fileUploadController.getMulterUploadObject().any(), async (req, res: any) => { - const { error: errQ, value: query } = executeProgramRawValidation(req.query) - const { error: errB, value: body } = executeProgramRawValidation(req.body) + // below validations are moved to preUploadMiddleware + // const { error: errQ, value: query } = executeProgramRawValidation(req.query) + // const { error: errB, value: body } = executeProgramRawValidation(req.body) - if (errQ && errB) return res.status(400).send(errB.details[0].message) + // if (errQ && errB) return res.status(400).send(errB.details[0].message) try { const response = await controller.executeReturnJson( req, - body, - query?._program + req.body, + req.query?._program as string ) // TODO: investigate if this code is required diff --git a/api/src/types/system/process.d.ts b/api/src/types/system/process.d.ts index 2d345c1..e8e83b4 100644 --- a/api/src/types/system/process.d.ts +++ b/api/src/types/system/process.d.ts @@ -1,11 +1,11 @@ declare namespace NodeJS { export interface Process { - runTimes: string[] sasLoc: string driveLoc: string sasSessionController?: import('../../controllers/internal').SASSessionController jsSessionController?: import('../../controllers/internal').JSSessionController appStreamConfig: import('../').AppStreamConfig logger: import('@sasjs/utils/logger').Logger + runTimes: import('../../utils').RunTimeType[] } } diff --git a/api/src/utils/getRunTimeAndFilePath.ts b/api/src/utils/getRunTimeAndFilePath.ts new file mode 100644 index 0000000..f25edf6 --- /dev/null +++ b/api/src/utils/getRunTimeAndFilePath.ts @@ -0,0 +1,18 @@ +import path from 'path' +import { fileExists } from '@sasjs/utils' +import { getFilesFolder } from './file' + +export const getRunTimeAndFilePath = async (programPath: string) => { + for (const runTime of process.runTimes) { + const codePath = + path + .join(getFilesFolder(), programPath) + .replace(new RegExp('/', 'g'), path.sep) + + '.' + + runTime + + if (await fileExists(codePath)) return { codePath, runTime } + } + + throw `The Program at (${programPath}) does not exist.` +} diff --git a/api/src/utils/index.ts b/api/src/utils/index.ts index 09254ba..b520778 100644 --- a/api/src/utils/index.ts +++ b/api/src/utils/index.ts @@ -10,6 +10,7 @@ export * from './generateRefreshToken' export * from './getCertificates' export * from './getDesktopFields' export * from './getPreProgramVariables' +export * from './getRunTimeAndFilePath' export * from './getServerUrl' export * from './instantiateLogger' export * from './isDebugOn' diff --git a/api/src/utils/setProcessVariables.ts b/api/src/utils/setProcessVariables.ts index 22205b3..43725a3 100644 --- a/api/src/utils/setProcessVariables.ts +++ b/api/src/utils/setProcessVariables.ts @@ -1,7 +1,7 @@ import path from 'path' import { createFolder, getAbsolutePath, getRealPath } from '@sasjs/utils' -import { getDesktopFields, ModeType } from '.' +import { getDesktopFields, ModeType, RunTimeType } from '.' export const setProcessVariables = async () => { if (process.env.NODE_ENV === 'test') { @@ -19,18 +19,15 @@ export const setProcessVariables = async () => { process.sasLoc = sasLoc } - const { SASJS_RUNTIMES } = process.env - - const runTimes = SASJS_RUNTIMES - ? SASJS_RUNTIMES.split(',').map((runTime) => runTime.toLowerCase()) - : ['sas'] - process.runTimes = runTimes - const { SASJS_ROOT } = process.env const absPath = getAbsolutePath(SASJS_ROOT ?? 'sasjs_root', process.cwd()) await createFolder(absPath) process.driveLoc = getRealPath(absPath) + const { RUN_TIMES } = process.env + process.runTimes = (RUN_TIMES as string).split(',') as RunTimeType[] + console.log('sasLoc: ', process.sasLoc) console.log('sasDrive: ', process.driveLoc) + console.log('runTimes: ', process.runTimes) } diff --git a/api/src/utils/verifyEnvVariables.ts b/api/src/utils/verifyEnvVariables.ts index 469c90f..e596074 100644 --- a/api/src/utils/verifyEnvVariables.ts +++ b/api/src/utils/verifyEnvVariables.ts @@ -26,7 +26,7 @@ export enum LOG_FORMAT_MORGANType { tiny = 'tiny' } -export enum SASJSRunTimes { +export enum RunTimeType { SAS = 'sas', JS = 'js' } @@ -51,7 +51,7 @@ export const verifyEnvVariables = (): ReturnCode => { errors.push(...verifyLOG_FORMAT_MORGAN()) - errors.push(...verifySASJSRunTimes()) + errors.push(...verifyRUN_TIMES()) if (errors.length) { process.logger?.error( @@ -209,26 +209,24 @@ const verifyLOG_FORMAT_MORGAN = (): string[] => { return errors } -const verifySASJSRunTimes = (): string[] => { +const verifyRUN_TIMES = (): string[] => { const errors: string[] = [] - const { SASJS_RUNTIMES } = process.env + const { RUN_TIMES } = process.env - if (SASJS_RUNTIMES) { - const runTimes = SASJS_RUNTIMES.split(',').map((runTime) => - runTime.toLowerCase() - ) + if (RUN_TIMES) { + const runTimes = RUN_TIMES.split(',') - const possibleRunTimes = Object.values(SASJSRunTimes) + const runTimeTypes = Object.values(RunTimeType) runTimes.forEach((runTime) => { - if (!possibleRunTimes.includes(runTime as SASJSRunTimes)) { + if (!runTimeTypes.includes(runTime.toLowerCase() as RunTimeType)) { errors.push( - `- Invalid '${runTime}' runtime\n - valid options ${possibleRunTimes}` + `- Invalid '${runTime}' runtime\n - valid options ${runTimeTypes}` ) } }) } else { - process.env.SASJS_RUNTIMES = DEFAULTS.SASJS_RUNTIMES + process.env.RUN_TIMES = DEFAULTS.RUN_TIMES } return errors } @@ -239,5 +237,5 @@ const DEFAULTS = { PORT: '5000', HELMET_COEP: HelmetCoepType.TRUE, LOG_FORMAT_MORGAN: LOG_FORMAT_MORGANType.Common, - SASJS_RUNTIMES: SASJSRunTimes.SAS + RUN_TIMES: RunTimeType.SAS }