diff --git a/README.md b/README.md index 409ccc1..233f7b3 100644 --- a/README.md +++ b/README.md @@ -64,12 +64,22 @@ Example contents of a `.env` file: # Server mode is multi-user and suitable for intranet / internet use MODE= +# A comma separated string that defines the available runTimes. +# Priority is given to the runtime that comes first in the string. +# Possible options at the moment are sas and js + +# options: [sas|js|py|sas,js|js,sas|py,sas|sas,py|py,js|js,py|sas,js,py|sas,py,js|js,sas,py|js,py,sas|py,sas,js|py,js,sas] default:sas +RUN_TIMES= + # Path to SAS executable (sas.exe / sas.sh) SAS_PATH=/path/to/sas/executable.exe # Path to Node.js executable NODE_PATH=~/.nvm/versions/node/v16.14.0/bin/node +# Path to Python executable +PYTHON_PATH=/usr/bin/python + # Path to working directory # This location is for SAS WORK, staged files, DRIVE, configuration etc SASJS_ROOT=./sasjs_root @@ -139,13 +149,6 @@ LOG_FORMAT_MORGAN= # This location is for server logs with classical UNIX logrotate behavior LOG_LOCATION=./sasjs_root/logs -# A comma separated string that defines the available runTimes. -# Priority is given to the runtime that comes first in the string. -# Possible options at the moment are sas and js - -# options: [sas,js|js,sas|sas|js] default:sas -RUN_TIMES= - ``` ## Persisting the Session diff --git a/api/.env.example b/api/.env.example index e585aaf..705e0d2 100644 --- a/api/.env.example +++ b/api/.env.example @@ -14,9 +14,10 @@ HELMET_COEP=[true|false] if omitted HELMET default will be used DB_CONNECT=mongodb+srv://:@/?retryWrites=true&w=majority -RUN_TIMES=[sas|js|sas,js|js,sas] default considered as sas +RUN_TIMES=[sas|js|py|sas,js|js,sas|py,sas|sas,py|py,js|js,py|sas,js,py|sas,py,js|js,sas,py|js,py,sas|py,sas,js|py,js,sas] default considered as sas SAS_PATH=/opt/sas/sas9/SASHome/SASFoundation/9.4/sas NODE_PATH=~/.nvm/versions/node/v16.14.0/bin/node +PYTHON_PATH=/usr/bin/python SASJS_ROOT=./sasjs_root diff --git a/api/src/controllers/internal/Session.ts b/api/src/controllers/internal/Session.ts index 2db668c..3cbf4d9 100644 --- a/api/src/controllers/internal/Session.ts +++ b/api/src/controllers/internal/Session.ts @@ -197,9 +197,41 @@ export class JSSessionController extends SessionController { } } +export class PythonSessionController extends SessionController { + protected async createSession(): Promise { + const sessionId = generateUniqueFileName(generateTimestamp()) + const sessionFolder = path.join(getSessionsFolder(), sessionId) + + const creationTimeStamp = sessionId.split('-').pop() as string + // death time of session is 15 mins from creation + const deathTimeStamp = ( + parseInt(creationTimeStamp) + + 15 * 60 * 1000 - + 1000 + ).toString() + + const session: Session = { + id: sessionId, + ready: true, + inUse: true, + consumed: false, + completed: false, + creationTimeStamp, + deathTimeStamp, + path: sessionFolder + } + + const headersPath = path.join(session.path, 'stpsrv_header.txt') + await createFile(headersPath, 'Content-type: application/json') + + this.sessions.push(session) + return session + } +} + export const getSessionController = ( runTime: RunTimeType -): SASSessionController | JSSessionController => { +): SASSessionController | JSSessionController | PythonSessionController => { if (runTime === RunTimeType.SAS) { return getSASSessionController() } @@ -208,6 +240,10 @@ export const getSessionController = ( return getJSSessionController() } + if (runTime === RunTimeType.PY) { + return getPythonSessionController() + } + throw new Error('No Runtime is configured') } @@ -227,6 +263,14 @@ const getJSSessionController = (): JSSessionController => { return process.jsSessionController } +const getPythonSessionController = (): PythonSessionController => { + if (process.pythonSessionController) return process.pythonSessionController + + process.pythonSessionController = new PythonSessionController() + + return process.pythonSessionController +} + const autoExecContent = ` data _null_; /* remove the dummy SYSIN */ diff --git a/api/src/controllers/internal/createPythonProgram.ts b/api/src/controllers/internal/createPythonProgram.ts new file mode 100644 index 0000000..497ab27 --- /dev/null +++ b/api/src/controllers/internal/createPythonProgram.ts @@ -0,0 +1,65 @@ +import { isWindows } from '@sasjs/utils' +import { PreProgramVars, Session } from '../../types' +import { generateFileUploadPythonCode } from '../../utils' +import { ExecutionVars } from './' + +export const createPythonProgram = async ( + program: string, + preProgramVariables: PreProgramVars, + vars: ExecutionVars, + session: Session, + weboutPath: string, + tokenFile: string, + otherArgs?: any +) => { + const varStatments = Object.keys(vars).reduce( + (computed: string, key: string) => `${computed}${key} = '${vars[key]}';\n`, + '' + ) + + const preProgramVarStatments = ` +_SASJS_SESSION_PATH = '${ + isWindows() ? session.path.replace(/\\/g, '\\\\') : session.path + }'; +_WEBOUT = '${isWindows() ? weboutPath.replace(/\\/g, '\\\\') : weboutPath}'; +_SASJS_TOKENFILE = '${ + isWindows() ? tokenFile.replace(/\\/g, '\\\\') : tokenFile + }'; +_SASJS_USERNAME = '${preProgramVariables?.username}'; +_SASJS_USERID = '${preProgramVariables?.userId}'; +_SASJS_DISPLAYNAME = '${preProgramVariables?.displayName}'; +_METAPERSON = _SASJS_DISPLAYNAME; +_METAUSER = _SASJS_USERNAME; +SASJSPROCESSMODE = 'Stored Program'; +` + + const requiredModules = `import os` + + program = ` +# runtime vars +${varStatments} + +# dynamic user-provided vars +${preProgramVarStatments} + +# change working directory to session folder +os.chdir(_SASJS_SESSION_PATH) + +# actual job code +${program} + +` + // if no files are uploaded filesNamesMap will be undefined + if (otherArgs?.filesNamesMap) { + const uploadJSCode = await generateFileUploadPythonCode( + otherArgs.filesNamesMap, + session.path + ) + + //If js code for the file is generated it will be appended to the top of jsCode + if (uploadJSCode.length > 0) { + program = `${uploadJSCode}\n` + program + } + } + return requiredModules + program +} diff --git a/api/src/controllers/internal/index.ts b/api/src/controllers/internal/index.ts index 50672d7..2a64677 100644 --- a/api/src/controllers/internal/index.ts +++ b/api/src/controllers/internal/index.ts @@ -4,4 +4,5 @@ export * from './Execution' export * from './FileUploadController' export * from './createSASProgram' export * from './createJSProgram' +export * from './createPythonProgram' export * from './processProgram' diff --git a/api/src/controllers/internal/processProgram.ts b/api/src/controllers/internal/processProgram.ts index c059797..9f541c7 100644 --- a/api/src/controllers/internal/processProgram.ts +++ b/api/src/controllers/internal/processProgram.ts @@ -5,7 +5,12 @@ import { once } from 'stream' import { createFile, moveFile } from '@sasjs/utils' import { PreProgramVars, Session } from '../../types' import { RunTimeType } from '../../utils' -import { ExecutionVars, createSASProgram, createJSProgram } from './' +import { + ExecutionVars, + createSASProgram, + createJSProgram, + createPythonProgram +} from './' export const processProgram = async ( program: string, @@ -47,6 +52,42 @@ export const processProgram = async ( // copy the code.js program to log and end write stream writeStream.end(program) + session.completed = true + console.log('session completed', session) + } catch (err: any) { + session.completed = true + session.crashed = err.toString() + console.log('session crashed', session.id, session.crashed) + } + } else if (runTime === RunTimeType.PY) { + program = await createPythonProgram( + program, + preProgramVariables, + vars, + session, + weboutPath, + tokenFile, + otherArgs + ) + + const codePath = path.join(session.path, 'code.py') + + try { + await createFile(codePath, program) + + // create a stream that will write to console outputs to log file + const writeStream = fs.createWriteStream(logPath) + + // waiting for the open event so that we can have underlying file descriptor + await once(writeStream, 'open') + + execFileSync(process.pythonLoc!, [codePath], { + stdio: ['ignore', writeStream, writeStream] + }) + + // copy the code.py program to log and end write stream + writeStream.end(program) + session.completed = true console.log('session completed', session) } catch (err: any) { diff --git a/api/src/types/system/process.d.ts b/api/src/types/system/process.d.ts index bf29e16..f635d46 100644 --- a/api/src/types/system/process.d.ts +++ b/api/src/types/system/process.d.ts @@ -2,10 +2,12 @@ declare namespace NodeJS { export interface Process { sasLoc?: string nodeLoc?: string + pythonLoc?: string driveLoc: string logsLoc: string sasSessionController?: import('../../controllers/internal').SASSessionController jsSessionController?: import('../../controllers/internal').JSSessionController + pythonSessionController?: import('../../controllers/internal').PythonSessionController appStreamConfig: import('../').AppStreamConfig logger: import('@sasjs/utils/logger').Logger runTimes: import('../../utils').RunTimeType[] diff --git a/api/src/utils/getDesktopFields.ts b/api/src/utils/getDesktopFields.ts index e1e0966..bbf2391 100644 --- a/api/src/utils/getDesktopFields.ts +++ b/api/src/utils/getDesktopFields.ts @@ -4,9 +4,9 @@ import { createFolder, fileExists, folderExists, isWindows } from '@sasjs/utils' import { RunTimeType } from './verifyEnvVariables' export const getDesktopFields = async () => { - const { SAS_PATH, NODE_PATH } = process.env + const { SAS_PATH, NODE_PATH, PYTHON_PATH } = process.env - let sasLoc, nodeLoc + let sasLoc, nodeLoc, pythonLoc if (process.runTimes.includes(RunTimeType.SAS)) { sasLoc = SAS_PATH ?? (await getSASLocation()) @@ -16,7 +16,11 @@ export const getDesktopFields = async () => { nodeLoc = NODE_PATH ?? (await getNodeLocation()) } - return { sasLoc, nodeLoc } + if (process.runTimes.includes(RunTimeType.JS)) { + pythonLoc = PYTHON_PATH ?? (await getPythonLocation()) + } + + return { sasLoc, nodeLoc, pythonLoc } } const getDriveLocation = async (): Promise => { @@ -91,3 +95,25 @@ const getNodeLocation = async (): Promise => { return targetName } + +const getPythonLocation = async (): Promise => { + const validator = async (filePath: string) => { + if (!filePath) return 'Path to Python executable is required.' + + if (!(await fileExists(filePath))) { + return 'No file found at provided path.' + } + + return true + } + + const defaultLocation = isWindows() ? 'C:\\Python' : '/usr/bin/python' + + const targetName = await getString( + 'Please enter full path to a Python executable: ', + validator, + defaultLocation + ) + + return targetName +} diff --git a/api/src/utils/getRunTimeAndFilePath.ts b/api/src/utils/getRunTimeAndFilePath.ts index 259b0a9..e27fb2e 100644 --- a/api/src/utils/getRunTimeAndFilePath.ts +++ b/api/src/utils/getRunTimeAndFilePath.ts @@ -5,7 +5,7 @@ import { RunTimeType } from '.' export const getRunTimeAndFilePath = async (programPath: string) => { const ext = path.extname(programPath) - // If programPath (_program) is provided with a ".sas" or ".js" extension + // If programPath (_program) is provided with a ".sas", ".js" or ".py" extension // we should use that extension to determine the appropriate runTime if (ext && Object.values(RunTimeType).includes(ext.slice(1) as RunTimeType)) { const runTime = ext.slice(1) diff --git a/api/src/utils/setProcessVariables.ts b/api/src/utils/setProcessVariables.ts index 9fc3616..07c03e2 100644 --- a/api/src/utils/setProcessVariables.ts +++ b/api/src/utils/setProcessVariables.ts @@ -28,11 +28,13 @@ export const setProcessVariables = async () => { if (MODE === ModeType.Server) { process.sasLoc = process.env.SAS_PATH process.nodeLoc = process.env.NODE_PATH + process.pythonLoc = process.env.PYTHON_PATH } else { - const { sasLoc, nodeLoc } = await getDesktopFields() + const { sasLoc, nodeLoc, pythonLoc } = await getDesktopFields() process.sasLoc = sasLoc process.nodeLoc = nodeLoc + process.pythonLoc = pythonLoc } const { SASJS_ROOT } = process.env diff --git a/api/src/utils/upload.ts b/api/src/utils/upload.ts index 8a64e62..648f961 100644 --- a/api/src/utils/upload.ts +++ b/api/src/utils/upload.ts @@ -126,9 +126,34 @@ export const generateFileUploadJSCode = async ( } }) - if (fileCount) { - uploadCode = `\nconst _WEBIN_FILE_COUNT = ${fileCount}` + uploadCode - } + uploadCode += `\nconst _WEBIN_FILE_COUNT = ${fileCount}` + + return uploadCode +} + +/** + * Generates the python code that references uploaded files in the concurrent request + * @param filesNamesMap object that maps hashed file names and original file names + * @param sessionFolder name of the folder that is created for the purpose of files in concurrent request + * @returns generated python code + */ +export const generateFileUploadPythonCode = async ( + filesNamesMap: FilenamesMap, + sessionFolder: string +) => { + let uploadCode = '' + let fileCount = 0 + + const sessionFolderList: string[] = await listFilesInFolder(sessionFolder) + sessionFolderList.forEach(async (fileName) => { + if (fileName.includes('req_file')) { + fileCount++ + uploadCode += `\n_WEBIN_FILENAME${fileCount} = '${filesNamesMap[fileName].originalName}'` + uploadCode += `\n_WEBIN_NAME${fileCount} = '${filesNamesMap[fileName].fieldName}'` + } + }) + + uploadCode += `\n_WEBIN_FILE_COUNT = ${fileCount}` return uploadCode } diff --git a/api/src/utils/verifyEnvVariables.ts b/api/src/utils/verifyEnvVariables.ts index b0a40b2..023cf05 100644 --- a/api/src/utils/verifyEnvVariables.ts +++ b/api/src/utils/verifyEnvVariables.ts @@ -28,7 +28,8 @@ export enum LOG_FORMAT_MORGANType { export enum RunTimeType { SAS = 'sas', - JS = 'js' + JS = 'js', + PY = 'py' } export enum ReturnCode { @@ -228,7 +229,7 @@ const verifyRUN_TIMES = (): string[] => { const verifyExecutablePaths = () => { const errors: string[] = [] - const { RUN_TIMES, SAS_PATH, NODE_PATH, MODE } = process.env + const { RUN_TIMES, SAS_PATH, NODE_PATH, PYTHON_PATH, MODE } = process.env if (MODE === ModeType.Server) { const runTimes = RUN_TIMES?.split(',') @@ -240,6 +241,10 @@ const verifyExecutablePaths = () => { if (runTimes?.includes(RunTimeType.JS) && !NODE_PATH) { errors.push(`- NODE_PATH is required for ${RunTimeType.JS} run time`) } + + if (runTimes?.includes(RunTimeType.PY) && !PYTHON_PATH) { + errors.push(`- PYTHON_PATH is required for ${RunTimeType.PY} run time`) + } } return errors