Merge pull request #259 from sasjs/python-runtime

feat: implement the logic for running python stored programs

README.md

@@ -64,12 +64,27 @@ Example contents of a `.env` file:
 # Server mode is multi-user and suitable for intranet / internet use
 MODE=
 
+# A comma separated string that defines the available runTimes.
+# Priority is given to the runtime that comes first in the string.
+# Possible options at the moment are sas and js
+
+# This string sets the priority of the available analytic runtimes
+# Valid runtimes are SAS (sas), JavaScript (js) and Python (py)
+# For each option provided, there should be a corresponding path,
+# eg SAS_PATH, NODE_PATH or PYTHON_PATH
+# Priority is given to runtimes earlier in the string
+# Example options:  [sas,js,py | js,py | sas | sas,js]
+RUN_TIMES=
+
 # Path to SAS executable (sas.exe / sas.sh)
 SAS_PATH=/path/to/sas/executable.exe
 
 # Path to Node.js executable
 NODE_PATH=~/.nvm/versions/node/v16.14.0/bin/node
 
+# Path to Python executable
+PYTHON_PATH=/usr/bin/python
+
 # Path to working directory
 # This location is for SAS WORK, staged files, DRIVE, configuration etc
 SASJS_ROOT=./sasjs_root
@@ -139,13 +154,6 @@ LOG_FORMAT_MORGAN=
 # This location is for server logs with classical UNIX logrotate behavior
 LOG_LOCATION=./sasjs_root/logs
 
-# A comma separated string that defines the available runTimes.
-# Priority is given to the runtime that comes first in the string.
-# Possible options at the moment are sas and js
-
-# options: [sas,js|js,sas|sas|js] default:sas
-RUN_TIMES=
-
 ```
 
 ## Persisting the Session

api/.env.example

@@ -14,9 +14,10 @@ HELMET_COEP=[true|false] if omitted HELMET default will be used
 
 DB_CONNECT=mongodb+srv://<DB_USERNAME>:<DB_PASSWORD>@<CLUSTER>/<DB_NAME>?retryWrites=true&w=majority
 
-RUN_TIMES=[sas|js|sas,js|js,sas] default considered as sas
+RUN_TIMES=[sas,js,py | js,py | sas | sas,js] default considered as sas
 SAS_PATH=/opt/sas/sas9/SASHome/SASFoundation/9.4/sas
 NODE_PATH=~/.nvm/versions/node/v16.14.0/bin/node
+PYTHON_PATH=/usr/bin/python
 
 SASJS_ROOT=./sasjs_root
 

api/src/controllers/internal/Execution.ts

@@ -89,6 +89,8 @@ export class ExecutionController {
       tokenFile,
       preProgramVariables?.httpHeaders.join('\n') ?? ''
     )
+    if (returnJson)
+      await createFile(headersPath, 'Content-type: application/json')
 
     await processProgram(
       program,
@@ -96,6 +98,7 @@ export class ExecutionController {
       vars,
       session,
       weboutPath,
+      headersPath,
       tokenFile,
       runTime,
       logPath,

api/src/controllers/internal/Session.ts

@@ -190,7 +190,39 @@ export class JSSessionController extends SessionController {
     }
 
     const headersPath = path.join(session.path, 'stpsrv_header.txt')
-    await createFile(headersPath, 'Content-type: application/json')
+    await createFile(headersPath, 'Content-type: text/plain')
+
+    this.sessions.push(session)
+    return session
+  }
+}
+
+export class PythonSessionController extends SessionController {
+  protected async createSession(): Promise<Session> {
+    const sessionId = generateUniqueFileName(generateTimestamp())
+    const sessionFolder = path.join(getSessionsFolder(), sessionId)
+
+    const creationTimeStamp = sessionId.split('-').pop() as string
+    // death time of session is 15 mins from creation
+    const deathTimeStamp = (
+      parseInt(creationTimeStamp) +
+      15 * 60 * 1000 -
+      1000
+    ).toString()
+
+    const session: Session = {
+      id: sessionId,
+      ready: true,
+      inUse: true,
+      consumed: false,
+      completed: false,
+      creationTimeStamp,
+      deathTimeStamp,
+      path: sessionFolder
+    }
+
+    const headersPath = path.join(session.path, 'stpsrv_header.txt')
+    await createFile(headersPath, 'Content-type: text/plain')
 
     this.sessions.push(session)
     return session
@@ -199,7 +231,7 @@ export class JSSessionController extends SessionController {
 
 export const getSessionController = (
   runTime: RunTimeType
-): SASSessionController | JSSessionController => {
+): SASSessionController | JSSessionController | PythonSessionController => {
   if (runTime === RunTimeType.SAS) {
     return getSASSessionController()
   }
@@ -208,6 +240,10 @@ export const getSessionController = (
     return getJSSessionController()
   }
 
+  if (runTime === RunTimeType.PY) {
+    return getPythonSessionController()
+  }
+
   throw new Error('No Runtime is configured')
 }
 
@@ -227,6 +263,14 @@ const getJSSessionController = (): JSSessionController => {
   return process.jsSessionController
 }
 
+const getPythonSessionController = (): PythonSessionController => {
+  if (process.pythonSessionController) return process.pythonSessionController
+
+  process.pythonSessionController = new PythonSessionController()
+
+  return process.pythonSessionController
+}
+
 const autoExecContent = `
 data _null_;
   /* remove the dummy SYSIN */

api/src/controllers/internal/createJSProgram.ts

@@ -9,6 +9,7 @@ export const createJSProgram = async (
   vars: ExecutionVars,
   session: Session,
   weboutPath: string,
+  headersPath: string,
   tokenFile: string,
   otherArgs?: any
 ) => {
@@ -23,15 +24,16 @@ let _webout = '';
 const weboutPath = '${
     isWindows() ? weboutPath.replace(/\\/g, '\\\\') : weboutPath
   }'; 
-const _sasjs_tokenfile = '${
+const _SASJS_TOKENFILE = '${
     isWindows() ? tokenFile.replace(/\\/g, '\\\\') : tokenFile
   }';
-const _sasjs_username = '${preProgramVariables?.username}';
+const _SASJS_WEBOUT_HEADERS = '${headersPath}';
-const _sasjs_userid = '${preProgramVariables?.userId}';
+const _SASJS_USERNAME = '${preProgramVariables?.username}';
-const _sasjs_displayname = '${preProgramVariables?.displayName}';
+const _SASJS_USERID = '${preProgramVariables?.userId}';
-const _metaperson = _sasjs_displayname;
+const _SASJS_DISPLAYNAME = '${preProgramVariables?.displayName}';
-const _metauser = _sasjs_username;
+const _METAPERSON = _SASJS_DISPLAYNAME;
-const sasjsprocessmode = 'Stored Program';
+const _METAUSER = _SASJS_USERNAME;
+const SASJSPROCESSMODE = 'Stored Program';
 `
 
   const requiredModules = `const fs = require('fs')`
@@ -55,14 +57,15 @@ if (_webout) {
 `
   // if no files are uploaded filesNamesMap will be undefined
   if (otherArgs?.filesNamesMap) {
-    const uploadJSCode = await generateFileUploadJSCode(
+    const uploadJsCode = await generateFileUploadJSCode(
       otherArgs.filesNamesMap,
       session.path
     )
 
-    //If js code for the file is generated it will be appended to the top of jsCode
+    // If any files are uploaded, the program needs to be updated with some
-    if (uploadJSCode.length > 0) {
+    // dynamically generated variables (pointers) for ease of ingestion
-      program = `${uploadJSCode}\n` + program
+    if (uploadJsCode.length > 0) {
+      program = `${uploadJsCode}\n` + program
     }
   }
   return requiredModules + program

api/src/controllers/internal/createPythonProgram.ts

@@ -0,0 +1,68 @@
+import { isWindows } from '@sasjs/utils'
+import { PreProgramVars, Session } from '../../types'
+import { generateFileUploadPythonCode } from '../../utils'
+import { ExecutionVars } from './'
+
+export const createPythonProgram = async (
+  program: string,
+  preProgramVariables: PreProgramVars,
+  vars: ExecutionVars,
+  session: Session,
+  weboutPath: string,
+  headersPath: string,
+  tokenFile: string,
+  otherArgs?: any
+) => {
+  const varStatments = Object.keys(vars).reduce(
+    (computed: string, key: string) => `${computed}${key} = '${vars[key]}';\n`,
+    ''
+  )
+
+  const preProgramVarStatments = `
+_SASJS_SESSION_PATH = '${
+    isWindows() ? session.path.replace(/\\/g, '\\\\') : session.path
+  }';
+_WEBOUT = '${isWindows() ? weboutPath.replace(/\\/g, '\\\\') : weboutPath}'; 
+_SASJS_WEBOUT_HEADERS = '${headersPath}';
+_SASJS_TOKENFILE = '${
+    isWindows() ? tokenFile.replace(/\\/g, '\\\\') : tokenFile
+  }';
+_SASJS_USERNAME = '${preProgramVariables?.username}';
+_SASJS_USERID = '${preProgramVariables?.userId}';
+_SASJS_DISPLAYNAME = '${preProgramVariables?.displayName}';
+_METAPERSON = _SASJS_DISPLAYNAME;
+_METAUSER = _SASJS_USERNAME;
+SASJSPROCESSMODE = 'Stored Program';
+`
+
+  const requiredModules = `import os`
+
+  program = `
+# runtime vars
+${varStatments}
+
+# dynamic user-provided vars
+${preProgramVarStatments}
+
+# change working directory to  session folder
+os.chdir(_SASJS_SESSION_PATH)
+
+# actual job code
+${program}
+
+`
+  // if no files are uploaded filesNamesMap will be undefined
+  if (otherArgs?.filesNamesMap) {
+    const uploadPythonCode = await generateFileUploadPythonCode(
+      otherArgs.filesNamesMap,
+      session.path
+    )
+
+    // If any files are uploaded, the program needs to be updated with some
+    // dynamically generated variables (pointers) for ease of ingestion
+    if (uploadPythonCode.length > 0) {
+      program = `${uploadPythonCode}\n` + program
+    }
+  }
+  return requiredModules + program
+}

api/src/controllers/internal/createSASProgram.ts

@@ -23,10 +23,14 @@ export const createSASProgram = async (
 %let _sasjs_displayname=${preProgramVariables?.displayName};
 %let _sasjs_apiserverurl=${preProgramVariables?.serverUrl};
 %let _sasjs_apipath=/SASjsApi/stp/execute;
+%let _sasjs_webout_headers=%sysfunc(pathname(work))/../stpsrv_header.txt;
 %let _metaperson=&_sasjs_displayname;
 %let _metauser=&_sasjs_username;
+
+/* the below is here for compatibility and will be removed in a future release */
+%let sasjs_stpsrv_header_loc=&_sasjs_webout_headers;
+
 %let sasjsprocessmode=Stored Program;
-%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/../stpsrv_header.txt;
 
 %global SYSPROCESSMODE SYSTCPIPHOSTNAME SYSHOSTINFOLONG;
 %macro _sasjs_server_init();
@@ -63,7 +67,8 @@ ${program}`
       session.path
     )
 
-    //If sas code for the file is generated it will be appended to the top of sasCode
+    // If any files are uploaded, the program needs to be updated with some
+    // dynamically generated variables (pointers) for ease of ingestion
     if (uploadSasCode.length > 0) {
       program = `${uploadSasCode}` + program
     }

api/src/controllers/internal/index.ts

@@ -4,4 +4,5 @@ export * from './Execution'
 export * from './FileUploadController'
 export * from './createSASProgram'
 export * from './createJSProgram'
+export * from './createPythonProgram'
 export * from './processProgram'

api/src/controllers/internal/processProgram.ts

@@ -5,7 +5,12 @@ import { once } from 'stream'
 import { createFile, moveFile } from '@sasjs/utils'
 import { PreProgramVars, Session } from '../../types'
 import { RunTimeType } from '../../utils'
-import { ExecutionVars, createSASProgram, createJSProgram } from './'
+import {
+  ExecutionVars,
+  createSASProgram,
+  createJSProgram,
+  createPythonProgram
+} from './'
 
 export const processProgram = async (
   program: string,
@@ -13,6 +18,7 @@ export const processProgram = async (
   vars: ExecutionVars,
   session: Session,
   weboutPath: string,
+  headersPath: string,
   tokenFile: string,
   runTime: RunTimeType,
   logPath: string,
@@ -25,6 +31,7 @@ export const processProgram = async (
       vars,
       session,
       weboutPath,
+      headersPath,
       tokenFile,
       otherArgs
     )
@@ -47,6 +54,43 @@ export const processProgram = async (
       // copy the code.js program to log and end write stream
       writeStream.end(program)
 
+      session.completed = true
+      console.log('session completed', session)
+    } catch (err: any) {
+      session.completed = true
+      session.crashed = err.toString()
+      console.log('session crashed', session.id, session.crashed)
+    }
+  } else if (runTime === RunTimeType.PY) {
+    program = await createPythonProgram(
+      program,
+      preProgramVariables,
+      vars,
+      session,
+      weboutPath,
+      headersPath,
+      tokenFile,
+      otherArgs
+    )
+
+    const codePath = path.join(session.path, 'code.py')
+
+    try {
+      await createFile(codePath, program)
+
+      // create a stream that will write to console outputs to log file
+      const writeStream = fs.createWriteStream(logPath)
+
+      // waiting for the open event so that we can have underlying file descriptor
+      await once(writeStream, 'open')
+
+      execFileSync(process.pythonLoc!, [codePath], {
+        stdio: ['ignore', writeStream, writeStream]
+      })
+
+      // copy the code.py program to log and end write stream
+      writeStream.end(program)
+
       session.completed = true
       console.log('session completed', session)
     } catch (err: any) {

api/src/routes/api/spec/stp.spec.ts

@@ -22,7 +22,8 @@ import {
 import { createFile, generateTimestamp, deleteFolder } from '@sasjs/utils'
 import {
   SASSessionController,
-  JSSessionController
+  JSSessionController,
+  PythonSessionController
 } from '../../../controllers/internal'
 import * as ProcessProgramModule from '../../../controllers/internal/processProgram'
 import { Session } from '../../../types'
@@ -39,14 +40,17 @@ const user = {
 
 const sampleSasProgram = '%put hello world!;'
 const sampleJsProgram = `console.log('hello world!/')`
+const samplePyProgram = `print('hello world!/')`
 
 const filesFolder = getFilesFolder()
+const testFilesFolder = `test-stp-${generateTimestamp()}`
+
+let app: Express
+let accessToken: string
 
 describe('stp', () => {
-  let app: Express
   let con: Mongoose
   let mongoServer: MongoMemoryServer
-  let accessToken: string
   const userController = new UserController()
   const permissionController = new PermissionController()
 
@@ -72,8 +76,6 @@ describe('stp', () => {
   })
 
   describe('execute', () => {
-    const testFilesFolder = `test-stp-${generateTimestamp()}`
-
     describe('get', () => {
       describe('with runtime js', () => {
         const testFilesFolder = `test-stp-${generateTimestamp()}`
@@ -93,41 +95,45 @@ describe('stp', () => {
         })
 
         it('should execute js program when both js and sas program are present', async () => {
-          const programPath = path.join(testFilesFolder, 'program')
+          await makeRequestAndAssert(
-          const sasProgramPath = path.join(filesFolder, `${programPath}.sas`)
+            [RunTimeType.JS, RunTimeType.SAS],
-          const jsProgramPath = path.join(filesFolder, `${programPath}.js`)
+            200,
-          await createFile(sasProgramPath, sampleSasProgram)
+            RunTimeType.JS
-          await createFile(jsProgramPath, sampleJsProgram)
-
-          await request(app)
-            .get(`/SASjsApi/stp/execute?_program=${programPath}`)
-            .auth(accessToken, { type: 'bearer' })
-            .send()
-            .expect(200)
-
-          expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith(
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            RunTimeType.JS,
-            expect.anything(),
-            undefined
           )
         })
 
         it('should throw error when js program is not present but sas program exists', async () => {
-          const programPath = path.join(testFilesFolder, 'program')
+          await makeRequestAndAssert([], 400)
-          const sasProgramPath = path.join(filesFolder, `${programPath}.sas`)
+        })
-          await createFile(sasProgramPath, sampleSasProgram)
+      })
 
-          await request(app)
+      describe('with runtime py', () => {
-            .get(`/SASjsApi/stp/execute?_program=${programPath}`)
+        const testFilesFolder = `test-stp-${generateTimestamp()}`
-            .auth(accessToken, { type: 'bearer' })
+
-            .send()
+        beforeAll(() => {
-            .expect(400)
+          process.runTimes = [RunTimeType.PY]
+        })
+
+        beforeEach(() => {
+          jest.resetModules() // it clears the cache
+          setupMocks()
+        })
+
+        afterEach(async () => {
+          jest.resetAllMocks()
+          await deleteFolder(path.join(filesFolder, testFilesFolder))
+        })
+
+        it('should execute python program when python, js and sas programs are present', async () => {
+          await makeRequestAndAssert(
+            [RunTimeType.PY, RunTimeType.SAS, RunTimeType.JS],
+            200,
+            RunTimeType.PY
+          )
+        })
+
+        it('should throw error when py program is not present but js or sas program exists', async () => {
+          await makeRequestAndAssert([], 400)
         })
       })
 
@@ -147,41 +153,11 @@ describe('stp', () => {
         })
 
         it('should execute sas program when both sas and js programs are present', async () => {
-          const programPath = path.join(testFilesFolder, 'program')
+          await makeRequestAndAssert([RunTimeType.SAS], 200, RunTimeType.SAS)
-          const sasProgramPath = path.join(filesFolder, `${programPath}.sas`)
-          const jsProgramPath = path.join(filesFolder, `${programPath}.js`)
-          await createFile(sasProgramPath, sampleSasProgram)
-          await createFile(jsProgramPath, sampleJsProgram)
-
-          await request(app)
-            .get(`/SASjsApi/stp/execute?_program=${programPath}`)
-            .auth(accessToken, { type: 'bearer' })
-            .send()
-            .expect(200)
-
-          expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith(
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            RunTimeType.SAS,
-            expect.anything(),
-            undefined
-          )
         })
 
         it('should throw error when sas program do not exit but js exists', async () => {
-          const programPath = path.join(testFilesFolder, 'program')
+          await makeRequestAndAssert([], 400)
-          const jsProgramPath = path.join(filesFolder, `${programPath}.js`)
-          await createFile(jsProgramPath, sampleJsProgram)
-
-          await request(app)
-            .get(`/SASjsApi/stp/execute?_program=${programPath}`)
-            .auth(accessToken, { type: 'bearer' })
-            .send()
-            .expect(400)
         })
       })
 
@@ -201,63 +177,51 @@ describe('stp', () => {
         })
 
         it('should execute js program when both js and sas program are present', async () => {
-          const programPath = path.join(testFilesFolder, 'program')
+          await makeRequestAndAssert(
-          const sasProgramPath = path.join(filesFolder, `${programPath}.sas`)
+            [RunTimeType.SAS, RunTimeType.JS],
-          const jsProgramPath = path.join(filesFolder, `${programPath}.js`)
+            200,
-          await createFile(sasProgramPath, sampleSasProgram)
+            RunTimeType.JS
-          await createFile(jsProgramPath, sampleJsProgram)
-
-          await request(app)
-            .get(`/SASjsApi/stp/execute?_program=${programPath}`)
-            .auth(accessToken, { type: 'bearer' })
-            .send()
-            .expect(200)
-
-          expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith(
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            RunTimeType.JS,
-            expect.anything(),
-            undefined
           )
         })
 
         it('should execute sas program when js program is not present but sas program exists', async () => {
-          const programPath = path.join(testFilesFolder, 'program')
+          await makeRequestAndAssert([RunTimeType.SAS], 200, RunTimeType.SAS)
-          const sasProgramPath = path.join(filesFolder, `${programPath}.sas`)
-          await createFile(sasProgramPath, sampleSasProgram)
-
-          await request(app)
-            .get(`/SASjsApi/stp/execute?_program=${programPath}`)
-            .auth(accessToken, { type: 'bearer' })
-            .send()
-            .expect(200)
-
-          expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith(
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            RunTimeType.SAS,
-            expect.anything(),
-            undefined
-          )
         })
 
         it('should throw error when both sas and js programs do not exist', async () => {
-          const programPath = path.join(testFilesFolder, 'program')
+          await makeRequestAndAssert([], 400)
+        })
+      })
 
-          await request(app)
+      describe('with runtime py and sas', () => {
-            .get(`/SASjsApi/stp/execute?_program=${programPath}`)
+        beforeAll(() => {
-            .auth(accessToken, { type: 'bearer' })
+          process.runTimes = [RunTimeType.PY, RunTimeType.SAS]
-            .send()
+        })
-            .expect(400)
+
+        beforeEach(() => {
+          jest.resetModules() // it clears the cache
+          setupMocks()
+        })
+
+        afterEach(async () => {
+          jest.resetAllMocks()
+          await deleteFolder(path.join(filesFolder, testFilesFolder))
+        })
+
+        it('should execute python program when both python and sas program are present', async () => {
+          await makeRequestAndAssert(
+            [RunTimeType.PY, RunTimeType.SAS],
+            200,
+            RunTimeType.PY
+          )
+        })
+
+        it('should execute sas program when python program is not present but sas program exists', async () => {
+          await makeRequestAndAssert([RunTimeType.SAS], 200, RunTimeType.SAS)
+        })
+
+        it('should throw error when both sas and js programs do not exist', async () => {
+          await makeRequestAndAssert([], 400)
         })
       })
 
@@ -277,76 +241,220 @@ describe('stp', () => {
         })
 
         it('should execute sas program when both sas and js programs  exist', async () => {
-          const programPath = path.join(testFilesFolder, 'program')
+          await makeRequestAndAssert(
-          const sasProgramPath = path.join(filesFolder, `${programPath}.sas`)
+            [RunTimeType.SAS, RunTimeType.JS],
-          const jsProgramPath = path.join(filesFolder, `${programPath}.js`)
+            200,
-          await createFile(sasProgramPath, sampleSasProgram)
+            RunTimeType.SAS
-          await createFile(jsProgramPath, sampleJsProgram)
-
-          await request(app)
-            .get(`/SASjsApi/stp/execute?_program=${programPath}`)
-            .auth(accessToken, { type: 'bearer' })
-            .send()
-            .expect(200)
-
-          expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith(
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            RunTimeType.SAS,
-            expect.anything(),
-            undefined
           )
         })
 
         it('should execute js program when sas program is not present but js program exists', async () => {
-          const programPath = path.join(testFilesFolder, 'program')
+          await makeRequestAndAssert([RunTimeType.JS], 200, RunTimeType.JS)
-          const jsProgramPath = path.join(filesFolder, `${programPath}.js`)
-          await createFile(jsProgramPath, sampleJsProgram)
-
-          await request(app)
-            .get(`/SASjsApi/stp/execute?_program=${programPath}`)
-            .auth(accessToken, { type: 'bearer' })
-            .send()
-            .expect(200)
-
-          expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith(
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            expect.anything(),
-            RunTimeType.JS,
-            expect.anything(),
-            undefined
-          )
         })
 
         it('should throw error when both sas and js programs do not exist', async () => {
-          const programPath = path.join(testFilesFolder, 'program')
+          await makeRequestAndAssert([], 400)
+        })
+      })
 
-          await request(app)
+      describe('with runtime sas and py', () => {
-            .get(`/SASjsApi/stp/execute?_program=${programPath}`)
+        beforeAll(() => {
-            .auth(accessToken, { type: 'bearer' })
+          process.runTimes = [RunTimeType.SAS, RunTimeType.PY]
-            .send()
+        })
-            .expect(400)
+
+        beforeEach(() => {
+          jest.resetModules() // it clears the cache
+          setupMocks()
+        })
+
+        afterEach(async () => {
+          jest.resetAllMocks()
+          await deleteFolder(path.join(filesFolder, testFilesFolder))
+        })
+
+        it('should execute sas program when both sas and python programs exist', async () => {
+          await makeRequestAndAssert(
+            [RunTimeType.SAS, RunTimeType.PY],
+            200,
+            RunTimeType.SAS
+          )
+        })
+
+        it('should execute python program when sas program is not present but python program exists', async () => {
+          await makeRequestAndAssert([RunTimeType.PY], 200, RunTimeType.PY)
+        })
+
+        it('should throw error when both sas and python programs do not exist', async () => {
+          await makeRequestAndAssert([], 400)
+        })
+      })
+
+      describe('with runtime sas, js and py', () => {
+        beforeAll(() => {
+          process.runTimes = [RunTimeType.SAS, RunTimeType.JS, RunTimeType.PY]
+        })
+
+        beforeEach(() => {
+          jest.resetModules() // it clears the cache
+          setupMocks()
+        })
+
+        afterEach(async () => {
+          jest.resetAllMocks()
+          await deleteFolder(path.join(filesFolder, testFilesFolder))
+        })
+
+        it('should execute sas program when it exists, no matter js and python programs exist or not', async () => {
+          await makeRequestAndAssert(
+            [RunTimeType.SAS, RunTimeType.PY, RunTimeType.JS],
+            200,
+            RunTimeType.SAS
+          )
+        })
+
+        it('should execute js program when sas program is absent but js and python programs are present', async () => {
+          await makeRequestAndAssert(
+            [RunTimeType.JS, RunTimeType.PY],
+            200,
+            RunTimeType.JS
+          )
+        })
+
+        it('should execute python program when both sas and js programs are not present', async () => {
+          await makeRequestAndAssert([RunTimeType.PY], 200, RunTimeType.PY)
+        })
+
+        it('should throw error when no program exists', async () => {
+          await makeRequestAndAssert([], 400)
+        })
+      })
+
+      describe('with runtime js, sas and py', () => {
+        beforeAll(() => {
+          process.runTimes = [RunTimeType.JS, RunTimeType.SAS, RunTimeType.PY]
+        })
+
+        beforeEach(() => {
+          jest.resetModules() // it clears the cache
+          setupMocks()
+        })
+
+        afterEach(async () => {
+          jest.resetAllMocks()
+          await deleteFolder(path.join(filesFolder, testFilesFolder))
+        })
+
+        it('should execute js program when it exists, no matter sas and python programs exist or not', async () => {
+          await makeRequestAndAssert(
+            [RunTimeType.JS, RunTimeType.SAS, RunTimeType.PY],
+            200,
+            RunTimeType.JS
+          )
+        })
+
+        it('should execute sas program when js program is absent but sas and python programs are present', async () => {
+          await makeRequestAndAssert(
+            [RunTimeType.SAS, RunTimeType.PY],
+            200,
+            RunTimeType.SAS
+          )
+        })
+
+        it('should execute python program when both sas and js programs are not present', async () => {
+          await makeRequestAndAssert([RunTimeType.PY], 200, RunTimeType.PY)
+        })
+
+        it('should throw error when no program exists', async () => {
+          await makeRequestAndAssert([], 400)
+        })
+      })
+
+      describe('with runtime py, sas and js', () => {
+        beforeAll(() => {
+          process.runTimes = [RunTimeType.PY, RunTimeType.SAS, RunTimeType.JS]
+        })
+
+        beforeEach(() => {
+          jest.resetModules() // it clears the cache
+          setupMocks()
+        })
+
+        afterEach(async () => {
+          jest.resetAllMocks()
+          await deleteFolder(path.join(filesFolder, testFilesFolder))
+        })
+
+        it('should execute python program when it exists, no matter sas and js programs exist or not', async () => {
+          await makeRequestAndAssert(
+            [RunTimeType.PY, RunTimeType.SAS, RunTimeType.JS],
+            200,
+            RunTimeType.PY
+          )
+        })
+
+        it('should execute sas program when python program is absent but sas and js programs are present', async () => {
+          await makeRequestAndAssert(
+            [RunTimeType.SAS, RunTimeType.JS],
+            200,
+            RunTimeType.SAS
+          )
+        })
+
+        it('should execute js program when both sas and python programs are not present', async () => {
+          await makeRequestAndAssert([RunTimeType.JS], 200, RunTimeType.JS)
+        })
+
+        it('should throw error when no program exists', async () => {
+          await makeRequestAndAssert([], 400)
         })
       })
     })
   })
 })
 
-const generateSaveTokenAndCreateUser = async (
+const makeRequestAndAssert = async (
-  someUser: any
+  programTypes: RunTimeType[],
-): Promise<string> => {
+  expectedStatusCode: number,
-  const userController = new UserController()
+  expectedRuntime?: RunTimeType
-  const dbUser = await userController.createUser(someUser)
+) => {
+  const programPath = path.join(testFilesFolder, 'program')
+  for (const programType of programTypes) {
+    if (programType === RunTimeType.JS)
+      await createFile(
+        path.join(filesFolder, `${programPath}.js`),
+        sampleJsProgram
+      )
+    else if (programType === RunTimeType.PY)
+      await createFile(
+        path.join(filesFolder, `${programPath}.py`),
+        samplePyProgram
+      )
+    else if (programType === RunTimeType.SAS)
+      await createFile(
+        path.join(filesFolder, `${programPath}.sas`),
+        sampleSasProgram
+      )
+  }
 
-  return generateAndSaveToken(dbUser.id)
+  await request(app)
+    .get(`/SASjsApi/stp/execute?_program=${programPath}`)
+    .auth(accessToken, { type: 'bearer' })
+    .send()
+    .expect(expectedStatusCode)
+
+  if (expectedRuntime)
+    expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith(
+      expect.anything(),
+      expect.anything(),
+      expect.anything(),
+      expect.anything(),
+      expect.anything(),
+      expect.anything(),
+      expect.anything(),
+      expectedRuntime,
+      expect.anything(),
+      undefined
+    )
 }
 
 const generateAndSaveToken = async (userId: number) => {
@@ -367,6 +475,10 @@ const setupMocks = async () => {
     .spyOn(JSSessionController.prototype, 'getSession')
     .mockImplementation(mockedGetSession)
 
+  jest
+    .spyOn(PythonSessionController.prototype, 'getSession')
+    .mockImplementation(mockedGetSession)
+
   jest
     .spyOn(ProcessProgramModule, 'processProgram')
     .mockImplementation(() => Promise.resolve())

api/src/types/system/process.d.ts

@@ -2,10 +2,12 @@ declare namespace NodeJS {
   export interface Process {
     sasLoc?: string
     nodeLoc?: string
+    pythonLoc?: string
     driveLoc: string
     logsLoc: string
     sasSessionController?: import('../../controllers/internal').SASSessionController
     jsSessionController?: import('../../controllers/internal').JSSessionController
+    pythonSessionController?: import('../../controllers/internal').PythonSessionController
     appStreamConfig: import('../').AppStreamConfig
     logger: import('@sasjs/utils/logger').Logger
     runTimes: import('../../utils').RunTimeType[]

api/src/utils/getDesktopFields.ts

@@ -4,9 +4,9 @@ import { createFolder, fileExists, folderExists, isWindows } from '@sasjs/utils'
 import { RunTimeType } from './verifyEnvVariables'
 
 export const getDesktopFields = async () => {
-  const { SAS_PATH, NODE_PATH } = process.env
+  const { SAS_PATH, NODE_PATH, PYTHON_PATH } = process.env
 
-  let sasLoc, nodeLoc
+  let sasLoc, nodeLoc, pythonLoc
 
   if (process.runTimes.includes(RunTimeType.SAS)) {
     sasLoc = SAS_PATH ?? (await getSASLocation())
@@ -16,7 +16,11 @@ export const getDesktopFields = async () => {
     nodeLoc = NODE_PATH ?? (await getNodeLocation())
   }
 
-  return { sasLoc, nodeLoc }
+  if (process.runTimes.includes(RunTimeType.JS)) {
+    pythonLoc = PYTHON_PATH ?? (await getPythonLocation())
+  }
+
+  return { sasLoc, nodeLoc, pythonLoc }
 }
 
 const getDriveLocation = async (): Promise<string> => {
@@ -91,3 +95,25 @@ const getNodeLocation = async (): Promise<string> => {
 
   return targetName
 }
+
+const getPythonLocation = async (): Promise<string> => {
+  const validator = async (filePath: string) => {
+    if (!filePath) return 'Path to Python executable is required.'
+
+    if (!(await fileExists(filePath))) {
+      return 'No file found at provided path.'
+    }
+
+    return true
+  }
+
+  const defaultLocation = isWindows() ? 'C:\\Python' : '/usr/bin/python'
+
+  const targetName = await getString(
+    'Please enter full path to a Python executable: ',
+    validator,
+    defaultLocation
+  )
+
+  return targetName
+}

api/src/utils/getRunTimeAndFilePath.ts

@@ -5,7 +5,7 @@ import { RunTimeType } from '.'
 
 export const getRunTimeAndFilePath = async (programPath: string) => {
   const ext = path.extname(programPath)
-  // If programPath (_program) is provided with a ".sas" or ".js" extension
+  // If programPath (_program) is provided with a ".sas", ".js" or ".py" extension
   // we should use that extension to determine the appropriate runTime
   if (ext && Object.values(RunTimeType).includes(ext.slice(1) as RunTimeType)) {
     const runTime = ext.slice(1)

api/src/utils/setProcessVariables.ts

@@ -28,11 +28,13 @@ export const setProcessVariables = async () => {
   if (MODE === ModeType.Server) {
     process.sasLoc = process.env.SAS_PATH
     process.nodeLoc = process.env.NODE_PATH
+    process.pythonLoc = process.env.PYTHON_PATH
   } else {
-    const { sasLoc, nodeLoc } = await getDesktopFields()
+    const { sasLoc, nodeLoc, pythonLoc } = await getDesktopFields()
 
     process.sasLoc = sasLoc
     process.nodeLoc = nodeLoc
+    process.pythonLoc = pythonLoc
   }
 
   const { SASJS_ROOT } = process.env

api/src/utils/upload.ts

@@ -126,9 +126,34 @@ export const generateFileUploadJSCode = async (
     }
   })
 
-  if (fileCount) {
+  uploadCode += `\nconst _WEBIN_FILE_COUNT = ${fileCount}`
-    uploadCode = `\nconst _WEBIN_FILE_COUNT = ${fileCount}` + uploadCode
+
-  }
+  return uploadCode
+}
+
+/**
+ * Generates the python code that references uploaded files in the concurrent request
+ * @param filesNamesMap object that maps hashed file names and original file names
+ * @param sessionFolder name of the folder that is created for the purpose of files in concurrent request
+ * @returns generated python code
+ */
+export const generateFileUploadPythonCode = async (
+  filesNamesMap: FilenamesMap,
+  sessionFolder: string
+) => {
+  let uploadCode = ''
+  let fileCount = 0
+
+  const sessionFolderList: string[] = await listFilesInFolder(sessionFolder)
+  sessionFolderList.forEach(async (fileName) => {
+    if (fileName.includes('req_file')) {
+      fileCount++
+      uploadCode += `\n_WEBIN_FILENAME${fileCount} = '${filesNamesMap[fileName].originalName}'`
+      uploadCode += `\n_WEBIN_NAME${fileCount} = '${filesNamesMap[fileName].fieldName}'`
+    }
+  })
+
+  uploadCode += `\n_WEBIN_FILE_COUNT = ${fileCount}`
 
   return uploadCode
 }

api/src/utils/verifyEnvVariables.ts

@@ -28,7 +28,8 @@ export enum LOG_FORMAT_MORGANType {
 
 export enum RunTimeType {
   SAS = 'sas',
-  JS = 'js'
+  JS = 'js',
+  PY = 'py'
 }
 
 export enum ReturnCode {
@@ -228,7 +229,7 @@ const verifyRUN_TIMES = (): string[] => {
 
 const verifyExecutablePaths = () => {
   const errors: string[] = []
-  const { RUN_TIMES, SAS_PATH, NODE_PATH, MODE } = process.env
+  const { RUN_TIMES, SAS_PATH, NODE_PATH, PYTHON_PATH, MODE } = process.env
 
   if (MODE === ModeType.Server) {
     const runTimes = RUN_TIMES?.split(',')
@@ -240,6 +241,10 @@ const verifyExecutablePaths = () => {
     if (runTimes?.includes(RunTimeType.JS) && !NODE_PATH) {
       errors.push(`- NODE_PATH is required for ${RunTimeType.JS} run time`)
     }
+
+    if (runTimes?.includes(RunTimeType.PY) && !PYTHON_PATH) {
+      errors.push(`- PYTHON_PATH is required for ${RunTimeType.PY} run time`)
+    }
   }
 
   return errors