feat: implement the logic for running python stored programs

README.md

@@ -64,12 +64,22 @@ Example contents of a `.env` file:
 # Server mode is multi-user and suitable for intranet / internet use
 MODE=
 
+# A comma separated string that defines the available runTimes.
+# Priority is given to the runtime that comes first in the string.
+# Possible options at the moment are sas and js
+
+# options: [sas|js|py|sas,js|js,sas|py,sas|sas,py|py,js|js,py|sas,js,py|sas,py,js|js,sas,py|js,py,sas|py,sas,js|py,js,sas] default:sas
+RUN_TIMES=
+
 # Path to SAS executable (sas.exe / sas.sh)
 SAS_PATH=/path/to/sas/executable.exe
 
 # Path to Node.js executable
 NODE_PATH=~/.nvm/versions/node/v16.14.0/bin/node
 
+# Path to Python executable
+PYTHON_PATH=/usr/bin/python
+
 # Path to working directory
 # This location is for SAS WORK, staged files, DRIVE, configuration etc
 SASJS_ROOT=./sasjs_root
@@ -139,13 +149,6 @@ LOG_FORMAT_MORGAN=
 # This location is for server logs with classical UNIX logrotate behavior
 LOG_LOCATION=./sasjs_root/logs
 
-# A comma separated string that defines the available runTimes.
-# Priority is given to the runtime that comes first in the string.
-# Possible options at the moment are sas and js
-
-# options: [sas,js|js,sas|sas|js] default:sas
-RUN_TIMES=
-
 ```
 
 ## Persisting the Session

api/.env.example

@@ -14,9 +14,10 @@ HELMET_COEP=[true|false] if omitted HELMET default will be used
 
 DB_CONNECT=mongodb+srv://<DB_USERNAME>:<DB_PASSWORD>@<CLUSTER>/<DB_NAME>?retryWrites=true&w=majority
 
-RUN_TIMES=[sas|js|sas,js|js,sas] default considered as sas
+RUN_TIMES=[sas|js|py|sas,js|js,sas|py,sas|sas,py|py,js|js,py|sas,js,py|sas,py,js|js,sas,py|js,py,sas|py,sas,js|py,js,sas] default considered as sas
 SAS_PATH=/opt/sas/sas9/SASHome/SASFoundation/9.4/sas
 NODE_PATH=~/.nvm/versions/node/v16.14.0/bin/node
+PYTHON_PATH=/usr/bin/python
 
 SASJS_ROOT=./sasjs_root
 

api/src/controllers/internal/Session.ts

@@ -197,9 +197,41 @@ export class JSSessionController extends SessionController {
   }
 }
 
+export class PythonSessionController extends SessionController {
+  protected async createSession(): Promise<Session> {
+    const sessionId = generateUniqueFileName(generateTimestamp())
+    const sessionFolder = path.join(getSessionsFolder(), sessionId)
+
+    const creationTimeStamp = sessionId.split('-').pop() as string
+    // death time of session is 15 mins from creation
+    const deathTimeStamp = (
+      parseInt(creationTimeStamp) +
+      15 * 60 * 1000 -
+      1000
+    ).toString()
+
+    const session: Session = {
+      id: sessionId,
+      ready: true,
+      inUse: true,
+      consumed: false,
+      completed: false,
+      creationTimeStamp,
+      deathTimeStamp,
+      path: sessionFolder
+    }
+
+    const headersPath = path.join(session.path, 'stpsrv_header.txt')
+    await createFile(headersPath, 'Content-type: application/json')
+
+    this.sessions.push(session)
+    return session
+  }
+}
+
 export const getSessionController = (
   runTime: RunTimeType
-): SASSessionController | JSSessionController => {
+): SASSessionController | JSSessionController | PythonSessionController => {
   if (runTime === RunTimeType.SAS) {
     return getSASSessionController()
   }
@@ -208,6 +240,10 @@ export const getSessionController = (
     return getJSSessionController()
   }
 
+  if (runTime === RunTimeType.PY) {
+    return getPythonSessionController()
+  }
+
   throw new Error('No Runtime is configured')
 }
 
@@ -227,6 +263,14 @@ const getJSSessionController = (): JSSessionController => {
   return process.jsSessionController
 }
 
+const getPythonSessionController = (): PythonSessionController => {
+  if (process.pythonSessionController) return process.pythonSessionController
+
+  process.pythonSessionController = new PythonSessionController()
+
+  return process.pythonSessionController
+}
+
 const autoExecContent = `
 data _null_;
   /* remove the dummy SYSIN */

api/src/controllers/internal/createPythonProgram.ts

@@ -0,0 +1,65 @@
+import { isWindows } from '@sasjs/utils'
+import { PreProgramVars, Session } from '../../types'
+import { generateFileUploadPythonCode } from '../../utils'
+import { ExecutionVars } from './'
+
+export const createPythonProgram = async (
+  program: string,
+  preProgramVariables: PreProgramVars,
+  vars: ExecutionVars,
+  session: Session,
+  weboutPath: string,
+  tokenFile: string,
+  otherArgs?: any
+) => {
+  const varStatments = Object.keys(vars).reduce(
+    (computed: string, key: string) => `${computed}${key} = '${vars[key]}';\n`,
+    ''
+  )
+
+  const preProgramVarStatments = `
+_SASJS_SESSION_PATH = '${
+    isWindows() ? session.path.replace(/\\/g, '\\\\') : session.path
+  }';
+_WEBOUT = '${isWindows() ? weboutPath.replace(/\\/g, '\\\\') : weboutPath}'; 
+_SASJS_TOKENFILE = '${
+    isWindows() ? tokenFile.replace(/\\/g, '\\\\') : tokenFile
+  }';
+_SASJS_USERNAME = '${preProgramVariables?.username}';
+_SASJS_USERID = '${preProgramVariables?.userId}';
+_SASJS_DISPLAYNAME = '${preProgramVariables?.displayName}';
+_METAPERSON = _SASJS_DISPLAYNAME;
+_METAUSER = _SASJS_USERNAME;
+SASJSPROCESSMODE = 'Stored Program';
+`
+
+  const requiredModules = `import os`
+
+  program = `
+# runtime vars
+${varStatments}
+
+# dynamic user-provided vars
+${preProgramVarStatments}
+
+# change working directory to  session folder
+os.chdir(_SASJS_SESSION_PATH)
+
+# actual job code
+${program}
+
+`
+  // if no files are uploaded filesNamesMap will be undefined
+  if (otherArgs?.filesNamesMap) {
+    const uploadJSCode = await generateFileUploadPythonCode(
+      otherArgs.filesNamesMap,
+      session.path
+    )
+
+    //If js code for the file is generated it will be appended to the top of jsCode
+    if (uploadJSCode.length > 0) {
+      program = `${uploadJSCode}\n` + program
+    }
+  }
+  return requiredModules + program
+}

api/src/controllers/internal/index.ts

@@ -4,4 +4,5 @@ export * from './Execution'
 export * from './FileUploadController'
 export * from './createSASProgram'
 export * from './createJSProgram'
+export * from './createPythonProgram'
 export * from './processProgram'

api/src/controllers/internal/processProgram.ts

@@ -5,7 +5,12 @@ import { once } from 'stream'
 import { createFile, moveFile } from '@sasjs/utils'
 import { PreProgramVars, Session } from '../../types'
 import { RunTimeType } from '../../utils'
-import { ExecutionVars, createSASProgram, createJSProgram } from './'
+import {
+  ExecutionVars,
+  createSASProgram,
+  createJSProgram,
+  createPythonProgram
+} from './'
 
 export const processProgram = async (
   program: string,
@@ -47,6 +52,42 @@ export const processProgram = async (
       // copy the code.js program to log and end write stream
       writeStream.end(program)
 
+      session.completed = true
+      console.log('session completed', session)
+    } catch (err: any) {
+      session.completed = true
+      session.crashed = err.toString()
+      console.log('session crashed', session.id, session.crashed)
+    }
+  } else if (runTime === RunTimeType.PY) {
+    program = await createPythonProgram(
+      program,
+      preProgramVariables,
+      vars,
+      session,
+      weboutPath,
+      tokenFile,
+      otherArgs
+    )
+
+    const codePath = path.join(session.path, 'code.py')
+
+    try {
+      await createFile(codePath, program)
+
+      // create a stream that will write to console outputs to log file
+      const writeStream = fs.createWriteStream(logPath)
+
+      // waiting for the open event so that we can have underlying file descriptor
+      await once(writeStream, 'open')
+
+      execFileSync(process.pythonLoc!, [codePath], {
+        stdio: ['ignore', writeStream, writeStream]
+      })
+
+      // copy the code.py program to log and end write stream
+      writeStream.end(program)
+
       session.completed = true
       console.log('session completed', session)
     } catch (err: any) {

api/src/types/system/process.d.ts

@@ -2,10 +2,12 @@ declare namespace NodeJS {
   export interface Process {
     sasLoc?: string
     nodeLoc?: string
+    pythonLoc?: string
     driveLoc: string
     logsLoc: string
     sasSessionController?: import('../../controllers/internal').SASSessionController
     jsSessionController?: import('../../controllers/internal').JSSessionController
+    pythonSessionController?: import('../../controllers/internal').PythonSessionController
     appStreamConfig: import('../').AppStreamConfig
     logger: import('@sasjs/utils/logger').Logger
     runTimes: import('../../utils').RunTimeType[]

api/src/utils/getDesktopFields.ts

@@ -4,9 +4,9 @@ import { createFolder, fileExists, folderExists, isWindows } from '@sasjs/utils'
 import { RunTimeType } from './verifyEnvVariables'
 
 export const getDesktopFields = async () => {
-  const { SAS_PATH, NODE_PATH } = process.env
+  const { SAS_PATH, NODE_PATH, PYTHON_PATH } = process.env
 
-  let sasLoc, nodeLoc
+  let sasLoc, nodeLoc, pythonLoc
 
   if (process.runTimes.includes(RunTimeType.SAS)) {
     sasLoc = SAS_PATH ?? (await getSASLocation())
@@ -16,7 +16,11 @@ export const getDesktopFields = async () => {
     nodeLoc = NODE_PATH ?? (await getNodeLocation())
   }
 
-  return { sasLoc, nodeLoc }
+  if (process.runTimes.includes(RunTimeType.JS)) {
+    pythonLoc = PYTHON_PATH ?? (await getPythonLocation())
+  }
+
+  return { sasLoc, nodeLoc, pythonLoc }
 }
 
 const getDriveLocation = async (): Promise<string> => {
@@ -91,3 +95,25 @@ const getNodeLocation = async (): Promise<string> => {
 
   return targetName
 }
+
+const getPythonLocation = async (): Promise<string> => {
+  const validator = async (filePath: string) => {
+    if (!filePath) return 'Path to Python executable is required.'
+
+    if (!(await fileExists(filePath))) {
+      return 'No file found at provided path.'
+    }
+
+    return true
+  }
+
+  const defaultLocation = isWindows() ? 'C:\\Python' : '/usr/bin/python'
+
+  const targetName = await getString(
+    'Please enter full path to a Python executable: ',
+    validator,
+    defaultLocation
+  )
+
+  return targetName
+}

api/src/utils/getRunTimeAndFilePath.ts

@@ -5,7 +5,7 @@ import { RunTimeType } from '.'
 
 export const getRunTimeAndFilePath = async (programPath: string) => {
   const ext = path.extname(programPath)
-  // If programPath (_program) is provided with a ".sas" or ".js" extension
+  // If programPath (_program) is provided with a ".sas", ".js" or ".py" extension
   // we should use that extension to determine the appropriate runTime
   if (ext && Object.values(RunTimeType).includes(ext.slice(1) as RunTimeType)) {
     const runTime = ext.slice(1)

api/src/utils/setProcessVariables.ts

@@ -28,11 +28,13 @@ export const setProcessVariables = async () => {
   if (MODE === ModeType.Server) {
     process.sasLoc = process.env.SAS_PATH
     process.nodeLoc = process.env.NODE_PATH
+    process.pythonLoc = process.env.PYTHON_PATH
   } else {
-    const { sasLoc, nodeLoc } = await getDesktopFields()
+    const { sasLoc, nodeLoc, pythonLoc } = await getDesktopFields()
 
     process.sasLoc = sasLoc
     process.nodeLoc = nodeLoc
+    process.pythonLoc = pythonLoc
   }
 
   const { SASJS_ROOT } = process.env

api/src/utils/upload.ts

@@ -126,9 +126,34 @@ export const generateFileUploadJSCode = async (
     }
   })
 
-  if (fileCount) {
-    uploadCode = `\nconst _WEBIN_FILE_COUNT = ${fileCount}` + uploadCode
-  }
+  uploadCode += `\nconst _WEBIN_FILE_COUNT = ${fileCount}`
+
+  return uploadCode
+}
+
+/**
+ * Generates the python code that references uploaded files in the concurrent request
+ * @param filesNamesMap object that maps hashed file names and original file names
+ * @param sessionFolder name of the folder that is created for the purpose of files in concurrent request
+ * @returns generated python code
+ */
+export const generateFileUploadPythonCode = async (
+  filesNamesMap: FilenamesMap,
+  sessionFolder: string
+) => {
+  let uploadCode = ''
+  let fileCount = 0
+
+  const sessionFolderList: string[] = await listFilesInFolder(sessionFolder)
+  sessionFolderList.forEach(async (fileName) => {
+    if (fileName.includes('req_file')) {
+      fileCount++
+      uploadCode += `\n_WEBIN_FILENAME${fileCount} = '${filesNamesMap[fileName].originalName}'`
+      uploadCode += `\n_WEBIN_NAME${fileCount} = '${filesNamesMap[fileName].fieldName}'`
+    }
+  })
+
+  uploadCode += `\n_WEBIN_FILE_COUNT = ${fileCount}`
 
   return uploadCode
 }

api/src/utils/verifyEnvVariables.ts

@@ -28,7 +28,8 @@ export enum LOG_FORMAT_MORGANType {
 
 export enum RunTimeType {
   SAS = 'sas',
-  JS = 'js'
+  JS = 'js',
+  PY = 'py'
 }
 
 export enum ReturnCode {
@@ -228,7 +229,7 @@ const verifyRUN_TIMES = (): string[] => {
 
 const verifyExecutablePaths = () => {
   const errors: string[] = []
-  const { RUN_TIMES, SAS_PATH, NODE_PATH, MODE } = process.env
+  const { RUN_TIMES, SAS_PATH, NODE_PATH, PYTHON_PATH, MODE } = process.env
 
   if (MODE === ModeType.Server) {
     const runTimes = RUN_TIMES?.split(',')
@@ -240,6 +241,10 @@ const verifyExecutablePaths = () => {
     if (runTimes?.includes(RunTimeType.JS) && !NODE_PATH) {
       errors.push(`- NODE_PATH is required for ${RunTimeType.JS} run time`)
     }
+
+    if (runTimes?.includes(RunTimeType.PY) && !PYTHON_PATH) {
+      errors.push(`- PYTHON_PATH is required for ${RunTimeType.PY} run time`)
+    }
   }
 
   return errors