diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e93774d..6f74e93 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -55,6 +55,9 @@ jobs: REFRESH_TOKEN_SECRET: ${{secrets.REFRESH_TOKEN_SECRET}} AUTH_CODE_SECRET: ${{secrets.AUTH_CODE_SECRET}} SESSION_SECRET: ${{secrets.SESSION_SECRET}} + RUN_TIMES: 'sas,js' + SAS_PATH: '/some/path/to/sas' + NODE_PATH: '/some/path/to/node' - name: Build Package working-directory: ./api diff --git a/.nvmrc b/.nvmrc index e2838c8..0a2cd98 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -v16.14.0 \ No newline at end of file +v16.15.1 \ No newline at end of file diff --git a/README.md b/README.md index 992bede..365b4c2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ # SASjs Server + + [![All Contributors](https://img.shields.io/badge/all_contributors-7-orange.svg?style=flat-square)](#contributors-) + SASjs Server provides a NodeJS wrapper for calling the SAS binary executable. It can be installed on an actual SAS server, or locally on your desktop. It provides: @@ -64,10 +67,14 @@ MODE= # Path to SAS executable (sas.exe / sas.sh) SAS_PATH=/path/to/sas/executable.exe +# Path to Node.js executable +NODE_PATH=~/.nvm/versions/node/v16.14.0/bin/node + # Path to working directory # This location is for SAS WORK, staged files, DRIVE, configuration etc SASJS_ROOT=./sasjs_root + # options: [http|https] default: http PROTOCOL= @@ -132,6 +139,13 @@ HELMET_CSP_CONFIG_PATH=./csp.config.json # Docs: https://www.npmjs.com/package/morgan#predefined-formats LOG_FORMAT_MORGAN= +# A comma separated string that defines the available runTimes. +# Priority is given to the runtime that cSAS_PATHomes first in string. +# Possible options at the moment are sas and js + +# options: [sas,js|js,sas|sas|js] default:sas,js +RUN_TIMES= + ``` ## Persisting the Session @@ -213,4 +227,4 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d -This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! \ No newline at end of file +This project follows the [all-contributors](https://github.com/all-contributors/all-contributors) specification. Contributions of any kind welcome! diff --git a/api/.env.example b/api/.env.example index ab6de3e..cdb2431 100644 --- a/api/.env.example +++ b/api/.env.example @@ -17,7 +17,10 @@ AUTH_CODE_SECRET= SESSION_SECRET= DB_CONNECT=mongodb+srv://:@/?retryWrites=true&w=majority +RUN_TIMES=[sas|js|sas,js|js,sas] default considered as sas,js SAS_PATH=/opt/sas/sas9/SASHome/SASFoundation/9.4/sas +NODE_PATH=~/.nvm/versions/node/v16.14.0/bin/node + SASJS_ROOT=./sasjs_root LOG_FORMAT_MORGAN=common \ No newline at end of file diff --git a/api/.nvmrc b/api/.nvmrc index e2838c8..0a2cd98 100644 --- a/api/.nvmrc +++ b/api/.nvmrc @@ -1 +1 @@ -v16.14.0 \ No newline at end of file +v16.15.1 \ No newline at end of file diff --git a/api/public/swagger.yaml b/api/public/swagger.yaml index 832e335..0ab1c0e 100644 --- a/api/public/swagger.yaml +++ b/api/public/swagger.yaml @@ -139,14 +139,24 @@ components: - httpHeaders type: object additionalProperties: false + RunTimeType: + enum: + - sas + - js + type: string ExecuteSASCodePayload: properties: code: type: string - description: 'Code of SAS program' - example: '* SAS Code HERE;' + description: 'Code of program' + example: '* Code HERE;' + runTime: + $ref: '#/components/schemas/RunTimeType' + description: 'runtime for program' + example: js required: - code + - runTime type: object additionalProperties: false MemberType.folder: @@ -458,11 +468,16 @@ components: type: array protocol: type: string + runTimes: + items: + type: string + type: array required: - mode - cors - whiteList - protocol + - runTimes type: object additionalProperties: false ExecuteReturnJsonPayload: @@ -1401,7 +1416,7 @@ paths: $ref: '#/components/schemas/InfoResponse' examples: 'Example 1': - value: {mode: desktop, cors: enable, whiteList: ['http://example.com', 'http://example2.com'], protocol: http} + value: {mode: desktop, cors: enable, whiteList: ['http://example.com', 'http://example2.com'], protocol: http, runTimes: [sas, js]} summary: 'Get server info (mode, cors, whiteList, protocol).' tags: - Info @@ -1439,8 +1454,8 @@ paths: anyOf: - {type: string} - {type: string, format: byte} - description: "Trigger a SAS program using it's location in the _program URL parameter.\nEnable debugging using the _debug URL parameter. Setting _debug=131 will\ncause the log to be streamed in the output.\n\nAdditional URL parameters are turned into SAS macro variables.\n\nAny files provided in the request body are placed into the SAS session with\ncorresponding _WEBIN_XXX variables created.\n\nThe response headers can be adjusted using the mfs_httpheader() macro. Any\nfile type can be returned, including binary files such as zip or xls.\n\nIf _debug is >= 131, response headers will contain Content-Type: 'text/plain'\n\nThis behaviour differs for POST requests, in which case the response is\nalways JSON." - summary: 'Execute Stored Program, return raw _webout content.' + description: "Trigger a SAS or JS program using the _program URL parameter.\n\nAccepts URL parameters and file uploads. For more details, see docs:\n\nhttps://server.sasjs.io/storedprograms" + summary: 'Execute a Stored Program, returns raw _webout content.' tags: - STP security: @@ -1448,13 +1463,13 @@ paths: bearerAuth: [] parameters: - - description: 'Location of SAS program' + description: 'Location of SAS or JS code' in: query name: _program required: true schema: type: string - example: /Public/somefolder/some.file + example: /Projects/myApp/some/program post: operationId: ExecuteReturnJson responses: @@ -1467,8 +1482,8 @@ paths: examples: 'Example 1': value: {status: success, _webout: 'webout content', log: [], httpHeaders: {Content-type: application/zip, Cache-Control: 'public, max-age=1000'}} - description: "Trigger a SAS program using it's location in the _program URL parameter.\nEnable debugging using the _debug URL parameter. In any case, the log is\nalways returned in the log object.\n\nAdditional URL parameters are turned into SAS macro variables.\n\nAny files provided in the request body are placed into the SAS session with\ncorresponding _WEBIN_XXX variables created.\n\nThe response will be a JSON object with the following root attributes: log,\nwebout, headers.\n\nThe webout will be a nested JSON object ONLY if the response-header\ncontains a content-type of application/json AND it is valid JSON.\nOtherwise it will be a stringified version of the webout content.\n\nResponse headers from the mfs_httpheader macro are simply listed in the\nheaders object, for POST requests they have no effect on the actual\nresponse header." - summary: 'Execute Stored Program, return JSON' + description: "Trigger a SAS or JS program using the _program URL parameter.\n\nAccepts URL parameters and file uploads. For more details, see docs:\n\nhttps://server.sasjs.io/storedprograms\n\nThe response will be a JSON object with the following root attributes:\nlog, webout, headers.\n\nThe webout attribute will be nested JSON ONLY if the response-header\ncontains a content-type of application/json AND it is valid JSON.\nOtherwise it will be a stringified version of the webout content." + summary: 'Execute a Stored Program, return a JSON object' tags: - STP security: @@ -1476,13 +1491,13 @@ paths: bearerAuth: [] parameters: - - description: 'Location of SAS program' + description: 'Location of SAS or JS code' in: query name: _program required: false schema: type: string - example: /Public/somefolder/some.file + example: /Projects/myApp/some/program requestBody: required: false content: diff --git a/api/src/controllers/code.ts b/api/src/controllers/code.ts index a4b1d74..1e09c4d 100644 --- a/api/src/controllers/code.ts +++ b/api/src/controllers/code.ts @@ -6,15 +6,21 @@ import { getPreProgramVariables, getUserAutoExec, ModeType, - parseLogToArray + parseLogToArray, + RunTimeType } from '../utils' -interface ExecuteSASCodePayload { +interface ExecuteCodePayload { /** - * Code of SAS program - * @example "* SAS Code HERE;" + * Code of program + * @example "* Code HERE;" */ code: string + /** + * runtime for program + * @example "js" + */ + runTime: RunTimeType } @Security('bearerAuth') @@ -26,17 +32,17 @@ export class CodeController { * @summary Run SAS Code and returns log */ @Post('/execute') - public async executeSASCode( + public async executeCode( @Request() request: express.Request, - @Body() body: ExecuteSASCodePayload + @Body() body: ExecuteCodePayload ): Promise { - return executeSASCode(request, body) + return executeCode(request, body) } } -const executeSASCode = async ( +const executeCode = async ( req: express.Request, - { code }: ExecuteSASCodePayload + { code, runTime }: ExecuteCodePayload ) => { const { user } = req const userAutoExec = @@ -46,13 +52,14 @@ const executeSASCode = async ( try { const { webout, log, httpHeaders } = - (await new ExecutionController().executeProgram( - code, - getPreProgramVariables(req), - { ...req.query, _debug: 131 }, - { userAutoExec }, - true - )) as ExecuteReturnJson + (await new ExecutionController().executeProgram({ + program: code, + preProgramVariables: getPreProgramVariables(req), + vars: { ...req.query, _debug: 131 }, + otherArgs: { userAutoExec }, + returnJson: true, + runTime: runTime + })) as ExecuteReturnJson return { status: 'success', diff --git a/api/src/controllers/info.ts b/api/src/controllers/info.ts index fca2604..5c4cf86 100644 --- a/api/src/controllers/info.ts +++ b/api/src/controllers/info.ts @@ -5,6 +5,7 @@ export interface InfoResponse { cors: string whiteList: string[] protocol: string + runTimes: string[] } @Route('SASjsApi/info') @@ -18,7 +19,8 @@ export class InfoController { mode: 'desktop', cors: 'enable', whiteList: ['http://example.com', 'http://example2.com'], - protocol: 'http' + protocol: 'http', + runTimes: ['sas', 'js'] }) @Get('/') public info(): InfoResponse { @@ -29,7 +31,8 @@ export class InfoController { (process.env.MODE === 'server' ? 'disable' : 'enable'), whiteList: process.env.WHITELIST?.split(' ')?.filter((url) => !!url) ?? [], - protocol: process.env.PROTOCOL ?? 'http' + protocol: process.env.PROTOCOL ?? 'http', + runTimes: process.runTimes } return response } diff --git a/api/src/controllers/internal/Execution.ts b/api/src/controllers/internal/Execution.ts index 141942a..b742402 100644 --- a/api/src/controllers/internal/Execution.ts +++ b/api/src/controllers/internal/Execution.ts @@ -1,21 +1,18 @@ import path from 'path' import fs from 'fs' -import { getSessionController } from './' import { - readFile, - fileExists, - createFile, - moveFile, - readFileBinary -} from '@sasjs/utils' + getSASSessionController, + getJSSessionController, + processProgram +} from './' +import { readFile, fileExists, createFile, readFileBinary } from '@sasjs/utils' import { PreProgramVars, Session, TreeNode } from '../../types' import { extractHeaders, - generateFileUploadSasCode, getFilesFolder, - getMacrosFolder, HTTPHeaders, - isDebugOn + isDebugOn, + RunTimeType } from '../../utils' export interface ExecutionVars { @@ -33,39 +30,56 @@ export interface ExecuteReturnJson { log?: string } -export class ExecutionController { - async executeFile( - programPath: string, - preProgramVariables: PreProgramVars, - vars: ExecutionVars, - otherArgs?: any, - returnJson?: boolean, - session?: Session - ) { - if (!(await fileExists(programPath))) - throw `The Stored Program at (${vars._program}) does not exist, or you do not have permission to view it.` +interface ExecuteFileParams { + programPath: string + preProgramVariables: PreProgramVars + vars: ExecutionVars + otherArgs?: any + returnJson?: boolean + session?: Session + runTime: RunTimeType +} +interface ExecuteProgramParams extends Omit { + program: string +} + +export class ExecutionController { + async executeFile({ + programPath, + preProgramVariables, + vars, + otherArgs, + returnJson, + session, + runTime + }: ExecuteFileParams) { const program = await readFile(programPath) - return this.executeProgram( + return this.executeProgram({ program, preProgramVariables, vars, otherArgs, returnJson, - session - ) + session, + runTime + }) } - async executeProgram( - program: string, - preProgramVariables: PreProgramVars, - vars: ExecutionVars, - otherArgs?: any, - returnJson?: boolean, - sessionByFileUpload?: Session - ): Promise { - const sessionController = getSessionController() + async executeProgram({ + program, + preProgramVariables, + vars, + otherArgs, + returnJson, + session: sessionByFileUpload, + runTime + }: ExecuteProgramParams): Promise { + const sessionController = + runTime === RunTimeType.SAS + ? getSASSessionController() + : getJSSessionController() const session = sessionByFileUpload ?? (await sessionController.getSession()) @@ -83,78 +97,18 @@ export class ExecutionController { preProgramVariables?.httpHeaders.join('\n') ?? '' ) - const varStatments = Object.keys(vars).reduce( - (computed: string, key: string) => - `${computed}%let ${key}=${vars[key]};\n`, - '' + await processProgram( + program, + preProgramVariables, + vars, + session, + weboutPath, + tokenFile, + runTime, + logPath, + otherArgs ) - const preProgramVarStatments = ` -%let _sasjs_tokenfile=${tokenFile}; -%let _sasjs_username=${preProgramVariables?.username}; -%let _sasjs_userid=${preProgramVariables?.userId}; -%let _sasjs_displayname=${preProgramVariables?.displayName}; -%let _sasjs_apiserverurl=${preProgramVariables?.serverUrl}; -%let _sasjs_apipath=/SASjsApi/stp/execute; -%let _metaperson=&_sasjs_displayname; -%let _metauser=&_sasjs_username; -%let sasjsprocessmode=Stored Program; -%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/../stpsrv_header.txt; - -%global SYSPROCESSMODE SYSTCPIPHOSTNAME SYSHOSTINFOLONG; -%macro _sasjs_server_init(); - %if "&SYSPROCESSMODE"="" %then %let SYSPROCESSMODE=&sasjsprocessmode; - %if "&SYSTCPIPHOSTNAME"="" %then %let SYSTCPIPHOSTNAME=&_sasjs_apiserverurl; -%mend; -%_sasjs_server_init() -` - - program = ` -options insert=(SASAUTOS="${getMacrosFolder()}"); - -/* runtime vars */ -${varStatments} -filename _webout "${weboutPath}" mod; - -/* dynamic user-provided vars */ -${preProgramVarStatments} - -/* user autoexec starts */ -${otherArgs?.userAutoExec ?? ''} -/* user autoexec ends */ - -/* actual job code */ -${program}` - - // if no files are uploaded filesNamesMap will be undefined - if (otherArgs?.filesNamesMap) { - const uploadSasCode = await generateFileUploadSasCode( - otherArgs.filesNamesMap, - session.path - ) - - //If sas code for the file is generated it will be appended to the top of sasCode - if (uploadSasCode.length > 0) { - program = `${uploadSasCode}` + program - } - } - - const codePath = path.join(session.path, 'code.sas') - - // Creating this file in a RUNNING session will break out - // the autoexec loop and actually execute the program - // but - given it will take several milliseconds to create - // (which can mean SAS trying to run a partial program, or - // failing due to file lock) we first create the file THEN - // we rename it. - await createFile(codePath + '.bkp', program) - await moveFile(codePath + '.bkp', codePath) - - // we now need to poll the session status - while (!session.completed) { - await delay(50) - } - const log = (await fileExists(logPath)) ? await readFile(logPath) : '' const headersContent = (await fileExists(headersPath)) ? await readFile(headersPath) @@ -228,5 +182,3 @@ ${program}` return root } } - -const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) diff --git a/api/src/controllers/internal/FileUploadController.ts b/api/src/controllers/internal/FileUploadController.ts index 9593b68..7c6737c 100644 --- a/api/src/controllers/internal/FileUploadController.ts +++ b/api/src/controllers/internal/FileUploadController.ts @@ -1,13 +1,18 @@ import { Request, RequestHandler } from 'express' import multer from 'multer' import { uuidv4 } from '@sasjs/utils' -import { getSessionController } from '.' +import { getSASSessionController, getJSSessionController } from '.' +import { + executeProgramRawValidation, + getRunTimeAndFilePath, + RunTimeType +} from '../../utils' export class FileUploadController { private storage = multer.diskStorage({ destination: function (req: Request, file: any, cb: any) { //Sending the intercepted files to the sessions subfolder - cb(null, req.sasSession?.path) + cb(null, req.sasjsSession?.path) }, filename: function (req: Request, file: any, cb: any) { //req_file prefix + unique hash added to sas request files @@ -20,15 +25,36 @@ export class FileUploadController { //It will intercept request and generate unique uuid to be used as a subfolder name //that will store the files uploaded public preUploadMiddleware: RequestHandler = async (req, res, next) => { - let session + const { error: errQ, value: query } = executeProgramRawValidation(req.query) + const { error: errB, value: body } = executeProgramRawValidation(req.body) - const sessionController = getSessionController() - session = await sessionController.getSession() + if (errQ && errB) return res.status(400).send(errB.details[0].message) + + const programPath = (query?._program ?? body?._program) as string + + let runTime + + try { + ;({ runTime } = await getRunTimeAndFilePath(programPath)) + } catch (err: any) { + res.status(400).send({ + status: 'failure', + message: 'Job execution failed', + error: typeof err === 'object' ? err.toString() : err + }) + } + + const sessionController = + runTime === RunTimeType.SAS + ? getSASSessionController() + : getJSSessionController() + + const session = await sessionController.getSession() // marking consumed true, so that it's not available // as readySession for any other request session.consumed = true - req.sasSession = session + req.sasjsSession = session next() } diff --git a/api/src/controllers/internal/Session.ts b/api/src/controllers/internal/Session.ts index 39c86e2..623ec85 100644 --- a/api/src/controllers/internal/Session.ts +++ b/api/src/controllers/internal/Session.ts @@ -17,12 +17,14 @@ import { const execFilePromise = promisify(execFile) -export class SessionController { - private sessions: Session[] = [] +abstract class SessionController { + protected sessions: Session[] = [] - private getReadySessions = (): Session[] => + protected getReadySessions = (): Session[] => this.sessions.filter((sess: Session) => sess.ready && !sess.consumed) + protected abstract createSession(): Promise + public async getSession() { const readySessions = this.getReadySessions() @@ -34,8 +36,10 @@ export class SessionController { return session } +} - private async createSession(): Promise { +export class SASSessionController extends SessionController { + protected async createSession(): Promise { const sessionId = generateUniqueFileName(generateTimestamp()) const sessionFolder = path.join(getSessionsFolder(), sessionId) @@ -158,12 +162,52 @@ ${autoExecContent}` } } -export const getSessionController = (): SessionController => { - if (process.sessionController) return process.sessionController +export class JSSessionController extends SessionController { + protected async createSession(): Promise { + const sessionId = generateUniqueFileName(generateTimestamp()) + const sessionFolder = path.join(getSessionsFolder(), sessionId) - process.sessionController = new SessionController() + const creationTimeStamp = sessionId.split('-').pop() as string + // death time of session is 15 mins from creation + const deathTimeStamp = ( + parseInt(creationTimeStamp) + + 15 * 60 * 1000 - + 1000 + ).toString() - return process.sessionController + const session: Session = { + id: sessionId, + ready: true, + inUse: true, + consumed: false, + completed: false, + creationTimeStamp, + deathTimeStamp, + path: sessionFolder + } + + const headersPath = path.join(session.path, 'stpsrv_header.txt') + await createFile(headersPath, 'Content-type: application/json') + + this.sessions.push(session) + return session + } +} + +export const getSASSessionController = (): SASSessionController => { + if (process.sasSessionController) return process.sasSessionController + + process.sasSessionController = new SASSessionController() + + return process.sasSessionController +} + +export const getJSSessionController = (): JSSessionController => { + if (process.jsSessionController) return process.jsSessionController + + process.jsSessionController = new JSSessionController() + + return process.jsSessionController } const autoExecContent = ` diff --git a/api/src/controllers/internal/createJSProgram.ts b/api/src/controllers/internal/createJSProgram.ts new file mode 100644 index 0000000..f2e3333 --- /dev/null +++ b/api/src/controllers/internal/createJSProgram.ts @@ -0,0 +1,64 @@ +import { PreProgramVars, Session } from '../../types' +import { generateFileUploadJSCode } from '../../utils' +import { ExecutionVars } from './' + +export const createJSProgram = async ( + program: string, + preProgramVariables: PreProgramVars, + vars: ExecutionVars, + session: Session, + weboutPath: string, + tokenFile: string, + otherArgs?: any +) => { + const varStatments = Object.keys(vars).reduce( + (computed: string, key: string) => + `${computed}const ${key} = '${vars[key]}';\n`, + '' + ) + + const preProgramVarStatments = ` +let _webout = ''; +const weboutPath = '${weboutPath}'; +const _sasjs_tokenfile = '${tokenFile}'; +const _sasjs_username = '${preProgramVariables?.username}'; +const _sasjs_userid = '${preProgramVariables?.userId}'; +const _sasjs_displayname = '${preProgramVariables?.displayName}'; +const _metaperson = _sasjs_displayname; +const _metauser = _sasjs_username; +const sasjsprocessmode = 'Stored Program'; +` + + const requiredModules = `const fs = require('fs')` + + program = ` +/* runtime vars */ +${varStatments} + +/* dynamic user-provided vars */ +${preProgramVarStatments} + +/* actual job code */ +${program} + +/* write webout file only if webout exists*/ +if (_webout) { + fs.writeFile(weboutPath, _webout, function (err) { + if (err) throw err; + }) +} +` + // if no files are uploaded filesNamesMap will be undefined + if (otherArgs?.filesNamesMap) { + const uploadJSCode = await generateFileUploadJSCode( + otherArgs.filesNamesMap, + session.path + ) + + //If js code for the file is generated it will be appended to the top of jsCode + if (uploadJSCode.length > 0) { + program = `${uploadJSCode}\n` + program + } + } + return requiredModules + program +} diff --git a/api/src/controllers/internal/createSASProgram.ts b/api/src/controllers/internal/createSASProgram.ts new file mode 100644 index 0000000..8669586 --- /dev/null +++ b/api/src/controllers/internal/createSASProgram.ts @@ -0,0 +1,69 @@ +import { PreProgramVars, Session } from '../../types' +import { generateFileUploadSasCode, getMacrosFolder } from '../../utils' +import { ExecutionVars } from './' + +export const createSASProgram = async ( + program: string, + preProgramVariables: PreProgramVars, + vars: ExecutionVars, + session: Session, + weboutPath: string, + tokenFile: string, + otherArgs?: any +) => { + const varStatments = Object.keys(vars).reduce( + (computed: string, key: string) => `${computed}%let ${key}=${vars[key]};\n`, + '' + ) + + const preProgramVarStatments = ` +%let _sasjs_tokenfile=${tokenFile}; +%let _sasjs_username=${preProgramVariables?.username}; +%let _sasjs_userid=${preProgramVariables?.userId}; +%let _sasjs_displayname=${preProgramVariables?.displayName}; +%let _sasjs_apiserverurl=${preProgramVariables?.serverUrl}; +%let _sasjs_apipath=/SASjsApi/stp/execute; +%let _metaperson=&_sasjs_displayname; +%let _metauser=&_sasjs_username; +%let sasjsprocessmode=Stored Program; +%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/../stpsrv_header.txt; + +%global SYSPROCESSMODE SYSTCPIPHOSTNAME SYSHOSTINFOLONG; +%macro _sasjs_server_init(); +%if "&SYSPROCESSMODE"="" %then %let SYSPROCESSMODE=&sasjsprocessmode; +%if "&SYSTCPIPHOSTNAME"="" %then %let SYSTCPIPHOSTNAME=&_sasjs_apiserverurl; +%mend; +%_sasjs_server_init() +` + + program = ` +options insert=(SASAUTOS="${getMacrosFolder()}"); + +/* runtime vars */ +${varStatments} +filename _webout "${weboutPath}" mod; + +/* dynamic user-provided vars */ +${preProgramVarStatments} + +/* user autoexec starts */ +${otherArgs?.userAutoExec ?? ''} +/* user autoexec ends */ + +/* actual job code */ +${program}` + + // if no files are uploaded filesNamesMap will be undefined + if (otherArgs?.filesNamesMap) { + const uploadSasCode = await generateFileUploadSasCode( + otherArgs.filesNamesMap, + session.path + ) + + //If sas code for the file is generated it will be appended to the top of sasCode + if (uploadSasCode.length > 0) { + program = `${uploadSasCode}` + program + } + } + return program +} diff --git a/api/src/controllers/internal/index.ts b/api/src/controllers/internal/index.ts index 86939a2..50672d7 100644 --- a/api/src/controllers/internal/index.ts +++ b/api/src/controllers/internal/index.ts @@ -2,3 +2,6 @@ export * from './deploy' export * from './Session' export * from './Execution' export * from './FileUploadController' +export * from './createSASProgram' +export * from './createJSProgram' +export * from './processProgram' diff --git a/api/src/controllers/internal/processProgram.ts b/api/src/controllers/internal/processProgram.ts new file mode 100644 index 0000000..f9f3f6b --- /dev/null +++ b/api/src/controllers/internal/processProgram.ts @@ -0,0 +1,86 @@ +import path from 'path' +import fs from 'fs' +import { execFileSync } from 'child_process' +import { once } from 'stream' +import { createFile, moveFile } from '@sasjs/utils' +import { PreProgramVars, Session } from '../../types' +import { RunTimeType } from '../../utils' +import { ExecutionVars, createSASProgram, createJSProgram } from './' + +export const processProgram = async ( + program: string, + preProgramVariables: PreProgramVars, + vars: ExecutionVars, + session: Session, + weboutPath: string, + tokenFile: string, + runTime: RunTimeType, + logPath: string, + otherArgs?: any +) => { + if (runTime === RunTimeType.JS) { + program = await createJSProgram( + program, + preProgramVariables, + vars, + session, + weboutPath, + tokenFile, + otherArgs + ) + + const codePath = path.join(session.path, 'code.js') + + try { + await createFile(codePath, program) + + // create a stream that will write to console outputs to log file + const writeStream = fs.createWriteStream(logPath) + + // waiting for the open event so that we can have underlying file descriptor + await once(writeStream, 'open') + + execFileSync(process.nodeLoc, [codePath], { + stdio: ['ignore', writeStream, writeStream] + }) + + // copy the code.js program to log and end write stream + writeStream.end(program) + + session.completed = true + console.log('session completed', session) + } catch (err: any) { + session.completed = true + session.crashed = err.toString() + console.log('session crashed', session.id, session.crashed) + } + } else { + program = await createSASProgram( + program, + preProgramVariables, + vars, + session, + weboutPath, + tokenFile, + otherArgs + ) + + const codePath = path.join(session.path, 'code.sas') + + // Creating this file in a RUNNING session will break out + // the autoexec loop and actually execute the program + // but - given it will take several milliseconds to create + // (which can mean SAS trying to run a partial program, or + // failing due to file lock) we first create the file THEN + // we rename it. + await createFile(codePath + '.bkp', program) + await moveFile(codePath + '.bkp', codePath) + + // we now need to poll the session status + while (!session.completed) { + await delay(50) + } + } +} + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) diff --git a/api/src/controllers/stp.ts b/api/src/controllers/stp.ts index 6a5d378..a32c706 100644 --- a/api/src/controllers/stp.ts +++ b/api/src/controllers/stp.ts @@ -1,5 +1,4 @@ import express from 'express' -import path from 'path' import { Request, Security, @@ -19,12 +18,12 @@ import { } from './internal' import { getPreProgramVariables, - getFilesFolder, HTTPHeaders, isDebugOn, LogLine, makeFilesNamesMap, - parseLogToArray + parseLogToArray, + getRunTimeAndFilePath } from '../utils' import { MulterFile } from '../types/Upload' @@ -52,26 +51,15 @@ export interface ExecuteReturnJsonResponse { @Tags('STP') export class STPController { /** - * Trigger a SAS program using it's location in the _program URL parameter. - * Enable debugging using the _debug URL parameter. Setting _debug=131 will - * cause the log to be streamed in the output. + * Trigger a SAS or JS program using the _program URL parameter. * - * Additional URL parameters are turned into SAS macro variables. + * Accepts URL parameters and file uploads. For more details, see docs: * - * Any files provided in the request body are placed into the SAS session with - * corresponding _WEBIN_XXX variables created. + * https://server.sasjs.io/storedprograms * - * The response headers can be adjusted using the mfs_httpheader() macro. Any - * file type can be returned, including binary files such as zip or xls. - * - * If _debug is >= 131, response headers will contain Content-Type: 'text/plain' - * - * This behaviour differs for POST requests, in which case the response is - * always JSON. - * - * @summary Execute Stored Program, return raw _webout content. - * @param _program Location of SAS program - * @example _program "/Public/somefolder/some.file" + * @summary Execute a Stored Program, returns raw _webout content. + * @param _program Location of SAS or JS code + * @example _program "/Projects/myApp/some/program" */ @Get('/execute') public async executeReturnRaw( @@ -82,29 +70,22 @@ export class STPController { } /** - * Trigger a SAS program using it's location in the _program URL parameter. - * Enable debugging using the _debug URL parameter. In any case, the log is - * always returned in the log object. + * Trigger a SAS or JS program using the _program URL parameter. * - * Additional URL parameters are turned into SAS macro variables. + * Accepts URL parameters and file uploads. For more details, see docs: * - * Any files provided in the request body are placed into the SAS session with - * corresponding _WEBIN_XXX variables created. + * https://server.sasjs.io/storedprograms * - * The response will be a JSON object with the following root attributes: log, - * webout, headers. + * The response will be a JSON object with the following root attributes: + * log, webout, headers. * - * The webout will be a nested JSON object ONLY if the response-header + * The webout attribute will be nested JSON ONLY if the response-header * contains a content-type of application/json AND it is valid JSON. * Otherwise it will be a stringified version of the webout content. * - * Response headers from the mfs_httpheader macro are simply listed in the - * headers object, for POST requests they have no effect on the actual - * response header. - * - * @summary Execute Stored Program, return JSON - * @param _program Location of SAS program - * @example _program "/Public/somefolder/some.file" + * @summary Execute a Stored Program, return a JSON object + * @param _program Location of SAS or JS code + * @example _program "/Projects/myApp/some/program" */ @Example({ status: 'success', @@ -131,18 +112,17 @@ const executeReturnRaw = async ( _program: string ): Promise => { const query = req.query as ExecutionVars - const sasCodePath = - path - .join(getFilesFolder(), _program) - .replace(new RegExp('/', 'g'), path.sep) + '.sas' try { + const { codePath, runTime } = await getRunTimeAndFilePath(_program) + const { result, httpHeaders } = - (await new ExecutionController().executeFile( - sasCodePath, - getPreProgramVariables(req), - query - )) as ExecuteReturnRaw + (await new ExecutionController().executeFile({ + programPath: codePath, + preProgramVariables: getPreProgramVariables(req), + vars: query, + runTime + })) as ExecuteReturnRaw // Should over-ride response header for debug // on GET request to see entire log rendering on browser. @@ -171,25 +151,23 @@ const executeReturnJson = async ( req: express.Request, _program: string ): Promise => { - const sasCodePath = - path - .join(getFilesFolder(), _program) - .replace(new RegExp('/', 'g'), path.sep) + '.sas' - const filesNamesMap = req.files?.length ? makeFilesNamesMap(req.files as MulterFile[]) : null try { + const { codePath, runTime } = await getRunTimeAndFilePath(_program) + const { webout, log, httpHeaders } = - (await new ExecutionController().executeFile( - sasCodePath, - getPreProgramVariables(req), - { ...req.query, ...req.body }, - { filesNamesMap: filesNamesMap }, - true, - req.sasSession - )) as ExecuteReturnJson + (await new ExecutionController().executeFile({ + programPath: codePath, + preProgramVariables: getPreProgramVariables(req), + vars: { ...req.query, ...req.body }, + otherArgs: { filesNamesMap: filesNamesMap }, + returnJson: true, + session: req.sasjsSession, + runTime + })) as ExecuteReturnJson let weboutRes: string | IRecordOfAny = webout if (httpHeaders['content-type']?.toLowerCase() === 'application/json') { diff --git a/api/src/routes/api/code.ts b/api/src/routes/api/code.ts index efeaccd..09171c0 100644 --- a/api/src/routes/api/code.ts +++ b/api/src/routes/api/code.ts @@ -1,5 +1,5 @@ import express from 'express' -import { runSASValidation } from '../../utils' +import { runCodeValidation } from '../../utils' import { CodeController } from '../../controllers/' const runRouter = express.Router() @@ -7,11 +7,11 @@ const runRouter = express.Router() const controller = new CodeController() runRouter.post('/execute', async (req, res) => { - const { error, value: body } = runSASValidation(req.body) + const { error, value: body } = runCodeValidation(req.body) if (error) return res.status(400).send(error.details[0].message) try { - const response = await controller.executeSASCode(req, body) + const response = await controller.executeCode(req, body) if (response instanceof Buffer) { res.writeHead(200, (req as any).sasHeaders) diff --git a/api/src/routes/api/spec/stp.spec.ts b/api/src/routes/api/spec/stp.spec.ts new file mode 100644 index 0000000..f80ab8b --- /dev/null +++ b/api/src/routes/api/spec/stp.spec.ts @@ -0,0 +1,383 @@ +import path from 'path' +import { Express } from 'express' +import mongoose, { Mongoose } from 'mongoose' +import { MongoMemoryServer } from 'mongodb-memory-server' +import request from 'supertest' +import appPromise from '../../../app' +import { UserController } from '../../../controllers/' +import { + generateAccessToken, + saveTokensInDB, + getFilesFolder, + RunTimeType, + generateUniqueFileName, + getSessionsFolder +} from '../../../utils' +import { createFile, generateTimestamp, deleteFolder } from '@sasjs/utils' +import { + SASSessionController, + JSSessionController +} from '../../../controllers/internal' +import * as ProcessProgramModule from '../../../controllers/internal/processProgram' +import { Session } from '../../../types' + +const clientId = 'someclientID' + +const user = { + displayName: 'Test User', + username: 'testUsername', + password: '87654321', + isAdmin: false, + isActive: true +} + +const sampleSasProgram = '%put hello world!;' +const sampleJsProgram = `console.log('hello world!/')` + +const filesFolder = getFilesFolder() + +describe('stp', () => { + let app: Express + let con: Mongoose + let mongoServer: MongoMemoryServer + let accessToken: string + + beforeAll(async () => { + app = await appPromise + mongoServer = await MongoMemoryServer.create() + con = await mongoose.connect(mongoServer.getUri()) + accessToken = await generateSaveTokenAndCreateUser(user) + }) + + afterAll(async () => { + await con.connection.dropDatabase() + await con.connection.close() + await mongoServer.stop() + }) + + describe('execute', () => { + const testFilesFolder = `test-stp-${generateTimestamp()}` + + describe('get', () => { + describe('with runtime js', () => { + const testFilesFolder = `test-stp-${generateTimestamp()}` + + beforeAll(() => { + process.runTimes = [RunTimeType.JS] + }) + + beforeEach(() => { + jest.resetModules() // it clears the cache + setupMocks() + }) + + afterEach(async () => { + jest.resetAllMocks() + await deleteFolder(path.join(filesFolder, testFilesFolder)) + }) + + it('should execute js program when both js and sas program are present', async () => { + const programPath = path.join(testFilesFolder, 'program') + const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) + const jsProgramPath = path.join(filesFolder, `${programPath}.js`) + await createFile(sasProgramPath, sampleSasProgram) + await createFile(jsProgramPath, sampleJsProgram) + + await request(app) + .get(`/SASjsApi/stp/execute?_program=${programPath}`) + .auth(accessToken, { type: 'bearer' }) + .send() + .expect(200) + + expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + RunTimeType.JS, + expect.anything(), + undefined + ) + }) + + it('should throw error when js program is not present but sas program exists', async () => { + const programPath = path.join(testFilesFolder, 'program') + const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) + await createFile(sasProgramPath, sampleSasProgram) + + await request(app) + .get(`/SASjsApi/stp/execute?_program=${programPath}`) + .auth(accessToken, { type: 'bearer' }) + .send() + .expect(400) + }) + }) + + describe('with runtime sas', () => { + beforeAll(() => { + process.runTimes = [RunTimeType.SAS] + }) + + beforeEach(() => { + jest.resetModules() // it clears the cache + setupMocks() + }) + + afterEach(async () => { + jest.resetAllMocks() + await deleteFolder(path.join(filesFolder, testFilesFolder)) + }) + + it('should execute sas program when both sas and js programs are present', async () => { + const programPath = path.join(testFilesFolder, 'program') + const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) + const jsProgramPath = path.join(filesFolder, `${programPath}.js`) + await createFile(sasProgramPath, sampleSasProgram) + await createFile(jsProgramPath, sampleJsProgram) + + await request(app) + .get(`/SASjsApi/stp/execute?_program=${programPath}`) + .auth(accessToken, { type: 'bearer' }) + .send() + .expect(200) + + expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + RunTimeType.SAS, + expect.anything(), + undefined + ) + }) + + it('should throw error when sas program do not exit but js exists', async () => { + const programPath = path.join(testFilesFolder, 'program') + const jsProgramPath = path.join(filesFolder, `${programPath}.js`) + await createFile(jsProgramPath, sampleJsProgram) + + await request(app) + .get(`/SASjsApi/stp/execute?_program=${programPath}`) + .auth(accessToken, { type: 'bearer' }) + .send() + .expect(400) + }) + }) + + describe('with runtime js and sas', () => { + beforeAll(() => { + process.runTimes = [RunTimeType.JS, RunTimeType.SAS] + }) + + beforeEach(() => { + jest.resetModules() // it clears the cache + setupMocks() + }) + + afterEach(async () => { + jest.resetAllMocks() + await deleteFolder(path.join(filesFolder, testFilesFolder)) + }) + + it('should execute js program when both js and sas program are present', async () => { + const programPath = path.join(testFilesFolder, 'program') + const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) + const jsProgramPath = path.join(filesFolder, `${programPath}.js`) + await createFile(sasProgramPath, sampleSasProgram) + await createFile(jsProgramPath, sampleJsProgram) + + await request(app) + .get(`/SASjsApi/stp/execute?_program=${programPath}`) + .auth(accessToken, { type: 'bearer' }) + .send() + .expect(200) + + expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + RunTimeType.JS, + expect.anything(), + undefined + ) + }) + + it('should execute sas program when js program is not present but sas program exists', async () => { + const programPath = path.join(testFilesFolder, 'program') + const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) + await createFile(sasProgramPath, sampleSasProgram) + + await request(app) + .get(`/SASjsApi/stp/execute?_program=${programPath}`) + .auth(accessToken, { type: 'bearer' }) + .send() + .expect(200) + + expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + RunTimeType.SAS, + expect.anything(), + undefined + ) + }) + + it('should throw error when both sas and js programs do not exist', async () => { + const programPath = path.join(testFilesFolder, 'program') + + await request(app) + .get(`/SASjsApi/stp/execute?_program=${programPath}`) + .auth(accessToken, { type: 'bearer' }) + .send() + .expect(400) + }) + }) + + describe('with runtime sas and js', () => { + beforeAll(() => { + process.runTimes = [RunTimeType.SAS, RunTimeType.JS] + }) + + beforeEach(() => { + jest.resetModules() // it clears the cache + setupMocks() + }) + + afterEach(async () => { + jest.resetAllMocks() + await deleteFolder(path.join(filesFolder, testFilesFolder)) + }) + + it('should execute sas program when both sas and js programs exist', async () => { + const programPath = path.join(testFilesFolder, 'program') + const sasProgramPath = path.join(filesFolder, `${programPath}.sas`) + const jsProgramPath = path.join(filesFolder, `${programPath}.js`) + await createFile(sasProgramPath, sampleSasProgram) + await createFile(jsProgramPath, sampleJsProgram) + + await request(app) + .get(`/SASjsApi/stp/execute?_program=${programPath}`) + .auth(accessToken, { type: 'bearer' }) + .send() + .expect(200) + + expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + RunTimeType.SAS, + expect.anything(), + undefined + ) + }) + + it('should execute js program when sas program is not present but js program exists', async () => { + const programPath = path.join(testFilesFolder, 'program') + const jsProgramPath = path.join(filesFolder, `${programPath}.js`) + await createFile(jsProgramPath, sampleJsProgram) + + await request(app) + .get(`/SASjsApi/stp/execute?_program=${programPath}`) + .auth(accessToken, { type: 'bearer' }) + .send() + .expect(200) + + expect(ProcessProgramModule.processProgram).toHaveBeenCalledWith( + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + expect.anything(), + RunTimeType.JS, + expect.anything(), + undefined + ) + }) + + it('should throw error when both sas and js programs do not exist', async () => { + const programPath = path.join(testFilesFolder, 'program') + + await request(app) + .get(`/SASjsApi/stp/execute?_program=${programPath}`) + .auth(accessToken, { type: 'bearer' }) + .send() + .expect(400) + }) + }) + }) + }) +}) + +const generateSaveTokenAndCreateUser = async ( + someUser: any +): Promise => { + const userController = new UserController() + const dbUser = await userController.createUser(someUser) + + return generateAndSaveToken(dbUser.id) +} + +const generateAndSaveToken = async (userId: number) => { + const accessToken = generateAccessToken({ + clientId, + userId + }) + await saveTokensInDB(userId, clientId, accessToken, 'refreshToken') + return accessToken +} + +const setupMocks = async () => { + jest + .spyOn(SASSessionController.prototype, 'getSession') + .mockImplementation(mockedGetSession) + + jest + .spyOn(JSSessionController.prototype, 'getSession') + .mockImplementation(mockedGetSession) + + jest + .spyOn(ProcessProgramModule, 'processProgram') + .mockImplementation(() => Promise.resolve()) +} + +const mockedGetSession = async () => { + const sessionId = generateUniqueFileName(generateTimestamp()) + const sessionFolder = path.join(getSessionsFolder(), sessionId) + + const creationTimeStamp = sessionId.split('-').pop() as string + // death time of session is 15 mins from creation + const deathTimeStamp = ( + parseInt(creationTimeStamp) + + 15 * 60 * 1000 - + 1000 + ).toString() + + const session: Session = { + id: sessionId, + ready: true, + inUse: true, + consumed: false, + completed: false, + creationTimeStamp, + deathTimeStamp, + path: sessionFolder + } + + return session +} diff --git a/api/src/routes/api/stp.ts b/api/src/routes/api/stp.ts index 5ea37db..858feb5 100644 --- a/api/src/routes/api/stp.ts +++ b/api/src/routes/api/stp.ts @@ -35,16 +35,17 @@ stpRouter.post( fileUploadController.preUploadMiddleware, fileUploadController.getMulterUploadObject().any(), async (req, res: any) => { - const { error: errQ, value: query } = executeProgramRawValidation(req.query) - const { error: errB, value: body } = executeProgramRawValidation(req.body) + // below validations are moved to preUploadMiddleware + // const { error: errQ, value: query } = executeProgramRawValidation(req.query) + // const { error: errB, value: body } = executeProgramRawValidation(req.body) - if (errQ && errB) return res.status(400).send(errB.details[0].message) + // if (errQ && errB) return res.status(400).send(errB.details[0].message) try { const response = await controller.executeReturnJson( req, - body, - query?._program + req.body, + req.query?._program as string ) // TODO: investigate if this code is required diff --git a/api/src/types/system/express.d.ts b/api/src/types/system/express.d.ts index 7ed4e79..d02d220 100644 --- a/api/src/types/system/express.d.ts +++ b/api/src/types/system/express.d.ts @@ -2,6 +2,6 @@ declare namespace Express { export interface Request { accessToken?: string user?: import('../').RequestUser - sasSession?: import('../').Session + sasjsSession?: import('../').Session } } diff --git a/api/src/types/system/process.d.ts b/api/src/types/system/process.d.ts index f7845bc..45f9572 100644 --- a/api/src/types/system/process.d.ts +++ b/api/src/types/system/process.d.ts @@ -1,9 +1,12 @@ declare namespace NodeJS { export interface Process { sasLoc: string + nodeLoc: string driveLoc: string - sessionController?: import('../../controllers/internal').SessionController + sasSessionController?: import('../../controllers/internal').SASSessionController + jsSessionController?: import('../../controllers/internal').JSSessionController appStreamConfig: import('../').AppStreamConfig logger: import('@sasjs/utils/logger').Logger + runTimes: import('../../utils').RunTimeType[] } } diff --git a/api/src/utils/getDesktopFields.ts b/api/src/utils/getDesktopFields.ts index af499a7..74999fa 100644 --- a/api/src/utils/getDesktopFields.ts +++ b/api/src/utils/getDesktopFields.ts @@ -5,12 +5,13 @@ import { createFolder, fileExists, folderExists } from '@sasjs/utils' const isWindows = () => process.platform === 'win32' export const getDesktopFields = async () => { - const { SAS_PATH } = process.env + const { SAS_PATH, NODE_PATH } = process.env const sasLoc = SAS_PATH ?? (await getSASLocation()) + const nodeLoc = NODE_PATH ?? (await getNodeLocation()) // const driveLoc = DRIVE_PATH ?? (await getDriveLocation()) - return { sasLoc } + return { sasLoc, nodeLoc } } const getDriveLocation = async (): Promise => { @@ -61,3 +62,27 @@ const getSASLocation = async (): Promise => { return targetName } + +const getNodeLocation = async (): Promise => { + const validator = async (filePath: string) => { + if (!filePath) return 'Path to NodeJS executable is required.' + + if (!(await fileExists(filePath))) { + return 'No file found at provided path.' + } + + return true + } + + const defaultLocation = isWindows() + ? 'C:\\Program Files\\nodejs\\' + : '/usr/local/nodejs/bin' + + const targetName = await getString( + 'Please enter path to nodejs executable (absolute path): ', + validator, + defaultLocation + ) + + return targetName +} diff --git a/api/src/utils/getRunTimeAndFilePath.ts b/api/src/utils/getRunTimeAndFilePath.ts new file mode 100644 index 0000000..e36c931 --- /dev/null +++ b/api/src/utils/getRunTimeAndFilePath.ts @@ -0,0 +1,37 @@ +import path from 'path' +import { fileExists } from '@sasjs/utils' +import { getFilesFolder } from './file' +import { RunTimeType } from '.' + +export const getRunTimeAndFilePath = async (programPath: string) => { + const ext = path.extname(programPath) + // if program path is provided with extension we should split that into code path and ext as run time + if (ext) { + const runTime = ext.slice(1) + const runTimeTypes = Object.values(RunTimeType) + + if (!runTimeTypes.includes(runTime as RunTimeType)) { + throw `The '${runTime}' runtime is not supported.` + } + + const codePath = path + .join(getFilesFolder(), programPath) + .replace(new RegExp('/', 'g'), path.sep) + + if (await fileExists(codePath)) { + return { codePath, runTime: runTime as RunTimeType } + } + } else { + for (const runTime of process.runTimes) { + const codePath = + path + .join(getFilesFolder(), programPath) + .replace(new RegExp('/', 'g'), path.sep) + + '.' + + runTime + + if (await fileExists(codePath)) return { codePath, runTime } + } + } + throw `The Program at (${programPath}) does not exist.` +} diff --git a/api/src/utils/index.ts b/api/src/utils/index.ts index 4fc0f8e..36e1efc 100644 --- a/api/src/utils/index.ts +++ b/api/src/utils/index.ts @@ -11,6 +11,7 @@ export * from './generateRefreshToken' export * from './getCertificates' export * from './getDesktopFields' export * from './getPreProgramVariables' +export * from './getRunTimeAndFilePath' export * from './getServerUrl' export * from './instantiateLogger' export * from './isDebugOn' diff --git a/api/src/utils/setProcessVariables.ts b/api/src/utils/setProcessVariables.ts index 4edac7c..d365ec1 100644 --- a/api/src/utils/setProcessVariables.ts +++ b/api/src/utils/setProcessVariables.ts @@ -1,7 +1,7 @@ import path from 'path' import { createFolder, getAbsolutePath, getRealPath } from '@sasjs/utils' -import { getDesktopFields, ModeType } from '.' +import { getDesktopFields, ModeType, RunTimeType } from '.' export const setProcessVariables = async () => { if (process.env.NODE_ENV === 'test') { @@ -13,10 +13,12 @@ export const setProcessVariables = async () => { if (MODE === ModeType.Server) { process.sasLoc = process.env.SAS_PATH as string + process.nodeLoc = process.env.NODE_PATH as string } else { - const { sasLoc } = await getDesktopFields() + const { sasLoc, nodeLoc } = await getDesktopFields() process.sasLoc = sasLoc + process.nodeLoc = nodeLoc } const { SASJS_ROOT } = process.env @@ -24,6 +26,10 @@ export const setProcessVariables = async () => { await createFolder(absPath) process.driveLoc = getRealPath(absPath) + const { RUN_TIMES } = process.env + process.runTimes = (RUN_TIMES as string).split(',') as RunTimeType[] + console.log('sasLoc: ', process.sasLoc) console.log('sasDrive: ', process.driveLoc) + console.log('runTimes: ', process.runTimes) } diff --git a/api/src/utils/upload.ts b/api/src/utils/upload.ts index 640464c..cc9a47f 100644 --- a/api/src/utils/upload.ts +++ b/api/src/utils/upload.ts @@ -1,5 +1,6 @@ +import path from 'path' import { MulterFile } from '../types/Upload' -import { listFilesInFolder } from '@sasjs/utils' +import { listFilesInFolder, readFileBinary } from '@sasjs/utils' interface FilenameMapSingle { fieldName: string @@ -98,3 +99,34 @@ export const generateFileUploadSasCode = async ( return uploadSasCode } + +/** + * Generates the js code that references uploaded files in the concurrent request + * @param filesNamesMap object that maps hashed file names and original file names + * @param sessionFolder name of the folder that is created for the purpose of files in concurrent request + * @returns generated js code + */ +export const generateFileUploadJSCode = async ( + filesNamesMap: FilenamesMap, + sessionFolder: string +) => { + let uploadCode = '' + let fileCount = 0 + + const sessionFolderList: string[] = await listFilesInFolder(sessionFolder) + sessionFolderList.forEach(async (fileName) => { + if (fileName.includes('req_file')) { + fileCount++ + const filePath = path.join(sessionFolder, fileName) + uploadCode += `\nconst _WEBIN_FILEREF${fileCount} = fs.readFileSync('${filePath}')` + uploadCode += `\nconst _WEBIN_FILENAME${fileCount} = '${filesNamesMap[fileName].originalName}'` + uploadCode += `\nconst _WEBIN_NAME${fileCount} = '${filesNamesMap[fileName].fieldName}'` + } + }) + + if (fileCount) { + uploadCode = `\nconst _WEBIN_FILE_COUNT = ${fileCount}` + uploadCode + } + + return uploadCode +} diff --git a/api/src/utils/validation.ts b/api/src/utils/validation.ts index d69ce12..c406b4b 100644 --- a/api/src/utils/validation.ts +++ b/api/src/utils/validation.ts @@ -1,4 +1,5 @@ import Joi from 'joi' +import { RunTimeType } from '.' const usernameSchema = Joi.string().lowercase().alphanum().min(3).max(16) const passwordSchema = Joi.string().min(6).max(1024) @@ -120,9 +121,10 @@ export const folderParamValidation = (data: any): Joi.ValidationResult => _folderPath: Joi.string() }).validate(data) -export const runSASValidation = (data: any): Joi.ValidationResult => +export const runCodeValidation = (data: any): Joi.ValidationResult => Joi.object({ - code: Joi.string().required() + code: Joi.string().required(), + runTime: Joi.string().valid(...Object.values(RunTimeType)) }).validate(data) export const executeProgramRawValidation = (data: any): Joi.ValidationResult => diff --git a/api/src/utils/verifyEnvVariables.ts b/api/src/utils/verifyEnvVariables.ts index 9d82069..3c51363 100644 --- a/api/src/utils/verifyEnvVariables.ts +++ b/api/src/utils/verifyEnvVariables.ts @@ -26,6 +26,11 @@ export enum LOG_FORMAT_MORGANType { tiny = 'tiny' } +export enum RunTimeType { + SAS = 'sas', + JS = 'js' +} + export enum ReturnCode { Success, InvalidEnv @@ -46,6 +51,10 @@ export const verifyEnvVariables = (): ReturnCode => { errors.push(...verifyLOG_FORMAT_MORGAN()) + errors.push(...verifyRUN_TIMES()) + + errors.push(...verifyExecutablePaths()) + if (errors.length) { process.logger?.error( `Invalid environment variable(s) provided: \n${errors.join('\n')}` @@ -202,10 +211,52 @@ const verifyLOG_FORMAT_MORGAN = (): string[] => { return errors } +const verifyRUN_TIMES = (): string[] => { + const errors: string[] = [] + const { RUN_TIMES } = process.env + + if (RUN_TIMES) { + const runTimes = RUN_TIMES.split(',') + + const runTimeTypes = Object.values(RunTimeType) + + runTimes.forEach((runTime) => { + if (!runTimeTypes.includes(runTime as RunTimeType)) { + errors.push( + `- Invalid '${runTime}' runtime\n - valid options ${runTimeTypes}` + ) + } + }) + } else { + process.env.RUN_TIMES = DEFAULTS.RUN_TIMES + } + return errors +} + +const verifyExecutablePaths = () => { + const errors: string[] = [] + const { RUN_TIMES, SAS_PATH, NODE_PATH, MODE } = process.env + + if (MODE === ModeType.Server) { + const runTimes = RUN_TIMES?.split(',') + + if (runTimes?.includes(RunTimeType.SAS) && !SAS_PATH) { + errors.push(`- SAS_PATH is required for ${RunTimeType.SAS} run time`) + } + + if (runTimes?.includes(RunTimeType.JS) && !NODE_PATH) { + errors.push(`- NODE_PATH is required for ${RunTimeType.JS} run time`) + } + } + + return errors +} + const DEFAULTS = { MODE: ModeType.Desktop, PROTOCOL: ProtocolType.HTTP, PORT: '5000', HELMET_COEP: HelmetCoepType.TRUE, - LOG_FORMAT_MORGAN: LOG_FORMAT_MORGANType.Common + LOG_FORMAT_MORGAN: LOG_FORMAT_MORGANType.Common, + RUN_TIMES: `${RunTimeType.SAS},${RunTimeType.JS}` } diff --git a/web/.nvmrc b/web/.nvmrc index e2838c8..0a2cd98 100644 --- a/web/.nvmrc +++ b/web/.nvmrc @@ -1 +1 @@ -v16.14.0 \ No newline at end of file +v16.15.1 \ No newline at end of file diff --git a/web/src/components/home.tsx b/web/src/components/home.tsx index 74c6f90..9a9b547 100644 --- a/web/src/components/home.tsx +++ b/web/src/components/home.tsx @@ -9,8 +9,8 @@ const Home = () => {

Welcome to SASjs Server!

- SASjs Server provides a REST interface for executing Stored Programs - and ad hoc code (studio) against SAS and JS executables. The source is + SASjs Server provides a REST interface for executing Stored Programs and + ad hoc code (studio) against SAS and JS executables. The source is available on{' '} { setEditMode(false) } else { window.open( - `${baseUrl}/SASjsApi/stp/execute?_program=${props.selectedFilePath.replace( - /.sas$/, - '' - )}` + `${baseUrl}/SASjsApi/stp/execute?_program=${props.selectedFilePath}` ) } } diff --git a/web/src/containers/Studio/index.tsx b/web/src/containers/Studio/index.tsx index c043dd0..10bd25d 100644 --- a/web/src/containers/Studio/index.tsx +++ b/web/src/containers/Studio/index.tsx @@ -1,13 +1,24 @@ -import React, { useEffect, useRef, useState } from 'react' +import React, { useEffect, useRef, useState, useContext } from 'react' import axios from 'axios' -import Box from '@mui/material/Box' -import { Button, Paper, Stack, Tab, Tooltip } from '@mui/material' +import { + Box, + MenuItem, + FormControl, + Select, + SelectChangeEvent, + Button, + Paper, + Tab, + Tooltip +} from '@mui/material' import { makeStyles } from '@mui/styles' import Editor, { EditorDidMount } from 'react-monaco-editor' import { useLocation } from 'react-router-dom' import { TabContext, TabList, TabPanel } from '@mui/lab' +import { AppContext, RunTimeType } from '../../context/appContext' + const useStyles = makeStyles(() => ({ root: { fontSize: '1rem', @@ -30,12 +41,14 @@ const useStyles = makeStyles(() => ({ })) const Studio = () => { + const appContext = useContext(AppContext) const location = useLocation() const [fileContent, setFileContent] = useState('') const [log, setLog] = useState('') const [ctrlPressed, setCtrlPressed] = useState(false) const [webout, setWebout] = useState('') - const [tab, setTab] = React.useState('1') + const [tab, setTab] = useState('1') + const [selectedRunTime, setSelectedRunTime] = useState(RunTimeType.SAS) const handleTabChange = (_e: any, newValue: string) => { setTab(newValue) @@ -57,7 +70,7 @@ const Studio = () => { const runCode = (code: string) => { axios - .post(`/SASjsApi/code/execute`, { code }) + .post(`/SASjsApi/code/execute`, { code, runTime: selectedRunTime }) .then((res: any) => { const parsedLog = res?.data?.log .map((logLine: any) => logLine.line) @@ -89,6 +102,10 @@ const Studio = () => { if (!event.ctrlKey && ctrlPressed) setCtrlPressed(false) } + const handleChangeRunTime = (event: SelectChangeEvent) => { + setSelectedRunTime(event.target.value as RunTimeType) + } + useEffect(() => { const content = localStorage.getItem('fileContent') ?? '' setFileContent(content) @@ -149,8 +166,21 @@ const Studio = () => { RUN + + + + + - {/* */} > | null mode: ModeType + runTimes: RunTimeType[] logout: (() => void) | null } @@ -39,6 +45,7 @@ export const AppContext = createContext({ displayName: '', setDisplayName: null, mode: ModeType.Server, + runTimes: [], logout: null }) @@ -50,6 +57,7 @@ const AppContextProvider = (props: { children: ReactNode }) => { const [username, setUsername] = useState('') const [displayName, setDisplayName] = useState('') const [mode, setMode] = useState(ModeType.Server) + const [runTimes, setRunTimes] = useState([]) useEffect(() => { setCheckingSession(true) @@ -74,6 +82,7 @@ const AppContextProvider = (props: { children: ReactNode }) => { .then((res) => res.data) .then((data: any) => { setMode(data.mode) + setRunTimes(data.runTimes) }) .catch(() => {}) }, []) @@ -99,6 +108,7 @@ const AppContextProvider = (props: { children: ReactNode }) => { displayName, setDisplayName, mode, + runTimes, logout }} >