sasjs/server
1
0
Fork 0
mirror of https://github.com/sasjs/server.git synced 2025-12-10 11:24:35 +00:00
Code Issues Projects Releases Wiki Activity

chore: code fixes

Browse Source
This commit is contained in:
Saad Jutt
2022-06-14 16:48:58 +05:00
parent de9ed15286
commit c830f44e29
9 changed files with 343 additions and 310 deletions
Download Patch File Download Diff File Expand all files Collapse all files

497
api/src/controllers/internal/Execution.ts
View File

@@ -19,7 +19,7 @@ import {
getMacrosFolder,
HTTPHeaders,
isDebugOn,
SASJSRunTimes
RunTimeType
} from '../../utils'
export interface ExecutionVars {
@@ -37,64 +37,56 @@ export interface ExecuteReturnJson {
log?: string
}
export class ExecutionController {
async executeFile(
programPath: string,
preProgramVariables: PreProgramVars,
vars: ExecutionVars,
otherArgs?: any,
returnJson?: boolean,
session?: Session
) {
for (const runTime of process.runTimes) {
const codePath =
path
.join(getFilesFolder(), programPath)
.replace(new RegExp('/', 'g'), path.sep) +
'.' +
runTime
if (await fileExists(codePath)) {
const program = await readFile(codePath)
interface ExecuteFileParams {
programPath: string
preProgramVariables: PreProgramVars
vars: ExecutionVars
otherArgs?: any
returnJson?: boolean
session?: Session
runTime: RunTimeType
}
if (runTime === SASJSRunTimes.JS) {
return this.executeProgram(
program,
preProgramVariables,
vars,
otherArgs,
returnJson,
session,
runTime
)
} else {
return this.executeProgram(
program,
preProgramVariables,
vars,
otherArgs,
returnJson,
session,
runTime
)
}
}
}
throw `ExecutionController: The Stored Program at "${programPath}" does not exist, or you do not have permission to view it.`
interface ExecuteProgramParams extends Omit<ExecuteFileParams, 'programPath'> {
program: string
}
export class ExecutionController {
async executeFile({
programPath,
preProgramVariables,
vars,
otherArgs,
returnJson,
session,
runTime
}: ExecuteFileParams) {
const program = await readFile(programPath)
return this.executeProgram({
program,
preProgramVariables,
vars,
otherArgs,
returnJson,
session,
runTime
})
}
async executeProgram(
program: string,
preProgramVariables: PreProgramVars,
vars: ExecutionVars,
otherArgs?: any,
returnJson?: boolean,
sessionByFileUpload?: Session,
runTime: string = 'sas'
): Promise<ExecuteReturnRaw | ExecuteReturnJson> {
async executeProgram({
program,
preProgramVariables,
vars,
otherArgs,
returnJson,
session: sessionByFileUpload,
runTime
}: ExecuteProgramParams): Promise<ExecuteReturnRaw | ExecuteReturnJson> {
const sessionController =
runTime === SASJSRunTimes.JS
? getJSSessionController()
: getSASSessionController()
runTime === RunTimeType.SAS
? getSASSessionController()
: getJSSessionController()
const session =
sessionByFileUpload ?? (await sessionController.getSession())
@@ -112,69 +104,17 @@ export class ExecutionController {
preProgramVariables?.httpHeaders.join('\n') ?? ''
)
if (runTime === SASJSRunTimes.JS) {
program = await this.createJSProgram(
program,
preProgramVariables,
vars,
session,
weboutPath,
tokenFile,
otherArgs
)
const codePath = path.join(session.path, 'code.js')
try {
await createFile(codePath, program)
// create a stream that will write to console outputs to log file
const writeStream = fs.createWriteStream(logPath)
// waiting for the open event so that we can have underlying file descriptor
await once(writeStream, 'open')
execFileSync('node', [codePath], {
stdio: ['ignore', writeStream, writeStream]
})
// copy the code.js program to log and end write stream
writeStream.end(program)
session.completed = true
console.log('session completed', session)
} catch (err: any) {
session.completed = true
session.crashed = err.toString()
console.log('session crashed', session.id, session.crashed)
}
} else {
program = await this.createSASProgram(
program,
preProgramVariables,
vars,
session,
weboutPath,
tokenFile,
otherArgs
)
const codePath = path.join(session.path, 'code.sas')
// Creating this file in a RUNNING session will break out
// the autoexec loop and actually execute the program
// but - given it will take several milliseconds to create
// (which can mean SAS trying to run a partial program, or
// failing due to file lock) we first create the file THEN
// we rename it.
await createFile(codePath + '.bkp', program)
await moveFile(codePath + '.bkp', codePath)
// we now need to poll the session status
while (!session.completed) {
await delay(50)
}
}
await processProgram(
program,
preProgramVariables,
vars,
session,
weboutPath,
tokenFile,
runTime,
logPath,
otherArgs
)
const log = (await fileExists(logPath)) ? await readFile(logPath) : ''
const headersContent = (await fileExists(headersPath))
@@ -212,129 +152,6 @@ export class ExecutionController {
}
}
private async createSASProgram(
program: string,
preProgramVariables: PreProgramVars,
vars: ExecutionVars,
session: Session,
weboutPath: string,
tokenFile: string,
otherArgs?: any
) {
const varStatments = Object.keys(vars).reduce(
(computed: string, key: string) =>
`${computed}%let ${key}=${vars[key]};\n`,
''
)
const preProgramVarStatments = `
%let _sasjs_tokenfile=${tokenFile};
%let _sasjs_username=${preProgramVariables?.username};
%let _sasjs_userid=${preProgramVariables?.userId};
%let _sasjs_displayname=${preProgramVariables?.displayName};
%let _sasjs_apiserverurl=${preProgramVariables?.serverUrl};
%let _sasjs_apipath=/SASjsApi/stp/execute;
%let _metaperson=&_sasjs_displayname;
%let _metauser=&_sasjs_username;
%let sasjsprocessmode=Stored Program;
%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/../stpsrv_header.txt;
%global SYSPROCESSMODE SYSTCPIPHOSTNAME SYSHOSTINFOLONG;
%macro _sasjs_server_init();
%if "&SYSPROCESSMODE"="" %then %let SYSPROCESSMODE=&sasjsprocessmode;
%if "&SYSTCPIPHOSTNAME"="" %then %let SYSTCPIPHOSTNAME=&_sasjs_apiserverurl;
%mend;
%_sasjs_server_init()
`
program = `
options insert=(SASAUTOS="${getMacrosFolder()}");
/* runtime vars */
${varStatments}
filename _webout "${weboutPath}" mod;
/* dynamic user-provided vars */
${preProgramVarStatments}
/* user autoexec starts */
${otherArgs?.userAutoExec ?? ''}
/* user autoexec ends */
/* actual job code */
${program}`
// if no files are uploaded filesNamesMap will be undefined
if (otherArgs?.filesNamesMap) {
const uploadSasCode = await generateFileUploadSasCode(
otherArgs.filesNamesMap,
session.path
)
//If sas code for the file is generated it will be appended to the top of sasCode
if (uploadSasCode.length > 0) {
program = `${uploadSasCode}` + program
}
}
return program
}
private async createJSProgram(
program: string,
preProgramVariables: PreProgramVars,
vars: ExecutionVars,
session: Session,
weboutPath: string,
tokenFile: string,
otherArgs?: any
) {
const varStatments = Object.keys(vars).reduce(
(computed: string, key: string) =>
`${computed}const ${key} = '${vars[key]}';\n`,
''
)
const preProgramVarStatments = `
const weboutPath = '${weboutPath}';
const _sasjs_tokenfile = '${tokenFile}';
const _sasjs_username = '${preProgramVariables?.username}';
const _sasjs_userid = '${preProgramVariables?.userId}';
const _sasjs_displayname = '${preProgramVariables?.displayName}';
const _metaperson = _sasjs_displayname;
const _metauser = _sasjs_username;
const sasjsprocessmode = 'Stored Program';
`
const requiredModules = `const fs = require('fs-extra')`
program = `
/* runtime vars */
${varStatments}
/* dynamic user-provided vars */
${preProgramVarStatments}
/* actual job code */
${program}
/* write webout file*/
fs.promises.writeFile(weboutPath, _webout)
`
// if no files are uploaded filesNamesMap will be undefined
if (otherArgs?.filesNamesMap) {
const uploadJSCode = await generateFileUploadJSCode(
otherArgs.filesNamesMap,
session.path
)
//If js code for the file is generated it will be appended to the top of jsCode
if (uploadJSCode.length > 0) {
program = `${uploadJSCode}\n` + program
}
}
return requiredModules + program
}
buildDirectoryTree() {
const root: TreeNode = {
name: 'files',
@@ -374,3 +191,201 @@ fs.promises.writeFile(weboutPath, _webout)
}
const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
const createSASProgram = async (
program: string,
preProgramVariables: PreProgramVars,
vars: ExecutionVars,
session: Session,
weboutPath: string,
tokenFile: string,
otherArgs?: any
) => {
const varStatments = Object.keys(vars).reduce(
(computed: string, key: string) => `${computed}%let ${key}=${vars[key]};\n`,
''
)
const preProgramVarStatments = `
%let _sasjs_tokenfile=${tokenFile};
%let _sasjs_username=${preProgramVariables?.username};
%let _sasjs_userid=${preProgramVariables?.userId};
%let _sasjs_displayname=${preProgramVariables?.displayName};
%let _sasjs_apiserverurl=${preProgramVariables?.serverUrl};
%let _sasjs_apipath=/SASjsApi/stp/execute;
%let _metaperson=&_sasjs_displayname;
%let _metauser=&_sasjs_username;
%let sasjsprocessmode=Stored Program;
%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/../stpsrv_header.txt;
%global SYSPROCESSMODE SYSTCPIPHOSTNAME SYSHOSTINFOLONG;
%macro _sasjs_server_init();
%if "&SYSPROCESSMODE"="" %then %let SYSPROCESSMODE=&sasjsprocessmode;
%if "&SYSTCPIPHOSTNAME"="" %then %let SYSTCPIPHOSTNAME=&_sasjs_apiserverurl;
%mend;
%_sasjs_server_init()
`
program = `
options insert=(SASAUTOS="${getMacrosFolder()}");
/* runtime vars */
${varStatments}
filename _webout "${weboutPath}" mod;
/* dynamic user-provided vars */
${preProgramVarStatments}
/* user autoexec starts */
${otherArgs?.userAutoExec ?? ''}
/* user autoexec ends */
/* actual job code */
${program}`
// if no files are uploaded filesNamesMap will be undefined
if (otherArgs?.filesNamesMap) {
const uploadSasCode = await generateFileUploadSasCode(
otherArgs.filesNamesMap,
session.path
)
//If sas code for the file is generated it will be appended to the top of sasCode
if (uploadSasCode.length > 0) {
program = `${uploadSasCode}` + program
}
}
return program
}
const createJSProgram = async (
program: string,
preProgramVariables: PreProgramVars,
vars: ExecutionVars,
session: Session,
weboutPath: string,
tokenFile: string,
otherArgs?: any
) => {
const varStatments = Object.keys(vars).reduce(
(computed: string, key: string) =>
`${computed}const ${key} = '${vars[key]}';\n`,
''
)
const preProgramVarStatments = `
const weboutPath = '${weboutPath}';
const _sasjs_tokenfile = '${tokenFile}';
const _sasjs_username = '${preProgramVariables?.username}';
const _sasjs_userid = '${preProgramVariables?.userId}';
const _sasjs_displayname = '${preProgramVariables?.displayName}';
const _metaperson = _sasjs_displayname;
const _metauser = _sasjs_username;
const sasjsprocessmode = 'Stored Program';
`
const requiredModules = `const fs = require('fs-extra')`
program = `
/* runtime vars */
${varStatments}
/* dynamic user-provided vars */
${preProgramVarStatments}
/* actual job code */
${program}
/* write webout file*/
fs.promises.writeFile(weboutPath, _webout)
`
// if no files are uploaded filesNamesMap will be undefined
if (otherArgs?.filesNamesMap) {
const uploadJSCode = await generateFileUploadJSCode(
otherArgs.filesNamesMap,
session.path
)
//If js code for the file is generated it will be appended to the top of jsCode
if (uploadJSCode.length > 0) {
program = `${uploadJSCode}\n` + program
}
}
return requiredModules + program
}
const processProgram = async (
program: string,
preProgramVariables: PreProgramVars,
vars: ExecutionVars,
session: Session,
weboutPath: string,
tokenFile: string,
runTime: RunTimeType,
logPath: string,
otherArgs?: any
) => {
if (runTime === RunTimeType.JS) {
program = await createJSProgram(
program,
preProgramVariables,
vars,
session,
weboutPath,
tokenFile,
otherArgs
)
const codePath = path.join(session.path, 'code.js')
try {
await createFile(codePath, program)
// create a stream that will write to console outputs to log file
const writeStream = fs.createWriteStream(logPath)
// waiting for the open event so that we can have underlying file descriptor
await once(writeStream, 'open')
execFileSync('node', [codePath], {
stdio: ['ignore', writeStream, writeStream]
})
// copy the code.js program to log and end write stream
writeStream.end(program)
session.completed = true
console.log('session completed', session)
} catch (err: any) {
session.completed = true
session.crashed = err.toString()
console.log('session crashed', session.id, session.crashed)
}
} else {
program = await createSASProgram(
program,
preProgramVariables,
vars,
session,
weboutPath,
tokenFile,
otherArgs
)
const codePath = path.join(session.path, 'code.sas')
// Creating this file in a RUNNING session will break out
// the autoexec loop and actually execute the program
// but - given it will take several milliseconds to create
// (which can mean SAS trying to run a partial program, or
// failing due to file lock) we first create the file THEN
// we rename it.
await createFile(codePath + '.bkp', program)
await moveFile(codePath + '.bkp', codePath)
// we now need to poll the session status
while (!session.completed) {
await delay(50)
}
}
}

50
api/src/controllers/internal/FileUploadController.ts
View File

@@ -1,9 +1,12 @@
import path from 'path'
import { Request, RequestHandler } from 'express'
import multer from 'multer'
import { uuidv4, fileExists } from '@sasjs/utils'
import { uuidv4 } from '@sasjs/utils'
import { getSASSessionController, getJSSessionController } from '.'
import { getFilesFolder, SASJSRunTimes } from '../../utils'
import {
executeProgramRawValidation,
getRunTimeAndFilePath,
RunTimeType
} from '../../utils'
export class FileUploadController {
private storage = multer.diskStorage({
@@ -22,31 +25,26 @@ export class FileUploadController {
//It will intercept request and generate unique uuid to be used as a subfolder name
//that will store the files uploaded
public preUploadMiddleware: RequestHandler = async (req, res, next) => {
const programPath = req.query._program as string
const { error: errQ, value: query } = executeProgramRawValidation(req.query)
const { error: errB, value: body } = executeProgramRawValidation(req.body)
for (const runTime of process.runTimes) {
const codePath =
path
.join(getFilesFolder(), programPath)
.replace(new RegExp('/', 'g'), path.sep) +
'.' +
runTime
if (errQ && errB) return res.status(400).send(errB.details[0].message)
if (await fileExists(codePath)) {
let sessionController
if (runTime === SASJSRunTimes.JS) {
sessionController = getJSSessionController()
} else {
sessionController = getSASSessionController()
}
const session = await sessionController.getSession()
// marking consumed true, so that it's not available
// as readySession for any other request
session.consumed = true
req.sasjsSession = session
break
}
}
const programPath = (query?._program ?? body?._program) as string
const { runTime } = await getRunTimeAndFilePath(programPath)
const sessionController =
runTime === RunTimeType.SAS
? getSASSessionController()
: getJSSessionController()
const session = await sessionController.getSession()
// marking consumed true, so that it's not available
// as readySession for any other request
session.consumed = true
req.sasjsSession = session
next()
}

37
api/src/controllers/stp.ts
View File

@@ -1,5 +1,4 @@
import express from 'express'
import path from 'path'
import {
Request,
Security,
@@ -19,12 +18,12 @@ import {
} from './internal'
import {
getPreProgramVariables,
getFilesFolder,
HTTPHeaders,
isDebugOn,
LogLine,
makeFilesNamesMap,
parseLogToArray
parseLogToArray,
getRunTimeAndFilePath
} from '../utils'
import { MulterFile } from '../types/Upload'
@@ -132,13 +131,16 @@ const executeReturnRaw = async (
): Promise<string | Buffer> => {
const query = req.query as ExecutionVars
const { codePath, runTime } = await getRunTimeAndFilePath(_program)
try {
const { result, httpHeaders } =
(await new ExecutionController().executeFile(
_program,
getPreProgramVariables(req),
query
)) as ExecuteReturnRaw
(await new ExecutionController().executeFile({
programPath: codePath,
preProgramVariables: getPreProgramVariables(req),
vars: query,
runTime
})) as ExecuteReturnRaw
// Should over-ride response header for debug
// on GET request to see entire log rendering on browser.
@@ -167,20 +169,23 @@ const executeReturnJson = async (
req: express.Request,
_program: string
): Promise<ExecuteReturnJsonResponse> => {
const { codePath, runTime } = await getRunTimeAndFilePath(_program)
const filesNamesMap = req.files?.length
? makeFilesNamesMap(req.files as MulterFile[])
: null
try {
const { webout, log, httpHeaders } =
(await new ExecutionController().executeFile(
_program,
getPreProgramVariables(req),
{ ...req.query, ...req.body },
{ filesNamesMap: filesNamesMap },
true,
req.sasjsSession
)) as ExecuteReturnJson
(await new ExecutionController().executeFile({
programPath: codePath,
preProgramVariables: getPreProgramVariables(req),
vars: { ...req.query, ...req.body },
otherArgs: { filesNamesMap: filesNamesMap },
returnJson: true,
session: req.sasjsSession,
runTime
})) as ExecuteReturnJson
let weboutRes: string | IRecordOfAny = webout
if (httpHeaders['content-type']?.toLowerCase() === 'application/json') {

11
api/src/routes/api/stp.ts
View File

@@ -35,16 +35,17 @@ stpRouter.post(
fileUploadController.preUploadMiddleware,
fileUploadController.getMulterUploadObject().any(),
async (req, res: any) => {
const { error: errQ, value: query } = executeProgramRawValidation(req.query)
const { error: errB, value: body } = executeProgramRawValidation(req.body)
// below validations are moved to preUploadMiddleware
// const { error: errQ, value: query } = executeProgramRawValidation(req.query)
// const { error: errB, value: body } = executeProgramRawValidation(req.body)
if (errQ && errB) return res.status(400).send(errB.details[0].message)
// if (errQ && errB) return res.status(400).send(errB.details[0].message)
try {
const response = await controller.executeReturnJson(
req,
body,
query?._program
req.body,
req.query?._program as string
)
// TODO: investigate if this code is required

2
api/src/types/system/process.d.ts vendored
View File

@@ -1,11 +1,11 @@
declare namespace NodeJS {
export interface Process {
runTimes: string[]
sasLoc: string
driveLoc: string
sasSessionController?: import('../../controllers/internal').SASSessionController
jsSessionController?: import('../../controllers/internal').JSSessionController
appStreamConfig: import('../').AppStreamConfig
logger: import('@sasjs/utils/logger').Logger
runTimes: import('../../utils').RunTimeType[]
}
}

18
api/src/utils/getRunTimeAndFilePath.ts Normal file
View File

@@ -0,0 +1,18 @@
import path from 'path'
import { fileExists } from '@sasjs/utils'
import { getFilesFolder } from './file'
export const getRunTimeAndFilePath = async (programPath: string) => {
for (const runTime of process.runTimes) {
const codePath =
path
.join(getFilesFolder(), programPath)
.replace(new RegExp('/', 'g'), path.sep) +
'.' +
runTime
if (await fileExists(codePath)) return { codePath, runTime }
}
throw `The Program at (${programPath}) does not exist.`
}

1
api/src/utils/index.ts
View File

@@ -10,6 +10,7 @@ export * from './generateRefreshToken'
export * from './getCertificates'
export * from './getDesktopFields'
export * from './getPreProgramVariables'
export * from './getRunTimeAndFilePath'
export * from './getServerUrl'
export * from './instantiateLogger'
export * from './isDebugOn'

13
api/src/utils/setProcessVariables.ts
View File

@@ -1,7 +1,7 @@
import path from 'path'
import { createFolder, getAbsolutePath, getRealPath } from '@sasjs/utils'
import { getDesktopFields, ModeType } from '.'
import { getDesktopFields, ModeType, RunTimeType } from '.'
export const setProcessVariables = async () => {
if (process.env.NODE_ENV === 'test') {
@@ -19,18 +19,15 @@ export const setProcessVariables = async () => {
process.sasLoc = sasLoc
}
const { SASJS_RUNTIMES } = process.env
const runTimes = SASJS_RUNTIMES
? SASJS_RUNTIMES.split(',').map((runTime) => runTime.toLowerCase())
: ['sas']
process.runTimes = runTimes
const { SASJS_ROOT } = process.env
const absPath = getAbsolutePath(SASJS_ROOT ?? 'sasjs_root', process.cwd())
await createFolder(absPath)
process.driveLoc = getRealPath(absPath)
const { RUN_TIMES } = process.env
process.runTimes = (RUN_TIMES as string).split(',') as RunTimeType[]
console.log('sasLoc: ', process.sasLoc)
console.log('sasDrive: ', process.driveLoc)
console.log('runTimes: ', process.runTimes)
}

24
api/src/utils/verifyEnvVariables.ts
View File

@@ -26,7 +26,7 @@ export enum LOG_FORMAT_MORGANType {
tiny = 'tiny'
}
export enum SASJSRunTimes {
export enum RunTimeType {
SAS = 'sas',
JS = 'js'
}
@@ -51,7 +51,7 @@ export const verifyEnvVariables = (): ReturnCode => {
errors.push(...verifyLOG_FORMAT_MORGAN())
errors.push(...verifySASJSRunTimes())
errors.push(...verifyRUN_TIMES())
if (errors.length) {
process.logger?.error(
@@ -209,26 +209,24 @@ const verifyLOG_FORMAT_MORGAN = (): string[] => {
return errors
}
const verifySASJSRunTimes = (): string[] => {
const verifyRUN_TIMES = (): string[] => {
const errors: string[] = []
const { SASJS_RUNTIMES } = process.env
const { RUN_TIMES } = process.env
if (SASJS_RUNTIMES) {
const runTimes = SASJS_RUNTIMES.split(',').map((runTime) =>
runTime.toLowerCase()
)
if (RUN_TIMES) {
const runTimes = RUN_TIMES.split(',')
const possibleRunTimes = Object.values(SASJSRunTimes)
const runTimeTypes = Object.values(RunTimeType)
runTimes.forEach((runTime) => {
if (!possibleRunTimes.includes(runTime as SASJSRunTimes)) {
if (!runTimeTypes.includes(runTime.toLowerCase() as RunTimeType)) {
errors.push(
`- Invalid '${runTime}' runtime\n - valid options ${possibleRunTimes}`
`- Invalid '${runTime}' runtime\n - valid options ${runTimeTypes}`
)
}
})
} else {
process.env.SASJS_RUNTIMES = DEFAULTS.SASJS_RUNTIMES
process.env.RUN_TIMES = DEFAULTS.RUN_TIMES
}
return errors
}
@@ -239,5 +237,5 @@ const DEFAULTS = {
PORT: '5000',
HELMET_COEP: HelmetCoepType.TRUE,
LOG_FORMAT_MORGAN: LOG_FORMAT_MORGANType.Common,
SASJS_RUNTIMES: SASJSRunTimes.SAS
RUN_TIMES: RunTimeType.SAS
}