From f738a6d7a326f950d85b1031ae636e3102254f4a Mon Sep 17 00:00:00 2001 From: Sabir Hassan Date: Sun, 19 Jun 2022 07:07:39 +0500 Subject: [PATCH] chore: splitted functions into different files --- api/src/controllers/internal/Execution.ts | 222 +----------------- .../controllers/internal/createJSProgram.ts | 64 +++++ .../controllers/internal/createSASProgram.ts | 69 ++++++ api/src/controllers/internal/index.ts | 3 + .../controllers/internal/processProgram.ts | 86 +++++++ 5 files changed, 227 insertions(+), 217 deletions(-) create mode 100644 api/src/controllers/internal/createJSProgram.ts create mode 100644 api/src/controllers/internal/createSASProgram.ts create mode 100644 api/src/controllers/internal/processProgram.ts diff --git a/api/src/controllers/internal/Execution.ts b/api/src/controllers/internal/Execution.ts index 723341f..b742402 100644 --- a/api/src/controllers/internal/Execution.ts +++ b/api/src/controllers/internal/Execution.ts @@ -1,22 +1,15 @@ import path from 'path' import fs from 'fs' -import { execFileSync } from 'child_process' -import { once } from 'stream' -import { getSASSessionController, getJSSessionController } from './' import { - readFile, - fileExists, - createFile, - moveFile, - readFileBinary -} from '@sasjs/utils' + getSASSessionController, + getJSSessionController, + processProgram +} from './' +import { readFile, fileExists, createFile, readFileBinary } from '@sasjs/utils' import { PreProgramVars, Session, TreeNode } from '../../types' import { extractHeaders, - generateFileUploadSasCode, - generateFileUploadJSCode, getFilesFolder, - getMacrosFolder, HTTPHeaders, isDebugOn, RunTimeType @@ -189,208 +182,3 @@ export class ExecutionController { return root } } - -const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) - -const createSASProgram = async ( - program: string, - preProgramVariables: PreProgramVars, - vars: ExecutionVars, - session: Session, - weboutPath: string, - tokenFile: string, - otherArgs?: any -) => { - const varStatments = Object.keys(vars).reduce( - (computed: string, key: string) => `${computed}%let ${key}=${vars[key]};\n`, - '' - ) - - const preProgramVarStatments = ` -%let _sasjs_tokenfile=${tokenFile}; -%let _sasjs_username=${preProgramVariables?.username}; -%let _sasjs_userid=${preProgramVariables?.userId}; -%let _sasjs_displayname=${preProgramVariables?.displayName}; -%let _sasjs_apiserverurl=${preProgramVariables?.serverUrl}; -%let _sasjs_apipath=/SASjsApi/stp/execute; -%let _metaperson=&_sasjs_displayname; -%let _metauser=&_sasjs_username; -%let sasjsprocessmode=Stored Program; -%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/../stpsrv_header.txt; - -%global SYSPROCESSMODE SYSTCPIPHOSTNAME SYSHOSTINFOLONG; -%macro _sasjs_server_init(); -%if "&SYSPROCESSMODE"="" %then %let SYSPROCESSMODE=&sasjsprocessmode; -%if "&SYSTCPIPHOSTNAME"="" %then %let SYSTCPIPHOSTNAME=&_sasjs_apiserverurl; -%mend; -%_sasjs_server_init() -` - - program = ` -options insert=(SASAUTOS="${getMacrosFolder()}"); - -/* runtime vars */ -${varStatments} -filename _webout "${weboutPath}" mod; - -/* dynamic user-provided vars */ -${preProgramVarStatments} - -/* user autoexec starts */ -${otherArgs?.userAutoExec ?? ''} -/* user autoexec ends */ - -/* actual job code */ -${program}` - - // if no files are uploaded filesNamesMap will be undefined - if (otherArgs?.filesNamesMap) { - const uploadSasCode = await generateFileUploadSasCode( - otherArgs.filesNamesMap, - session.path - ) - - //If sas code for the file is generated it will be appended to the top of sasCode - if (uploadSasCode.length > 0) { - program = `${uploadSasCode}` + program - } - } - return program -} - -const createJSProgram = async ( - program: string, - preProgramVariables: PreProgramVars, - vars: ExecutionVars, - session: Session, - weboutPath: string, - tokenFile: string, - otherArgs?: any -) => { - const varStatments = Object.keys(vars).reduce( - (computed: string, key: string) => - `${computed}const ${key} = '${vars[key]}';\n`, - '' - ) - - const preProgramVarStatments = ` -let _webout = ''; -const weboutPath = '${weboutPath}'; -const _sasjs_tokenfile = '${tokenFile}'; -const _sasjs_username = '${preProgramVariables?.username}'; -const _sasjs_userid = '${preProgramVariables?.userId}'; -const _sasjs_displayname = '${preProgramVariables?.displayName}'; -const _metaperson = _sasjs_displayname; -const _metauser = _sasjs_username; -const sasjsprocessmode = 'Stored Program'; -` - - const requiredModules = `const fs = require('fs')` - - program = ` -/* runtime vars */ -${varStatments} - -/* dynamic user-provided vars */ -${preProgramVarStatments} - -/* actual job code */ -${program} - -/* write webout file only if webout exists*/ -if (_webout) { - fs.writeFile(weboutPath, _webout, function (err) { - if (err) throw err; - }) -} -` - // if no files are uploaded filesNamesMap will be undefined - if (otherArgs?.filesNamesMap) { - const uploadJSCode = await generateFileUploadJSCode( - otherArgs.filesNamesMap, - session.path - ) - - //If js code for the file is generated it will be appended to the top of jsCode - if (uploadJSCode.length > 0) { - program = `${uploadJSCode}\n` + program - } - } - return requiredModules + program -} - -const processProgram = async ( - program: string, - preProgramVariables: PreProgramVars, - vars: ExecutionVars, - session: Session, - weboutPath: string, - tokenFile: string, - runTime: RunTimeType, - logPath: string, - otherArgs?: any -) => { - if (runTime === RunTimeType.JS) { - program = await createJSProgram( - program, - preProgramVariables, - vars, - session, - weboutPath, - tokenFile, - otherArgs - ) - - const codePath = path.join(session.path, 'code.js') - - try { - await createFile(codePath, program) - - // create a stream that will write to console outputs to log file - const writeStream = fs.createWriteStream(logPath) - - // waiting for the open event so that we can have underlying file descriptor - await once(writeStream, 'open') - - execFileSync(process.nodeLoc, [codePath], { - stdio: ['ignore', writeStream, writeStream] - }) - - // copy the code.js program to log and end write stream - writeStream.end(program) - - session.completed = true - console.log('session completed', session) - } catch (err: any) { - session.completed = true - session.crashed = err.toString() - console.log('session crashed', session.id, session.crashed) - } - } else { - program = await createSASProgram( - program, - preProgramVariables, - vars, - session, - weboutPath, - tokenFile, - otherArgs - ) - - const codePath = path.join(session.path, 'code.sas') - - // Creating this file in a RUNNING session will break out - // the autoexec loop and actually execute the program - // but - given it will take several milliseconds to create - // (which can mean SAS trying to run a partial program, or - // failing due to file lock) we first create the file THEN - // we rename it. - await createFile(codePath + '.bkp', program) - await moveFile(codePath + '.bkp', codePath) - - // we now need to poll the session status - while (!session.completed) { - await delay(50) - } - } -} diff --git a/api/src/controllers/internal/createJSProgram.ts b/api/src/controllers/internal/createJSProgram.ts new file mode 100644 index 0000000..f2e3333 --- /dev/null +++ b/api/src/controllers/internal/createJSProgram.ts @@ -0,0 +1,64 @@ +import { PreProgramVars, Session } from '../../types' +import { generateFileUploadJSCode } from '../../utils' +import { ExecutionVars } from './' + +export const createJSProgram = async ( + program: string, + preProgramVariables: PreProgramVars, + vars: ExecutionVars, + session: Session, + weboutPath: string, + tokenFile: string, + otherArgs?: any +) => { + const varStatments = Object.keys(vars).reduce( + (computed: string, key: string) => + `${computed}const ${key} = '${vars[key]}';\n`, + '' + ) + + const preProgramVarStatments = ` +let _webout = ''; +const weboutPath = '${weboutPath}'; +const _sasjs_tokenfile = '${tokenFile}'; +const _sasjs_username = '${preProgramVariables?.username}'; +const _sasjs_userid = '${preProgramVariables?.userId}'; +const _sasjs_displayname = '${preProgramVariables?.displayName}'; +const _metaperson = _sasjs_displayname; +const _metauser = _sasjs_username; +const sasjsprocessmode = 'Stored Program'; +` + + const requiredModules = `const fs = require('fs')` + + program = ` +/* runtime vars */ +${varStatments} + +/* dynamic user-provided vars */ +${preProgramVarStatments} + +/* actual job code */ +${program} + +/* write webout file only if webout exists*/ +if (_webout) { + fs.writeFile(weboutPath, _webout, function (err) { + if (err) throw err; + }) +} +` + // if no files are uploaded filesNamesMap will be undefined + if (otherArgs?.filesNamesMap) { + const uploadJSCode = await generateFileUploadJSCode( + otherArgs.filesNamesMap, + session.path + ) + + //If js code for the file is generated it will be appended to the top of jsCode + if (uploadJSCode.length > 0) { + program = `${uploadJSCode}\n` + program + } + } + return requiredModules + program +} diff --git a/api/src/controllers/internal/createSASProgram.ts b/api/src/controllers/internal/createSASProgram.ts new file mode 100644 index 0000000..8669586 --- /dev/null +++ b/api/src/controllers/internal/createSASProgram.ts @@ -0,0 +1,69 @@ +import { PreProgramVars, Session } from '../../types' +import { generateFileUploadSasCode, getMacrosFolder } from '../../utils' +import { ExecutionVars } from './' + +export const createSASProgram = async ( + program: string, + preProgramVariables: PreProgramVars, + vars: ExecutionVars, + session: Session, + weboutPath: string, + tokenFile: string, + otherArgs?: any +) => { + const varStatments = Object.keys(vars).reduce( + (computed: string, key: string) => `${computed}%let ${key}=${vars[key]};\n`, + '' + ) + + const preProgramVarStatments = ` +%let _sasjs_tokenfile=${tokenFile}; +%let _sasjs_username=${preProgramVariables?.username}; +%let _sasjs_userid=${preProgramVariables?.userId}; +%let _sasjs_displayname=${preProgramVariables?.displayName}; +%let _sasjs_apiserverurl=${preProgramVariables?.serverUrl}; +%let _sasjs_apipath=/SASjsApi/stp/execute; +%let _metaperson=&_sasjs_displayname; +%let _metauser=&_sasjs_username; +%let sasjsprocessmode=Stored Program; +%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/../stpsrv_header.txt; + +%global SYSPROCESSMODE SYSTCPIPHOSTNAME SYSHOSTINFOLONG; +%macro _sasjs_server_init(); +%if "&SYSPROCESSMODE"="" %then %let SYSPROCESSMODE=&sasjsprocessmode; +%if "&SYSTCPIPHOSTNAME"="" %then %let SYSTCPIPHOSTNAME=&_sasjs_apiserverurl; +%mend; +%_sasjs_server_init() +` + + program = ` +options insert=(SASAUTOS="${getMacrosFolder()}"); + +/* runtime vars */ +${varStatments} +filename _webout "${weboutPath}" mod; + +/* dynamic user-provided vars */ +${preProgramVarStatments} + +/* user autoexec starts */ +${otherArgs?.userAutoExec ?? ''} +/* user autoexec ends */ + +/* actual job code */ +${program}` + + // if no files are uploaded filesNamesMap will be undefined + if (otherArgs?.filesNamesMap) { + const uploadSasCode = await generateFileUploadSasCode( + otherArgs.filesNamesMap, + session.path + ) + + //If sas code for the file is generated it will be appended to the top of sasCode + if (uploadSasCode.length > 0) { + program = `${uploadSasCode}` + program + } + } + return program +} diff --git a/api/src/controllers/internal/index.ts b/api/src/controllers/internal/index.ts index 86939a2..50672d7 100644 --- a/api/src/controllers/internal/index.ts +++ b/api/src/controllers/internal/index.ts @@ -2,3 +2,6 @@ export * from './deploy' export * from './Session' export * from './Execution' export * from './FileUploadController' +export * from './createSASProgram' +export * from './createJSProgram' +export * from './processProgram' diff --git a/api/src/controllers/internal/processProgram.ts b/api/src/controllers/internal/processProgram.ts new file mode 100644 index 0000000..f9f3f6b --- /dev/null +++ b/api/src/controllers/internal/processProgram.ts @@ -0,0 +1,86 @@ +import path from 'path' +import fs from 'fs' +import { execFileSync } from 'child_process' +import { once } from 'stream' +import { createFile, moveFile } from '@sasjs/utils' +import { PreProgramVars, Session } from '../../types' +import { RunTimeType } from '../../utils' +import { ExecutionVars, createSASProgram, createJSProgram } from './' + +export const processProgram = async ( + program: string, + preProgramVariables: PreProgramVars, + vars: ExecutionVars, + session: Session, + weboutPath: string, + tokenFile: string, + runTime: RunTimeType, + logPath: string, + otherArgs?: any +) => { + if (runTime === RunTimeType.JS) { + program = await createJSProgram( + program, + preProgramVariables, + vars, + session, + weboutPath, + tokenFile, + otherArgs + ) + + const codePath = path.join(session.path, 'code.js') + + try { + await createFile(codePath, program) + + // create a stream that will write to console outputs to log file + const writeStream = fs.createWriteStream(logPath) + + // waiting for the open event so that we can have underlying file descriptor + await once(writeStream, 'open') + + execFileSync(process.nodeLoc, [codePath], { + stdio: ['ignore', writeStream, writeStream] + }) + + // copy the code.js program to log and end write stream + writeStream.end(program) + + session.completed = true + console.log('session completed', session) + } catch (err: any) { + session.completed = true + session.crashed = err.toString() + console.log('session crashed', session.id, session.crashed) + } + } else { + program = await createSASProgram( + program, + preProgramVariables, + vars, + session, + weboutPath, + tokenFile, + otherArgs + ) + + const codePath = path.join(session.path, 'code.sas') + + // Creating this file in a RUNNING session will break out + // the autoexec loop and actually execute the program + // but - given it will take several milliseconds to create + // (which can mean SAS trying to run a partial program, or + // failing due to file lock) we first create the file THEN + // we rename it. + await createFile(codePath + '.bkp', program) + await moveFile(codePath + '.bkp', codePath) + + // we now need to poll the session status + while (!session.completed) { + await delay(50) + } + } +} + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))