sasjs/server
1
0
Fork 0
mirror of https://github.com/sasjs/server.git synced 2025-12-10 19:34:34 +00:00
Code Issues Projects Releases Wiki Activity

chore: merging with sessions - sas request with file upload

Browse Source
This commit is contained in:
Mihajlo Medjedovic
2021-10-14 15:44:28 +00:00
parent 3f39cab357
commit 17a7a26fc3
8 changed files with 355 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -6,3 +6,4 @@ node_modules/
sas/
tmp/
build/
certificates/

180
package-lock.json generated
View File

@@ -1518,6 +1518,12 @@
"integrity": "sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog==",
"dev": true
},
"@types/cors": {
"version": "2.8.12",
"resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz",
"integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==",
"dev": true
},
"@types/express": {
"version": "4.17.12",
"resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.12.tgz",
@@ -1952,6 +1958,11 @@
"picomatch": "^2.0.4"
}
},
"append-field": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
"integrity": "sha1-HjRA6RXwsSA9I3SOeO3XubW0PlY="
},
"arg": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
@@ -2216,8 +2227,39 @@
"buffer-from": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==",
"dev": true
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
},
"busboy": {
"version": "0.2.14",
"resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz",
"integrity": "sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=",
"requires": {
"dicer": "0.2.5",
"readable-stream": "1.1.x"
},
"dependencies": {
"isarray": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
},
"readable-stream": {
"version": "1.1.14",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
"integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.1",
"isarray": "0.0.1",
"string_decoder": "~0.10.x"
}
},
"string_decoder": {
"version": "0.10.31",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
"integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
}
}
},
"bytes": {
"version": "3.1.0",
@@ -2490,6 +2532,41 @@
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
"dev": true
},
"concat-stream": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
"requires": {
"buffer-from": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^2.2.2",
"typedarray": "^0.0.6"
},
"dependencies": {
"readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"requires": {
"safe-buffer": "~5.1.0"
}
}
}
},
"configstore": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz",
@@ -2611,8 +2688,16 @@
"core-util-is": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
"dev": true
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"cors": {
"version": "2.8.5",
"resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz",
"integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==",
"requires": {
"object-assign": "^4",
"vary": "^1"
}
},
"cosmiconfig": {
"version": "7.0.0",
@@ -2822,6 +2907,38 @@
"integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==",
"dev": true
},
"dicer": {
"version": "0.2.5",
"resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz",
"integrity": "sha1-WZbAhrszIYyBLAkL3cCc0S+stw8=",
"requires": {
"readable-stream": "1.1.x",
"streamsearch": "0.1.2"
},
"dependencies": {
"isarray": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
},
"readable-stream": {
"version": "1.1.14",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
"integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.1",
"isarray": "0.0.1",
"string_decoder": "~0.10.x"
}
},
"string_decoder": {
"version": "0.10.31",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
"integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
}
}
},
"diff": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
@@ -3929,8 +4046,7 @@
"isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
"dev": true
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
},
"isexe": {
"version": "2.0.0",
@@ -5439,8 +5555,7 @@
"minimist": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==",
"dev": true
"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
},
"minimist-options": {
"version": "4.1.0",
@@ -5470,6 +5585,31 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"multer": {
"version": "1.4.3",
"resolved": "https://registry.npmjs.org/multer/-/multer-1.4.3.tgz",
"integrity": "sha512-np0YLKncuZoTzufbkM6wEKp68EhWJXcU6fq6QqrSwkckd2LlMgd1UqhUJLj6NS/5sZ8dE8LYDWslsltJznnXlg==",
"requires": {
"append-field": "^1.0.0",
"busboy": "^0.2.11",
"concat-stream": "^1.5.2",
"mkdirp": "^0.5.4",
"object-assign": "^4.1.1",
"on-finished": "^2.3.0",
"type-is": "^1.6.4",
"xtend": "^4.0.0"
},
"dependencies": {
"mkdirp": {
"version": "0.5.5",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
"integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
"requires": {
"minimist": "^1.2.5"
}
}
}
},
"natural-compare": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
@@ -7664,6 +7804,11 @@
"integrity": "sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ==",
"dev": true
},
"object-assign": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
},
"object-inspect": {
"version": "1.10.3",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.10.3.tgz",
@@ -8008,8 +8153,7 @@
"process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
"dev": true
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"prompts": {
"version": "2.4.1",
@@ -8694,6 +8838,11 @@
}
}
},
"streamsearch": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz",
"integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo="
},
"string-length": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz",
@@ -9171,6 +9320,11 @@
"mime-types": "~2.1.24"
}
},
"typedarray": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
"integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
},
"typedarray-to-buffer": {
"version": "3.1.5",
"resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz",
@@ -9266,8 +9420,7 @@
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=",
"dev": true
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"utils-merge": {
"version": "1.0.1",
@@ -9475,8 +9628,7 @@
"xtend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
"dev": true
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
},
"y18n": {
"version": "5.0.8",

7
package.json
View File

@@ -22,7 +22,8 @@
"author": "Analytium Ltd",
"dependencies": {
"@sasjs/utils": "^2.23.3",
"express": "^4.17.1"
"express": "^4.17.1",
"multer": "^1.4.3"
},
"devDependencies": {
"@types/express": "^4.17.12",
@@ -40,7 +41,7 @@
"typescript": "^4.3.2"
},
"configuration": {
"sasPath": "/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe/sas",
"sasJsPort": 5000
"sasPath": "/opt/sas/sas9/SASHome/SASFoundation/9.4/sas",
"sasJsPort": 5005
}
}

32
src/controllers/Execution.ts
View File

@@ -5,6 +5,7 @@ import { configuration } from '../../package.json'
import { promisify } from 'util'
import { execFile } from 'child_process'
import { Session } from '../types'
import { generateFileUploadSasCode } from '../utils'
const execFilePromise = promisify(execFile)
export class ExecutionController {
@@ -12,7 +13,8 @@ export class ExecutionController {
program = '',
autoExec?: string,
session?: Session,
vars?: any
vars?: any,
otherArgs?: any
) {
if (program) {
if (!(await fileExists(program))) {
@@ -39,10 +41,26 @@ export class ExecutionController {
let webout = path.join(session.path, 'webout.txt')
await createFile(webout, '')
program = `
%let sasjsprocessmode=Stored Program;
filename _webout "${webout}";
${program}`
program = `filename _webout "${webout}";\n${program}`
// if no files are uploaded filesNamesMap will be undefined
if (otherArgs && otherArgs.filesNamesMap) {
const uploadSasCode = generateFileUploadSasCode(
otherArgs.filesNamesMap,
session.path
)
const code = path.join(session.path, 'code.sas')
//If sas code for the file is generated it will be appended to the top of sasCode
if (uploadSasCode.length > 0) {
program = `${uploadSasCode}` + program
}
}
let code = path.join(session.path, 'code.sas')
if (!(await fileExists(code))) {
await createFile(code, program)
}
@@ -64,7 +82,9 @@ export class ExecutionController {
if (await fileExists(log)) log = await readFile(log)
else log = ''
if (stderr) return Promise.reject({ error: stderr, log: log })
// if (stderr) {
// return Promise.reject({ error: stderr, log: log })
// }
if (await fileExists(webout)) webout = await readFile(webout)
else webout = ''
@@ -73,7 +93,7 @@ export class ExecutionController {
(key: string) => key.toLowerCase() === '_debug'
)
if (debug && vars[debug] >= 131) {
if (debug && vars[debug] >= 131 || stderr) {
webout = `<html><body>
${webout}
<div style="text-align:left">
@@ -82,7 +102,7 @@ ${webout}
</div>
</body></html>`
}
session.inUse = false
sessionController.deleteSession(session)

2
src/controllers/Session.ts
View File

@@ -70,7 +70,7 @@ export class SessionController {
this.scheduleSessionDestroy(session)
this.executionController.execute('', autoExec, session).catch(() => {})
this.executionController.execute('', autoExec, session).catch((err) => {})
this.sessions.push(session)

73
src/routes/index.ts
View File

@@ -1,12 +1,41 @@
import express from 'express'
import { createFileTree, getTreeExample } from '../controllers'
import { createFileTree, getSessionController, getTreeExample } from '../controllers'
import { ExecutionResult, isRequestQuery, isFileTree } from '../types'
import path from 'path'
import { getTmpFilesFolderPath } from '../utils'
import { getTmpFilesFolderPath, getTmpFolderPath, makeFilesNamesMap } from '../utils'
import { ExecutionController } from '../controllers'
import { uuidv4 } from '@sasjs/utils'
const multer = require('multer')
const router = express.Router()
const storage = multer.diskStorage({
destination: function (req: any, file: any, cb: any) {
//Sending the intercepted files to the sessions subfolder
cb(null, req.sasSession.path)
},
filename: function (req: any, file: any, cb: any) {
//req_file prefix + unique hash added to sas request files
cb(null, `req_file_${uuidv4().replace(/-/gm, '')}`)
}
})
const upload = multer({ storage: storage })
//It will intercept request and generate uniqe uuid to be used as a subfolder name
//that will store the files uploaded
const preuploadMiddleware = async (req: any, res: any, next: any) => {
let session
const sessionController = getSessionController()
session = await sessionController.getSession()
session.inUse = true
req.sasSession = session
next()
}
router.get('/', async (_, res) => {
res.status(200).send('Welcome to @sasjs/server API')
})
@@ -46,9 +75,12 @@ router.get('/SASjsExecutor', async (req, res) => {
router.get('/SASjsExecutor/do', async (req, res) => {
if (isRequestQuery(req.query)) {
const sasCodePath = path
let sasCodePath = path
.join(getTmpFilesFolderPath(), req.query._program)
.replace(new RegExp('/', 'g'), path.sep)
// If no extension provided, add .sas extension
sasCodePath += !sasCodePath.includes('.') ? '.sas' : ''
await new ExecutionController()
.execute(sasCodePath, undefined, undefined, { ...req.query })
@@ -70,4 +102,39 @@ router.get('/SASjsExecutor/do', async (req, res) => {
}
})
router.post('/SASjsExecutor/do', preuploadMiddleware, upload.any(), async (req: any, res: any) => {
if (isRequestQuery(req.query)) {
let sasCodePath = path
.join(getTmpFilesFolderPath(), req.query._program)
.replace(new RegExp('/', 'g'), path.sep)
// If no extension provided, add .sas extension
sasCodePath += !sasCodePath.includes('.') ? '.sas' : ''
let filesNamesMap = null
if (req.files && req.files.length > 0) {
filesNamesMap = makeFilesNamesMap(req.files)
}
await new ExecutionController()
.execute(sasCodePath, undefined, req.sasSession, { ...req.query }, { filesNamesMap: filesNamesMap })
.then((result: {}) => {
res.status(200).send(result)
})
.catch((err: {} | string) => {
res.status(400).send({
status: 'failure',
message: 'Job execution failed.',
...(typeof err === 'object' ? err : { details: err })
})
})
} else {
res.status(400).send({
status: 'failure',
message: `Please provide the location of SAS code`
})
}
})
export default router

1
src/utils/index.ts
View File

@@ -1,2 +1,3 @@
export * from './file'
export * from './sleep'
export * from './upload'

86
src/utils/upload.ts Normal file
View File

@@ -0,0 +1,86 @@
import path from 'path'
import fs from 'fs'
import { getTmpSessionsFolderPath } from '.'
/**
* It will create a object that maps hashed file names to the original names
* @param files array of files to be mapped
* @returns object
*/
export const makeFilesNamesMap = (files: any) => {
if (!files) return null
const filesNamesMap: any = {}
for (let file of files) {
filesNamesMap[file.filename] = file.fieldname
}
return filesNamesMap
}
/**
* Generates the sas code that reference uploaded files in the concurrent request
* @param filesNamesMap object that maps hashed file names and original file names
* @param sasUploadFolder name of the folder that is created for the purpose of files in concurrent request
* @returns generated sas code
*/
export const generateFileUploadSasCode = (
filesNamesMap: any,
sasSessionFolder: string
): string => {
const uploadFilesDirPath = sasSessionFolder
let uploadSasCode = ''
let fileCount = 0
let uploadedFilesMap: {
fileref: string
filepath: string
filename: string
count: number
}[] = []
fs.readdirSync(uploadFilesDirPath).forEach((fileName) => {
let fileCountString = fileCount < 100 ? '0' + fileCount : fileCount
fileCountString = fileCount < 10 ? '00' + fileCount : fileCount
if (fileName.includes('req_file')) {
fileCount++
uploadedFilesMap.push({
fileref: `_sjs${fileCountString}`,
filepath: `${uploadFilesDirPath}/${fileName}`,
filename: filesNamesMap[fileName],
count: fileCount
})
}
})
for (let uploadedMap of uploadedFilesMap) {
uploadSasCode += `\nfilename ${uploadedMap.fileref} "${uploadedMap.filepath}";`
}
uploadSasCode += `\n%let _WEBIN_FILE_COUNT=${fileCount};`
for (let uploadedMap of uploadedFilesMap) {
uploadSasCode += `\n%let _WEBIN_FILENAME${uploadedMap.count}=${uploadedMap.filepath};`
}
for (let uploadedMap of uploadedFilesMap) {
uploadSasCode += `\n%let _WEBIN_FILEREF${uploadedMap.count}=${uploadedMap.fileref};`
}
for (let uploadedMap of uploadedFilesMap) {
uploadSasCode += `\n%let _WEBIN_NAME${uploadedMap.count}=${uploadedMap.filename};`
}
if (fileCount > 0) {
uploadSasCode += `\n%let _WEBIN_NAME=&_WEBIN_NAME1;`
uploadSasCode += `\n%let _WEBIN_FILEREF=&_WEBIN_FILEREF1;`
uploadSasCode += `\n%let _WEBIN_FILENAME=&_WEBIN_FILENAME1;`
}
uploadSasCode += `\n`
return uploadSasCode
}