sasjs/server
1
0
Fork 0
mirror of https://github.com/sasjs/server.git synced 2025-12-10 19:34:34 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #23 from sasjs/issue16-session

Browse Source 
feat: uploading files in sas request
This commit is contained in:
Yury Shkoda
2021-10-19 13:45:13 +03:00
committed by GitHub
parent 3f39cab357 38db48d59f
commit a1de0ae888
10 changed files with 388 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -6,3 +6,4 @@ node_modules/
sas/
tmp/
build/
certificates/

174
package-lock.json generated
View File

@@ -1687,6 +1687,15 @@
"integrity": "sha512-fZQQafSREFyuZcdWFAExYjBiCL7AUCdgsk80iO0q4yihYYdcIiH28CcuPTGFgLOCC8RlW49GSQxdHwZP+I7CNg==",
"dev": true
},
"@types/multer": {
"version": "1.4.7",
"resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.7.tgz",
"integrity": "sha512-/SNsDidUFCvqqcWDwxv2feww/yqhNeTRL5CVoL3jU4Goc4kKEL10T7Eye65ZqPNi4HRx8sAEX59pV1aEH7drNA==",
"dev": true,
"requires": {
"@types/express": "*"
}
},
"@types/node": {
"version": "15.12.2",
"resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.2.tgz",
@@ -1952,6 +1961,11 @@
"picomatch": "^2.0.4"
}
},
"append-field": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz",
"integrity": "sha1-HjRA6RXwsSA9I3SOeO3XubW0PlY="
},
"arg": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz",
@@ -2216,8 +2230,39 @@
"buffer-from": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz",
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==",
"dev": true
"integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A=="
},
"busboy": {
"version": "0.2.14",
"resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz",
"integrity": "sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=",
"requires": {
"dicer": "0.2.5",
"readable-stream": "1.1.x"
},
"dependencies": {
"isarray": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
},
"readable-stream": {
"version": "1.1.14",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
"integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.1",
"isarray": "0.0.1",
"string_decoder": "~0.10.x"
}
},
"string_decoder": {
"version": "0.10.31",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
"integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
}
}
},
"bytes": {
"version": "3.1.0",
@@ -2490,6 +2535,41 @@
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
"dev": true
},
"concat-stream": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
"requires": {
"buffer-from": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^2.2.2",
"typedarray": "^0.0.6"
},
"dependencies": {
"readable-stream": {
"version": "2.3.7",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz",
"integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.3",
"isarray": "~1.0.0",
"process-nextick-args": "~2.0.0",
"safe-buffer": "~5.1.1",
"string_decoder": "~1.1.1",
"util-deprecate": "~1.0.1"
}
},
"string_decoder": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz",
"integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==",
"requires": {
"safe-buffer": "~5.1.0"
}
}
}
},
"configstore": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz",
@@ -2611,8 +2691,7 @@
"core-util-is": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=",
"dev": true
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"cosmiconfig": {
"version": "7.0.0",
@@ -2822,6 +2901,38 @@
"integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==",
"dev": true
},
"dicer": {
"version": "0.2.5",
"resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz",
"integrity": "sha1-WZbAhrszIYyBLAkL3cCc0S+stw8=",
"requires": {
"readable-stream": "1.1.x",
"streamsearch": "0.1.2"
},
"dependencies": {
"isarray": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
"integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8="
},
"readable-stream": {
"version": "1.1.14",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
"integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=",
"requires": {
"core-util-is": "~1.0.0",
"inherits": "~2.0.1",
"isarray": "0.0.1",
"string_decoder": "~0.10.x"
}
},
"string_decoder": {
"version": "0.10.31",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz",
"integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ="
}
}
},
"diff": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
@@ -3929,8 +4040,7 @@
"isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=",
"dev": true
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
},
"isexe": {
"version": "2.0.0",
@@ -5439,8 +5549,7 @@
"minimist": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz",
"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==",
"dev": true
"integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw=="
},
"minimist-options": {
"version": "4.1.0",
@@ -5470,6 +5579,31 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"multer": {
"version": "1.4.3",
"resolved": "https://registry.npmjs.org/multer/-/multer-1.4.3.tgz",
"integrity": "sha512-np0YLKncuZoTzufbkM6wEKp68EhWJXcU6fq6QqrSwkckd2LlMgd1UqhUJLj6NS/5sZ8dE8LYDWslsltJznnXlg==",
"requires": {
"append-field": "^1.0.0",
"busboy": "^0.2.11",
"concat-stream": "^1.5.2",
"mkdirp": "^0.5.4",
"object-assign": "^4.1.1",
"on-finished": "^2.3.0",
"type-is": "^1.6.4",
"xtend": "^4.0.0"
},
"dependencies": {
"mkdirp": {
"version": "0.5.5",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz",
"integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==",
"requires": {
"minimist": "^1.2.5"
}
}
}
},
"natural-compare": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
@@ -7664,6 +7798,11 @@
"integrity": "sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ==",
"dev": true
},
"object-assign": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
"integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM="
},
"object-inspect": {
"version": "1.10.3",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.10.3.tgz",
@@ -8008,8 +8147,7 @@
"process-nextick-args": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
"dev": true
"integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
},
"prompts": {
"version": "2.4.1",
@@ -8694,6 +8832,11 @@
}
}
},
"streamsearch": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz",
"integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo="
},
"string-length": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz",
@@ -9171,6 +9314,11 @@
"mime-types": "~2.1.24"
}
},
"typedarray": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
"integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
},
"typedarray-to-buffer": {
"version": "3.1.5",
"resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz",
@@ -9266,8 +9414,7 @@
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=",
"dev": true
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"utils-merge": {
"version": "1.0.1",
@@ -9475,8 +9622,7 @@
"xtend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==",
"dev": true
"integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="
},
"y18n": {
"version": "5.0.8",

8
package.json
View File

@@ -22,11 +22,13 @@
"author": "Analytium Ltd",
"dependencies": {
"@sasjs/utils": "^2.23.3",
"express": "^4.17.1"
"express": "^4.17.1",
"multer": "^1.4.3"
},
"devDependencies": {
"@types/express": "^4.17.12",
"@types/jest": "^26.0.24",
"@types/multer": "^1.4.7",
"@types/node": "^15.12.2",
"@types/supertest": "^2.0.11",
"jest": "^27.0.6",
@@ -40,7 +42,7 @@
"typescript": "^4.3.2"
},
"configuration": {
"sasPath": "/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe/sas",
"sasJsPort": 5000
"sasPath": "/opt/sas/sas9/SASHome/SASFoundation/9.4/sas",
"sasJsPort": 5005
}
}

26
src/controllers/Execution.ts
View File

@@ -5,6 +5,7 @@ import { configuration } from '../../package.json'
import { promisify } from 'util'
import { execFile } from 'child_process'
import { Session } from '../types'
import { generateFileUploadSasCode } from '../utils'
const execFilePromise = promisify(execFile)
export class ExecutionController {
@@ -12,7 +13,8 @@ export class ExecutionController {
program = '',
autoExec?: string,
session?: Session,
vars?: any
vars?: any,
otherArgs?: any
) {
if (program) {
if (!(await fileExists(program))) {
@@ -40,7 +42,23 @@ export class ExecutionController {
let webout = path.join(session.path, 'webout.txt')
await createFile(webout, '')
program = `filename _webout "${webout}";\n${program}`
program = `
%let sasjsprocessmode=Stored Program;
filename _webout "${webout}";
${program}`
// if no files are uploaded filesNamesMap will be undefined
if (otherArgs && otherArgs.filesNamesMap) {
const uploadSasCode = await generateFileUploadSasCode(
otherArgs.filesNamesMap,
session.path
)
//If sas code for the file is generated it will be appended to the top of sasCode
if (uploadSasCode.length > 0) {
program = `${uploadSasCode}` + program
}
}
const code = path.join(session.path, 'code.sas')
if (!(await fileExists(code))) {
@@ -64,8 +82,6 @@ export class ExecutionController {
if (await fileExists(log)) log = await readFile(log)
else log = ''
if (stderr) return Promise.reject({ error: stderr, log: log })
if (await fileExists(webout)) webout = await readFile(webout)
else webout = ''
@@ -73,7 +89,7 @@ export class ExecutionController {
(key: string) => key.toLowerCase() === '_debug'
)
if (debug && vars[debug] >= 131) {
if ((debug && vars[debug] >= 131) || stderr) {
webout = `<html><body>
${webout}
<div style="text-align:left">

36
src/controllers/FileUploadController.ts Normal file
View File

@@ -0,0 +1,36 @@
import { uuidv4 } from '@sasjs/utils'
import { getSessionController } from '.'
const multer = require('multer')
export class FileUploadController {
private storage = multer.diskStorage({
destination: function (req: any, file: any, cb: any) {
//Sending the intercepted files to the sessions subfolder
cb(null, req.sasSession.path)
},
filename: function (req: any, file: any, cb: any) {
//req_file prefix + unique hash added to sas request files
cb(null, `req_file_${uuidv4().replace(/-/gm, '')}`)
}
})
private upload = multer({ storage: this.storage })
//It will intercept request and generate unique uuid to be used as a subfolder name
//that will store the files uploaded
public preuploadMiddleware = async (req: any, res: any, next: any) => {
let session
const sessionController = getSessionController()
session = await sessionController.getSession()
session.inUse = true
req.sasSession = session
next()
}
public getMulterUploadObject() {
return this.upload
}
}

1
src/controllers/index.ts
View File

@@ -1,3 +1,4 @@
export * from './deploy'
export * from './Session'
export * from './Execution'
export * from './FileUploadController'

68
src/routes/index.ts
View File

@@ -1,12 +1,23 @@
import express from 'express'
import { createFileTree, getTreeExample } from '../controllers'
import {
createFileTree,
getSessionController,
getTreeExample
} from '../controllers'
import { ExecutionResult, isRequestQuery, isFileTree } from '../types'
import path from 'path'
import { getTmpFilesFolderPath } from '../utils'
import { ExecutionController } from '../controllers'
import {
getTmpFilesFolderPath,
getTmpFolderPath,
makeFilesNamesMap
} from '../utils'
import { ExecutionController, FileUploadController } from '../controllers'
import { uuidv4 } from '@sasjs/utils'
const router = express.Router()
const fileUploadController = new FileUploadController()
router.get('/', async (_, res) => {
res.status(200).send('Welcome to @sasjs/server API')
})
@@ -46,10 +57,13 @@ router.get('/SASjsExecutor', async (req, res) => {
router.get('/SASjsExecutor/do', async (req, res) => {
if (isRequestQuery(req.query)) {
const sasCodePath = path
let sasCodePath = path
.join(getTmpFilesFolderPath(), req.query._program)
.replace(new RegExp('/', 'g'), path.sep)
// If no extension provided, add .sas extension
sasCodePath += '.sas'
await new ExecutionController()
.execute(sasCodePath, undefined, undefined, { ...req.query })
.then((result: {}) => {
@@ -70,4 +84,50 @@ router.get('/SASjsExecutor/do', async (req, res) => {
}
})
router.post(
'/SASjsExecutor/do',
fileUploadController.preuploadMiddleware,
fileUploadController.getMulterUploadObject().any(),
async (req: any, res: any) => {
if (isRequestQuery(req.query)) {
let sasCodePath = path
.join(getTmpFilesFolderPath(), req.query._program)
.replace(new RegExp('/', 'g'), path.sep)
// If no extension provided, add .sas extension
sasCodePath += '.sas'
let filesNamesMap = null
if (req.files && req.files.length > 0) {
filesNamesMap = makeFilesNamesMap(req.files)
}
await new ExecutionController()
.execute(
sasCodePath,
undefined,
req.sasSession,
{ ...req.query, ...req.body },
{ filesNamesMap: filesNamesMap }
)
.then((result: {}) => {
res.status(200).send(result)
})
.catch((err: {} | string) => {
res.status(400).send({
status: 'failure',
message: 'Job execution failed.',
...(typeof err === 'object' ? err : { details: err })
})
})
} else {
res.status(400).send({
status: 'failure',
message: `Please provide the location of SAS code`
})
}
}
)
export default router

10
src/types/Upload.ts Normal file
View File

@@ -0,0 +1,10 @@
export interface MulterFile {
fieldname: string
originalname: string
encoding: string
mimetype: string
destination: string
filename: string
path: string
size: number
}

1
src/utils/index.ts
View File

@@ -1,2 +1,3 @@
export * from './file'
export * from './sleep'
export * from './upload'

89
src/utils/upload.ts Normal file
View File

@@ -0,0 +1,89 @@
import path from 'path'
import fs from 'fs'
import { getTmpSessionsFolderPath } from '.'
import { MulterFile } from '../types/Upload'
import { listFilesInFolder } from '@sasjs/utils'
/**
* It will create an object that maps hashed file names to the original names
* @param files array of files to be mapped
* @returns object
*/
export const makeFilesNamesMap = (files: MulterFile[]) => {
if (!files) return null
const filesNamesMap: { [key: string]: string } = {}
for (let file of files) {
filesNamesMap[file.filename] = file.fieldname
}
return filesNamesMap
}
/**
* Generates the sas code that references uploaded files in the concurrent request
* @param filesNamesMap object that maps hashed file names and original file names
* @param sasUploadFolder name of the folder that is created for the purpose of files in concurrent request
* @returns generated sas code
*/
export const generateFileUploadSasCode = async (
filesNamesMap: any,
sasSessionFolder: string
): Promise<string> => {
let uploadSasCode = ''
let fileCount = 0
let uploadedFilesMap: {
fileref: string
filepath: string
filename: string
count: number
}[] = []
const sasSessionFolderList: string[] = await listFilesInFolder(
sasSessionFolder
)
sasSessionFolderList.forEach((fileName) => {
let fileCountString = fileCount < 100 ? '0' + fileCount : fileCount
fileCountString = fileCount < 10 ? '00' + fileCount : fileCount
if (fileName.includes('req_file')) {
fileCount++
uploadedFilesMap.push({
fileref: `_sjs${fileCountString}`,
filepath: `${sasSessionFolder}/${fileName}`,
filename: filesNamesMap[fileName],
count: fileCount
})
}
})
for (let uploadedMap of uploadedFilesMap) {
uploadSasCode += `\nfilename ${uploadedMap.fileref} "${uploadedMap.filepath}";`
}
uploadSasCode += `\n%let _WEBIN_FILE_COUNT=${fileCount};`
for (let uploadedMap of uploadedFilesMap) {
uploadSasCode += `\n%let _WEBIN_FILENAME${uploadedMap.count}=${uploadedMap.filepath};`
}
for (let uploadedMap of uploadedFilesMap) {
uploadSasCode += `\n%let _WEBIN_FILEREF${uploadedMap.count}=${uploadedMap.fileref};`
}
for (let uploadedMap of uploadedFilesMap) {
uploadSasCode += `\n%let _WEBIN_NAME${uploadedMap.count}=${uploadedMap.filename};`
}
if (fileCount > 0) {
uploadSasCode += `\n%let _WEBIN_NAME=&_WEBIN_NAME1;`
uploadSasCode += `\n%let _WEBIN_FILEREF=&_WEBIN_FILEREF1;`
uploadSasCode += `\n%let _WEBIN_FILENAME=&_WEBIN_FILENAME1;`
}
uploadSasCode += `\n`
return uploadSasCode
}