diff --git a/.gitignore b/.gitignore index f0a8b6c..13323df 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ node_modules/ sas/ tmp/ build/ +certificates/ diff --git a/package-lock.json b/package-lock.json index 1f9eeaa..7f6be90 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1687,6 +1687,15 @@ "integrity": "sha512-fZQQafSREFyuZcdWFAExYjBiCL7AUCdgsk80iO0q4yihYYdcIiH28CcuPTGFgLOCC8RlW49GSQxdHwZP+I7CNg==", "dev": true }, + "@types/multer": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.7.tgz", + "integrity": "sha512-/SNsDidUFCvqqcWDwxv2feww/yqhNeTRL5CVoL3jU4Goc4kKEL10T7Eye65ZqPNi4HRx8sAEX59pV1aEH7drNA==", + "dev": true, + "requires": { + "@types/express": "*" + } + }, "@types/node": { "version": "15.12.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.2.tgz", @@ -1952,6 +1961,11 @@ "picomatch": "^2.0.4" } }, + "append-field": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", + "integrity": "sha1-HjRA6RXwsSA9I3SOeO3XubW0PlY=" + }, "arg": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", @@ -2216,8 +2230,39 @@ "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", - "dev": true + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" + }, + "busboy": { + "version": "0.2.14", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz", + "integrity": "sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=", + "requires": { + "dicer": "0.2.5", + "readable-stream": "1.1.x" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } }, "bytes": { "version": "3.1.0", @@ -2490,6 +2535,41 @@ "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true }, + "concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "requires": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "configstore": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz", @@ -2611,8 +2691,7 @@ "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", - "dev": true + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, "cosmiconfig": { "version": "7.0.0", @@ -2822,6 +2901,38 @@ "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", "dev": true }, + "dicer": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz", + "integrity": "sha1-WZbAhrszIYyBLAkL3cCc0S+stw8=", + "requires": { + "readable-stream": "1.1.x", + "streamsearch": "0.1.2" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -3929,8 +4040,7 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", - "dev": true + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, "isexe": { "version": "2.0.0", @@ -5439,8 +5549,7 @@ "minimist": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" }, "minimist-options": { "version": "4.1.0", @@ -5470,6 +5579,31 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "multer": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.3.tgz", + "integrity": "sha512-np0YLKncuZoTzufbkM6wEKp68EhWJXcU6fq6QqrSwkckd2LlMgd1UqhUJLj6NS/5sZ8dE8LYDWslsltJznnXlg==", + "requires": { + "append-field": "^1.0.0", + "busboy": "^0.2.11", + "concat-stream": "^1.5.2", + "mkdirp": "^0.5.4", + "object-assign": "^4.1.1", + "on-finished": "^2.3.0", + "type-is": "^1.6.4", + "xtend": "^4.0.0" + }, + "dependencies": { + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "requires": { + "minimist": "^1.2.5" + } + } + } + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -7664,6 +7798,11 @@ "integrity": "sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ==", "dev": true }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, "object-inspect": { "version": "1.10.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.10.3.tgz", @@ -8008,8 +8147,7 @@ "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "dev": true + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, "prompts": { "version": "2.4.1", @@ -8694,6 +8832,11 @@ } } }, + "streamsearch": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", + "integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=" + }, "string-length": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", @@ -9171,6 +9314,11 @@ "mime-types": "~2.1.24" } }, + "typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" + }, "typedarray-to-buffer": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", @@ -9266,8 +9414,7 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", - "dev": true + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "utils-merge": { "version": "1.0.1", @@ -9475,8 +9622,7 @@ "xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "dev": true + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" }, "y18n": { "version": "5.0.8", diff --git a/package.json b/package.json index 86c8e0f..b55e24d 100644 --- a/package.json +++ b/package.json @@ -22,11 +22,13 @@ "author": "Analytium Ltd", "dependencies": { "@sasjs/utils": "^2.23.3", - "express": "^4.17.1" + "express": "^4.17.1", + "multer": "^1.4.3" }, "devDependencies": { "@types/express": "^4.17.12", "@types/jest": "^26.0.24", + "@types/multer": "^1.4.7", "@types/node": "^15.12.2", "@types/supertest": "^2.0.11", "jest": "^27.0.6", @@ -40,7 +42,7 @@ "typescript": "^4.3.2" }, "configuration": { - "sasPath": "/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe/sas", - "sasJsPort": 5000 + "sasPath": "/opt/sas/sas9/SASHome/SASFoundation/9.4/sas", + "sasJsPort": 5005 } } diff --git a/src/controllers/Execution.ts b/src/controllers/Execution.ts index 04f835f..0410594 100644 --- a/src/controllers/Execution.ts +++ b/src/controllers/Execution.ts @@ -5,6 +5,7 @@ import { configuration } from '../../package.json' import { promisify } from 'util' import { execFile } from 'child_process' import { Session } from '../types' +import { generateFileUploadSasCode } from '../utils' const execFilePromise = promisify(execFile) export class ExecutionController { @@ -12,7 +13,8 @@ export class ExecutionController { program = '', autoExec?: string, session?: Session, - vars?: any + vars?: any, + otherArgs?: any ) { if (program) { if (!(await fileExists(program))) { @@ -40,7 +42,23 @@ export class ExecutionController { let webout = path.join(session.path, 'webout.txt') await createFile(webout, '') - program = `filename _webout "${webout}";\n${program}` + program = ` +%let sasjsprocessmode=Stored Program; +filename _webout "${webout}"; +${program}` + + // if no files are uploaded filesNamesMap will be undefined + if (otherArgs && otherArgs.filesNamesMap) { + const uploadSasCode = await generateFileUploadSasCode( + otherArgs.filesNamesMap, + session.path + ) + + //If sas code for the file is generated it will be appended to the top of sasCode + if (uploadSasCode.length > 0) { + program = `${uploadSasCode}` + program + } + } const code = path.join(session.path, 'code.sas') if (!(await fileExists(code))) { @@ -64,8 +82,6 @@ export class ExecutionController { if (await fileExists(log)) log = await readFile(log) else log = '' - if (stderr) return Promise.reject({ error: stderr, log: log }) - if (await fileExists(webout)) webout = await readFile(webout) else webout = '' @@ -73,7 +89,7 @@ export class ExecutionController { (key: string) => key.toLowerCase() === '_debug' ) - if (debug && vars[debug] >= 131) { + if ((debug && vars[debug] >= 131) || stderr) { webout = ` ${webout}
diff --git a/src/controllers/FileUploadController.ts b/src/controllers/FileUploadController.ts new file mode 100644 index 0000000..53d42de --- /dev/null +++ b/src/controllers/FileUploadController.ts @@ -0,0 +1,36 @@ +import { uuidv4 } from '@sasjs/utils' +import { getSessionController } from '.' +const multer = require('multer') + +export class FileUploadController { + private storage = multer.diskStorage({ + destination: function (req: any, file: any, cb: any) { + //Sending the intercepted files to the sessions subfolder + cb(null, req.sasSession.path) + }, + filename: function (req: any, file: any, cb: any) { + //req_file prefix + unique hash added to sas request files + cb(null, `req_file_${uuidv4().replace(/-/gm, '')}`) + } + }) + + private upload = multer({ storage: this.storage }) + + //It will intercept request and generate unique uuid to be used as a subfolder name + //that will store the files uploaded + public preuploadMiddleware = async (req: any, res: any, next: any) => { + let session + + const sessionController = getSessionController() + session = await sessionController.getSession() + session.inUse = true + + req.sasSession = session + + next() + } + + public getMulterUploadObject() { + return this.upload + } +} diff --git a/src/controllers/index.ts b/src/controllers/index.ts index 2990104..86939a2 100644 --- a/src/controllers/index.ts +++ b/src/controllers/index.ts @@ -1,3 +1,4 @@ export * from './deploy' export * from './Session' export * from './Execution' +export * from './FileUploadController' diff --git a/src/routes/index.ts b/src/routes/index.ts index 1fc3bc9..1854f7d 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -1,12 +1,23 @@ import express from 'express' -import { createFileTree, getTreeExample } from '../controllers' +import { + createFileTree, + getSessionController, + getTreeExample +} from '../controllers' import { ExecutionResult, isRequestQuery, isFileTree } from '../types' import path from 'path' -import { getTmpFilesFolderPath } from '../utils' -import { ExecutionController } from '../controllers' +import { + getTmpFilesFolderPath, + getTmpFolderPath, + makeFilesNamesMap +} from '../utils' +import { ExecutionController, FileUploadController } from '../controllers' +import { uuidv4 } from '@sasjs/utils' const router = express.Router() +const fileUploadController = new FileUploadController() + router.get('/', async (_, res) => { res.status(200).send('Welcome to @sasjs/server API') }) @@ -46,10 +57,13 @@ router.get('/SASjsExecutor', async (req, res) => { router.get('/SASjsExecutor/do', async (req, res) => { if (isRequestQuery(req.query)) { - const sasCodePath = path + let sasCodePath = path .join(getTmpFilesFolderPath(), req.query._program) .replace(new RegExp('/', 'g'), path.sep) + // If no extension provided, add .sas extension + sasCodePath += '.sas' + await new ExecutionController() .execute(sasCodePath, undefined, undefined, { ...req.query }) .then((result: {}) => { @@ -70,4 +84,50 @@ router.get('/SASjsExecutor/do', async (req, res) => { } }) +router.post( + '/SASjsExecutor/do', + fileUploadController.preuploadMiddleware, + fileUploadController.getMulterUploadObject().any(), + async (req: any, res: any) => { + if (isRequestQuery(req.query)) { + let sasCodePath = path + .join(getTmpFilesFolderPath(), req.query._program) + .replace(new RegExp('/', 'g'), path.sep) + + // If no extension provided, add .sas extension + sasCodePath += '.sas' + + let filesNamesMap = null + + if (req.files && req.files.length > 0) { + filesNamesMap = makeFilesNamesMap(req.files) + } + + await new ExecutionController() + .execute( + sasCodePath, + undefined, + req.sasSession, + { ...req.query, ...req.body }, + { filesNamesMap: filesNamesMap } + ) + .then((result: {}) => { + res.status(200).send(result) + }) + .catch((err: {} | string) => { + res.status(400).send({ + status: 'failure', + message: 'Job execution failed.', + ...(typeof err === 'object' ? err : { details: err }) + }) + }) + } else { + res.status(400).send({ + status: 'failure', + message: `Please provide the location of SAS code` + }) + } + } +) + export default router diff --git a/src/types/Upload.ts b/src/types/Upload.ts new file mode 100644 index 0000000..74628f9 --- /dev/null +++ b/src/types/Upload.ts @@ -0,0 +1,10 @@ +export interface MulterFile { + fieldname: string + originalname: string + encoding: string + mimetype: string + destination: string + filename: string + path: string + size: number +} diff --git a/src/utils/index.ts b/src/utils/index.ts index 31581eb..2f4a3cb 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,2 +1,3 @@ export * from './file' export * from './sleep' +export * from './upload' diff --git a/src/utils/upload.ts b/src/utils/upload.ts new file mode 100644 index 0000000..f671367 --- /dev/null +++ b/src/utils/upload.ts @@ -0,0 +1,89 @@ +import path from 'path' +import fs from 'fs' +import { getTmpSessionsFolderPath } from '.' +import { MulterFile } from '../types/Upload' +import { listFilesInFolder } from '@sasjs/utils' + +/** + * It will create an object that maps hashed file names to the original names + * @param files array of files to be mapped + * @returns object + */ +export const makeFilesNamesMap = (files: MulterFile[]) => { + if (!files) return null + + const filesNamesMap: { [key: string]: string } = {} + + for (let file of files) { + filesNamesMap[file.filename] = file.fieldname + } + + return filesNamesMap +} + +/** + * Generates the sas code that references uploaded files in the concurrent request + * @param filesNamesMap object that maps hashed file names and original file names + * @param sasUploadFolder name of the folder that is created for the purpose of files in concurrent request + * @returns generated sas code + */ +export const generateFileUploadSasCode = async ( + filesNamesMap: any, + sasSessionFolder: string +): Promise => { + let uploadSasCode = '' + let fileCount = 0 + let uploadedFilesMap: { + fileref: string + filepath: string + filename: string + count: number + }[] = [] + + const sasSessionFolderList: string[] = await listFilesInFolder( + sasSessionFolder + ) + sasSessionFolderList.forEach((fileName) => { + let fileCountString = fileCount < 100 ? '0' + fileCount : fileCount + fileCountString = fileCount < 10 ? '00' + fileCount : fileCount + + if (fileName.includes('req_file')) { + fileCount++ + + uploadedFilesMap.push({ + fileref: `_sjs${fileCountString}`, + filepath: `${sasSessionFolder}/${fileName}`, + filename: filesNamesMap[fileName], + count: fileCount + }) + } + }) + + for (let uploadedMap of uploadedFilesMap) { + uploadSasCode += `\nfilename ${uploadedMap.fileref} "${uploadedMap.filepath}";` + } + + uploadSasCode += `\n%let _WEBIN_FILE_COUNT=${fileCount};` + + for (let uploadedMap of uploadedFilesMap) { + uploadSasCode += `\n%let _WEBIN_FILENAME${uploadedMap.count}=${uploadedMap.filepath};` + } + + for (let uploadedMap of uploadedFilesMap) { + uploadSasCode += `\n%let _WEBIN_FILEREF${uploadedMap.count}=${uploadedMap.fileref};` + } + + for (let uploadedMap of uploadedFilesMap) { + uploadSasCode += `\n%let _WEBIN_NAME${uploadedMap.count}=${uploadedMap.filename};` + } + + if (fileCount > 0) { + uploadSasCode += `\n%let _WEBIN_NAME=&_WEBIN_NAME1;` + uploadSasCode += `\n%let _WEBIN_FILEREF=&_WEBIN_FILEREF1;` + uploadSasCode += `\n%let _WEBIN_FILENAME=&_WEBIN_FILENAME1;` + } + + uploadSasCode += `\n` + + return uploadSasCode +}