From 17a7a26fc34d992bf0fcbb301bbf707e83d89c03 Mon Sep 17 00:00:00 2001 From: Mihajlo Medjedovic Date: Thu, 14 Oct 2021 15:44:28 +0000 Subject: [PATCH 01/12] chore: merging with sessions - sas request with file upload --- .gitignore | 1 + package-lock.json | 180 ++++++++++++++++++++++++++++++++--- package.json | 7 +- src/controllers/Execution.ts | 32 +++++-- src/controllers/Session.ts | 2 +- src/routes/index.ts | 73 +++++++++++++- src/utils/index.ts | 1 + src/utils/upload.ts | 86 +++++++++++++++++ 8 files changed, 355 insertions(+), 27 deletions(-) create mode 100644 src/utils/upload.ts diff --git a/.gitignore b/.gitignore index f0a8b6c..7c3856c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ node_modules/ sas/ tmp/ build/ +certificates/ \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 1f9eeaa..6c31521 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1518,6 +1518,12 @@ "integrity": "sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog==", "dev": true }, + "@types/cors": { + "version": "2.8.12", + "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", + "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==", + "dev": true + }, "@types/express": { "version": "4.17.12", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.12.tgz", @@ -1952,6 +1958,11 @@ "picomatch": "^2.0.4" } }, + "append-field": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/append-field/-/append-field-1.0.0.tgz", + "integrity": "sha1-HjRA6RXwsSA9I3SOeO3XubW0PlY=" + }, "arg": { "version": "4.1.3", "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", @@ -2216,8 +2227,39 @@ "buffer-from": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz", - "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==", - "dev": true + "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==" + }, + "busboy": { + "version": "0.2.14", + "resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.14.tgz", + "integrity": "sha1-bCpiLvz0fFe7vh4qnDetNseSVFM=", + "requires": { + "dicer": "0.2.5", + "readable-stream": "1.1.x" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } }, "bytes": { "version": "3.1.0", @@ -2490,6 +2532,41 @@ "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", "dev": true }, + "concat-stream": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz", + "integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==", + "requires": { + "buffer-from": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^2.2.2", + "typedarray": "^0.0.6" + }, + "dependencies": { + "readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "requires": { + "safe-buffer": "~5.1.0" + } + } + } + }, "configstore": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/configstore/-/configstore-5.0.1.tgz", @@ -2611,8 +2688,16 @@ "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", - "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", - "dev": true + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "cors": { + "version": "2.8.5", + "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", + "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", + "requires": { + "object-assign": "^4", + "vary": "^1" + } }, "cosmiconfig": { "version": "7.0.0", @@ -2822,6 +2907,38 @@ "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", "dev": true }, + "dicer": { + "version": "0.2.5", + "resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz", + "integrity": "sha1-WZbAhrszIYyBLAkL3cCc0S+stw8=", + "requires": { + "readable-stream": "1.1.x", + "streamsearch": "0.1.2" + }, + "dependencies": { + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8=" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha1-fPTFTvZI44EwhMY23SB54WbAgdk=", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha1-YuIDvEF2bGwoyfyEMB2rHFMQ+pQ=" + } + } + }, "diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -3929,8 +4046,7 @@ "isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", - "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", - "dev": true + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" }, "isexe": { "version": "2.0.0", @@ -5439,8 +5555,7 @@ "minimist": { "version": "1.2.5", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", - "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", - "dev": true + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" }, "minimist-options": { "version": "4.1.0", @@ -5470,6 +5585,31 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" }, + "multer": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/multer/-/multer-1.4.3.tgz", + "integrity": "sha512-np0YLKncuZoTzufbkM6wEKp68EhWJXcU6fq6QqrSwkckd2LlMgd1UqhUJLj6NS/5sZ8dE8LYDWslsltJznnXlg==", + "requires": { + "append-field": "^1.0.0", + "busboy": "^0.2.11", + "concat-stream": "^1.5.2", + "mkdirp": "^0.5.4", + "object-assign": "^4.1.1", + "on-finished": "^2.3.0", + "type-is": "^1.6.4", + "xtend": "^4.0.0" + }, + "dependencies": { + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "requires": { + "minimist": "^1.2.5" + } + } + } + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -7664,6 +7804,11 @@ "integrity": "sha512-h2AatdwYH+JHiZpv7pt/gSX1XoRGb7L/qSIeuqA6GwYoF9w1vP1cw42TO0aI2pNyshRK5893hNSl+1//vHK7hQ==", "dev": true }, + "object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" + }, "object-inspect": { "version": "1.10.3", "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.10.3.tgz", @@ -8008,8 +8153,7 @@ "process-nextick-args": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", - "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", - "dev": true + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, "prompts": { "version": "2.4.1", @@ -8694,6 +8838,11 @@ } } }, + "streamsearch": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz", + "integrity": "sha1-gIudDlb8Jz2Am6VzOOkpkZoanxo=" + }, "string-length": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", @@ -9171,6 +9320,11 @@ "mime-types": "~2.1.24" } }, + "typedarray": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz", + "integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c=" + }, "typedarray-to-buffer": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz", @@ -9266,8 +9420,7 @@ "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", - "dev": true + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "utils-merge": { "version": "1.0.1", @@ -9475,8 +9628,7 @@ "xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", - "dev": true + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" }, "y18n": { "version": "5.0.8", diff --git a/package.json b/package.json index 86c8e0f..be13750 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,8 @@ "author": "Analytium Ltd", "dependencies": { "@sasjs/utils": "^2.23.3", - "express": "^4.17.1" + "express": "^4.17.1", + "multer": "^1.4.3" }, "devDependencies": { "@types/express": "^4.17.12", @@ -40,7 +41,7 @@ "typescript": "^4.3.2" }, "configuration": { - "sasPath": "/opt/sas/sas9/SASHome/SASFoundation/9.4/sasexe/sas", - "sasJsPort": 5000 + "sasPath": "/opt/sas/sas9/SASHome/SASFoundation/9.4/sas", + "sasJsPort": 5005 } } diff --git a/src/controllers/Execution.ts b/src/controllers/Execution.ts index 04f835f..af379c1 100644 --- a/src/controllers/Execution.ts +++ b/src/controllers/Execution.ts @@ -5,6 +5,7 @@ import { configuration } from '../../package.json' import { promisify } from 'util' import { execFile } from 'child_process' import { Session } from '../types' +import { generateFileUploadSasCode } from '../utils' const execFilePromise = promisify(execFile) export class ExecutionController { @@ -12,7 +13,8 @@ export class ExecutionController { program = '', autoExec?: string, session?: Session, - vars?: any + vars?: any, + otherArgs?: any ) { if (program) { if (!(await fileExists(program))) { @@ -39,10 +41,26 @@ export class ExecutionController { let webout = path.join(session.path, 'webout.txt') await createFile(webout, '') + + program = ` +%let sasjsprocessmode=Stored Program; +filename _webout "${webout}"; +${program}` - program = `filename _webout "${webout}";\n${program}` + // if no files are uploaded filesNamesMap will be undefined + if (otherArgs && otherArgs.filesNamesMap) { + const uploadSasCode = generateFileUploadSasCode( + otherArgs.filesNamesMap, + session.path + ) - const code = path.join(session.path, 'code.sas') + //If sas code for the file is generated it will be appended to the top of sasCode + if (uploadSasCode.length > 0) { + program = `${uploadSasCode}` + program + } + } + + let code = path.join(session.path, 'code.sas') if (!(await fileExists(code))) { await createFile(code, program) } @@ -64,7 +82,9 @@ export class ExecutionController { if (await fileExists(log)) log = await readFile(log) else log = '' - if (stderr) return Promise.reject({ error: stderr, log: log }) + // if (stderr) { + // return Promise.reject({ error: stderr, log: log }) + // } if (await fileExists(webout)) webout = await readFile(webout) else webout = '' @@ -73,7 +93,7 @@ export class ExecutionController { (key: string) => key.toLowerCase() === '_debug' ) - if (debug && vars[debug] >= 131) { + if (debug && vars[debug] >= 131 || stderr) { webout = ` ${webout}
@@ -82,7 +102,7 @@ ${webout}
` } - + session.inUse = false sessionController.deleteSession(session) diff --git a/src/controllers/Session.ts b/src/controllers/Session.ts index 3e9af19..57a4850 100644 --- a/src/controllers/Session.ts +++ b/src/controllers/Session.ts @@ -70,7 +70,7 @@ export class SessionController { this.scheduleSessionDestroy(session) - this.executionController.execute('', autoExec, session).catch(() => {}) + this.executionController.execute('', autoExec, session).catch((err) => {}) this.sessions.push(session) diff --git a/src/routes/index.ts b/src/routes/index.ts index 1fc3bc9..60f280f 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -1,12 +1,41 @@ import express from 'express' -import { createFileTree, getTreeExample } from '../controllers' +import { createFileTree, getSessionController, getTreeExample } from '../controllers' import { ExecutionResult, isRequestQuery, isFileTree } from '../types' import path from 'path' -import { getTmpFilesFolderPath } from '../utils' +import { getTmpFilesFolderPath, getTmpFolderPath, makeFilesNamesMap } from '../utils' import { ExecutionController } from '../controllers' +import { uuidv4 } from '@sasjs/utils' +const multer = require('multer') const router = express.Router() +const storage = multer.diskStorage({ + destination: function (req: any, file: any, cb: any) { + //Sending the intercepted files to the sessions subfolder + cb(null, req.sasSession.path) + }, + filename: function (req: any, file: any, cb: any) { + //req_file prefix + unique hash added to sas request files + cb(null, `req_file_${uuidv4().replace(/-/gm, '')}`) + } +}) + +const upload = multer({ storage: storage }) + +//It will intercept request and generate uniqe uuid to be used as a subfolder name +//that will store the files uploaded +const preuploadMiddleware = async (req: any, res: any, next: any) => { + let session + + const sessionController = getSessionController() + session = await sessionController.getSession() + session.inUse = true + + req.sasSession = session + + next() +} + router.get('/', async (_, res) => { res.status(200).send('Welcome to @sasjs/server API') }) @@ -46,9 +75,12 @@ router.get('/SASjsExecutor', async (req, res) => { router.get('/SASjsExecutor/do', async (req, res) => { if (isRequestQuery(req.query)) { - const sasCodePath = path + let sasCodePath = path .join(getTmpFilesFolderPath(), req.query._program) .replace(new RegExp('/', 'g'), path.sep) + + // If no extension provided, add .sas extension + sasCodePath += !sasCodePath.includes('.') ? '.sas' : '' await new ExecutionController() .execute(sasCodePath, undefined, undefined, { ...req.query }) @@ -70,4 +102,39 @@ router.get('/SASjsExecutor/do', async (req, res) => { } }) +router.post('/SASjsExecutor/do', preuploadMiddleware, upload.any(), async (req: any, res: any) => { + if (isRequestQuery(req.query)) { + let sasCodePath = path + .join(getTmpFilesFolderPath(), req.query._program) + .replace(new RegExp('/', 'g'), path.sep) + + // If no extension provided, add .sas extension + sasCodePath += !sasCodePath.includes('.') ? '.sas' : '' + + let filesNamesMap = null + + if (req.files && req.files.length > 0) { + filesNamesMap = makeFilesNamesMap(req.files) + } + + await new ExecutionController() + .execute(sasCodePath, undefined, req.sasSession, { ...req.query }, { filesNamesMap: filesNamesMap }) + .then((result: {}) => { + res.status(200).send(result) + }) + .catch((err: {} | string) => { + res.status(400).send({ + status: 'failure', + message: 'Job execution failed.', + ...(typeof err === 'object' ? err : { details: err }) + }) + }) + } else { + res.status(400).send({ + status: 'failure', + message: `Please provide the location of SAS code` + }) + } +}) + export default router diff --git a/src/utils/index.ts b/src/utils/index.ts index 31581eb..61fe8f3 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,2 +1,3 @@ export * from './file' export * from './sleep' +export * from './upload' \ No newline at end of file diff --git a/src/utils/upload.ts b/src/utils/upload.ts new file mode 100644 index 0000000..ce9e8a0 --- /dev/null +++ b/src/utils/upload.ts @@ -0,0 +1,86 @@ +import path from 'path' +import fs from 'fs' +import { getTmpSessionsFolderPath } from '.' + +/** + * It will create a object that maps hashed file names to the original names + * @param files array of files to be mapped + * @returns object + */ +export const makeFilesNamesMap = (files: any) => { + if (!files) return null + + const filesNamesMap: any = {} + + for (let file of files) { + filesNamesMap[file.filename] = file.fieldname + } + + return filesNamesMap +} + +/** + * Generates the sas code that reference uploaded files in the concurrent request + * @param filesNamesMap object that maps hashed file names and original file names + * @param sasUploadFolder name of the folder that is created for the purpose of files in concurrent request + * @returns generated sas code + */ + export const generateFileUploadSasCode = ( + filesNamesMap: any, + sasSessionFolder: string +): string => { + const uploadFilesDirPath = sasSessionFolder + + let uploadSasCode = '' + let fileCount = 0 + let uploadedFilesMap: { + fileref: string + filepath: string + filename: string + count: number + }[] = [] + + fs.readdirSync(uploadFilesDirPath).forEach((fileName) => { + let fileCountString = fileCount < 100 ? '0' + fileCount : fileCount + fileCountString = fileCount < 10 ? '00' + fileCount : fileCount + + if (fileName.includes('req_file')) { + fileCount++ + + uploadedFilesMap.push({ + fileref: `_sjs${fileCountString}`, + filepath: `${uploadFilesDirPath}/${fileName}`, + filename: filesNamesMap[fileName], + count: fileCount + }) + } + }) + + for (let uploadedMap of uploadedFilesMap) { + uploadSasCode += `\nfilename ${uploadedMap.fileref} "${uploadedMap.filepath}";` + } + + uploadSasCode += `\n%let _WEBIN_FILE_COUNT=${fileCount};` + + for (let uploadedMap of uploadedFilesMap) { + uploadSasCode += `\n%let _WEBIN_FILENAME${uploadedMap.count}=${uploadedMap.filepath};` + } + + for (let uploadedMap of uploadedFilesMap) { + uploadSasCode += `\n%let _WEBIN_FILEREF${uploadedMap.count}=${uploadedMap.fileref};` + } + + for (let uploadedMap of uploadedFilesMap) { + uploadSasCode += `\n%let _WEBIN_NAME${uploadedMap.count}=${uploadedMap.filename};` + } + + if (fileCount > 0) { + uploadSasCode += `\n%let _WEBIN_NAME=&_WEBIN_NAME1;` + uploadSasCode += `\n%let _WEBIN_FILEREF=&_WEBIN_FILEREF1;` + uploadSasCode += `\n%let _WEBIN_FILENAME=&_WEBIN_FILENAME1;` + } + + uploadSasCode += `\n` + + return uploadSasCode +} \ No newline at end of file From 716ae81d9293b42dd2a7047ac52d75401b3b8798 Mon Sep 17 00:00:00 2001 From: Allan Bowe Date: Fri, 15 Oct 2021 06:24:01 +0000 Subject: [PATCH 02/12] fix: prettier --- src/controllers/Execution.ts | 6 +-- src/routes/index.ts | 81 ++++++++++++++++++++++-------------- src/utils/index.ts | 2 +- src/utils/upload.ts | 16 +++---- 4 files changed, 62 insertions(+), 43 deletions(-) diff --git a/src/controllers/Execution.ts b/src/controllers/Execution.ts index af379c1..47b5218 100644 --- a/src/controllers/Execution.ts +++ b/src/controllers/Execution.ts @@ -41,7 +41,7 @@ export class ExecutionController { let webout = path.join(session.path, 'webout.txt') await createFile(webout, '') - + program = ` %let sasjsprocessmode=Stored Program; filename _webout "${webout}"; @@ -93,7 +93,7 @@ ${program}` (key: string) => key.toLowerCase() === '_debug' ) - if (debug && vars[debug] >= 131 || stderr) { + if ((debug && vars[debug] >= 131) || stderr) { webout = ` ${webout}
@@ -102,7 +102,7 @@ ${webout}
` } - + session.inUse = false sessionController.deleteSession(session) diff --git a/src/routes/index.ts b/src/routes/index.ts index 60f280f..ba7e3a4 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -1,8 +1,16 @@ import express from 'express' -import { createFileTree, getSessionController, getTreeExample } from '../controllers' +import { + createFileTree, + getSessionController, + getTreeExample +} from '../controllers' import { ExecutionResult, isRequestQuery, isFileTree } from '../types' import path from 'path' -import { getTmpFilesFolderPath, getTmpFolderPath, makeFilesNamesMap } from '../utils' +import { + getTmpFilesFolderPath, + getTmpFolderPath, + makeFilesNamesMap +} from '../utils' import { ExecutionController } from '../controllers' import { uuidv4 } from '@sasjs/utils' @@ -30,7 +38,7 @@ const preuploadMiddleware = async (req: any, res: any, next: any) => { const sessionController = getSessionController() session = await sessionController.getSession() session.inUse = true - + req.sasSession = session next() @@ -78,7 +86,7 @@ router.get('/SASjsExecutor/do', async (req, res) => { let sasCodePath = path .join(getTmpFilesFolderPath(), req.query._program) .replace(new RegExp('/', 'g'), path.sep) - + // If no extension provided, add .sas extension sasCodePath += !sasCodePath.includes('.') ? '.sas' : '' @@ -102,39 +110,50 @@ router.get('/SASjsExecutor/do', async (req, res) => { } }) -router.post('/SASjsExecutor/do', preuploadMiddleware, upload.any(), async (req: any, res: any) => { - if (isRequestQuery(req.query)) { - let sasCodePath = path - .join(getTmpFilesFolderPath(), req.query._program) - .replace(new RegExp('/', 'g'), path.sep) +router.post( + '/SASjsExecutor/do', + preuploadMiddleware, + upload.any(), + async (req: any, res: any) => { + if (isRequestQuery(req.query)) { + let sasCodePath = path + .join(getTmpFilesFolderPath(), req.query._program) + .replace(new RegExp('/', 'g'), path.sep) - // If no extension provided, add .sas extension - sasCodePath += !sasCodePath.includes('.') ? '.sas' : '' + // If no extension provided, add .sas extension + sasCodePath += !sasCodePath.includes('.') ? '.sas' : '' - let filesNamesMap = null + let filesNamesMap = null - if (req.files && req.files.length > 0) { - filesNamesMap = makeFilesNamesMap(req.files) - } + if (req.files && req.files.length > 0) { + filesNamesMap = makeFilesNamesMap(req.files) + } - await new ExecutionController() - .execute(sasCodePath, undefined, req.sasSession, { ...req.query }, { filesNamesMap: filesNamesMap }) - .then((result: {}) => { - res.status(200).send(result) - }) - .catch((err: {} | string) => { - res.status(400).send({ - status: 'failure', - message: 'Job execution failed.', - ...(typeof err === 'object' ? err : { details: err }) + await new ExecutionController() + .execute( + sasCodePath, + undefined, + req.sasSession, + { ...req.query }, + { filesNamesMap: filesNamesMap } + ) + .then((result: {}) => { + res.status(200).send(result) }) + .catch((err: {} | string) => { + res.status(400).send({ + status: 'failure', + message: 'Job execution failed.', + ...(typeof err === 'object' ? err : { details: err }) + }) + }) + } else { + res.status(400).send({ + status: 'failure', + message: `Please provide the location of SAS code` }) - } else { - res.status(400).send({ - status: 'failure', - message: `Please provide the location of SAS code` - }) + } } -}) +) export default router diff --git a/src/utils/index.ts b/src/utils/index.ts index 61fe8f3..2f4a3cb 100644 --- a/src/utils/index.ts +++ b/src/utils/index.ts @@ -1,3 +1,3 @@ export * from './file' export * from './sleep' -export * from './upload' \ No newline at end of file +export * from './upload' diff --git a/src/utils/upload.ts b/src/utils/upload.ts index ce9e8a0..b04a2ee 100644 --- a/src/utils/upload.ts +++ b/src/utils/upload.ts @@ -8,15 +8,15 @@ import { getTmpSessionsFolderPath } from '.' * @returns object */ export const makeFilesNamesMap = (files: any) => { - if (!files) return null + if (!files) return null - const filesNamesMap: any = {} + const filesNamesMap: any = {} - for (let file of files) { - filesNamesMap[file.filename] = file.fieldname - } + for (let file of files) { + filesNamesMap[file.filename] = file.fieldname + } - return filesNamesMap + return filesNamesMap } /** @@ -25,7 +25,7 @@ export const makeFilesNamesMap = (files: any) => { * @param sasUploadFolder name of the folder that is created for the purpose of files in concurrent request * @returns generated sas code */ - export const generateFileUploadSasCode = ( +export const generateFileUploadSasCode = ( filesNamesMap: any, sasSessionFolder: string ): string => { @@ -83,4 +83,4 @@ export const makeFilesNamesMap = (files: any) => { uploadSasCode += `\n` return uploadSasCode -} \ No newline at end of file +} From c0a4e1aa14096ab5358f965b8ca18c70c03ae564 Mon Sep 17 00:00:00 2001 From: Mihajlo Medjedovic Date: Fri, 15 Oct 2021 12:43:22 +0000 Subject: [PATCH 03/12] chore: comments addressing --- .gitignore | 2 +- code.lst | 10 +++++++ package-lock.json | 24 ++++++---------- package.json | 1 + src/controllers/Execution.ts | 6 +--- src/controllers/FileUploadController.ts | 36 ++++++++++++++++++++++++ src/controllers/Session.ts | 2 +- src/controllers/index.ts | 1 + src/routes/index.ts | 37 ++++--------------------- src/types/Upload.ts | 10 +++++++ src/utils/file.ts | 4 +++ src/utils/upload.ts | 17 ++++++------ 12 files changed, 88 insertions(+), 62 deletions(-) create mode 100644 code.lst create mode 100644 src/controllers/FileUploadController.ts create mode 100644 src/types/Upload.ts diff --git a/.gitignore b/.gitignore index 7c3856c..13323df 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ node_modules/ sas/ tmp/ build/ -certificates/ \ No newline at end of file +certificates/ diff --git a/code.lst b/code.lst new file mode 100644 index 0000000..afd66d7 --- /dev/null +++ b/code.lst @@ -0,0 +1,10 @@ + The SAS System Friday, 15 October 2021 11:20:00 1 + + + -------- + 0 + The SAS System Friday, 15 October 2021 11:20:00 2 + + + -------- + 41140 diff --git a/package-lock.json b/package-lock.json index 6c31521..7f6be90 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1518,12 +1518,6 @@ "integrity": "sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog==", "dev": true }, - "@types/cors": { - "version": "2.8.12", - "resolved": "https://registry.npmjs.org/@types/cors/-/cors-2.8.12.tgz", - "integrity": "sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==", - "dev": true - }, "@types/express": { "version": "4.17.12", "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.12.tgz", @@ -1693,6 +1687,15 @@ "integrity": "sha512-fZQQafSREFyuZcdWFAExYjBiCL7AUCdgsk80iO0q4yihYYdcIiH28CcuPTGFgLOCC8RlW49GSQxdHwZP+I7CNg==", "dev": true }, + "@types/multer": { + "version": "1.4.7", + "resolved": "https://registry.npmjs.org/@types/multer/-/multer-1.4.7.tgz", + "integrity": "sha512-/SNsDidUFCvqqcWDwxv2feww/yqhNeTRL5CVoL3jU4Goc4kKEL10T7Eye65ZqPNi4HRx8sAEX59pV1aEH7drNA==", + "dev": true, + "requires": { + "@types/express": "*" + } + }, "@types/node": { "version": "15.12.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-15.12.2.tgz", @@ -2690,15 +2693,6 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, - "cors": { - "version": "2.8.5", - "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", - "integrity": "sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g==", - "requires": { - "object-assign": "^4", - "vary": "^1" - } - }, "cosmiconfig": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.0.0.tgz", diff --git a/package.json b/package.json index be13750..b55e24d 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "devDependencies": { "@types/express": "^4.17.12", "@types/jest": "^26.0.24", + "@types/multer": "^1.4.7", "@types/node": "^15.12.2", "@types/supertest": "^2.0.11", "jest": "^27.0.6", diff --git a/src/controllers/Execution.ts b/src/controllers/Execution.ts index af379c1..5eb4383 100644 --- a/src/controllers/Execution.ts +++ b/src/controllers/Execution.ts @@ -60,7 +60,7 @@ ${program}` } } - let code = path.join(session.path, 'code.sas') + const code = path.join(session.path, 'code.sas') if (!(await fileExists(code))) { await createFile(code, program) } @@ -82,10 +82,6 @@ ${program}` if (await fileExists(log)) log = await readFile(log) else log = '' - // if (stderr) { - // return Promise.reject({ error: stderr, log: log }) - // } - if (await fileExists(webout)) webout = await readFile(webout) else webout = '' diff --git a/src/controllers/FileUploadController.ts b/src/controllers/FileUploadController.ts new file mode 100644 index 0000000..0bd015c --- /dev/null +++ b/src/controllers/FileUploadController.ts @@ -0,0 +1,36 @@ +import { uuidv4 } from '@sasjs/utils' +import { getSessionController } from '.' +const multer = require('multer') + +export class FileUploadController { + private storage = multer.diskStorage({ + destination: function (req: any, file: any, cb: any) { + //Sending the intercepted files to the sessions subfolder + cb(null, req.sasSession.path) + }, + filename: function (req: any, file: any, cb: any) { + //req_file prefix + unique hash added to sas request files + cb(null, `req_file_${uuidv4().replace(/-/gm, '')}`) + } + }) + + private upload = multer({ storage: this.storage }) + + //It will intercept request and generate uniqe uuid to be used as a subfolder name + //that will store the files uploaded + public preuploadMiddleware = async (req: any, res: any, next: any) => { + let session + + const sessionController = getSessionController() + session = await sessionController.getSession() + session.inUse = true + + req.sasSession = session + + next() + } + + public getMulterUploadObject() { + return this.upload + } +} \ No newline at end of file diff --git a/src/controllers/Session.ts b/src/controllers/Session.ts index 57a4850..3e9af19 100644 --- a/src/controllers/Session.ts +++ b/src/controllers/Session.ts @@ -70,7 +70,7 @@ export class SessionController { this.scheduleSessionDestroy(session) - this.executionController.execute('', autoExec, session).catch((err) => {}) + this.executionController.execute('', autoExec, session).catch(() => {}) this.sessions.push(session) diff --git a/src/controllers/index.ts b/src/controllers/index.ts index 2990104..60c5d59 100644 --- a/src/controllers/index.ts +++ b/src/controllers/index.ts @@ -1,3 +1,4 @@ export * from './deploy' export * from './Session' export * from './Execution' +export * from './FileUploadController' \ No newline at end of file diff --git a/src/routes/index.ts b/src/routes/index.ts index 60f280f..0f921bf 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -2,39 +2,12 @@ import express from 'express' import { createFileTree, getSessionController, getTreeExample } from '../controllers' import { ExecutionResult, isRequestQuery, isFileTree } from '../types' import path from 'path' -import { getTmpFilesFolderPath, getTmpFolderPath, makeFilesNamesMap } from '../utils' -import { ExecutionController } from '../controllers' -import { uuidv4 } from '@sasjs/utils' +import { addExtensionIfNotFound, getTmpFilesFolderPath, getTmpFolderPath, makeFilesNamesMap } from '../utils' +import { ExecutionController, FileUploadController } from '../controllers' -const multer = require('multer') const router = express.Router() -const storage = multer.diskStorage({ - destination: function (req: any, file: any, cb: any) { - //Sending the intercepted files to the sessions subfolder - cb(null, req.sasSession.path) - }, - filename: function (req: any, file: any, cb: any) { - //req_file prefix + unique hash added to sas request files - cb(null, `req_file_${uuidv4().replace(/-/gm, '')}`) - } -}) - -const upload = multer({ storage: storage }) - -//It will intercept request and generate uniqe uuid to be used as a subfolder name -//that will store the files uploaded -const preuploadMiddleware = async (req: any, res: any, next: any) => { - let session - - const sessionController = getSessionController() - session = await sessionController.getSession() - session.inUse = true - - req.sasSession = session - - next() -} +const fileUploadController = new FileUploadController() router.get('/', async (_, res) => { res.status(200).send('Welcome to @sasjs/server API') @@ -102,14 +75,14 @@ router.get('/SASjsExecutor/do', async (req, res) => { } }) -router.post('/SASjsExecutor/do', preuploadMiddleware, upload.any(), async (req: any, res: any) => { +router.post('/SASjsExecutor/do', fileUploadController.preuploadMiddleware, fileUploadController.getMulterUploadObject().any(), async (req: any, res: any) => { if (isRequestQuery(req.query)) { let sasCodePath = path .join(getTmpFilesFolderPath(), req.query._program) .replace(new RegExp('/', 'g'), path.sep) // If no extension provided, add .sas extension - sasCodePath += !sasCodePath.includes('.') ? '.sas' : '' + sasCodePath += addExtensionIfNotFound(sasCodePath, 'sas') let filesNamesMap = null diff --git a/src/types/Upload.ts b/src/types/Upload.ts new file mode 100644 index 0000000..d025a4e --- /dev/null +++ b/src/types/Upload.ts @@ -0,0 +1,10 @@ +export interface MulterFile { + fieldname: string + originalname: string + encoding: string + mimetype: string + destination: string + filename: string + path: string + size: number +} \ No newline at end of file diff --git a/src/utils/file.ts b/src/utils/file.ts index fe09816..3dfe57f 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -24,3 +24,7 @@ export const generateUniqueFileName = (fileName: string, extension = '') => new Date().getTime(), extension ].join('') + +export const addExtensionIfNotFound = (value: string, extension: string) => { + return !value.includes('.') ? `.${extension}` : '' +} \ No newline at end of file diff --git a/src/utils/upload.ts b/src/utils/upload.ts index ce9e8a0..b033494 100644 --- a/src/utils/upload.ts +++ b/src/utils/upload.ts @@ -1,16 +1,18 @@ import path from 'path' import fs from 'fs' import { getTmpSessionsFolderPath } from '.' +import { MulterFile } from '../types/Upload' +import { listFilesInFolder } from '@sasjs/utils' /** - * It will create a object that maps hashed file names to the original names + * It will create an object that maps hashed file names to the original names * @param files array of files to be mapped * @returns object */ -export const makeFilesNamesMap = (files: any) => { +export const makeFilesNamesMap = (files: MulterFile[]) => { if (!files) return null - const filesNamesMap: any = {} + const filesNamesMap: {[key: string]: string} = {} for (let file of files) { filesNamesMap[file.filename] = file.fieldname @@ -20,7 +22,7 @@ export const makeFilesNamesMap = (files: any) => { } /** - * Generates the sas code that reference uploaded files in the concurrent request + * Generates the sas code that references uploaded files in the concurrent request * @param filesNamesMap object that maps hashed file names and original file names * @param sasUploadFolder name of the folder that is created for the purpose of files in concurrent request * @returns generated sas code @@ -29,8 +31,6 @@ export const makeFilesNamesMap = (files: any) => { filesNamesMap: any, sasSessionFolder: string ): string => { - const uploadFilesDirPath = sasSessionFolder - let uploadSasCode = '' let fileCount = 0 let uploadedFilesMap: { @@ -40,7 +40,8 @@ export const makeFilesNamesMap = (files: any) => { count: number }[] = [] - fs.readdirSync(uploadFilesDirPath).forEach((fileName) => { + + fs.readdirSync(sasSessionFolder).forEach((fileName) => { let fileCountString = fileCount < 100 ? '0' + fileCount : fileCount fileCountString = fileCount < 10 ? '00' + fileCount : fileCount @@ -49,7 +50,7 @@ export const makeFilesNamesMap = (files: any) => { uploadedFilesMap.push({ fileref: `_sjs${fileCountString}`, - filepath: `${uploadFilesDirPath}/${fileName}`, + filepath: `${sasSessionFolder}/${fileName}`, filename: filesNamesMap[fileName], count: fileCount }) From 38ab27c1ed9cf3f87d76773e78203357acf156c2 Mon Sep 17 00:00:00 2001 From: Mihajlo Medjedovic Date: Fri, 15 Oct 2021 12:57:26 +0000 Subject: [PATCH 04/12] style: lint --- src/controllers/FileUploadController.ts | 60 +++++++++++------------ src/controllers/index.ts | 2 +- src/routes/index.ts | 65 +++++++++++++++---------- src/types/Upload.ts | 18 +++---- src/utils/file.ts | 2 +- src/utils/upload.ts | 5 +- 6 files changed, 81 insertions(+), 71 deletions(-) diff --git a/src/controllers/FileUploadController.ts b/src/controllers/FileUploadController.ts index 0bd015c..3a55fce 100644 --- a/src/controllers/FileUploadController.ts +++ b/src/controllers/FileUploadController.ts @@ -3,34 +3,34 @@ import { getSessionController } from '.' const multer = require('multer') export class FileUploadController { - private storage = multer.diskStorage({ - destination: function (req: any, file: any, cb: any) { - //Sending the intercepted files to the sessions subfolder - cb(null, req.sasSession.path) - }, - filename: function (req: any, file: any, cb: any) { - //req_file prefix + unique hash added to sas request files - cb(null, `req_file_${uuidv4().replace(/-/gm, '')}`) - } - }) - - private upload = multer({ storage: this.storage }) - - //It will intercept request and generate uniqe uuid to be used as a subfolder name - //that will store the files uploaded - public preuploadMiddleware = async (req: any, res: any, next: any) => { - let session - - const sessionController = getSessionController() - session = await sessionController.getSession() - session.inUse = true - - req.sasSession = session - - next() - } + private storage = multer.diskStorage({ + destination: function (req: any, file: any, cb: any) { + //Sending the intercepted files to the sessions subfolder + cb(null, req.sasSession.path) + }, + filename: function (req: any, file: any, cb: any) { + //req_file prefix + unique hash added to sas request files + cb(null, `req_file_${uuidv4().replace(/-/gm, '')}`) + } + }) - public getMulterUploadObject() { - return this.upload - } -} \ No newline at end of file + private upload = multer({ storage: this.storage }) + + //It will intercept request and generate uniqe uuid to be used as a subfolder name + //that will store the files uploaded + public preuploadMiddleware = async (req: any, res: any, next: any) => { + let session + + const sessionController = getSessionController() + session = await sessionController.getSession() + session.inUse = true + + req.sasSession = session + + next() + } + + public getMulterUploadObject() { + return this.upload + } +} diff --git a/src/controllers/index.ts b/src/controllers/index.ts index 60c5d59..86939a2 100644 --- a/src/controllers/index.ts +++ b/src/controllers/index.ts @@ -1,4 +1,4 @@ export * from './deploy' export * from './Session' export * from './Execution' -export * from './FileUploadController' \ No newline at end of file +export * from './FileUploadController' diff --git a/src/routes/index.ts b/src/routes/index.ts index 1ed3d4b..7a9302a 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -85,39 +85,50 @@ router.get('/SASjsExecutor/do', async (req, res) => { } }) -router.post('/SASjsExecutor/do', fileUploadController.preuploadMiddleware, fileUploadController.getMulterUploadObject().any(), async (req: any, res: any) => { - if (isRequestQuery(req.query)) { - let sasCodePath = path - .join(getTmpFilesFolderPath(), req.query._program) - .replace(new RegExp('/', 'g'), path.sep) +router.post( + '/SASjsExecutor/do', + fileUploadController.preuploadMiddleware, + fileUploadController.getMulterUploadObject().any(), + async (req: any, res: any) => { + if (isRequestQuery(req.query)) { + let sasCodePath = path + .join(getTmpFilesFolderPath(), req.query._program) + .replace(new RegExp('/', 'g'), path.sep) - // If no extension provided, add .sas extension - sasCodePath += addExtensionIfNotFound(sasCodePath, 'sas') + // If no extension provided, add .sas extension + sasCodePath += addExtensionIfNotFound(sasCodePath, 'sas') - let filesNamesMap = null + let filesNamesMap = null - if (req.files && req.files.length > 0) { - filesNamesMap = makeFilesNamesMap(req.files) - } + if (req.files && req.files.length > 0) { + filesNamesMap = makeFilesNamesMap(req.files) + } - await new ExecutionController() - .execute(sasCodePath, undefined, req.sasSession, { ...req.query }, { filesNamesMap: filesNamesMap }) - .then((result: {}) => { - res.status(200).send(result) - }) - .catch((err: {} | string) => { - res.status(400).send({ - status: 'failure', - message: 'Job execution failed.', - ...(typeof err === 'object' ? err : { details: err }) + await new ExecutionController() + .execute( + sasCodePath, + undefined, + req.sasSession, + { ...req.query }, + { filesNamesMap: filesNamesMap } + ) + .then((result: {}) => { + res.status(200).send(result) }) + .catch((err: {} | string) => { + res.status(400).send({ + status: 'failure', + message: 'Job execution failed.', + ...(typeof err === 'object' ? err : { details: err }) + }) + }) + } else { + res.status(400).send({ + status: 'failure', + message: `Please provide the location of SAS code` }) - } else { - res.status(400).send({ - status: 'failure', - message: `Please provide the location of SAS code` - }) + } } -}) +) export default router diff --git a/src/types/Upload.ts b/src/types/Upload.ts index d025a4e..74628f9 100644 --- a/src/types/Upload.ts +++ b/src/types/Upload.ts @@ -1,10 +1,10 @@ export interface MulterFile { - fieldname: string - originalname: string - encoding: string - mimetype: string - destination: string - filename: string - path: string - size: number -} \ No newline at end of file + fieldname: string + originalname: string + encoding: string + mimetype: string + destination: string + filename: string + path: string + size: number +} diff --git a/src/utils/file.ts b/src/utils/file.ts index 3dfe57f..cb29f48 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -27,4 +27,4 @@ export const generateUniqueFileName = (fileName: string, extension = '') => export const addExtensionIfNotFound = (value: string, extension: string) => { return !value.includes('.') ? `.${extension}` : '' -} \ No newline at end of file +} diff --git a/src/utils/upload.ts b/src/utils/upload.ts index 2c685f4..e0ce494 100644 --- a/src/utils/upload.ts +++ b/src/utils/upload.ts @@ -10,9 +10,9 @@ import { listFilesInFolder } from '@sasjs/utils' * @returns object */ export const makeFilesNamesMap = (files: MulterFile[]) => { - if (!files) return null + if (!files) return null - const filesNamesMap: {[key: string]: string} = {} + const filesNamesMap: { [key: string]: string } = {} for (let file of files) { filesNamesMap[file.filename] = file.fieldname @@ -40,7 +40,6 @@ export const generateFileUploadSasCode = ( count: number }[] = [] - fs.readdirSync(sasSessionFolder).forEach((fileName) => { let fileCountString = fileCount < 100 ? '0' + fileCount : fileCount fileCountString = fileCount < 10 ? '00' + fileCount : fileCount From d9555e151b0e1d1a4068efdf8ee9ed53b25b9b89 Mon Sep 17 00:00:00 2001 From: Mihajlo Medjedovic Date: Mon, 18 Oct 2021 10:42:21 +0000 Subject: [PATCH 05/12] fix: debug not passed --- src/controllers/Execution.ts | 4 ++-- src/controllers/FileUploadController.ts | 2 +- src/routes/index.ts | 4 ++-- src/utils/upload.ts | 7 ++++--- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/controllers/Execution.ts b/src/controllers/Execution.ts index 24827b7..e24b576 100644 --- a/src/controllers/Execution.ts +++ b/src/controllers/Execution.ts @@ -41,7 +41,7 @@ export class ExecutionController { let webout = path.join(session.path, 'webout.txt') await createFile(webout, '') - + program = ` %let sasjsprocessmode=Stored Program; filename _webout "${webout}"; @@ -49,7 +49,7 @@ ${program}` // if no files are uploaded filesNamesMap will be undefined if (otherArgs && otherArgs.filesNamesMap) { - const uploadSasCode = generateFileUploadSasCode( + const uploadSasCode = await generateFileUploadSasCode( otherArgs.filesNamesMap, session.path ) diff --git a/src/controllers/FileUploadController.ts b/src/controllers/FileUploadController.ts index 3a55fce..53d42de 100644 --- a/src/controllers/FileUploadController.ts +++ b/src/controllers/FileUploadController.ts @@ -16,7 +16,7 @@ export class FileUploadController { private upload = multer({ storage: this.storage }) - //It will intercept request and generate uniqe uuid to be used as a subfolder name + //It will intercept request and generate unique uuid to be used as a subfolder name //that will store the files uploaded public preuploadMiddleware = async (req: any, res: any, next: any) => { let session diff --git a/src/routes/index.ts b/src/routes/index.ts index 7a9302a..930ec44 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -103,13 +103,13 @@ router.post( if (req.files && req.files.length > 0) { filesNamesMap = makeFilesNamesMap(req.files) } - + await new ExecutionController() .execute( sasCodePath, undefined, req.sasSession, - { ...req.query }, + { ...req.query, ...req.body }, { filesNamesMap: filesNamesMap } ) .then((result: {}) => { diff --git a/src/utils/upload.ts b/src/utils/upload.ts index e0ce494..c216cd4 100644 --- a/src/utils/upload.ts +++ b/src/utils/upload.ts @@ -27,10 +27,10 @@ export const makeFilesNamesMap = (files: MulterFile[]) => { * @param sasUploadFolder name of the folder that is created for the purpose of files in concurrent request * @returns generated sas code */ -export const generateFileUploadSasCode = ( +export const generateFileUploadSasCode = async ( filesNamesMap: any, sasSessionFolder: string -): string => { +): Promise => { let uploadSasCode = '' let fileCount = 0 let uploadedFilesMap: { @@ -40,7 +40,8 @@ export const generateFileUploadSasCode = ( count: number }[] = [] - fs.readdirSync(sasSessionFolder).forEach((fileName) => { + const sasSessionFolderList: string[] = await listFilesInFolder(sasSessionFolder) + sasSessionFolderList.forEach((fileName) => { let fileCountString = fileCount < 100 ? '0' + fileCount : fileCount fileCountString = fileCount < 10 ? '00' + fileCount : fileCount From 125cf3572241a21536f57dd7d3ce5efd375583c0 Mon Sep 17 00:00:00 2001 From: Mihajlo Medjedovic Date: Mon, 18 Oct 2021 11:25:02 +0000 Subject: [PATCH 06/12] chore: file removed --- code.lst | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 code.lst diff --git a/code.lst b/code.lst deleted file mode 100644 index afd66d7..0000000 --- a/code.lst +++ /dev/null @@ -1,10 +0,0 @@ - The SAS System Friday, 15 October 2021 11:20:00 1 - - - -------- - 0 - The SAS System Friday, 15 October 2021 11:20:00 2 - - - -------- - 41140 From 6efb2d0b518c0b925c27b8c84771296cfd757d17 Mon Sep 17 00:00:00 2001 From: Mihajlo Medjedovic Date: Mon, 18 Oct 2021 12:07:42 +0000 Subject: [PATCH 07/12] style: lint --- src/controllers/Execution.ts | 2 +- src/routes/index.ts | 2 +- src/utils/upload.ts | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/controllers/Execution.ts b/src/controllers/Execution.ts index e24b576..0410594 100644 --- a/src/controllers/Execution.ts +++ b/src/controllers/Execution.ts @@ -41,7 +41,7 @@ export class ExecutionController { let webout = path.join(session.path, 'webout.txt') await createFile(webout, '') - + program = ` %let sasjsprocessmode=Stored Program; filename _webout "${webout}"; diff --git a/src/routes/index.ts b/src/routes/index.ts index 930ec44..263c3ff 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -103,7 +103,7 @@ router.post( if (req.files && req.files.length > 0) { filesNamesMap = makeFilesNamesMap(req.files) } - + await new ExecutionController() .execute( sasCodePath, diff --git a/src/utils/upload.ts b/src/utils/upload.ts index c216cd4..f671367 100644 --- a/src/utils/upload.ts +++ b/src/utils/upload.ts @@ -40,7 +40,9 @@ export const generateFileUploadSasCode = async ( count: number }[] = [] - const sasSessionFolderList: string[] = await listFilesInFolder(sasSessionFolder) + const sasSessionFolderList: string[] = await listFilesInFolder( + sasSessionFolder + ) sasSessionFolderList.forEach((fileName) => { let fileCountString = fileCount < 100 ? '0' + fileCount : fileCount fileCountString = fileCount < 10 ? '00' + fileCount : fileCount From 94fc242afe83315c9a29fed07c379fcb9b5da544 Mon Sep 17 00:00:00 2001 From: Mihajlo Medjedovic Date: Mon, 18 Oct 2021 16:24:10 +0000 Subject: [PATCH 08/12] chore: addExtension function fix and testing --- src/routes/index.ts | 2 +- src/utils/file.ts | 8 +++++++- src/utils/spec/file.spec.ts | 11 +++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) create mode 100644 src/utils/spec/file.spec.ts diff --git a/src/routes/index.ts b/src/routes/index.ts index 263c3ff..d6e5b12 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -63,7 +63,7 @@ router.get('/SASjsExecutor/do', async (req, res) => { .replace(new RegExp('/', 'g'), path.sep) // If no extension provided, add .sas extension - sasCodePath += !sasCodePath.includes('.') ? '.sas' : '' + sasCodePath += addExtensionIfNotFound(sasCodePath, 'sas') await new ExecutionController() .execute(sasCodePath, undefined, undefined, { ...req.query }) diff --git a/src/utils/file.ts b/src/utils/file.ts index cb29f48..c1de713 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -26,5 +26,11 @@ export const generateUniqueFileName = (fileName: string, extension = '') => ].join('') export const addExtensionIfNotFound = (value: string, extension: string) => { - return !value.includes('.') ? `.${extension}` : '' + const valueSplit = value.split('.') + + if (valueSplit.length < 2) return `.${extension}` + + const hasExt = valueSplit[valueSplit.length - 1].length === 3 + + return !hasExt ? `.${extension}` : '' } diff --git a/src/utils/spec/file.spec.ts b/src/utils/spec/file.spec.ts new file mode 100644 index 0000000..af353e3 --- /dev/null +++ b/src/utils/spec/file.spec.ts @@ -0,0 +1,11 @@ +import { addExtensionIfNotFound } from ".." + +describe('file utils', () => { + it('should add extension if missing', async () => { + expect(addExtensionIfNotFound('test', 'ext')).toEqual('.ext') + expect(addExtensionIfNotFound('test.test', 'ext')).toEqual('.ext') + expect(addExtensionIfNotFound('test.sas', 'ext')).toEqual('') + expect(addExtensionIfNotFound('test.test.test', 'ext')).toEqual('.ext') + expect(addExtensionIfNotFound('test.test.test.sas', 'ext')).toEqual('') + }) +}) \ No newline at end of file From 91c7c60dc9e1c1b558c8ac74b98468e4be631533 Mon Sep 17 00:00:00 2001 From: Mihajlo Medjedovic Date: Tue, 19 Oct 2021 09:19:03 +0000 Subject: [PATCH 09/12] style: lint --- src/utils/file.ts | 2 +- src/utils/spec/file.spec.ts | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/utils/file.ts b/src/utils/file.ts index c1de713..6271316 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -27,7 +27,7 @@ export const generateUniqueFileName = (fileName: string, extension = '') => export const addExtensionIfNotFound = (value: string, extension: string) => { const valueSplit = value.split('.') - + if (valueSplit.length < 2) return `.${extension}` const hasExt = valueSplit[valueSplit.length - 1].length === 3 diff --git a/src/utils/spec/file.spec.ts b/src/utils/spec/file.spec.ts index af353e3..06d7511 100644 --- a/src/utils/spec/file.spec.ts +++ b/src/utils/spec/file.spec.ts @@ -1,11 +1,11 @@ -import { addExtensionIfNotFound } from ".." +import { addExtensionIfNotFound } from '..' describe('file utils', () => { - it('should add extension if missing', async () => { - expect(addExtensionIfNotFound('test', 'ext')).toEqual('.ext') - expect(addExtensionIfNotFound('test.test', 'ext')).toEqual('.ext') - expect(addExtensionIfNotFound('test.sas', 'ext')).toEqual('') - expect(addExtensionIfNotFound('test.test.test', 'ext')).toEqual('.ext') - expect(addExtensionIfNotFound('test.test.test.sas', 'ext')).toEqual('') - }) -}) \ No newline at end of file + it('should add extension if missing', async () => { + expect(addExtensionIfNotFound('test', 'ext')).toEqual('.ext') + expect(addExtensionIfNotFound('test.test', 'ext')).toEqual('.ext') + expect(addExtensionIfNotFound('test.sas', 'ext')).toEqual('') + expect(addExtensionIfNotFound('test.test.test', 'ext')).toEqual('.ext') + expect(addExtensionIfNotFound('test.test.test.sas', 'ext')).toEqual('') + }) +}) From f374acae6e652ee39803cd4622fd2c93fc12cd51 Mon Sep 17 00:00:00 2001 From: Mihajlo Medjedovic Date: Tue, 19 Oct 2021 10:10:00 +0000 Subject: [PATCH 10/12] chore: extension util fix --- src/routes/index.ts | 6 +++--- src/utils/file.ts | 3 ++- src/utils/spec/file.spec.ts | 12 ++++++------ 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/routes/index.ts b/src/routes/index.ts index d6e5b12..1e9c681 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -7,7 +7,7 @@ import { import { ExecutionResult, isRequestQuery, isFileTree } from '../types' import path from 'path' import { - addExtensionIfNotFound, + addSasExtensionIfNotFound, getTmpFilesFolderPath, getTmpFolderPath, makeFilesNamesMap @@ -63,7 +63,7 @@ router.get('/SASjsExecutor/do', async (req, res) => { .replace(new RegExp('/', 'g'), path.sep) // If no extension provided, add .sas extension - sasCodePath += addExtensionIfNotFound(sasCodePath, 'sas') + sasCodePath += addSasExtensionIfNotFound(sasCodePath) await new ExecutionController() .execute(sasCodePath, undefined, undefined, { ...req.query }) @@ -96,7 +96,7 @@ router.post( .replace(new RegExp('/', 'g'), path.sep) // If no extension provided, add .sas extension - sasCodePath += addExtensionIfNotFound(sasCodePath, 'sas') + sasCodePath += addSasExtensionIfNotFound(sasCodePath) let filesNamesMap = null diff --git a/src/utils/file.ts b/src/utils/file.ts index 6271316..ead26e0 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -25,7 +25,8 @@ export const generateUniqueFileName = (fileName: string, extension = '') => extension ].join('') -export const addExtensionIfNotFound = (value: string, extension: string) => { +export const addSasExtensionIfNotFound = (value: string) => { + const extension = 'sas' const valueSplit = value.split('.') if (valueSplit.length < 2) return `.${extension}` diff --git a/src/utils/spec/file.spec.ts b/src/utils/spec/file.spec.ts index 06d7511..07616ac 100644 --- a/src/utils/spec/file.spec.ts +++ b/src/utils/spec/file.spec.ts @@ -1,11 +1,11 @@ -import { addExtensionIfNotFound } from '..' +import { addSasExtensionIfNotFound } from '..' describe('file utils', () => { it('should add extension if missing', async () => { - expect(addExtensionIfNotFound('test', 'ext')).toEqual('.ext') - expect(addExtensionIfNotFound('test.test', 'ext')).toEqual('.ext') - expect(addExtensionIfNotFound('test.sas', 'ext')).toEqual('') - expect(addExtensionIfNotFound('test.test.test', 'ext')).toEqual('.ext') - expect(addExtensionIfNotFound('test.test.test.sas', 'ext')).toEqual('') + expect(addSasExtensionIfNotFound('test')).toEqual('.sas') + expect(addSasExtensionIfNotFound('test.test')).toEqual('.sas') + expect(addSasExtensionIfNotFound('test.sas')).toEqual('') + expect(addSasExtensionIfNotFound('test.test.test')).toEqual('.sas') + expect(addSasExtensionIfNotFound('test.test.test.sas')).toEqual('') }) }) From a09e90b010e2b91be0114e17c414089d9af118f0 Mon Sep 17 00:00:00 2001 From: Mihajlo Medjedovic Date: Tue, 19 Oct 2021 10:37:23 +0000 Subject: [PATCH 11/12] chore: always adding sas extension --- src/routes/index.ts | 5 ++--- src/utils/file.ts | 13 +------------ src/utils/spec/file.spec.ts | 11 ----------- 3 files changed, 3 insertions(+), 26 deletions(-) delete mode 100644 src/utils/spec/file.spec.ts diff --git a/src/routes/index.ts b/src/routes/index.ts index 1e9c681..1854f7d 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -7,7 +7,6 @@ import { import { ExecutionResult, isRequestQuery, isFileTree } from '../types' import path from 'path' import { - addSasExtensionIfNotFound, getTmpFilesFolderPath, getTmpFolderPath, makeFilesNamesMap @@ -63,7 +62,7 @@ router.get('/SASjsExecutor/do', async (req, res) => { .replace(new RegExp('/', 'g'), path.sep) // If no extension provided, add .sas extension - sasCodePath += addSasExtensionIfNotFound(sasCodePath) + sasCodePath += '.sas' await new ExecutionController() .execute(sasCodePath, undefined, undefined, { ...req.query }) @@ -96,7 +95,7 @@ router.post( .replace(new RegExp('/', 'g'), path.sep) // If no extension provided, add .sas extension - sasCodePath += addSasExtensionIfNotFound(sasCodePath) + sasCodePath += '.sas' let filesNamesMap = null diff --git a/src/utils/file.ts b/src/utils/file.ts index ead26e0..f819129 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -23,15 +23,4 @@ export const generateUniqueFileName = (fileName: string, extension = '') => '-', new Date().getTime(), extension - ].join('') - -export const addSasExtensionIfNotFound = (value: string) => { - const extension = 'sas' - const valueSplit = value.split('.') - - if (valueSplit.length < 2) return `.${extension}` - - const hasExt = valueSplit[valueSplit.length - 1].length === 3 - - return !hasExt ? `.${extension}` : '' -} + ].join('') \ No newline at end of file diff --git a/src/utils/spec/file.spec.ts b/src/utils/spec/file.spec.ts deleted file mode 100644 index 07616ac..0000000 --- a/src/utils/spec/file.spec.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { addSasExtensionIfNotFound } from '..' - -describe('file utils', () => { - it('should add extension if missing', async () => { - expect(addSasExtensionIfNotFound('test')).toEqual('.sas') - expect(addSasExtensionIfNotFound('test.test')).toEqual('.sas') - expect(addSasExtensionIfNotFound('test.sas')).toEqual('') - expect(addSasExtensionIfNotFound('test.test.test')).toEqual('.sas') - expect(addSasExtensionIfNotFound('test.test.test.sas')).toEqual('') - }) -}) From 38db48d59fb777c15331272ff04e5e5b84af4aa3 Mon Sep 17 00:00:00 2001 From: Mihajlo Medjedovic Date: Tue, 19 Oct 2021 10:37:41 +0000 Subject: [PATCH 12/12] style: lint --- src/utils/file.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/file.ts b/src/utils/file.ts index f819129..fe09816 100644 --- a/src/utils/file.ts +++ b/src/utils/file.ts @@ -23,4 +23,4 @@ export const generateUniqueFileName = (fileName: string, extension = '') => '-', new Date().getTime(), extension - ].join('') \ No newline at end of file + ].join('')