Merge pull request #74 from sasjs/issue-67

File uploading multi-part

api/.vscode/launch.json

@@ -0,0 +1,13 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Launch via NPM",
+      "request": "launch",
+      "runtimeArgs": ["run-script", "start"],
+      "runtimeExecutable": "npm",
+      "skipFiles": ["<node_internals>/**"],
+      "type": "pwa-node"
+    }
+  ]
+}

api/public/swagger.yaml

@@ -636,7 +636,6 @@ paths:
                     bearerAuth: []
             parameters:
                 -
-                    description: 'Location of SAS program'
                     in: query
                     name: filePath
                     required: true
@@ -674,9 +673,18 @@ paths:
             requestBody:
                 required: true
                 content:
-                    application/json:
+                    multipart/form-data:
                         schema:
-                            $ref: '#/components/schemas/FilePayload'
+                            type: object
+                            properties:
+                                filePath:
+                                    type: string
+                                file:
+                                    type: string
+                                    format: binary
+                            required:
+                                - filePath
+                                - file
         patch:
             operationId: UpdateFile
             responses:
@@ -1066,8 +1074,8 @@ paths:
                                 anyOf:
                                     - {type: string}
                                     - {type: string, format: byte}
-            description: "Trigger a SAS program using it's location in the _program parameter.\nEnable debugging using the _debug parameter.\nAdditional URL parameters are turned into SAS macro variables.\nAny files provided are placed into the session and\ncorresponding _WEBIN_XXX variables are created."
-            summary: 'Execute Stored Program, return raw content'
+            description: "Trigger a SAS program using it's location in the _program URL parameter.\nEnable debugging using the _debug URL parameter.  Setting _debug=131 will\ncause the log to be streamed in the output.\n\nAdditional URL parameters are turned into SAS macro variables.\n\nAny files provided in the request body are placed into the SAS session with\ncorresponding _WEBIN_XXX variables created.\n\nThe response headers can be adjusted using the mfs_httpheader() macro.  Any\nfile type can be returned, including binary files such as zip or xls.\n\nThis behaviour differs for POST requests, in which case the reponse is\nalways JSON."
+            summary: 'Execute Stored Program, return raw _webout content.'
             tags:
                 - STP
             security:
@@ -1094,7 +1102,7 @@ paths:
                             examples:
                                 'Example 1':
                                     value: {status: success, _webout: 'webout content', log: [], httpHeaders: {Content-type: application/zip, Cache-Control: 'public, max-age=1000'}}
-            description: "Trigger a SAS program using it's location in the _program parameter.\nEnable debugging using the _debug parameter.\nAdditional URL parameters are turned into SAS macro variables.\nAny files provided are placed into the session and\ncorresponding _WEBIN_XXX variables are created."
+            description: "Trigger a SAS program using it's location in the _program URL parameter.\nEnable debugging using the _debug URL parameter.  In any case, the log is\nalways returned in the log object.\n\nAdditional URL parameters are turned into SAS macro variables.\n\nAny files provided in the request body are placed into the SAS session with\ncorresponding _WEBIN_XXX variables created.\n\nThe response will be a JSON object with the following root attributes: log,\nwebout, headers.\n\nThe webout will be a nested JSON object ONLY if the response-header\ncontains a content-type of application/json AND it is valid JSON.\nOtherwise it will be a stringified version of the webout content.\n\nResponse headers from the mfs_httpheader macro are simply listed in the\nheaders object, for POST requests they have no effect on the actual\nresponse header."
             summary: 'Execute Stored Program, return JSON'
             tags:
                 - STP

api/src/app.ts

@@ -4,9 +4,7 @@ import morgan from 'morgan'
 import dotenv from 'dotenv'
 import cors from 'cors'
 
-import webRouter from './routes/web'
-import apiRouter from './routes/api'
-import { connectDB, getWebBuildFolderPath } from './utils'
+import { connectDB, getWebBuildFolderPath, setProcessVariables } from './utils'
 
 dotenv.config()
 
@@ -28,7 +26,12 @@ app.use(morgan('tiny'))
 app.use(express.static(path.join(__dirname, '../public')))
 app.use(express.static(getWebBuildFolderPath()))
 
-app.use('/', webRouter)
-app.use('/SASjsApi', apiRouter)
+export default setProcessVariables().then(async () => {
+  // loading these modules after setting up variables due to
+  // multer's usage of process var process.driveLoc
+  const { setupRoutes } = await import('./routes/setupRoutes')
+  setupRoutes(app)
 
-export default connectDB().then(() => app)
+  await connectDB()
+  return app
+})

api/src/controllers/drive.ts

@@ -1,3 +1,5 @@
+import path from 'path'
+import { Express } from 'express'
 import {
   Security,
   Route,
@@ -8,13 +10,21 @@ import {
   Response,
   Query,
   Get,
-  Patch
+  Patch,
+  UploadedFile,
+  FormField
 } from 'tsoa'
-import { fileExists, readFile, createFile } from '@sasjs/utils'
+import {
+  fileExists,
+  readFile,
+  createFile,
+  moveFile,
+  createFolder,
+  deleteFile
+} from '@sasjs/utils'
 import { createFileTree, ExecutionController, getTreeExample } from './internal'
 
 import { FileTree, isFileTree, TreeNode } from '../types'
-import path from 'path'
 import { getTmpFilesFolderPath } from '../utils'
 
 interface DeployPayload {
@@ -90,7 +100,7 @@ export class DriveController {
 
   /**
    * @summary Get file from SASjs Drive
-   * @param filePath Location of SAS program
+   * @query filePath Location of SAS program
    * @example filePath "/Public/somefolder/some.file"
    */
   @Example<GetFileResponse>({
@@ -119,9 +129,10 @@ export class DriveController {
   })
   @Post('/file')
   public async saveFile(
-    @Body() body: FilePayload
+    @FormField() filePath: string,
+    @UploadedFile() file: Express.Multer.File
   ): Promise<UpdateFileResponse> {
-    return saveFile(body)
+    return saveFile(filePath, file)
   }
 
   /**
@@ -192,27 +203,32 @@ const getFile = async (filePath: string): Promise<GetFileResponse> => {
   }
 }
 
-const saveFile = async (body: FilePayload): Promise<GetFileResponse> => {
-  const { filePath, fileContent } = body
-  try {
-    const filePathFull = path
-      .join(getTmpFilesFolderPath(), filePath)
-      .replace(new RegExp('/', 'g'), path.sep)
+const saveFile = async (
+  filePath: string,
+  multerFile: Express.Multer.File
+): Promise<GetFileResponse> => {
+  const driveFilesPath = getTmpFilesFolderPath()
 
-    if (await fileExists(filePathFull)) {
-      throw 'DriveController: File already exists.'
-    }
-    await createFile(filePathFull, fileContent)
+  const filePathFull = path
+    .join(driveFilesPath, filePath)
+    .replace(new RegExp('/', 'g'), path.sep)
 
-    return { status: 'success' }
-  } catch (err: any) {
-    throw {
-      code: 400,
-      status: 'failure',
-      message: 'File request failed.',
-      error: typeof err === 'object' ? err.toString() : err
-    }
+  if (!filePathFull.includes(driveFilesPath)) {
+    await deleteFile(multerFile.path)
+    throw new Error('Cannot put file outside drive.')
   }
+
+  if (await fileExists(filePathFull)) {
+    await deleteFile(multerFile.path)
+    throw new Error('File already exists.')
+  }
+
+  const folderPath = path.dirname(filePathFull)
+  await createFolder(folderPath)
+
+  await moveFile(multerFile.path, filePathFull)
+
+  return { status: 'success' }
 }
 
 const updateFile = async (body: FilePayload): Promise<GetFileResponse> => {

api/src/middlewares/multer.ts

@@ -0,0 +1,77 @@
+import path from 'path'
+import { Request } from 'express'
+import multer, { FileFilterCallback, Options } from 'multer'
+import { getTmpUploadsPath } from '../utils'
+
+const acceptableExtensions = ['.sas']
+const fieldNameSize = 300
+const fileSize = 10485760 // 10 MB
+
+const storage = multer.diskStorage({
+  destination: getTmpUploadsPath(),
+  filename: function (
+    _req: Request,
+    file: Express.Multer.File,
+    callback: (error: Error | null, filename: string) => void
+  ) {
+    callback(
+      null,
+      file.fieldname + path.extname(file.originalname) + '-' + Date.now()
+    )
+  }
+})
+
+const limits: Options['limits'] = {
+  fieldNameSize,
+  fileSize
+}
+
+const fileFilter: Options['fileFilter'] = (
+  req: Request,
+  file: Express.Multer.File,
+  callback: FileFilterCallback
+) => {
+  const fileExtension = path.extname(file.originalname).toLocaleLowerCase()
+
+  if (!acceptableExtensions.includes(fileExtension)) {
+    return callback(
+      new Error(
+        `File extension '${fileExtension}' not acceptable. Valid extension(s): ${acceptableExtensions.join(
+          ', '
+        )}`
+      )
+    )
+  }
+
+  const uploadFileSize = parseInt(req.headers['content-length'] ?? '')
+  if (uploadFileSize > fileSize) {
+    return callback(
+      new Error(
+        `File size is over limit. File limit is: ${fileSize / 1024 / 1024} MB`
+      )
+    )
+  }
+
+  callback(null, true)
+}
+
+const options: Options = { storage, limits, fileFilter }
+
+const multerInstance = multer(options)
+
+export const multerSingle = (fileName: string, arg: any) => {
+  const [req, res, next] = arg
+  const upload = multerInstance.single(fileName)
+
+  upload(req, res, function (err) {
+    if (err instanceof multer.MulterError) {
+      return res.status(500).send(err.message)
+    } else if (err) {
+      return res.status(400).send(err.message)
+    }
+    // Everything went fine.
+    next()
+  })
+}
+
+export default multerInstance

api/src/routes/api/drive.ts

@@ -1,11 +1,13 @@
 import express from 'express'
+import multer, { multerSingle } from '../../middlewares/multer'
 import { DriveController } from '../../controllers/'
 import { getFileDriveValidation, updateFileDriveValidation } from '../../utils'
 
+const controller = new DriveController()
+
 const driveRouter = express.Router()
 
 driveRouter.post('/deploy', async (req, res) => {
-  const controller = new DriveController()
   try {
     const response = await controller.deploy(req.body)
     res.send(response)
@@ -22,7 +24,6 @@ driveRouter.get('/file', async (req, res) => {
   const { error, value: query } = getFileDriveValidation(req.query)
   if (error) return res.status(400).send(error.details[0].message)
 
-  const controller = new DriveController()
   try {
     const response = await controller.getFile(query.filePath)
     res.send(response)
@@ -35,28 +36,28 @@ driveRouter.get('/file', async (req, res) => {
   }
 })
 
-driveRouter.post('/file', async (req, res) => {
-  const { error, value: body } = updateFileDriveValidation(req.body)
-  if (error) return res.status(400).send(error.details[0].message)
+driveRouter.post(
+  '/file',
+  (...arg) => multerSingle('file', arg),
+  async (req, res) => {
+    const { error, value: body } = updateFileDriveValidation(req.body)
+    if (error) return res.status(400).send(error.details[0].message)
 
-  const controller = new DriveController()
-  try {
-    const response = await controller.saveFile(body)
-    res.send(response)
-  } catch (err: any) {
-    const statusCode = err.code
+    if (!req.file) return res.status(400).send('"file" is not present.')
 
-    delete err.code
-
-    res.status(statusCode).send(err)
+    try {
+      const response = await controller.saveFile(body.filePath, req.file)
+      res.send(response)
+    } catch (err: any) {
+      res.status(403).send(err.toString())
+    }
   }
-})
+)
 
 driveRouter.patch('/file', async (req, res) => {
   const { error, value: body } = updateFileDriveValidation(req.body)
   if (error) return res.status(400).send(error.details[0].message)
 
-  const controller = new DriveController()
   try {
     const response = await controller.updateFile(body)
     res.send(response)
@@ -70,7 +71,6 @@ driveRouter.patch('/file', async (req, res) => {
 })
 
 driveRouter.get('/fileTree', async (req, res) => {
-  const controller = new DriveController()
   try {
     const response = await controller.getFileTree()
     res.send(response)

api/src/routes/api/spec/drive.spec.ts

@@ -1,15 +1,34 @@
+import path from 'path'
 import { Express } from 'express'
 import mongoose, { Mongoose } from 'mongoose'
 import { MongoMemoryServer } from 'mongodb-memory-server'
 import request from 'supertest'
+
+import {
+  folderExists,
+  fileExists,
+  readFile,
+  deleteFolder,
+  generateTimestamp,
+  copy
+} from '@sasjs/utils'
+import * as fileUtilModules from '../../../utils/file'
+
+const timestamp = generateTimestamp()
+const tmpFolder = path.join(process.cwd(), `tmp-${timestamp}`)
+jest
+  .spyOn(fileUtilModules, 'getTmpFolderPath')
+  .mockImplementation(() => tmpFolder)
+jest
+  .spyOn(fileUtilModules, 'getTmpUploadsPath')
+  .mockImplementation(() => path.join(tmpFolder, 'uploads'))
+
 import appPromise from '../../../app'
 import { UserController } from '../../../controllers/'
 import { getTreeExample } from '../../../controllers/internal'
-import { getTmpFilesFolderPath } from '../../../utils/file'
-import { folderExists, fileExists, readFile, deleteFolder } from '@sasjs/utils'
-import path from 'path'
-import { generateAccessToken, saveTokensInDB } from '../../../utils'
 import { FolderMember, ServiceMember } from '../../../types'
+import { generateAccessToken, saveTokensInDB } from '../../../utils/'
+const { getTmpFilesFolderPath } = fileUtilModules
 
 let app: Express
 appPromise.then((_app) => {
@@ -30,28 +49,27 @@ describe('files', () => {
   let mongoServer: MongoMemoryServer
   const controller = new UserController()
 
+  let accessToken: string
+
   beforeAll(async () => {
     mongoServer = await MongoMemoryServer.create()
     con = await mongoose.connect(mongoServer.getUri())
+
+    const dbUser = await controller.createUser(user)
+    accessToken = generateAccessToken({
+      clientId,
+      userId: dbUser.id
+    })
+    await saveTokensInDB(dbUser.id, clientId, accessToken, 'refreshToken')
   })
 
   afterAll(async () => {
     await con.connection.dropDatabase()
     await con.connection.close()
     await mongoServer.stop()
+    await deleteFolder(tmpFolder)
   })
   describe('deploy', () => {
-    let accessToken: string
-    let dbUser: any
-
-    beforeAll(async () => {
-      dbUser = await controller.createUser(user)
-      accessToken = generateAccessToken({
-        clientId,
-        userId: dbUser.id
-      })
-      await saveTokensInDB(dbUser.id, clientId, accessToken, 'refreshToken')
-    })
     const shouldFailAssertion = async (payload: any) => {
       const res = await request(app)
         .post('/SASjsApi/drive/deploy')
@@ -144,8 +162,6 @@ describe('files', () => {
       const exampleService = getExampleService()
       const testJobFile = path.join(testJobFolder, exampleService.name) + '.sas'
 
-      console.log(`[testJobFile]`, testJobFile)
-
       await expect(fileExists(testJobFile)).resolves.toEqual(true)
 
       await expect(readFile(testJobFile)).resolves.toEqual(exampleService.code)
@@ -153,6 +169,146 @@ describe('files', () => {
       await deleteFolder(getTmpFilesFolderPath())
     })
   })
+
+  describe('file', () => {
+    describe('create', () => {
+      it('should create a SAS file on drive', async () => {
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', '/my/path/code.sas')
+          .attach('file', path.join(__dirname, 'files', 'sample.sas'))
+
+        expect(res.statusCode).toEqual(200)
+        expect(res.body).toEqual({
+          status: 'success'
+        })
+      })
+
+      it('should respond with Unauthorized if access token is not present', async () => {
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .field('filePath', '/my/path/code.sas')
+          .attach('file', path.join(__dirname, 'files', 'sample.sas'))
+          .expect(401)
+
+        expect(res.text).toEqual('Unauthorized')
+        expect(res.body).toEqual({})
+      })
+
+      it('should respond with Forbidden if file is already present', async () => {
+        const fileToAttachPath = path.join(__dirname, 'files', 'sample.sas')
+        const pathToUpload = '/my/path/code.sas'
+
+        const pathToCopy = path.join(
+          fileUtilModules.getTmpFilesFolderPath(),
+          pathToUpload
+        )
+        await copy(fileToAttachPath, pathToCopy)
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .attach('file', fileToAttachPath)
+          .expect(403)
+
+        expect(res.text).toEqual('Error: File already exists.')
+        expect(res.body).toEqual({})
+      })
+
+      it('should respond with Forbidden if filePath outside Drive', async () => {
+        const fileToAttachPath = path.join(__dirname, 'files', 'sample.sas')
+        const pathToUpload = '/../path/code.sas'
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .attach('file', fileToAttachPath)
+          .expect(403)
+
+        expect(res.text).toEqual('Error: Cannot put file outside drive.')
+        expect(res.body).toEqual({})
+      })
+
+      it('should respond with Bad Request if filePath is missing', async () => {
+        const fileToAttachPath = path.join(__dirname, 'files', 'sample.sas')
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .attach('file', fileToAttachPath)
+          .expect(400)
+
+        expect(res.text).toEqual(`"filePath" is required`)
+        expect(res.body).toEqual({})
+      })
+
+      it("should respond with Bad Request if filePath doesn't has correct extension", async () => {
+        const fileToAttachPath = path.join(__dirname, 'files', 'sample.sas')
+        const pathToUpload = '/my/path/code.oth'
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .attach('file', fileToAttachPath)
+          .expect(400)
+
+        expect(res.text).toEqual('Valid extensions for filePath: .sas')
+        expect(res.body).toEqual({})
+      })
+
+      it('should respond with Bad Request if file is missing', async () => {
+        const pathToUpload = '/my/path/code.sas'
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .expect(400)
+
+        expect(res.text).toEqual('"file" is not present.')
+        expect(res.body).toEqual({})
+      })
+
+      it("should respond with Bad Request if attached file doesn't has correct extension", async () => {
+        const fileToAttachPath = path.join(__dirname, 'files', 'sample.oth')
+        const pathToUpload = '/my/path/code.sas'
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .attach('file', fileToAttachPath)
+          .expect(400)
+
+        expect(res.text).toEqual(
+          `File extension '.oth' not acceptable. Valid extension(s): .sas`
+        )
+        expect(res.body).toEqual({})
+      })
+
+      it('should respond with Bad Request if attached file exceeds file limit', async () => {
+        const pathToUpload = '/my/path/code.sas'
+
+        const attachedFile = Buffer.from('.'.repeat(20 * 1024 * 1024))
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .attach('file', attachedFile, 'another.sas')
+          .expect(400)
+
+        expect(res.text).toEqual(
+          'File size is over limit. File limit is: 10 MB'
+        )
+        expect(res.body).toEqual({})
+      })
+    })
+  })
 })
 
 const getExampleService = (): ServiceMember =>

api/src/routes/api/spec/files/sample.oth

@@ -0,0 +1 @@
+some code of sas

api/src/routes/api/spec/files/sample.sas

@@ -0,0 +1 @@
+some code of sas

api/src/routes/setupRoutes.ts

@@ -0,0 +1,9 @@
+import { Express } from 'express'
+
+import webRouter from './web'
+import apiRouter from './api'
+
+export const setupRoutes = (app: Express) => {
+  app.use('/', webRouter)
+  app.use('/SASjsApi', apiRouter)
+}

api/src/utils/connectDB.ts

@@ -1,40 +1,19 @@
-import path from 'path'
 import mongoose from 'mongoose'
-import { configuration } from '../../package.json'
-import { getDesktopFields } from '.'
 import { populateClients } from '../routes/api/auth'
-import { getRealPath } from '@sasjs/utils'
 
 export const connectDB = async () => {
   // NOTE: when exporting app.js as agent for supertest
   // we should exclude connecting to the real database
   if (process.env.NODE_ENV === 'test') {
-    process.driveLoc = path.join(process.cwd(), 'tmp')
     return
   } else {
     const { MODE } = process.env
 
     if (MODE?.trim() !== 'server') {
       console.log('Running in Destop Mode, no DB to connect.')
-
-      const { sasLoc, driveLoc } = await getDesktopFields()
-
-      process.sasLoc = sasLoc
-      process.driveLoc = driveLoc
-    } else {
-      const { SAS_PATH, DRIVE_PATH } = process.env
-
-      process.sasLoc = SAS_PATH ?? configuration.sasPath
-      process.driveLoc = getRealPath(
-        path.join(process.cwd(), DRIVE_PATH ?? 'tmp')
-      )
+      return
     }
 
-    console.log('sasLoc: ', process.sasLoc)
-    console.log('sasDrive: ', process.driveLoc)
-
-    if (MODE?.trim() !== 'server') return
-
     mongoose.connect(process.env.DB_CONNECT as string, async (err) => {
       if (err) throw err
 

api/src/utils/file.ts

@@ -13,6 +13,8 @@ export const getWebBuildFolderPath = () =>
 
 export const getTmpFolderPath = () => process.driveLoc
 
+export const getTmpUploadsPath = () => path.join(getTmpFolderPath(), 'uploads')
+
 export const getTmpFilesFolderPath = () =>
   path.join(getTmpFolderPath(), 'files')
 

api/src/utils/index.ts

@@ -9,6 +9,7 @@ export * from './getDesktopFields'
 export * from './parseLogToArray'
 export * from './removeTokensInDB'
 export * from './saveTokensInDB'
+export * from './setProcessVariables'
 export * from './sleep'
 export * from './upload'
 export * from './validation'

api/src/utils/setProcessVariables.ts

@@ -0,0 +1,31 @@
+import path from 'path'
+import { getRealPath } from '@sasjs/utils'
+
+import { configuration } from '../../package.json'
+import { getDesktopFields } from '.'
+
+export const setProcessVariables = async () => {
+  if (process.env.NODE_ENV === 'test') {
+    process.driveLoc = path.join(process.cwd(), 'tmp')
+    return
+  }
+
+  const { MODE } = process.env
+
+  if (MODE?.trim() !== 'server') {
+    const { sasLoc, driveLoc } = await getDesktopFields()
+
+    process.sasLoc = sasLoc
+    process.driveLoc = driveLoc
+  } else {
+    const { SAS_PATH, DRIVE_PATH } = process.env
+
+    process.sasLoc = SAS_PATH ?? configuration.sasPath
+    process.driveLoc = getRealPath(
+      path.join(process.cwd(), DRIVE_PATH ?? 'tmp')
+    )
+  }
+
+  console.log('sasLoc: ', process.sasLoc)
+  console.log('sasDrive: ', process.driveLoc)
+}

api/src/utils/validation.ts

@@ -73,8 +73,9 @@ export const getFileDriveValidation = (data: any): Joi.ValidationResult =>
 
 export const updateFileDriveValidation = (data: any): Joi.ValidationResult =>
   Joi.object({
-    filePath: Joi.string().required(),
-    fileContent: Joi.string().required()
+    filePath: Joi.string().pattern(/.sas$/).required().messages({
+      'string.pattern.base': `Valid extensions for filePath: .sas`
+    })
   }).validate(data)
 
 export const runSASValidation = (data: any): Joi.ValidationResult =>