fix: multi-part file upload + validations + specs

api/src/controllers/drive.ts

@@ -1,3 +1,5 @@
+import path from 'path'
+import { Express } from 'express'
 import {
   Security,
   Route,
@@ -8,13 +10,21 @@ import {
   Response,
   Query,
   Get,
-  Patch
+  Patch,
+  UploadedFile,
+  FormField
 } from 'tsoa'
-import { fileExists, readFile, createFile } from '@sasjs/utils'
+import {
+  fileExists,
+  readFile,
+  createFile,
+  moveFile,
+  createFolder,
+  deleteFile
+} from '@sasjs/utils'
 import { createFileTree, ExecutionController, getTreeExample } from './internal'
 
 import { FileTree, isFileTree, TreeNode } from '../types'
-import path from 'path'
 import { getTmpFilesFolderPath } from '../utils'
 
 interface DeployPayload {
@@ -90,7 +100,7 @@ export class DriveController {
 
   /**
    * @summary Get file from SASjs Drive
-   * @param filePath Location of SAS program
+   * @query filePath Location of SAS program
    * @example filePath "/Public/somefolder/some.file"
    */
   @Example<GetFileResponse>({
@@ -119,9 +129,10 @@ export class DriveController {
   })
   @Post('/file')
   public async saveFile(
-    @Body() body: FilePayload
+    @FormField() filePath: string,
+    @UploadedFile() file: Express.Multer.File
   ): Promise<UpdateFileResponse> {
-    return saveFile(body)
+    return saveFile(filePath, file)
   }
 
   /**
@@ -192,27 +203,32 @@ const getFile = async (filePath: string): Promise<GetFileResponse> => {
   }
 }
 
-const saveFile = async (body: FilePayload): Promise<GetFileResponse> => {
-  const { filePath, fileContent } = body
-  try {
-    const filePathFull = path
-      .join(getTmpFilesFolderPath(), filePath)
-      .replace(new RegExp('/', 'g'), path.sep)
+const saveFile = async (
+  filePath: string,
+  multerFile: Express.Multer.File
+): Promise<GetFileResponse> => {
+  const driveFilesPath = getTmpFilesFolderPath()
 
-    if (await fileExists(filePathFull)) {
-      throw 'DriveController: File already exists.'
-    }
-    await createFile(filePathFull, fileContent)
+  const filePathFull = path
+    .join(driveFilesPath, filePath)
+    .replace(new RegExp('/', 'g'), path.sep)
 
-    return { status: 'success' }
-  } catch (err: any) {
-    throw {
-      code: 400,
-      status: 'failure',
-      message: 'File request failed.',
-      error: typeof err === 'object' ? err.toString() : err
-    }
+  if (!filePathFull.includes(driveFilesPath)) {
+    await deleteFile(multerFile.path)
+    throw new Error('Cannot put file outside drive.')
   }
+
+  if (await fileExists(filePathFull)) {
+    await deleteFile(multerFile.path)
+    throw new Error('File already exists.')
+  }
+
+  const folderPath = path.dirname(filePathFull)
+  await createFolder(folderPath)
+
+  await moveFile(multerFile.path, filePathFull)
+
+  return { status: 'success' }
 }
 
 const updateFile = async (body: FilePayload): Promise<GetFileResponse> => {

api/src/middlewares/multer.ts

@@ -0,0 +1,77 @@
+import path from 'path'
+import { Request } from 'express'
+import multer, { FileFilterCallback, Options } from 'multer'
+import { getTmpUploadsPath } from '../utils'
+
+const acceptableExtensions = ['.sas']
+const fieldNameSize = 300
+const fileSize = 10485760 // 10 MB
+
+const storage = multer.diskStorage({
+  destination: getTmpUploadsPath(),
+  filename: function (
+    _req: Request,
+    file: Express.Multer.File,
+    callback: (error: Error | null, filename: string) => void
+  ) {
+    callback(
+      null,
+      file.fieldname + path.extname(file.originalname) + '-' + Date.now()
+    )
+  }
+})
+
+const limits: Options['limits'] = {
+  fieldNameSize,
+  fileSize
+}
+
+const fileFilter: Options['fileFilter'] = (
+  req: Request,
+  file: Express.Multer.File,
+  callback: FileFilterCallback
+) => {
+  const fileExtension = path.extname(file.originalname).toLocaleLowerCase()
+
+  if (!acceptableExtensions.includes(fileExtension)) {
+    return callback(
+      new Error(
+        `File extension '${fileExtension}' not acceptable. Valid extension(s): ${acceptableExtensions.join(
+          ', '
+        )}`
+      )
+    )
+  }
+
+  const uploadFileSize = parseInt(req.headers['content-length'] ?? '')
+  if (uploadFileSize > fileSize) {
+    return callback(
+      new Error(
+        `File size is over limit. File limit is: ${fileSize / 1024 / 1024} MB`
+      )
+    )
+  }
+
+  callback(null, true)
+}
+
+const options: Options = { storage, limits, fileFilter }
+
+const multerInstance = multer(options)
+
+export const multerSingle = (fileName: string, arg: any) => {
+  const [req, res, next] = arg
+  const upload = multerInstance.single(fileName)
+
+  upload(req, res, function (err) {
+    if (err instanceof multer.MulterError) {
+      return res.status(500).send(err.message)
+    } else if (err) {
+      return res.status(400).send(err.message)
+    }
+    // Everything went fine.
+    next()
+  })
+}
+
+export default multerInstance

api/src/routes/api/drive.ts

@@ -1,11 +1,13 @@
 import express from 'express'
+import multer, { multerSingle } from '../../middlewares/multer'
 import { DriveController } from '../../controllers/'
 import { getFileDriveValidation, updateFileDriveValidation } from '../../utils'
 
+const controller = new DriveController()
+
 const driveRouter = express.Router()
 
 driveRouter.post('/deploy', async (req, res) => {
-  const controller = new DriveController()
   try {
     const response = await controller.deploy(req.body)
     res.send(response)
@@ -22,7 +24,6 @@ driveRouter.get('/file', async (req, res) => {
   const { error, value: query } = getFileDriveValidation(req.query)
   if (error) return res.status(400).send(error.details[0].message)
 
-  const controller = new DriveController()
   try {
     const response = await controller.getFile(query.filePath)
     res.send(response)
@@ -35,28 +36,28 @@ driveRouter.get('/file', async (req, res) => {
   }
 })
 
-driveRouter.post('/file', async (req, res) => {
-  const { error, value: body } = updateFileDriveValidation(req.body)
-  if (error) return res.status(400).send(error.details[0].message)
+driveRouter.post(
+  '/file',
+  (...arg) => multerSingle('file', arg),
+  async (req, res) => {
+    const { error, value: body } = updateFileDriveValidation(req.body)
+    if (error) return res.status(400).send(error.details[0].message)
 
-  const controller = new DriveController()
-  try {
-    const response = await controller.saveFile(body)
-    res.send(response)
-  } catch (err: any) {
-    const statusCode = err.code
+    if (!req.file) return res.status(400).send('"file" is not present.')
 
-    delete err.code
-
-    res.status(statusCode).send(err)
+    try {
+      const response = await controller.saveFile(body.filePath, req.file)
+      res.send(response)
+    } catch (err: any) {
+      res.status(403).send(err.toString())
+    }
   }
-})
+)
 
 driveRouter.patch('/file', async (req, res) => {
   const { error, value: body } = updateFileDriveValidation(req.body)
   if (error) return res.status(400).send(error.details[0].message)
 
-  const controller = new DriveController()
   try {
     const response = await controller.updateFile(body)
     res.send(response)
@@ -70,7 +71,6 @@ driveRouter.patch('/file', async (req, res) => {
 })
 
 driveRouter.get('/fileTree', async (req, res) => {
-  const controller = new DriveController()
   try {
     const response = await controller.getFileTree()
     res.send(response)

api/src/routes/api/spec/drive.spec.ts

@@ -1,15 +1,34 @@
+import path from 'path'
 import { Express } from 'express'
 import mongoose, { Mongoose } from 'mongoose'
 import { MongoMemoryServer } from 'mongodb-memory-server'
 import request from 'supertest'
+
+import {
+  folderExists,
+  fileExists,
+  readFile,
+  deleteFolder,
+  generateTimestamp,
+  copy
+} from '@sasjs/utils'
+import * as fileUtilModules from '../../../utils/file'
+
+const timestamp = generateTimestamp()
+const tmpFolder = path.join(process.cwd(), `tmp-${timestamp}`)
+jest
+  .spyOn(fileUtilModules, 'getTmpFolderPath')
+  .mockImplementation(() => tmpFolder)
+jest
+  .spyOn(fileUtilModules, 'getTmpUploadsPath')
+  .mockImplementation(() => path.join(tmpFolder, 'uploads'))
+
 import appPromise from '../../../app'
 import { UserController } from '../../../controllers/'
 import { getTreeExample } from '../../../controllers/internal'
-import { getTmpFilesFolderPath } from '../../../utils/file'
-import { folderExists, fileExists, readFile, deleteFolder } from '@sasjs/utils'
-import path from 'path'
-import { generateAccessToken, saveTokensInDB } from '../../../utils'
 import { FolderMember, ServiceMember } from '../../../types'
+import { generateAccessToken, saveTokensInDB } from '../../../utils/'
+const { getTmpFilesFolderPath } = fileUtilModules
 
 let app: Express
 appPromise.then((_app) => {
@@ -30,28 +49,27 @@ describe('files', () => {
   let mongoServer: MongoMemoryServer
   const controller = new UserController()
 
+  let accessToken: string
+
   beforeAll(async () => {
     mongoServer = await MongoMemoryServer.create()
     con = await mongoose.connect(mongoServer.getUri())
+
+    const dbUser = await controller.createUser(user)
+    accessToken = generateAccessToken({
+      clientId,
+      userId: dbUser.id
+    })
+    await saveTokensInDB(dbUser.id, clientId, accessToken, 'refreshToken')
   })
 
   afterAll(async () => {
     await con.connection.dropDatabase()
     await con.connection.close()
     await mongoServer.stop()
+    await deleteFolder(tmpFolder)
   })
   describe('deploy', () => {
-    let accessToken: string
-    let dbUser: any
-
-    beforeAll(async () => {
-      dbUser = await controller.createUser(user)
-      accessToken = generateAccessToken({
-        clientId,
-        userId: dbUser.id
-      })
-      await saveTokensInDB(dbUser.id, clientId, accessToken, 'refreshToken')
-    })
     const shouldFailAssertion = async (payload: any) => {
       const res = await request(app)
         .post('/SASjsApi/drive/deploy')
@@ -144,8 +162,6 @@ describe('files', () => {
       const exampleService = getExampleService()
       const testJobFile = path.join(testJobFolder, exampleService.name) + '.sas'
 
-      console.log(`[testJobFile]`, testJobFile)
-
       await expect(fileExists(testJobFile)).resolves.toEqual(true)
 
       await expect(readFile(testJobFile)).resolves.toEqual(exampleService.code)
@@ -153,6 +169,146 @@ describe('files', () => {
       await deleteFolder(getTmpFilesFolderPath())
     })
   })
+
+  describe('file', () => {
+    describe('create', () => {
+      it('should create a SAS file on drive', async () => {
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', '/my/path/code.sas')
+          .attach('file', path.join(__dirname, 'files', 'sample.sas'))
+
+        expect(res.statusCode).toEqual(200)
+        expect(res.body).toEqual({
+          status: 'success'
+        })
+      })
+
+      it('should respond with Unauthorized if access token is not present', async () => {
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .field('filePath', '/my/path/code.sas')
+          .attach('file', path.join(__dirname, 'files', 'sample.sas'))
+          .expect(401)
+
+        expect(res.text).toEqual('Unauthorized')
+        expect(res.body).toEqual({})
+      })
+
+      it('should respond with Forbidden if file is already present', async () => {
+        const fileToAttachPath = path.join(__dirname, 'files', 'sample.sas')
+        const pathToUpload = '/my/path/code.sas'
+
+        const pathToCopy = path.join(
+          fileUtilModules.getTmpFilesFolderPath(),
+          pathToUpload
+        )
+        await copy(fileToAttachPath, pathToCopy)
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .attach('file', fileToAttachPath)
+          .expect(403)
+
+        expect(res.text).toEqual('Error: File already exists.')
+        expect(res.body).toEqual({})
+      })
+
+      it('should respond with Forbidden if filePath outside Drive', async () => {
+        const fileToAttachPath = path.join(__dirname, 'files', 'sample.sas')
+        const pathToUpload = '/../path/code.sas'
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .attach('file', fileToAttachPath)
+          .expect(403)
+
+        expect(res.text).toEqual('Error: Cannot put file outside drive.')
+        expect(res.body).toEqual({})
+      })
+
+      it('should respond with Bad Request if filePath is missing', async () => {
+        const fileToAttachPath = path.join(__dirname, 'files', 'sample.sas')
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .attach('file', fileToAttachPath)
+          .expect(400)
+
+        expect(res.text).toEqual(`"filePath" is required`)
+        expect(res.body).toEqual({})
+      })
+
+      it("should respond with Bad Request if filePath doesn't has correct extension", async () => {
+        const fileToAttachPath = path.join(__dirname, 'files', 'sample.sas')
+        const pathToUpload = '/my/path/code.oth'
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .attach('file', fileToAttachPath)
+          .expect(400)
+
+        expect(res.text).toEqual('Valid extensions for filePath: .sas')
+        expect(res.body).toEqual({})
+      })
+
+      it('should respond with Bad Request if file is missing', async () => {
+        const pathToUpload = '/my/path/code.sas'
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .expect(400)
+
+        expect(res.text).toEqual('"file" is not present.')
+        expect(res.body).toEqual({})
+      })
+
+      it("should respond with Bad Request if attached file doesn't has correct extension", async () => {
+        const fileToAttachPath = path.join(__dirname, 'files', 'sample.oth')
+        const pathToUpload = '/my/path/code.sas'
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .attach('file', fileToAttachPath)
+          .expect(400)
+
+        expect(res.text).toEqual(
+          `File extension '.oth' not acceptable. Valid extension(s): .sas`
+        )
+        expect(res.body).toEqual({})
+      })
+
+      it('should respond with Bad Request if attached file exceeds file limit', async () => {
+        const pathToUpload = '/my/path/code.sas'
+
+        const attachedFile = Buffer.from('.'.repeat(20 * 1024 * 1024))
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .attach('file', attachedFile, 'another.sas')
+          .expect(400)
+
+        expect(res.text).toEqual(
+          'File size is over limit. File limit is: 10 MB'
+        )
+        expect(res.body).toEqual({})
+      })
+    })
+  })
 })
 
 const getExampleService = (): ServiceMember =>

api/src/routes/api/spec/files/sample.oth

@@ -0,0 +1 @@
+some code of sas

api/src/routes/api/spec/files/sample.sas

@@ -0,0 +1 @@
+some code of sas

api/src/utils/validation.ts

@@ -73,8 +73,9 @@ export const getFileDriveValidation = (data: any): Joi.ValidationResult =>
 
 export const updateFileDriveValidation = (data: any): Joi.ValidationResult =>
   Joi.object({
-    filePath: Joi.string().required(),
-    fileContent: Joi.string().required()
+    filePath: Joi.string().pattern(/.sas$/).required().messages({
+      'string.pattern.base': `Valid extensions for filePath: .sas`
+    })
   }).validate(data)
 
 export const runSASValidation = (data: any): Joi.ValidationResult =>