From 18d0604bdd0b20ad468f9345474b4de034ee3a67 Mon Sep 17 00:00:00 2001 From: Saad Jutt Date: Sat, 2 Apr 2022 05:23:25 +0500 Subject: [PATCH 1/5] feat(deploy): new route added for deploy with build.json --- api/package-lock.json | 14 +++--- api/package.json | 2 +- api/public/swagger.yaml | 50 +++++++++++++++++++++ api/scripts/compileSysInit.ts | 4 +- api/src/controllers/drive.ts | 25 +++++++++-- api/src/controllers/internal/deploy.ts | 14 +++--- api/src/routes/api/drive.ts | 46 ++++++++++++++++++- api/src/routes/api/spec/drive.spec.ts | 5 ++- api/src/types/FileTree.ts | 62 -------------------------- api/src/types/index.ts | 1 - 10 files changed, 138 insertions(+), 85 deletions(-) delete mode 100644 api/src/types/FileTree.ts diff --git a/api/package-lock.json b/api/package-lock.json index 37485a7..8869e77 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -9,7 +9,7 @@ "version": "0.0.2", "dependencies": { "@sasjs/core": "4.9.0", - "@sasjs/utils": "2.36.2", + "@sasjs/utils": "2.42.1", "bcryptjs": "^2.4.3", "cookie-parser": "^1.4.6", "cors": "^2.8.5", @@ -1384,9 +1384,9 @@ "integrity": "sha512-zc1Ey0ylHt/eRZAfK0mVG3EqNyq//wLxbiguiK0R6FhVqwYFEkprs3IiLGZ5M9ttKs2rHRIjOe/ckklHm+6HNQ==" }, "node_modules/@sasjs/utils": { - "version": "2.36.2", - "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.36.2.tgz", - "integrity": "sha512-r0O9vkNIK5+2peBiGbcKc3Ei62eAMDt+1SQl17U9Vv26LYqezxQBwIYYMUjnkZE8Q7XlTI/FUS+SIHTCZMr4Jg==", + "version": "2.42.1", + "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.42.1.tgz", + "integrity": "sha512-DzHNYjeoj2eUkwV7Sa4eHCKRoTrYaQ6eyv6c1U5qOYXwVdZpMoYA3HFsHj55UcMOn2U3CXI5nrR7PZlUmVwVbQ==", "hasInstallScript": true, "dependencies": { "@types/fs-extra": "9.0.13", @@ -11132,9 +11132,9 @@ "integrity": "sha512-zc1Ey0ylHt/eRZAfK0mVG3EqNyq//wLxbiguiK0R6FhVqwYFEkprs3IiLGZ5M9ttKs2rHRIjOe/ckklHm+6HNQ==" }, "@sasjs/utils": { - "version": "2.36.2", - "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.36.2.tgz", - "integrity": "sha512-r0O9vkNIK5+2peBiGbcKc3Ei62eAMDt+1SQl17U9Vv26LYqezxQBwIYYMUjnkZE8Q7XlTI/FUS+SIHTCZMr4Jg==", + "version": "2.42.1", + "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.42.1.tgz", + "integrity": "sha512-DzHNYjeoj2eUkwV7Sa4eHCKRoTrYaQ6eyv6c1U5qOYXwVdZpMoYA3HFsHj55UcMOn2U3CXI5nrR7PZlUmVwVbQ==", "requires": { "@types/fs-extra": "9.0.13", "@types/prompts": "2.0.13", diff --git a/api/package.json b/api/package.json index 09a7d56..58654bc 100644 --- a/api/package.json +++ b/api/package.json @@ -47,7 +47,7 @@ "author": "4GL Ltd", "dependencies": { "@sasjs/core": "4.9.0", - "@sasjs/utils": "2.36.2", + "@sasjs/utils": "2.42.1", "bcryptjs": "^2.4.3", "cookie-parser": "^1.4.6", "cors": "^2.8.5", diff --git a/api/public/swagger.yaml b/api/public/swagger.yaml index c3f1787..d4fb2eb 100644 --- a/api/public/swagger.yaml +++ b/api/public/swagger.yaml @@ -606,6 +606,56 @@ paths: application/json: schema: $ref: '#/components/schemas/DeployPayload' + /SASjsApi/drive/deploy/upload: + post: + operationId: DeployUpload + responses: + '200': + description: Ok + content: + application/json: + schema: + $ref: '#/components/schemas/DeployResponse' + examples: + 'Example 1': + value: {status: success, message: 'Files deployed successfully to @sasjs/server.'} + '400': + description: 'Invalid Format' + content: + application/json: + schema: + $ref: '#/components/schemas/DeployResponse' + examples: + 'Example 1': + value: {status: failure, message: 'Provided not supported data format.'} + '500': + description: 'Execution Error' + content: + application/json: + schema: + $ref: '#/components/schemas/DeployResponse' + examples: + 'Example 1': + value: {status: failure, message: 'Deployment failed!'} + summary: 'Creates/updates files within SASjs Drive using uploaded JSON file.' + tags: + - Drive + security: + - + bearerAuth: [] + parameters: [] + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + properties: + file: + type: string + format: binary + required: + - file /SASjsApi/drive/file: get: operationId: GetFile diff --git a/api/scripts/compileSysInit.ts b/api/scripts/compileSysInit.ts index 0d6498b..f094324 100644 --- a/api/scripts/compileSysInit.ts +++ b/api/scripts/compileSysInit.ts @@ -1,5 +1,6 @@ import path from 'path' import { + CompileTree, createFile, loadDependenciesFile, readFile, @@ -18,7 +19,8 @@ const compiledSystemInit = async (systemInit: string) => macroFolders: [], buildSourceFolder: '', binaryFolders: [], - macroCorePath + macroCorePath, + compileTree: new CompileTree('') // dummy compileTree })) const createSysInitFile = async () => { diff --git a/api/src/controllers/drive.ts b/api/src/controllers/drive.ts index 0955b6f..7f22522 100644 --- a/api/src/controllers/drive.ts +++ b/api/src/controllers/drive.ts @@ -14,7 +14,8 @@ import { Patch, UploadedFile, FormField, - Delete + Delete, + Hidden } from 'tsoa' import { fileExists, @@ -22,14 +23,15 @@ import { createFolder, deleteFile as deleteFileOnSystem, folderExists, - listFilesAndSubFoldersInFolder, listFilesInFolder, listSubFoldersInFolder, - isFolder + isFolder, + FileTree, + isFileTree } from '@sasjs/utils' import { createFileTree, ExecutionController, getTreeExample } from './internal' -import { FileTree, isFileTree, TreeNode } from '../types' +import { TreeNode } from '../types' import { getTmpFilesFolderPath } from '../utils' interface DeployPayload { @@ -93,6 +95,21 @@ export class DriveController { return deploy(body) } + /** + * @summary Creates/updates files within SASjs Drive using uploaded JSON file. + * + */ + @Example(successDeployResponse) + @Response(400, 'Invalid Format', invalidDeployFormatResponse) + @Response(500, 'Execution Error', execDeployErrorResponse) + @Post('/deploy/upload') + public async deployUpload( + @UploadedFile() file: Express.Multer.File, // passing here for API docs + @Query() @Hidden() body?: DeployPayload // Hidden decorator has be optional + ): Promise { + return deploy(body!) + } + /** * * @summary Get file from SASjs Drive diff --git a/api/src/controllers/internal/deploy.ts b/api/src/controllers/internal/deploy.ts index af6b830..05a1996 100644 --- a/api/src/controllers/internal/deploy.ts +++ b/api/src/controllers/internal/deploy.ts @@ -1,13 +1,15 @@ import path from 'path' +import { getTmpFilesFolderPath } from '../../utils/file' import { - MemberType, + createFolder, + createFile, + asyncForEach, FolderMember, ServiceMember, - FileTree, - FileMember -} from '../../types' -import { getTmpFilesFolderPath } from '../../utils/file' -import { createFolder, createFile, asyncForEach } from '@sasjs/utils' + FileMember, + MemberType, + FileTree +} from '@sasjs/utils' // REFACTOR: export FileTreeCpntroller export const createFileTree = async ( diff --git a/api/src/routes/api/drive.ts b/api/src/routes/api/drive.ts index cdd881b..bdb2e5f 100644 --- a/api/src/routes/api/drive.ts +++ b/api/src/routes/api/drive.ts @@ -1,5 +1,5 @@ import express from 'express' -import { deleteFile } from '@sasjs/utils' +import { deleteFile, readFile } from '@sasjs/utils' import { publishAppStream } from '../appStream' @@ -43,6 +43,50 @@ driveRouter.post('/deploy', async (req, res) => { } }) +driveRouter.post( + '/deploy/upload', + (...arg) => multerSingle('file', arg), + async (req, res) => { + if (!req.file) return res.status(400).send('"file" is not present.') + + const fileContent = await readFile(req.file.path) + + let jsonContent + try { + jsonContent = JSON.parse(fileContent) + } catch (err) { + return res.status(400).send('File containing invalid JSON content.') + } + + const { error, value: body } = deployValidation(jsonContent) + if (error) return res.status(400).send(error.details[0].message) + + try { + const response = await controller.deployUpload(req.file, body) + + if (body.streamWebFolder) { + const { streamServiceName } = await publishAppStream( + body.appLoc, + body.streamWebFolder, + body.streamServiceName, + body.streamLogo + ) + response.streamServiceName = streamServiceName + } + + res.send(response) + } catch (err: any) { + const statusCode = err.code + + delete err.code + + res.status(statusCode).send(err) + } finally { + await deleteFile(req.file.path) + } + } +) + driveRouter.get('/file', async (req, res) => { const { error: errQ, value: query } = fileParamValidation(req.query) diff --git a/api/src/routes/api/spec/drive.spec.ts b/api/src/routes/api/spec/drive.spec.ts index a236898..7f65069 100644 --- a/api/src/routes/api/spec/drive.spec.ts +++ b/api/src/routes/api/spec/drive.spec.ts @@ -12,7 +12,9 @@ import { generateTimestamp, copy, createFolder, - createFile + createFile, + ServiceMember, + FolderMember } from '@sasjs/utils' import * as fileUtilModules from '../../../utils/file' @@ -28,7 +30,6 @@ jest import appPromise from '../../../app' import { UserController } from '../../../controllers/' import { getTreeExample } from '../../../controllers/internal' -import { FolderMember, ServiceMember } from '../../../types' import { generateAccessToken, saveTokensInDB } from '../../../utils/' const { getTmpFilesFolderPath } = fileUtilModules diff --git a/api/src/types/FileTree.ts b/api/src/types/FileTree.ts deleted file mode 100644 index 2e95bc8..0000000 --- a/api/src/types/FileTree.ts +++ /dev/null @@ -1,62 +0,0 @@ -export enum MemberType { - service = 'service', - file = 'file', - folder = 'folder' -} - -export interface ServiceMember { - name: string - type: MemberType.service - code: string -} - -export interface FileMember { - name: string - type: MemberType.file - code: string -} - -export interface FolderMember { - name: string - type: MemberType.folder - members: (FolderMember | ServiceMember | FileMember)[] -} -export interface FileTree { - members: (FolderMember | ServiceMember | FileMember)[] -} - -export const isFileTree = (arg: any): arg is FileTree => - arg && - arg.members && - Array.isArray(arg.members) && - arg.members.filter( - (member: ServiceMember | FileMember | FolderMember) => - !isServiceMember(member, '-') && - !isFileMember(member, '-') && - !isFolderMember(member, '-') - ).length === 0 - -const isServiceMember = (arg: any, pre: string): arg is ServiceMember => - arg && - typeof arg.name === 'string' && - arg.type === MemberType.service && - typeof arg.code === 'string' - -const isFileMember = (arg: any, pre: string): arg is ServiceMember => - arg && - typeof arg.name === 'string' && - arg.type === MemberType.file && - typeof arg.code === 'string' - -const isFolderMember = (arg: any, pre: string): arg is FolderMember => - arg && - typeof arg.name === 'string' && - arg.type === MemberType.folder && - arg.members && - Array.isArray(arg.members) && - arg.members.filter( - (member: FolderMember | ServiceMember) => - !isServiceMember(member, pre + '-') && - !isFileMember(member, pre + '-') && - !isFolderMember(member, pre + '-') - ).length === 0 diff --git a/api/src/types/index.ts b/api/src/types/index.ts index fcd3176..682aee8 100644 --- a/api/src/types/index.ts +++ b/api/src/types/index.ts @@ -1,7 +1,6 @@ // TODO: uppercase types export * from './AppStreamConfig' export * from './Execution' -export * from './FileTree' export * from './InfoJWT' export * from './PreProgramVars' export * from './Request' From 9d167abe2adb743bca161862b4561bf573182c00 Mon Sep 17 00:00:00 2001 From: Saad Jutt Date: Sat, 2 Apr 2022 05:29:34 +0500 Subject: [PATCH 2/5] fix: remove uploaded build.json from temp folder in all cases --- api/src/routes/api/drive.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/api/src/routes/api/drive.ts b/api/src/routes/api/drive.ts index bdb2e5f..71ffa4a 100644 --- a/api/src/routes/api/drive.ts +++ b/api/src/routes/api/drive.ts @@ -55,11 +55,15 @@ driveRouter.post( try { jsonContent = JSON.parse(fileContent) } catch (err) { + deleteFile(req.file.path) return res.status(400).send('File containing invalid JSON content.') } const { error, value: body } = deployValidation(jsonContent) - if (error) return res.status(400).send(error.details[0].message) + if (error) { + deleteFile(req.file.path) + return res.status(400).send(error.details[0].message) + } try { const response = await controller.deployUpload(req.file, body) @@ -82,7 +86,7 @@ driveRouter.post( res.status(statusCode).send(err) } finally { - await deleteFile(req.file.path) + deleteFile(req.file.path) } } ) From 9d9769eef3bdd7614a639a547c28d5d52bbd1559 Mon Sep 17 00:00:00 2001 From: Saad Jutt Date: Sat, 2 Apr 2022 05:36:53 +0500 Subject: [PATCH 3/5] chore: increased file upload size to 100mb --- api/src/middlewares/multer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/src/middlewares/multer.ts b/api/src/middlewares/multer.ts index d46f466..21ab913 100644 --- a/api/src/middlewares/multer.ts +++ b/api/src/middlewares/multer.ts @@ -4,7 +4,7 @@ import multer, { FileFilterCallback, Options } from 'multer' import { blockFileRegex, getTmpUploadsPath } from '../utils' const fieldNameSize = 300 -const fileSize = 10485760 // 10 MB +const fileSize = 104857600 // 100 MB const storage = multer.diskStorage({ destination: getTmpUploadsPath(), From e430bdb0d4d6fe9b6638070da2a115103ca092f7 Mon Sep 17 00:00:00 2001 From: Saad Jutt Date: Sat, 2 Apr 2022 05:51:24 +0500 Subject: [PATCH 4/5] test(upload): spec updated for file upload exceeding limit --- api/src/routes/api/spec/drive.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/routes/api/spec/drive.spec.ts b/api/src/routes/api/spec/drive.spec.ts index 7f65069..ec054d1 100644 --- a/api/src/routes/api/spec/drive.spec.ts +++ b/api/src/routes/api/spec/drive.spec.ts @@ -425,7 +425,7 @@ describe('drive', () => { it('should respond with Bad Request if attached file exceeds file limit', async () => { const pathToUpload = '/my/path/code.sas' - const attachedFile = Buffer.from('.'.repeat(20 * 1024 * 1024)) + const attachedFile = Buffer.from('.'.repeat(110 * 1024 * 1024)) // 110mb const res = await request(app) .post('/SASjsApi/drive/file') @@ -435,7 +435,7 @@ describe('drive', () => { .expect(400) expect(res.text).toEqual( - 'File size is over limit. File limit is: 10 MB' + 'File size is over limit. File limit is: 100 MB' ) expect(res.body).toEqual({}) }) From e1ebbfd0873a66f404fbc0d2651358ee59387c68 Mon Sep 17 00:00:00 2001 From: Saad Jutt Date: Sat, 2 Apr 2022 06:04:34 +0500 Subject: [PATCH 5/5] chore: increased file upload size to 100mb --- api/src/routes/api/spec/drive.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/routes/api/spec/drive.spec.ts b/api/src/routes/api/spec/drive.spec.ts index ec054d1..ef05469 100644 --- a/api/src/routes/api/spec/drive.spec.ts +++ b/api/src/routes/api/spec/drive.spec.ts @@ -583,7 +583,7 @@ describe('drive', () => { it('should respond with Bad Request if attached file exceeds file limit', async () => { const pathToUpload = '/my/path/code.sas' - const attachedFile = Buffer.from('.'.repeat(20 * 1024 * 1024)) + const attachedFile = Buffer.from('.'.repeat(110 * 1024 * 1024)) // 110mb const res = await request(app) .patch('/SASjsApi/drive/file') @@ -593,7 +593,7 @@ describe('drive', () => { .expect(400) expect(res.text).toEqual( - 'File size is over limit. File limit is: 10 MB' + 'File size is over limit. File limit is: 100 MB' ) expect(res.body).toEqual({}) })