From 92fda183f3f0f3956b7c791669eb8dd52c389d1b Mon Sep 17 00:00:00 2001 From: Saad Jutt Date: Sat, 16 Jul 2022 04:42:54 +0500 Subject: [PATCH] feat(logs): logs to file with rotating + code split into files --- api/package-lock.json | 17 +++ api/package.json | 3 +- api/public/swagger.yaml | 12 +- api/src/app-modules/configureCors.ts | 21 ++++ .../app-modules/configureExpressSession.ts | 32 +++++ api/src/app-modules/configureLogger.ts | 33 +++++ api/src/app-modules/configureSecurity.ts | 26 ++++ api/src/app-modules/index.ts | 4 + api/src/app.ts | 113 +++++------------- 9 files changed, 168 insertions(+), 93 deletions(-) create mode 100644 api/src/app-modules/configureCors.ts create mode 100644 api/src/app-modules/configureExpressSession.ts create mode 100644 api/src/app-modules/configureLogger.ts create mode 100644 api/src/app-modules/configureSecurity.ts create mode 100644 api/src/app-modules/index.ts diff --git a/api/package-lock.json b/api/package-lock.json index 989732e..7ab0cd0 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -24,6 +24,7 @@ "mongoose-sequence": "^5.3.1", "morgan": "^1.10.0", "multer": "^1.4.3", + "rotating-file-stream": "^3.0.4", "swagger-ui-express": "4.3.0", "unzipper": "^0.10.11", "url": "^0.10.3" @@ -8435,6 +8436,17 @@ "resolved": "https://registry.npmjs.org/rndm/-/rndm-1.2.0.tgz", "integrity": "sha1-8z/pz7Urv9UgqhgyO8ZdsRCht2w=" }, + "node_modules/rotating-file-stream": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/rotating-file-stream/-/rotating-file-stream-3.0.4.tgz", + "integrity": "sha512-Zd+IA5soqBUKtCa7KfMRnF7GKEaRav/dJFU7UHc+r1FMd8VvMLUFE0q1HX5s6XJO1cZ6jAqtTa/ZFnjc7IqlJA==", + "engines": { + "node": ">=14.0" + }, + "funding": { + "url": "https://www.blockchain.com/btc/address/12p1p5q7sK75tPyuesZmssiMYr4TKzpSCN" + } + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -16465,6 +16477,11 @@ "resolved": "https://registry.npmjs.org/rndm/-/rndm-1.2.0.tgz", "integrity": "sha1-8z/pz7Urv9UgqhgyO8ZdsRCht2w=" }, + "rotating-file-stream": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/rotating-file-stream/-/rotating-file-stream-3.0.4.tgz", + "integrity": "sha512-Zd+IA5soqBUKtCa7KfMRnF7GKEaRav/dJFU7UHc+r1FMd8VvMLUFE0q1HX5s6XJO1cZ6jAqtTa/ZFnjc7IqlJA==" + }, "run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", diff --git a/api/package.json b/api/package.json index 372ce98..1d4b25b 100644 --- a/api/package.json +++ b/api/package.json @@ -7,7 +7,7 @@ "initial": "npm run swagger && npm run compileSysInit && npm run copySASjsCore", "prestart": "npm run initial", "prebuild": "npm run initial", - "start": "nodemon ./src/server.ts", + "start": "NODE_ENV=development nodemon ./src/server.ts", "start:prod": "node ./build/src/server.js", "build": "rimraf build && tsc", "postbuild": "npm run copy:files", @@ -63,6 +63,7 @@ "mongoose-sequence": "^5.3.1", "morgan": "^1.10.0", "multer": "^1.4.3", + "rotating-file-stream": "^3.0.4", "swagger-ui-express": "4.3.0", "unzipper": "^0.10.11", "url": "^0.10.3" diff --git a/api/public/swagger.yaml b/api/public/swagger.yaml index ec7814c..5acf997 100644 --- a/api/public/swagger.yaml +++ b/api/public/swagger.yaml @@ -1709,13 +1709,13 @@ servers: tags: - name: Info - description: 'Get Server Info' + description: 'Get Server Information' - name: Session description: 'Get Session information' - name: User - description: 'Operations about users' + description: 'Operations with users' - name: Permission description: 'Operations about permissions' @@ -1727,16 +1727,16 @@ tags: description: 'Operations about auth' - name: Drive - description: 'Operations about drive' + description: 'Operations on SASjs Drive' - name: Group - description: 'Operations about group' + description: 'Operations on groups and group memberships' - name: STP - description: 'Operations about STP' + description: 'Execution of Stored Programs' - name: CODE - description: 'Operations on SAS code' + description: 'Execution of code (various runtimes are supported)' - name: Web description: 'Operations on Web' diff --git a/api/src/app-modules/configureCors.ts b/api/src/app-modules/configureCors.ts new file mode 100644 index 0000000..631b166 --- /dev/null +++ b/api/src/app-modules/configureCors.ts @@ -0,0 +1,21 @@ +import { Express } from 'express' +import cors from 'cors' +import { CorsType } from '../utils' + +export const configureCors = (app: Express) => { + const { CORS, WHITELIST } = process.env + + if (CORS === CorsType.ENABLED) { + const whiteList: string[] = [] + WHITELIST?.split(' ') + ?.filter((url) => !!url) + .forEach((url) => { + if (url.startsWith('http')) + // removing trailing slash of URLs listing for CORS + whiteList.push(url.replace(/\/$/, '')) + }) + + console.log('All CORS Requests are enabled for:', whiteList) + app.use(cors({ credentials: true, origin: whiteList })) + } +} diff --git a/api/src/app-modules/configureExpressSession.ts b/api/src/app-modules/configureExpressSession.ts new file mode 100644 index 0000000..7e8992a --- /dev/null +++ b/api/src/app-modules/configureExpressSession.ts @@ -0,0 +1,32 @@ +import { Express } from 'express' +import mongoose from 'mongoose' +import session from 'express-session' +import MongoStore from 'connect-mongo' + +import { ModeType } from '../utils' +import { cookieOptions } from '../app' + +export const configureExpressSession = (app: Express) => { + const { MODE } = process.env + + if (MODE === ModeType.Server) { + let store: MongoStore | undefined + + if (process.env.NODE_ENV !== 'test') { + store = MongoStore.create({ + client: mongoose.connection!.getClient() as any, + collectionName: 'sessions' + }) + } + + app.use( + session({ + secret: process.secrets.SESSION_SECRET, + saveUninitialized: false, // don't create session until something stored + resave: false, //don't save session if unmodified + store, + cookie: cookieOptions + }) + ) + } +} diff --git a/api/src/app-modules/configureLogger.ts b/api/src/app-modules/configureLogger.ts new file mode 100644 index 0000000..6edb3ba --- /dev/null +++ b/api/src/app-modules/configureLogger.ts @@ -0,0 +1,33 @@ +import path from 'path' +import { Express } from 'express' +import morgan from 'morgan' +import { createStream } from 'rotating-file-stream' +import { generateTimestamp } from '@sasjs/utils' +import { getLogFolder } from '../utils' + +export const configureLogger = (app: Express) => { + const { LOG_FORMAT_MORGAN } = process.env + + let options + if ( + process.env.NODE_ENV !== 'development' && + process.env.NODE_ENV !== 'test' + ) { + const timestamp = generateTimestamp() + const filename = `${timestamp}.log` + const logsFolder = getLogFolder() + + // create a rotating write stream + var accessLogStream = createStream(filename, { + interval: '1d', // rotate daily + path: logsFolder + }) + + console.log('Writing Logs to :', path.join(logsFolder, filename)) + + options = { stream: accessLogStream } + } + + // setup the logger + app.use(morgan(LOG_FORMAT_MORGAN as string, options)) +} diff --git a/api/src/app-modules/configureSecurity.ts b/api/src/app-modules/configureSecurity.ts new file mode 100644 index 0000000..a3a787f --- /dev/null +++ b/api/src/app-modules/configureSecurity.ts @@ -0,0 +1,26 @@ +import { Express } from 'express' +import { getEnvCSPDirectives } from '../utils/parseHelmetConfig' +import { HelmetCoepType, ProtocolType } from '../utils' +import helmet from 'helmet' + +export const configureSecurity = (app: Express) => { + const { PROTOCOL, HELMET_CSP_CONFIG_PATH, HELMET_COEP } = process.env + + const cspConfigJson: { [key: string]: string[] | null } = getEnvCSPDirectives( + HELMET_CSP_CONFIG_PATH + ) + if (PROTOCOL === ProtocolType.HTTP) + cspConfigJson['upgrade-insecure-requests'] = null + + app.use( + helmet({ + contentSecurityPolicy: { + directives: { + ...helmet.contentSecurityPolicy.getDefaultDirectives(), + ...cspConfigJson + } + }, + crossOriginEmbedderPolicy: HELMET_COEP === HelmetCoepType.TRUE + }) + ) +} diff --git a/api/src/app-modules/index.ts b/api/src/app-modules/index.ts new file mode 100644 index 0000000..39d5216 --- /dev/null +++ b/api/src/app-modules/index.ts @@ -0,0 +1,4 @@ +export * from './configureCors' +export * from './configureExpressSession' +export * from './configureLogger' +export * from './configureSecurity' diff --git a/api/src/app.ts b/api/src/app.ts index 79d26c8..cab4690 100644 --- a/api/src/app.ts +++ b/api/src/app.ts @@ -1,30 +1,26 @@ import path from 'path' import express, { ErrorRequestHandler } from 'express' -import mongoose from 'mongoose' import csrf from 'csurf' -import session from 'express-session' -import MongoStore from 'connect-mongo' -import morgan from 'morgan' import cookieParser from 'cookie-parser' import dotenv from 'dotenv' -import cors from 'cors' -import helmet from 'helmet' import { copySASjsCore, - CorsType, getWebBuildFolder, - HelmetCoepType, instantiateLogger, loadAppStreamConfig, - ModeType, ProtocolType, ReturnCode, setProcessVariables, setupFolders, verifyEnvVariables } from './utils' -import { getEnvCSPDirectives } from './utils/parseHelmetConfig' +import { + configureCors, + configureExpressSession, + configureLogger, + configureSecurity +} from './app-modules' dotenv.config() @@ -34,19 +30,7 @@ if (verifyEnvVariables()) process.exit(ReturnCode.InvalidEnv) const app = express() -app.use(cookieParser()) - -const { - MODE, - CORS, - WHITELIST, - PROTOCOL, - HELMET_CSP_CONFIG_PATH, - HELMET_COEP, - LOG_FORMAT_MORGAN -} = process.env - -app.use(morgan(LOG_FORMAT_MORGAN as string)) +const { PROTOCOL } = process.env export const cookieOptions = { secure: PROTOCOL === ProtocolType.HTTPS, @@ -54,87 +38,44 @@ export const cookieOptions = { maxAge: 24 * 60 * 60 * 1000 // 24 hours } -const cspConfigJson: { [key: string]: string[] | null } = getEnvCSPDirectives( - HELMET_CSP_CONFIG_PATH -) -if (PROTOCOL === ProtocolType.HTTP) - cspConfigJson['upgrade-insecure-requests'] = null - /*********************************** * CSRF Protection * ***********************************/ export const csrfProtection = csrf({ cookie: cookieOptions }) -/*********************************** - * Handle security and origin * - ***********************************/ -app.use( - helmet({ - contentSecurityPolicy: { - directives: { - ...helmet.contentSecurityPolicy.getDefaultDirectives(), - ...cspConfigJson - } - }, - crossOriginEmbedderPolicy: HELMET_COEP === HelmetCoepType.TRUE - }) -) +const onError: ErrorRequestHandler = (err, req, res, next) => { + if (err.code === 'EBADCSRFTOKEN') + return res.status(400).send('Invalid CSRF token!') -/*********************************** - * Enabling CORS * - ***********************************/ -if (CORS === CorsType.ENABLED) { - const whiteList: string[] = [] - WHITELIST?.split(' ') - ?.filter((url) => !!url) - .forEach((url) => { - if (url.startsWith('http')) - // removing trailing slash of URLs listing for CORS - whiteList.push(url.replace(/\/$/, '')) - }) - - console.log('All CORS Requests are enabled for:', whiteList) - app.use(cors({ credentials: true, origin: whiteList })) + console.error(err.stack) + res.status(500).send('Something broke!') } export default setProcessVariables().then(async () => { + app.use(cookieParser()) + + configureLogger(app) + + /*********************************** + * Handle security and origin * + ***********************************/ + configureSecurity(app) + + /*********************************** + * Enabling CORS * + ***********************************/ + configureCors(app) + /*********************************** * DB Connection & * * Express Sessions * * With Mongo Store * ***********************************/ - if (MODE === ModeType.Server) { - let store: MongoStore | undefined - - if (process.env.NODE_ENV !== 'test') { - store = MongoStore.create({ - client: mongoose.connection!.getClient() as any, - collectionName: 'sessions' - }) - } - - app.use( - session({ - secret: process.secrets.SESSION_SECRET, - saveUninitialized: false, // don't create session until something stored - resave: false, //don't save session if unmodified - store, - cookie: cookieOptions - }) - ) - } + configureExpressSession(app) app.use(express.json({ limit: '100mb' })) app.use(express.static(path.join(__dirname, '../public'))) - const onError: ErrorRequestHandler = (err, req, res, next) => { - if (err.code === 'EBADCSRFTOKEN') - return res.status(400).send('Invalid CSRF token!') - - console.error(err.stack) - res.status(500).send('Something broke!') - } - await setupFolders() await copySASjsCore()