feat(logs): logs to file with rotating + code split into files

api/package-lock.json

@@ -24,6 +24,7 @@
         "mongoose-sequence": "^5.3.1",
         "morgan": "^1.10.0",
         "multer": "^1.4.3",
+        "rotating-file-stream": "^3.0.4",
         "swagger-ui-express": "4.3.0",
         "unzipper": "^0.10.11",
         "url": "^0.10.3"
@@ -8435,6 +8436,17 @@
       "resolved": "https://registry.npmjs.org/rndm/-/rndm-1.2.0.tgz",
       "integrity": "sha1-8z/pz7Urv9UgqhgyO8ZdsRCht2w="
     },
+    "node_modules/rotating-file-stream": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/rotating-file-stream/-/rotating-file-stream-3.0.4.tgz",
+      "integrity": "sha512-Zd+IA5soqBUKtCa7KfMRnF7GKEaRav/dJFU7UHc+r1FMd8VvMLUFE0q1HX5s6XJO1cZ6jAqtTa/ZFnjc7IqlJA==",
+      "engines": {
+        "node": ">=14.0"
+      },
+      "funding": {
+        "url": "https://www.blockchain.com/btc/address/12p1p5q7sK75tPyuesZmssiMYr4TKzpSCN"
+      }
+    },
     "node_modules/run-parallel": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
@@ -16465,6 +16477,11 @@
       "resolved": "https://registry.npmjs.org/rndm/-/rndm-1.2.0.tgz",
       "integrity": "sha1-8z/pz7Urv9UgqhgyO8ZdsRCht2w="
     },
+    "rotating-file-stream": {
+      "version": "3.0.4",
+      "resolved": "https://registry.npmjs.org/rotating-file-stream/-/rotating-file-stream-3.0.4.tgz",
+      "integrity": "sha512-Zd+IA5soqBUKtCa7KfMRnF7GKEaRav/dJFU7UHc+r1FMd8VvMLUFE0q1HX5s6XJO1cZ6jAqtTa/ZFnjc7IqlJA=="
+    },
     "run-parallel": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",

api/package.json

@@ -7,7 +7,7 @@
     "initial": "npm run swagger && npm run compileSysInit && npm run copySASjsCore",
     "prestart": "npm run initial",
     "prebuild": "npm run initial",
-    "start": "nodemon ./src/server.ts",
+    "start": "NODE_ENV=development nodemon ./src/server.ts",
     "start:prod": "node ./build/src/server.js",
     "build": "rimraf build && tsc",
     "postbuild": "npm run copy:files",
@@ -63,6 +63,7 @@
     "mongoose-sequence": "^5.3.1",
     "morgan": "^1.10.0",
     "multer": "^1.4.3",
+    "rotating-file-stream": "^3.0.4",
     "swagger-ui-express": "4.3.0",
     "unzipper": "^0.10.11",
     "url": "^0.10.3"

api/public/swagger.yaml

@@ -1709,13 +1709,13 @@ servers:
 tags:
     -
         name: Info
-        description: 'Get Server Info'
+        description: 'Get Server Information'
     -
         name: Session
         description: 'Get Session information'
     -
         name: User
-        description: 'Operations about users'
+        description: 'Operations with users'
     -
         name: Permission
         description: 'Operations about permissions'
@@ -1727,16 +1727,16 @@ tags:
         description: 'Operations about auth'
     -
         name: Drive
-        description: 'Operations about drive'
+        description: 'Operations on SASjs Drive'
     -
         name: Group
-        description: 'Operations about group'
+        description: 'Operations on groups and group memberships'
     -
         name: STP
-        description: 'Operations about STP'
+        description: 'Execution of Stored Programs'
     -
         name: CODE
-        description: 'Operations on SAS code'
+        description: 'Execution of code (various runtimes are supported)'
     -
         name: Web
         description: 'Operations on Web'

api/src/app-modules/configureCors.ts

@@ -0,0 +1,21 @@
+import { Express } from 'express'
+import cors from 'cors'
+import { CorsType } from '../utils'
+
+export const configureCors = (app: Express) => {
+  const { CORS, WHITELIST } = process.env
+
+  if (CORS === CorsType.ENABLED) {
+    const whiteList: string[] = []
+    WHITELIST?.split(' ')
+      ?.filter((url) => !!url)
+      .forEach((url) => {
+        if (url.startsWith('http'))
+          // removing trailing slash of URLs listing for CORS
+          whiteList.push(url.replace(/\/$/, ''))
+      })
+
+    console.log('All CORS Requests are enabled for:', whiteList)
+    app.use(cors({ credentials: true, origin: whiteList }))
+  }
+}

api/src/app-modules/configureExpressSession.ts

@@ -0,0 +1,32 @@
+import { Express } from 'express'
+import mongoose from 'mongoose'
+import session from 'express-session'
+import MongoStore from 'connect-mongo'
+
+import { ModeType } from '../utils'
+import { cookieOptions } from '../app'
+
+export const configureExpressSession = (app: Express) => {
+  const { MODE } = process.env
+
+  if (MODE === ModeType.Server) {
+    let store: MongoStore | undefined
+
+    if (process.env.NODE_ENV !== 'test') {
+      store = MongoStore.create({
+        client: mongoose.connection!.getClient() as any,
+        collectionName: 'sessions'
+      })
+    }
+
+    app.use(
+      session({
+        secret: process.secrets.SESSION_SECRET,
+        saveUninitialized: false, // don't create session until something stored
+        resave: false, //don't save session if unmodified
+        store,
+        cookie: cookieOptions
+      })
+    )
+  }
+}

api/src/app-modules/configureLogger.ts

@@ -0,0 +1,33 @@
+import path from 'path'
+import { Express } from 'express'
+import morgan from 'morgan'
+import { createStream } from 'rotating-file-stream'
+import { generateTimestamp } from '@sasjs/utils'
+import { getLogFolder } from '../utils'
+
+export const configureLogger = (app: Express) => {
+  const { LOG_FORMAT_MORGAN } = process.env
+
+  let options
+  if (
+    process.env.NODE_ENV !== 'development' &&
+    process.env.NODE_ENV !== 'test'
+  ) {
+    const timestamp = generateTimestamp()
+    const filename = `${timestamp}.log`
+    const logsFolder = getLogFolder()
+
+    // create a rotating write stream
+    var accessLogStream = createStream(filename, {
+      interval: '1d', // rotate daily
+      path: logsFolder
+    })
+
+    console.log('Writing Logs to :', path.join(logsFolder, filename))
+
+    options = { stream: accessLogStream }
+  }
+
+  // setup the logger
+  app.use(morgan(LOG_FORMAT_MORGAN as string, options))
+}

api/src/app-modules/configureSecurity.ts

@@ -0,0 +1,26 @@
+import { Express } from 'express'
+import { getEnvCSPDirectives } from '../utils/parseHelmetConfig'
+import { HelmetCoepType, ProtocolType } from '../utils'
+import helmet from 'helmet'
+
+export const configureSecurity = (app: Express) => {
+  const { PROTOCOL, HELMET_CSP_CONFIG_PATH, HELMET_COEP } = process.env
+
+  const cspConfigJson: { [key: string]: string[] | null } = getEnvCSPDirectives(
+    HELMET_CSP_CONFIG_PATH
+  )
+  if (PROTOCOL === ProtocolType.HTTP)
+    cspConfigJson['upgrade-insecure-requests'] = null
+
+  app.use(
+    helmet({
+      contentSecurityPolicy: {
+        directives: {
+          ...helmet.contentSecurityPolicy.getDefaultDirectives(),
+          ...cspConfigJson
+        }
+      },
+      crossOriginEmbedderPolicy: HELMET_COEP === HelmetCoepType.TRUE
+    })
+  )
+}

api/src/app-modules/index.ts

@@ -0,0 +1,4 @@
+export * from './configureCors'
+export * from './configureExpressSession'
+export * from './configureLogger'
+export * from './configureSecurity'

api/src/app.ts

@@ -1,30 +1,26 @@
 import path from 'path'
 import express, { ErrorRequestHandler } from 'express'
-import mongoose from 'mongoose'
 import csrf from 'csurf'
-import session from 'express-session'
-import MongoStore from 'connect-mongo'
-import morgan from 'morgan'
 import cookieParser from 'cookie-parser'
 import dotenv from 'dotenv'
-import cors from 'cors'
-import helmet from 'helmet'
 
 import {
   copySASjsCore,
-  CorsType,
   getWebBuildFolder,
-  HelmetCoepType,
   instantiateLogger,
   loadAppStreamConfig,
-  ModeType,
   ProtocolType,
   ReturnCode,
   setProcessVariables,
   setupFolders,
   verifyEnvVariables
 } from './utils'
-import { getEnvCSPDirectives } from './utils/parseHelmetConfig'
+import {
+  configureCors,
+  configureExpressSession,
+  configureLogger,
+  configureSecurity
+} from './app-modules'
 
 dotenv.config()
 
@@ -34,19 +30,7 @@ if (verifyEnvVariables()) process.exit(ReturnCode.InvalidEnv)
 
 const app = express()
 
-app.use(cookieParser())
+const { PROTOCOL } = process.env
-
-const {
-  MODE,
-  CORS,
-  WHITELIST,
-  PROTOCOL,
-  HELMET_CSP_CONFIG_PATH,
-  HELMET_COEP,
-  LOG_FORMAT_MORGAN
-} = process.env
-
-app.use(morgan(LOG_FORMAT_MORGAN as string))
 
 export const cookieOptions = {
   secure: PROTOCOL === ProtocolType.HTTPS,
@@ -54,87 +38,44 @@ export const cookieOptions = {
   maxAge: 24 * 60 * 60 * 1000 // 24 hours
 }
 
-const cspConfigJson: { [key: string]: string[] | null } = getEnvCSPDirectives(
-  HELMET_CSP_CONFIG_PATH
-)
-if (PROTOCOL === ProtocolType.HTTP)
-  cspConfigJson['upgrade-insecure-requests'] = null
-
 /***********************************
  *         CSRF Protection         *
  ***********************************/
 export const csrfProtection = csrf({ cookie: cookieOptions })
 
-/***********************************
+const onError: ErrorRequestHandler = (err, req, res, next) => {
- *   Handle security and origin    *
+  if (err.code === 'EBADCSRFTOKEN')
- ***********************************/
+    return res.status(400).send('Invalid CSRF token!')
-app.use(
-  helmet({
-    contentSecurityPolicy: {
-      directives: {
-        ...helmet.contentSecurityPolicy.getDefaultDirectives(),
-        ...cspConfigJson
-      }
-    },
-    crossOriginEmbedderPolicy: HELMET_COEP === HelmetCoepType.TRUE
-  })
-)
 
-/***********************************
+  console.error(err.stack)
- *         Enabling CORS           *
+  res.status(500).send('Something broke!')
- ***********************************/
-if (CORS === CorsType.ENABLED) {
-  const whiteList: string[] = []
-  WHITELIST?.split(' ')
-    ?.filter((url) => !!url)
-    .forEach((url) => {
-      if (url.startsWith('http'))
-        // removing trailing slash of URLs listing for CORS
-        whiteList.push(url.replace(/\/$/, ''))
-    })
-
-  console.log('All CORS Requests are enabled for:', whiteList)
-  app.use(cors({ credentials: true, origin: whiteList }))
 }
 
 export default setProcessVariables().then(async () => {
+  app.use(cookieParser())
+
+  configureLogger(app)
+
+  /***********************************
+   *   Handle security and origin    *
+   ***********************************/
+  configureSecurity(app)
+
+  /***********************************
+   *         Enabling CORS           *
+   ***********************************/
+  configureCors(app)
+
   /***********************************
    *         DB Connection &          *
    *        Express Sessions          *
    *        With Mongo Store          *
    ***********************************/
-  if (MODE === ModeType.Server) {
+  configureExpressSession(app)
-    let store: MongoStore | undefined
-
-    if (process.env.NODE_ENV !== 'test') {
-      store = MongoStore.create({
-        client: mongoose.connection!.getClient() as any,
-        collectionName: 'sessions'
-      })
-    }
-
-    app.use(
-      session({
-        secret: process.secrets.SESSION_SECRET,
-        saveUninitialized: false, // don't create session until something stored
-        resave: false, //don't save session if unmodified
-        store,
-        cookie: cookieOptions
-      })
-    )
-  }
 
   app.use(express.json({ limit: '100mb' }))
   app.use(express.static(path.join(__dirname, '../public')))
 
-  const onError: ErrorRequestHandler = (err, req, res, next) => {
-    if (err.code === 'EBADCSRFTOKEN')
-      return res.status(400).send('Invalid CSRF token!')
-
-    console.error(err.stack)
-    res.status(500).send('Something broke!')
-  }
-
   await setupFolders()
   await copySASjsCore()