refactor: specs + cleanup

2025-12-10 19:34:34 +00:00 · 2021-11-14 00:48:22 +05:00
parent 2bb10c7166
commit f2e50eb4cc
15 changed files with 130 additions and 14420 deletions
--- a/api/package-lock.json
+++ b/api/package-lock.json
@@ -18,12 +18,11 @@
        "mongoose-sequence": "^5.3.1",
        "morgan": "^1.10.0",
        "multer": "^1.4.3",
-        "open": "^8.4.0",
        "swagger-ui-express": "^4.1.6",
        "tsoa": "^3.14.0"
      },
      "bin": {
-        "api": "src/server.js"
+        "api": "build/src/server.js"
      },
      "devDependencies": {
        "@types/bcryptjs": "^2.4.2",
@@ -4112,14 +4111,6 @@
      "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==",
      "dev": true
    },
-    "node_modules/define-lazy-prop": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz",
-      "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==",
-      "engines": {
-        "node": ">=8"
-      }
-    },
    "node_modules/define-properties": {
      "version": "1.1.3",
      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
@@ -5933,20 +5924,6 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
-    "node_modules/is-docker": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz",
-      "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==",
-      "bin": {
-        "is-docker": "cli.js"
-      },
-      "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
    "node_modules/is-extglob": {
      "version": "2.1.1",
      "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
@@ -6200,17 +6177,6 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
-    "node_modules/is-wsl": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz",
-      "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==",
-      "dependencies": {
-        "is-docker": "^2.0.0"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
    "node_modules/is-yarn-global": {
      "version": "0.3.0",
      "resolved": "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.3.0.tgz",
@@ -11886,22 +11852,6 @@
        "url": "https://github.com/sponsors/sindresorhus"
      }
    },
-    "node_modules/open": {
-      "version": "8.4.0",
-      "resolved": "https://registry.npmjs.org/open/-/open-8.4.0.tgz",
-      "integrity": "sha512-XgFPPM+B28FtCCgSb9I+s9szOC1vZRSwgWsRUA5ylIxRTgKozqjOCrVOqGsYABPYK5qnfqClxZTFBa8PKt2v6Q==",
-      "dependencies": {
-        "define-lazy-prop": "^2.0.0",
-        "is-docker": "^2.1.1",
-        "is-wsl": "^2.2.0"
-      },
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
    "node_modules/optionator": {
      "version": "0.8.3",
      "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz",
@@ -18227,11 +18177,6 @@
      "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==",
      "dev": true
    },
-    "define-lazy-prop": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz",
-      "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og=="
-    },
    "define-properties": {
      "version": "1.1.3",
      "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
@@ -19609,11 +19554,6 @@
        "has-tostringtag": "^1.0.0"
      }
    },
-    "is-docker": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz",
-      "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ=="
-    },
    "is-extglob": {
      "version": "2.1.1",
      "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
@@ -19780,14 +19720,6 @@
        "call-bind": "^1.0.0"
      }
    },
-    "is-wsl": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz",
-      "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==",
-      "requires": {
-        "is-docker": "^2.0.0"
-      }
-    },
    "is-yarn-global": {
      "version": "0.3.0",
      "resolved": "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.3.0.tgz",
@@ -24005,16 +23937,6 @@
        "mimic-fn": "^2.1.0"
      }
    },
-    "open": {
-      "version": "8.4.0",
-      "resolved": "https://registry.npmjs.org/open/-/open-8.4.0.tgz",
-      "integrity": "sha512-XgFPPM+B28FtCCgSb9I+s9szOC1vZRSwgWsRUA5ylIxRTgKozqjOCrVOqGsYABPYK5qnfqClxZTFBa8PKt2v6Q==",
-      "requires": {
-        "define-lazy-prop": "^2.0.0",
-        "is-docker": "^2.1.1",
-        "is-wsl": "^2.2.0"
-      }
-    },
    "optionator": {
      "version": "0.8.3",
      "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz",
--- a/api/package.json
+++ b/api/package.json
@@ -47,7 +47,6 @@
    "mongoose-sequence": "^5.3.1",
    "morgan": "^1.10.0",
    "multer": "^1.4.3",
-    "open": "^8.4.0",
    "swagger-ui-express": "^4.1.6",
    "tsoa": "^3.14.0"
  },
--- a/api/src/controllers/internal/Execution.ts
+++ b/api/src/controllers/internal/Execution.ts
@@ -1,47 +1,36 @@
 import path from 'path'
 import fs from 'fs'
 import { getSessionController } from './'
-import { readFile, fileExists, createFile } from '@sasjs/utils'
-import { PreProgramVars, Session, TreeNode } from '../../types'
+import { readFile, fileExists, createFile, moveFile } from '@sasjs/utils'
+import { PreProgramVars, TreeNode } from '../../types'
 import { generateFileUploadSasCode, getTmpFilesFolderPath } from '../../utils'
-export const delay = (ms: number) =>
-new Promise((resolve) => setTimeout(resolve, ms))

 export class ExecutionController {
  async execute(
-    program = '',
-    preProgramVariables?: PreProgramVars,
-    autoExec?: string,
-    session?: Session,
-    vars?: any,
+    programPath: string,
+    preProgramVariables: PreProgramVars,
+    vars: { [key: string]: string | number | undefined },
    otherArgs?: any,
    returnJson?: boolean
  ) {
-    if (program) {
-      if (!(await fileExists(program))) {
-        throw 'ExecutionController: SAS file does not exist.'
-      }
+    if (!(await fileExists(programPath)))
+      throw 'ExecutionController: SAS file does not exist.'

-      program = await readFile(program)
+    let program = await readFile(programPath)

-      if (vars) {
-        Object.keys(vars).forEach(
-          (key: string) => (program = `%let ${key}=${vars[key]};\n${program}`)
-        )
-      }
-    }
+    Object.keys(vars).forEach(
+      (key: string) => (program = `%let ${key}=${vars[key]};\n${program}`)
+    )

    const sessionController = getSessionController()

-    if (!session) {
-      session = await sessionController.getSession()
-      session.inUse = true
-    }
+    const session = await sessionController.getSession()
+    session.inUse = true

-    let log = path.join(session.path, 'log.log')
+    const logPath = path.join(session.path, 'log.log')

-    let webout = path.join(session.path, 'webout.txt')
-    await createFile(webout, '')
+    const weboutPath = path.join(session.path, 'webout.txt')
+    await createFile(weboutPath, '')

    const tokenFile = path.join(session.path, 'accessToken.txt')
    await createFile(
@@ -57,7 +46,7 @@ export class ExecutionController {
 %let _sasjs_apiserverurl=${preProgramVariables?.serverUrl};
 %let _sasjs_apipath=/SASjsApi/stp/execute;
 %let sasjsprocessmode=Stored Program;
-filename _webout "${webout}";
+filename _webout "${weboutPath}";
 ${program}`

    // if no files are uploaded filesNamesMap will be undefined
@@ -74,7 +63,7 @@ ${program}`
    }

    const codePath = path.join(session.path, 'code.sas')
-    
+
    // Creating this file in a RUNNING session will break out
    // the autoexec loop and actually execute the program
    // but - given it will take several milliseconds to create
@@ -82,45 +71,31 @@ ${program}`
    // failing due to file lock) we first create the file THEN
    // we rename it.
    await createFile(codePath + '.bkp', program)
-    fs.renameSync(codePath + '.bkp',codePath)
+    await moveFile(codePath + '.bkp', codePath)

-    // we now need to poll the session array 
-    while (
-      !session.completed
-    ) { 
+    // we now need to poll the session array
+    while (!session.completed || !session.crashed) {
      await delay(50)
    }

+    const log = (await fileExists(logPath)) ? await readFile(logPath) : ''
+    const webout = (await fileExists(weboutPath))
+      ? await readFile(weboutPath)
+      : ''

+    const debugValue =
+      typeof vars._debug === 'string' ? parseInt(vars._debug) : vars._debug

-    if (await fileExists(log)) log = await readFile(log)
-    else log = ''
-
-    if (await fileExists(webout)) webout = await readFile(webout)
-    else webout = ''
-
-    const debug = Object.keys(vars).find(
-      (key: string) => key.toLowerCase() === '_debug'
-    )
-
-    let jsonResult
-    if ((debug && vars[debug] >= 131)) {
-      webout = `<html><body>
-${webout}
-<div style="text-align:left">
-<hr /><h2>SAS Log</h2>
-<pre>${log}</pre>
-</div>
-</body></html>`
-    } else if (returnJson) {
-      jsonResult = { result: webout, log: log }
+    let debugResponse: string | undefined
+    if ((debugValue && debugValue >= 131) || session.crashed) {
+      debugResponse = `<html><body>${webout}<div style="text-align:left"><hr /><h2>SAS Log</h2><pre>${log}</pre></div></body></html>`
    }

    session.inUse = false
-
    sessionController.deleteSession(session)

-    return Promise.resolve(jsonResult || webout)
+    if (returnJson) return { result: debugResponse ?? webout, log }
+    return debugResponse ?? webout
  }

  buildDirectorytree() {
@@ -160,3 +135,5 @@ ${webout}
    return root
  }
 }
+
+const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
--- a/api/src/controllers/internal/Session.ts
+++ b/api/src/controllers/internal/Session.ts
@@ -1,3 +1,4 @@
+import path from 'path'
 import { Session } from '../../types'
 import { configuration } from '../../../package.json'
 import { promisify } from 'util'
@@ -7,22 +8,13 @@ import {
  deleteFolder,
  createFile,
  fileExists,
-  deleteFile,
  generateTimestamp
 } from '@sasjs/utils'
-import path from 'path'
-import { ExecutionController } from './Execution'
-import { date } from 'joi'

 const execFilePromise = promisify(execFile)

 export class SessionController {
  private sessions: Session[] = []
-  private executionController: ExecutionController
-
-  constructor() {
-    this.executionController = new ExecutionController()
-  }

  public async getSession() {
    const readySessions = this.sessions.filter((sess: Session) => sess.ready)
@@ -40,59 +32,36 @@ export class SessionController {
    const sessionId = generateUniqueFileName(generateTimestamp())
    const sessionFolder = path.join(getTmpSessionsFolderPath(), sessionId)

-    const autoExecContent = `
-  data _null_;
-    /* remove the dummy SYSIN */
-    length fname $8;
-    rc=filename(fname,getoption('SYSIN') );
-    if rc = 0 and fexist(fname) then rc=fdelete(fname);
-    rc=filename(fname);
-    /* now wait for the real SYSIN */
-    slept=0;
-    do until ( fileexist(getoption('SYSIN')) or slept>(60*15) );
-      slept=slept+sleep(0.01,1);
-    end;
-    stop;
-  run;
-`
-    // the autoexec file is executed on SAS startup
-    const autoExec = path.join(sessionFolder, 'autoexec.sas')
-    await createFile(autoExec, autoExecContent)
-
-    // a dummy SYSIN code.sas file is necessary to start SAS
-    await createFile(path.join(sessionFolder, 'code.sas'), '')
-
    const creationTimeStamp = sessionId.split('-').pop() as string
+    const deathTimeStamp = (
+      parseInt(creationTimeStamp) +
+      15 * 60 * 1000 -
+      1000
+    ).toString()

    const session: Session = {
      id: sessionId,
      ready: false,
-      creationTimeStamp: creationTimeStamp,
-      deathTimeStamp: (
-        parseInt(creationTimeStamp) +
-        15 * 60 * 1000 -
-        1000
-      ).toString(),
-      path: sessionFolder,
      inUse: false,
-      completed: false
+      completed: false,
+      creationTimeStamp,
+      deathTimeStamp,
+      path: sessionFolder
    }

-    // we do not want to leave sessions running forever 
+    // we do not want to leave sessions running forever
    // we clean them up after a predefined period, if unused
    this.scheduleSessionDestroy(session)

+    // the autoexec file is executed on SAS startup
+    const autoExecPath = path.join(sessionFolder, 'autoexec.sas')
+    await createFile(autoExecPath, autoExecContent)
+
    // create empty code.sas as SAS will not start without a SYSIN
    const codePath = path.join(session.path, 'code.sas')
    await createFile(codePath, '')

-
-    // this.executionController
-    //   .execute('', undefined, autoExec, session)
-    //   .catch((err) => {console.log(err)})
-    
-
-    // trigger SAS but don't wait for completion - we need to 
+    // trigger SAS but don't wait for completion - we need to
    // update the session array to say that it is currently running
    // however we also need a promise so that we can update the
    // session array to say that it has (eventually) finished.
@@ -104,18 +73,24 @@ export class SessionController {
      path.join(session.path, 'log.log'),
      '-WORK',
      session.path,
-      '-AUTOEXEC', 
-      path.join(session.path, 'autoexec.sas'),
+      '-AUTOEXEC',
+      autoExecPath,
      process.platform === 'win32' ? '-nosplash' : ''
-    ]).then(() => {
-      session.completed=true
-      console.log('session completed', session)
-    }).catch((err) => {})
+    ])
+      .then(() => {
+        session.completed = true
+        console.log('session completed', session)
+      })
+      .catch((err) => {
+        session.completed = true
+        session.crashed = true
+        console.log('session crashed', session.id, err)
+      })

    // we have a triggered session - add to array
    this.sessions.push(session)

-    // SAS has been triggered but we can't use it until 
+    // SAS has been triggered but we can't use it until
    // the autoexec deletes the code.sas file
    await this.waitForSession(session)

@@ -123,17 +98,12 @@ export class SessionController {
  }

  public async waitForSession(session: Session) {
-    if (await fileExists(path.join(session.path, 'code.sas'))) {
-      while (await fileExists(path.join(session.path, 'code.sas'))) {}
+    const codeFilePath = path.join(session.path, 'code.sas')

-      session.ready = true
+    while (await fileExists(codeFilePath)) {}

-      return Promise.resolve(session)
-    } else {
-      session.ready = true
-
-      return Promise.resolve(session)
-    }
+    session.ready = true
+    return Promise.resolve(session)
  }

  public async deleteSession(session: Session) {
@@ -168,3 +138,19 @@ export const getSessionController = (): SessionController => {

  return process.sessionController
 }
+
+const autoExecContent = `
+data _null_;
+  /* remove the dummy SYSIN */
+  length fname $8;
+  rc=filename(fname,getoption('SYSIN') );
+  if rc = 0 and fexist(fname) then rc=fdelete(fname);
+  rc=filename(fname);
+  /* now wait for the real SYSIN */
+  slept=0;
+  do until ( fileexist(getoption('SYSIN')) or slept>(60*15) );
+    slept=slept+sleep(0.01,1);
+  end;
+  stop;
+run;
+`
--- a/api/src/controllers/stp.ts
+++ b/api/src/controllers/stp.ts
@@ -14,6 +14,7 @@ import {
 import { ExecutionController } from './internal'
 import { PreProgramVars } from '../types'
 import { getTmpFilesFolderPath, makeFilesNamesMap } from '../utils'
+import { request } from 'https'

 interface ExecuteReturnJsonPayload {
  /**
@@ -75,6 +76,7 @@ const executeReturnRaw = async (
  req: express.Request,
  _program: string
 ): Promise<string> => {
+  const query = req.query as { [key: string]: string | number | undefined }
  const sasCodePath =
    path
      .join(getTmpFilesFolderPath(), _program)
@@ -84,11 +86,7 @@ const executeReturnRaw = async (
    const result = await new ExecutionController().execute(
      sasCodePath,
      getPreProgramVariables(req),
-      undefined,
-      undefined,
-      {
-        ...req.query
-      }
+      query
    )

    return result as string
@@ -117,8 +115,6 @@ const executeReturnJson = async (
    const jsonResult: any = await new ExecutionController().execute(
      sasCodePath,
      getPreProgramVariables(req),
-      undefined,
-      req.sasSession,
      { ...req.query, ...req.body },
      { filesNamesMap: filesNamesMap },
      true
--- a/api/src/routes/api/spec/auth.spec.ts
+++ b/api/src/routes/api/spec/auth.spec.ts
@@ -1,7 +1,8 @@
+import { Express } from 'express'
 import mongoose, { Mongoose } from 'mongoose'
 import { MongoMemoryServer } from 'mongodb-memory-server'
 import request from 'supertest'
-import app from '../../../app'
+import appPromise from '../../../app'
 import {
  UserController,
  ClientController,
@@ -17,6 +18,11 @@ import {
  verifyTokenInDB
 } from '../../../utils'

+let app: Express
+appPromise.then((_app) => {
+  app = _app
+})
+
 const clientId = 'someclientID'
 const clientSecret = 'someclientSecret'
 const user = {
--- a/api/src/routes/api/spec/client.spec.ts
+++ b/api/src/routes/api/spec/client.spec.ts
@@ -1,10 +1,16 @@
+import { Express } from 'express'
 import mongoose, { Mongoose } from 'mongoose'
 import { MongoMemoryServer } from 'mongodb-memory-server'
 import request from 'supertest'
-import app from '../../../app'
+import appPromise from '../../../app'
 import { UserController, ClientController } from '../../../controllers/'
 import { generateAccessToken, saveTokensInDB } from '../../../utils'

+let app: Express
+appPromise.then((_app) => {
+  app = _app
+})
+
 const client = {
  clientId: 'someclientID',
  clientSecret: 'someclientSecret'
--- a/api/src/routes/api/spec/drive.spec.ts
+++ b/api/src/routes/api/spec/drive.spec.ts
@@ -1,7 +1,8 @@
+import { Express } from 'express'
 import mongoose, { Mongoose } from 'mongoose'
 import { MongoMemoryServer } from 'mongodb-memory-server'
 import request from 'supertest'
-import app from '../../../app'
+import appPromise from '../../../app'
 import { UserController } from '../../../controllers/'
 import { getTreeExample } from '../../../controllers/internal'
 import { getTmpFilesFolderPath } from '../../../utils/file'
@@ -10,6 +11,11 @@ import path from 'path'
 import { generateAccessToken, saveTokensInDB } from '../../../utils'
 import { FolderMember, ServiceMember } from '../../../types'

+let app: Express
+appPromise.then((_app) => {
+  app = _app
+})
+
 const clientId = 'someclientID'
 const user = {
  displayName: 'Test User',
--- a/api/src/routes/api/spec/group.spec.ts
+++ b/api/src/routes/api/spec/group.spec.ts
@@ -1,10 +1,16 @@
+import { Express } from 'express'
 import mongoose, { Mongoose } from 'mongoose'
 import { MongoMemoryServer } from 'mongodb-memory-server'
 import request from 'supertest'
-import app from '../../../app'
+import appPromise from '../../../app'
 import { UserController, GroupController } from '../../../controllers/'
 import { generateAccessToken, saveTokensInDB } from '../../../utils'

+let app: Express
+appPromise.then((_app) => {
+  app = _app
+})
+
 const clientId = 'someclientID'
 const adminUser = {
  displayName: 'Test Admin',
--- a/api/src/routes/api/spec/user.spec.ts
+++ b/api/src/routes/api/spec/user.spec.ts
@@ -1,10 +1,16 @@
+import { Express } from 'express'
 import mongoose, { Mongoose } from 'mongoose'
 import { MongoMemoryServer } from 'mongodb-memory-server'
 import request from 'supertest'
-import app from '../../../app'
+import appPromise from '../../../app'
 import { UserController } from '../../../controllers/'
 import { generateAccessToken, saveTokensInDB } from '../../../utils'

+let app: Express
+appPromise.then((_app) => {
+  app = _app
+})
+
 const clientId = 'someclientID'
 const adminUser = {
  displayName: 'Test Admin',
--- a/api/src/server.ts
+++ b/api/src/server.ts
@@ -1,4 +1,3 @@
-import open from 'open'
 import appPromise from './app'
 import { configuration } from '../package.json'

@@ -7,9 +6,5 @@ appPromise.then((app) => {
    console.log(
      `⚡️[server]: Server is running at http://localhost:${configuration.sasJsPort}`
    )
-    const { MODE } = process.env
-    if (MODE?.trim() !== 'server') {
-      open(`http://localhost:${configuration.sasJsPort}`)
-    }
  })
 })
--- a/api/src/types/Session.ts
+++ b/api/src/types/Session.ts
@@ -5,5 +5,6 @@ export interface Session {
  deathTimeStamp: string
  path: string
  inUse: boolean
-  completed?: boolean
+  completed: boolean
+  crashed?: boolean
 }
--- a/api/src/utils/validation.ts
+++ b/api/src/utils/validation.ts
@@ -81,5 +81,5 @@ export const executeProgramRawValidation = (data: any): Joi.ValidationResult =>
  Joi.object({
    _program: Joi.string().required()
  })
-    .pattern(/^/, Joi.string())
+    .pattern(/^/, Joi.alternatives(Joi.string(), Joi.number()))
    .validate(data)