refactor: specs + cleanup

.github/workflows/build.yml

@@ -50,6 +50,7 @@ jobs:
         run: npm test
         env:
           CI: true
+          MODE: 'server'
           ACCESS_TOKEN_SECRET: ${{secrets.ACCESS_TOKEN_SECRET}}
           REFRESH_TOKEN_SECRET: ${{secrets.REFRESH_TOKEN_SECRET}}
           AUTH_CODE_SECRET: ${{secrets.AUTH_CODE_SECRET}}

api/package-lock.json

@@ -18,12 +18,11 @@
         "mongoose-sequence": "^5.3.1",
         "morgan": "^1.10.0",
         "multer": "^1.4.3",
-        "open": "^8.4.0",
         "swagger-ui-express": "^4.1.6",
         "tsoa": "^3.14.0"
       },
       "bin": {
-        "api": "src/server.js"
+        "api": "build/src/server.js"
       },
       "devDependencies": {
         "@types/bcryptjs": "^2.4.2",
@@ -4112,14 +4111,6 @@
       "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==",
       "dev": true
     },
-    "node_modules/define-lazy-prop": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz",
-      "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==",
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/define-properties": {
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
@@ -5933,20 +5924,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/is-docker": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz",
-      "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==",
-      "bin": {
-        "is-docker": "cli.js"
-      },
-      "engines": {
-        "node": ">=8"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
     "node_modules/is-extglob": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
@@ -6200,17 +6177,6 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/is-wsl": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz",
-      "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==",
-      "dependencies": {
-        "is-docker": "^2.0.0"
-      },
-      "engines": {
-        "node": ">=8"
-      }
-    },
     "node_modules/is-yarn-global": {
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.3.0.tgz",
@@ -11886,22 +11852,6 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
-    "node_modules/open": {
-      "version": "8.4.0",
-      "resolved": "https://registry.npmjs.org/open/-/open-8.4.0.tgz",
-      "integrity": "sha512-XgFPPM+B28FtCCgSb9I+s9szOC1vZRSwgWsRUA5ylIxRTgKozqjOCrVOqGsYABPYK5qnfqClxZTFBa8PKt2v6Q==",
-      "dependencies": {
-        "define-lazy-prop": "^2.0.0",
-        "is-docker": "^2.1.1",
-        "is-wsl": "^2.2.0"
-      },
-      "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/sindresorhus"
-      }
-    },
     "node_modules/optionator": {
       "version": "0.8.3",
       "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz",
@@ -18227,11 +18177,6 @@
       "integrity": "sha512-0ISdNousHvZT2EiFlZeZAHBUvSxmKswVCEf8hW7KWgG4a8MVEu/3Vb6uWYozkjylyCxe0JBIiRB1jV45S70WVQ==",
       "dev": true
     },
-    "define-lazy-prop": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz",
-      "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og=="
-    },
     "define-properties": {
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
@@ -19609,11 +19554,6 @@
         "has-tostringtag": "^1.0.0"
       }
     },
-    "is-docker": {
-      "version": "2.2.1",
-      "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz",
-      "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ=="
-    },
     "is-extglob": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
@@ -19780,14 +19720,6 @@
         "call-bind": "^1.0.0"
       }
     },
-    "is-wsl": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz",
-      "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==",
-      "requires": {
-        "is-docker": "^2.0.0"
-      }
-    },
     "is-yarn-global": {
       "version": "0.3.0",
       "resolved": "https://registry.npmjs.org/is-yarn-global/-/is-yarn-global-0.3.0.tgz",
@@ -24005,16 +23937,6 @@
         "mimic-fn": "^2.1.0"
       }
     },
-    "open": {
-      "version": "8.4.0",
-      "resolved": "https://registry.npmjs.org/open/-/open-8.4.0.tgz",
-      "integrity": "sha512-XgFPPM+B28FtCCgSb9I+s9szOC1vZRSwgWsRUA5ylIxRTgKozqjOCrVOqGsYABPYK5qnfqClxZTFBa8PKt2v6Q==",
-      "requires": {
-        "define-lazy-prop": "^2.0.0",
-        "is-docker": "^2.1.1",
-        "is-wsl": "^2.2.0"
-      }
-    },
     "optionator": {
       "version": "0.8.3",
       "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz",

api/package.json

@@ -47,7 +47,6 @@
     "mongoose-sequence": "^5.3.1",
     "morgan": "^1.10.0",
     "multer": "^1.4.3",
-    "open": "^8.4.0",
     "swagger-ui-express": "^4.1.6",
     "tsoa": "^3.14.0"
   },

api/src/controllers/internal/Execution.ts

@@ -1,47 +1,36 @@
 import path from 'path'
 import fs from 'fs'
 import { getSessionController } from './'
-import { readFile, fileExists, createFile } from '@sasjs/utils'
+import { readFile, fileExists, createFile, moveFile } from '@sasjs/utils'
-import { PreProgramVars, Session, TreeNode } from '../../types'
+import { PreProgramVars, TreeNode } from '../../types'
 import { generateFileUploadSasCode, getTmpFilesFolderPath } from '../../utils'
-export const delay = (ms: number) =>
-new Promise((resolve) => setTimeout(resolve, ms))
 
 export class ExecutionController {
   async execute(
-    program = '',
+    programPath: string,
-    preProgramVariables?: PreProgramVars,
+    preProgramVariables: PreProgramVars,
-    autoExec?: string,
+    vars: { [key: string]: string | number | undefined },
-    session?: Session,
-    vars?: any,
     otherArgs?: any,
     returnJson?: boolean
   ) {
-    if (program) {
+    if (!(await fileExists(programPath)))
-      if (!(await fileExists(program))) {
+      throw 'ExecutionController: SAS file does not exist.'
-        throw 'ExecutionController: SAS file does not exist.'
-      }
 
-      program = await readFile(program)
+    let program = await readFile(programPath)
 
-      if (vars) {
+    Object.keys(vars).forEach(
-        Object.keys(vars).forEach(
+      (key: string) => (program = `%let ${key}=${vars[key]};\n${program}`)
-          (key: string) => (program = `%let ${key}=${vars[key]};\n${program}`)
+    )
-        )
-      }
-    }
 
     const sessionController = getSessionController()
 
-    if (!session) {
+    const session = await sessionController.getSession()
-      session = await sessionController.getSession()
+    session.inUse = true
-      session.inUse = true
-    }
 
-    let log = path.join(session.path, 'log.log')
+    const logPath = path.join(session.path, 'log.log')
 
-    let webout = path.join(session.path, 'webout.txt')
+    const weboutPath = path.join(session.path, 'webout.txt')
-    await createFile(webout, '')
+    await createFile(weboutPath, '')
 
     const tokenFile = path.join(session.path, 'accessToken.txt')
     await createFile(
@@ -57,7 +46,7 @@ export class ExecutionController {
 %let _sasjs_apiserverurl=${preProgramVariables?.serverUrl};
 %let _sasjs_apipath=/SASjsApi/stp/execute;
 %let sasjsprocessmode=Stored Program;
-filename _webout "${webout}";
+filename _webout "${weboutPath}";
 ${program}`
 
     // if no files are uploaded filesNamesMap will be undefined
@@ -74,7 +63,7 @@ ${program}`
     }
 
     const codePath = path.join(session.path, 'code.sas')
-    
+
     // Creating this file in a RUNNING session will break out
     // the autoexec loop and actually execute the program
     // but - given it will take several milliseconds to create
@@ -82,45 +71,31 @@ ${program}`
     // failing due to file lock) we first create the file THEN
     // we rename it.
     await createFile(codePath + '.bkp', program)
-    fs.renameSync(codePath + '.bkp',codePath)
+    await moveFile(codePath + '.bkp', codePath)
 
-    // we now need to poll the session array 
+    // we now need to poll the session array
-    while (
+    while (!session.completed || !session.crashed) {
-      !session.completed
-    ) { 
       await delay(50)
     }
 
+    const log = (await fileExists(logPath)) ? await readFile(logPath) : ''
+    const webout = (await fileExists(weboutPath))
+      ? await readFile(weboutPath)
+      : ''
 
+    const debugValue =
+      typeof vars._debug === 'string' ? parseInt(vars._debug) : vars._debug
 
-    if (await fileExists(log)) log = await readFile(log)
+    let debugResponse: string | undefined
-    else log = ''
+    if ((debugValue && debugValue >= 131) || session.crashed) {
-
+      debugResponse = `<html><body>${webout}<div style="text-align:left"><hr /><h2>SAS Log</h2><pre>${log}</pre></div></body></html>`
-    if (await fileExists(webout)) webout = await readFile(webout)
-    else webout = ''
-
-    const debug = Object.keys(vars).find(
-      (key: string) => key.toLowerCase() === '_debug'
-    )
-
-    let jsonResult
-    if ((debug && vars[debug] >= 131)) {
-      webout = `<html><body>
-${webout}
-<div style="text-align:left">
-<hr /><h2>SAS Log</h2>
-<pre>${log}</pre>
-</div>
-</body></html>`
-    } else if (returnJson) {
-      jsonResult = { result: webout, log: log }
     }
 
     session.inUse = false
-
     sessionController.deleteSession(session)
 
-    return Promise.resolve(jsonResult || webout)
+    if (returnJson) return { result: debugResponse ?? webout, log }
+    return debugResponse ?? webout
   }
 
   buildDirectorytree() {
@@ -160,3 +135,5 @@ ${webout}
     return root
   }
 }
+
+const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))

api/src/controllers/internal/Session.ts

@@ -1,3 +1,4 @@
+import path from 'path'
 import { Session } from '../../types'
 import { configuration } from '../../../package.json'
 import { promisify } from 'util'
@@ -7,22 +8,13 @@ import {
   deleteFolder,
   createFile,
   fileExists,
-  deleteFile,
   generateTimestamp
 } from '@sasjs/utils'
-import path from 'path'
-import { ExecutionController } from './Execution'
-import { date } from 'joi'
 
 const execFilePromise = promisify(execFile)
 
 export class SessionController {
   private sessions: Session[] = []
-  private executionController: ExecutionController
-
-  constructor() {
-    this.executionController = new ExecutionController()
-  }
 
   public async getSession() {
     const readySessions = this.sessions.filter((sess: Session) => sess.ready)
@@ -40,59 +32,36 @@ export class SessionController {
     const sessionId = generateUniqueFileName(generateTimestamp())
     const sessionFolder = path.join(getTmpSessionsFolderPath(), sessionId)
 
-    const autoExecContent = `
-  data _null_;
-    /* remove the dummy SYSIN */
-    length fname $8;
-    rc=filename(fname,getoption('SYSIN') );
-    if rc = 0 and fexist(fname) then rc=fdelete(fname);
-    rc=filename(fname);
-    /* now wait for the real SYSIN */
-    slept=0;
-    do until ( fileexist(getoption('SYSIN')) or slept>(60*15) );
-      slept=slept+sleep(0.01,1);
-    end;
-    stop;
-  run;
-`
-    // the autoexec file is executed on SAS startup
-    const autoExec = path.join(sessionFolder, 'autoexec.sas')
-    await createFile(autoExec, autoExecContent)
-
-    // a dummy SYSIN code.sas file is necessary to start SAS
-    await createFile(path.join(sessionFolder, 'code.sas'), '')
-
     const creationTimeStamp = sessionId.split('-').pop() as string
+    const deathTimeStamp = (
+      parseInt(creationTimeStamp) +
+      15 * 60 * 1000 -
+      1000
+    ).toString()
 
     const session: Session = {
       id: sessionId,
       ready: false,
-      creationTimeStamp: creationTimeStamp,
-      deathTimeStamp: (
-        parseInt(creationTimeStamp) +
-        15 * 60 * 1000 -
-        1000
-      ).toString(),
-      path: sessionFolder,
       inUse: false,
-      completed: false
+      completed: false,
+      creationTimeStamp,
+      deathTimeStamp,
+      path: sessionFolder
     }
 
-    // we do not want to leave sessions running forever 
+    // we do not want to leave sessions running forever
     // we clean them up after a predefined period, if unused
     this.scheduleSessionDestroy(session)
 
+    // the autoexec file is executed on SAS startup
+    const autoExecPath = path.join(sessionFolder, 'autoexec.sas')
+    await createFile(autoExecPath, autoExecContent)
+
     // create empty code.sas as SAS will not start without a SYSIN
     const codePath = path.join(session.path, 'code.sas')
     await createFile(codePath, '')
 
-
+    // trigger SAS but don't wait for completion - we need to
-    // this.executionController
-    //   .execute('', undefined, autoExec, session)
-    //   .catch((err) => {console.log(err)})
-    
-
-    // trigger SAS but don't wait for completion - we need to 
     // update the session array to say that it is currently running
     // however we also need a promise so that we can update the
     // session array to say that it has (eventually) finished.
@@ -104,18 +73,24 @@ export class SessionController {
       path.join(session.path, 'log.log'),
       '-WORK',
       session.path,
-      '-AUTOEXEC', 
+      '-AUTOEXEC',
-      path.join(session.path, 'autoexec.sas'),
+      autoExecPath,
       process.platform === 'win32' ? '-nosplash' : ''
-    ]).then(() => {
+    ])
-      session.completed=true
+      .then(() => {
-      console.log('session completed', session)
+        session.completed = true
-    }).catch((err) => {})
+        console.log('session completed', session)
+      })
+      .catch((err) => {
+        session.completed = true
+        session.crashed = true
+        console.log('session crashed', session.id, err)
+      })
 
     // we have a triggered session - add to array
     this.sessions.push(session)
 
-    // SAS has been triggered but we can't use it until 
+    // SAS has been triggered but we can't use it until
     // the autoexec deletes the code.sas file
     await this.waitForSession(session)
 
@@ -123,17 +98,12 @@ export class SessionController {
   }
 
   public async waitForSession(session: Session) {
-    if (await fileExists(path.join(session.path, 'code.sas'))) {
+    const codeFilePath = path.join(session.path, 'code.sas')
-      while (await fileExists(path.join(session.path, 'code.sas'))) {}
 
-      session.ready = true
+    while (await fileExists(codeFilePath)) {}
 
-      return Promise.resolve(session)
+    session.ready = true
-    } else {
+    return Promise.resolve(session)
-      session.ready = true
-
-      return Promise.resolve(session)
-    }
   }
 
   public async deleteSession(session: Session) {
@@ -168,3 +138,19 @@ export const getSessionController = (): SessionController => {
 
   return process.sessionController
 }
+
+const autoExecContent = `
+data _null_;
+  /* remove the dummy SYSIN */
+  length fname $8;
+  rc=filename(fname,getoption('SYSIN') );
+  if rc = 0 and fexist(fname) then rc=fdelete(fname);
+  rc=filename(fname);
+  /* now wait for the real SYSIN */
+  slept=0;
+  do until ( fileexist(getoption('SYSIN')) or slept>(60*15) );
+    slept=slept+sleep(0.01,1);
+  end;
+  stop;
+run;
+`

api/src/controllers/stp.ts

@@ -14,6 +14,7 @@ import {
 import { ExecutionController } from './internal'
 import { PreProgramVars } from '../types'
 import { getTmpFilesFolderPath, makeFilesNamesMap } from '../utils'
+import { request } from 'https'
 
 interface ExecuteReturnJsonPayload {
   /**
@@ -75,6 +76,7 @@ const executeReturnRaw = async (
   req: express.Request,
   _program: string
 ): Promise<string> => {
+  const query = req.query as { [key: string]: string | number | undefined }
   const sasCodePath =
     path
       .join(getTmpFilesFolderPath(), _program)
@@ -84,11 +86,7 @@ const executeReturnRaw = async (
     const result = await new ExecutionController().execute(
       sasCodePath,
       getPreProgramVariables(req),
-      undefined,
+      query
-      undefined,
-      {
-        ...req.query
-      }
     )
 
     return result as string
@@ -117,8 +115,6 @@ const executeReturnJson = async (
     const jsonResult: any = await new ExecutionController().execute(
       sasCodePath,
       getPreProgramVariables(req),
-      undefined,
-      req.sasSession,
       { ...req.query, ...req.body },
       { filesNamesMap: filesNamesMap },
       true

api/src/routes/api/spec/auth.spec.ts

@@ -1,7 +1,8 @@
+import { Express } from 'express'
 import mongoose, { Mongoose } from 'mongoose'
 import { MongoMemoryServer } from 'mongodb-memory-server'
 import request from 'supertest'
-import app from '../../../app'
+import appPromise from '../../../app'
 import {
   UserController,
   ClientController,
@@ -17,6 +18,11 @@ import {
   verifyTokenInDB
 } from '../../../utils'
 
+let app: Express
+appPromise.then((_app) => {
+  app = _app
+})
+
 const clientId = 'someclientID'
 const clientSecret = 'someclientSecret'
 const user = {

api/src/routes/api/spec/client.spec.ts

@@ -1,10 +1,16 @@
+import { Express } from 'express'
 import mongoose, { Mongoose } from 'mongoose'
 import { MongoMemoryServer } from 'mongodb-memory-server'
 import request from 'supertest'
-import app from '../../../app'
+import appPromise from '../../../app'
 import { UserController, ClientController } from '../../../controllers/'
 import { generateAccessToken, saveTokensInDB } from '../../../utils'
 
+let app: Express
+appPromise.then((_app) => {
+  app = _app
+})
+
 const client = {
   clientId: 'someclientID',
   clientSecret: 'someclientSecret'

api/src/routes/api/spec/drive.spec.ts

@@ -1,7 +1,8 @@
+import { Express } from 'express'
 import mongoose, { Mongoose } from 'mongoose'
 import { MongoMemoryServer } from 'mongodb-memory-server'
 import request from 'supertest'
-import app from '../../../app'
+import appPromise from '../../../app'
 import { UserController } from '../../../controllers/'
 import { getTreeExample } from '../../../controllers/internal'
 import { getTmpFilesFolderPath } from '../../../utils/file'
@@ -10,6 +11,11 @@ import path from 'path'
 import { generateAccessToken, saveTokensInDB } from '../../../utils'
 import { FolderMember, ServiceMember } from '../../../types'
 
+let app: Express
+appPromise.then((_app) => {
+  app = _app
+})
+
 const clientId = 'someclientID'
 const user = {
   displayName: 'Test User',

api/src/routes/api/spec/group.spec.ts

@@ -1,10 +1,16 @@
+import { Express } from 'express'
 import mongoose, { Mongoose } from 'mongoose'
 import { MongoMemoryServer } from 'mongodb-memory-server'
 import request from 'supertest'
-import app from '../../../app'
+import appPromise from '../../../app'
 import { UserController, GroupController } from '../../../controllers/'
 import { generateAccessToken, saveTokensInDB } from '../../../utils'
 
+let app: Express
+appPromise.then((_app) => {
+  app = _app
+})
+
 const clientId = 'someclientID'
 const adminUser = {
   displayName: 'Test Admin',

api/src/routes/api/spec/user.spec.ts

@@ -1,10 +1,16 @@
+import { Express } from 'express'
 import mongoose, { Mongoose } from 'mongoose'
 import { MongoMemoryServer } from 'mongodb-memory-server'
 import request from 'supertest'
-import app from '../../../app'
+import appPromise from '../../../app'
 import { UserController } from '../../../controllers/'
 import { generateAccessToken, saveTokensInDB } from '../../../utils'
 
+let app: Express
+appPromise.then((_app) => {
+  app = _app
+})
+
 const clientId = 'someclientID'
 const adminUser = {
   displayName: 'Test Admin',

api/src/server.ts

@@ -1,4 +1,3 @@
-import open from 'open'
 import appPromise from './app'
 import { configuration } from '../package.json'
 
@@ -7,9 +6,5 @@ appPromise.then((app) => {
     console.log(
       `⚡️[server]: Server is running at http://localhost:${configuration.sasJsPort}`
     )
-    const { MODE } = process.env
-    if (MODE?.trim() !== 'server') {
-      open(`http://localhost:${configuration.sasJsPort}`)
-    }
   })
 })

api/src/types/Session.ts

@@ -5,5 +5,6 @@ export interface Session {
   deathTimeStamp: string
   path: string
   inUse: boolean
-  completed?: boolean
+  completed: boolean
+  crashed?: boolean
 }

api/src/utils/validation.ts

@@ -81,5 +81,5 @@ export const executeProgramRawValidation = (data: any): Joi.ValidationResult =>
   Joi.object({
     _program: Joi.string().required()
   })
-    .pattern(/^/, Joi.string())
+    .pattern(/^/, Joi.alternatives(Joi.string(), Joi.number()))
     .validate(data)