Merge pull request #34 from sasjs/issue-9

fix: Issue 9

.github/workflows/release.yml

@@ -0,0 +1,47 @@
+name: SASjs Server Executable Release
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install Dependencies WEB
+        working-directory: ./web
+        run: npm ci
+
+      - name: Build WEB
+        working-directory: ./web
+        run: npm run build
+        env:
+          CI: true
+
+      - name: Install Dependencies API
+        working-directory: ./api
+        run: npm ci
+
+      - name: Build Executables
+        working-directory: ./api
+        run: npm run exe
+        env:
+          CI: true
+
+      - name: Create Tag with Release
+      - uses: Klemensas/action-autotag@stable
+        with:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            ./executables/api-linux
+            ./executables/api-macos
+            ./executables/api-win.exe
+          token: ${{ secrets.GH_TOKEN }}

.gitignore

@@ -6,6 +6,7 @@ node_modules/
 sas/
 tmp/
 build/
+sasjsbuild/
 certificates/
 executables/
 .env

README.md

@@ -17,8 +17,11 @@ Configuration is made in the `configuration` section of `package.json`:
 ### Using dockers:
 
 There is `.env.example` file present at root of the project. [for Production]
+
 There is `.env.example` file present at `./api` of the project. [for Development]
+
 There is `.env.example` file present at `./web` of the project. [for Development]
+
 Remember to provide enviornment variables.
 
 #### Development

api/package.json

@@ -4,26 +4,32 @@
   "description": "Api of SASjs server",
   "main": "./src/server.ts",
   "scripts": {
-    "prestart": "npm run swagger",
-    "prestart:prod": "npm run swagger",
+    "initial": "npm run swagger && npm run compileSysInit",
+    "prestart": "npm run initial",
+    "prestart:prod": "npm run initial",
+    "prebuild": "npm run initial",
     "start": "nodemon ./src/server.ts",
     "start:prod": "nodemon ./src/prod-server.ts",
     "build": "rimraf build && tsc",
     "swagger": "tsoa spec",
     "semantic-release": "semantic-release -d",
     "prepare": "[ -d .git ] && git config core.hooksPath ./.git-hooks || true",
-    "test": "mkdir -p tmp && mkdir -p ../web/build && jest --coverage",
+    "test": "mkdir -p tmp && mkdir -p ../web/build && jest --silent --coverage",
     "lint:fix": "npx prettier --write \"src/**/*.{ts,tsx,js,jsx,html,css,sass,less,yml,md,graphql}\"",
     "lint": "npx prettier --check \"src/**/*.{ts,tsx,js,jsx,html,css,sass,less,yml,md,graphql}\"",
     "package:lib": "npm run build && cp ./package.json build && cp README.md build && cd build && npm version \"5.0.0\" && npm pack",
-    "exe": "npm run build && npm run public:copy && npm run web:copy && pkg .",
+    "exe": "npm run build && npm run exe:copy && pkg .",
+    "exe:copy": "npm run public:copy && npm run sasjsbuild:copy && npm run web:copy",
     "public:copy": "cp -r ./public/ ./build/public/",
-    "web:copy": "rimraf web && mkdir web && cp -r ../web/build/ ./web/build/"
+    "sasjsbuild:copy": "cp -r ./sasjsbuild/ ./build/sasjsbuild/",
+    "web:copy": "rimraf web && mkdir web && cp -r ../web/build/ ./web/build/",
+    "compileSysInit": "ts-node ./scripts/compileSysInit.ts"
   },
   "bin": "./build/src/server.js",
   "pkg": {
     "assets": [
       "./build/public/**/*",
+      "./build/sasjsbuild/**/*",
       "./web/build/**/*"
     ],
     "targets": [
@@ -40,7 +46,8 @@
   },
   "author": "Analytium Ltd",
   "dependencies": {
-    "@sasjs/utils": "^2.33.1",
+    "@sasjs/core": "^2.48.6",
+    "@sasjs/utils": "2.34.1",
     "bcryptjs": "^2.4.3",
     "cors": "^2.8.5",
     "express": "^4.17.1",
@@ -50,7 +57,6 @@
     "mongoose-sequence": "^5.3.1",
     "morgan": "^1.10.0",
     "multer": "^1.4.3",
-    "shelljs": "^0.8.4",
     "swagger-ui-express": "^4.1.6",
     "tsoa": "^3.14.0"
   },
@@ -64,7 +70,6 @@
     "@types/morgan": "^1.9.3",
     "@types/multer": "^1.4.7",
     "@types/node": "^15.12.2",
-    "@types/shelljs": "^0.8.9",
     "@types/supertest": "^2.0.11",
     "@types/swagger-ui-express": "^4.1.3",
     "dotenv": "^10.0.0",

api/public/swagger.yaml

@@ -364,7 +364,7 @@ components:
                     type: string
                 log:
                     type: string
-                result:
+                _webout:
                     type: string
                 message:
                     type: string
@@ -981,7 +981,7 @@ paths:
                         application/json:
                             schema:
                                 type: string
-            description: "Trigger a SAS program using it's location in the _program parameter.\nEnable debugging using the _debug parameter.\nAdditional URL parameters are turned into SAS macro variables.\nAny files provided are placed into the session and\ncorresponding _WEBIN_XXX variables are created."
+            description: "Trigger a SAS program using it's location in the _program parameter.\r\nEnable debugging using the _debug parameter.\r\nAdditional URL parameters are turned into SAS macro variables.\r\nAny files provided are placed into the session and\r\ncorresponding _WEBIN_XXX variables are created."
             summary: 'Execute Stored Program, return raw content'
             tags:
                 - STP
@@ -1005,7 +1005,7 @@ paths:
                         application/json:
                             schema:
                                 $ref: '#/components/schemas/ExecuteReturnJsonResponse'
-            description: "Trigger a SAS program using it's location in the _program parameter.\nEnable debugging using the _debug parameter.\nAdditional URL parameters are turned into SAS macro variables.\nAny files provided are placed into the session and\ncorresponding _WEBIN_XXX variables are created."
+            description: "Trigger a SAS program using it's location in the _program parameter.\r\nEnable debugging using the _debug parameter.\r\nAdditional URL parameters are turned into SAS macro variables.\r\nAny files provided are placed into the session and\r\ncorresponding _WEBIN_XXX variables are created."
             summary: 'Execute Stored Program, return JSON'
             tags:
                 - STP

api/scripts/compileSysInit.ts

@@ -0,0 +1,35 @@
+import path from 'path'
+import {
+  createFile,
+  loadDependenciesFile,
+  readFile,
+  SASJsFileType
+} from '@sasjs/utils'
+import { apiRoot, sysInitCompiledPath } from '../src/utils'
+
+const macroCorePath = path.join(apiRoot, 'node_modules', '@sasjs', 'core')
+
+const compiledSystemInit = async (systemInit: string) =>
+  'options ps=max;\n' +
+  (await loadDependenciesFile({
+    fileContent: systemInit,
+    type: SASJsFileType.job,
+    programFolders: [],
+    macroFolders: [],
+    buildSourceFolder: '',
+    macroCorePath
+  }))
+
+const createSysInitFile = async () => {
+  console.log('macroCorePath', macroCorePath)
+  const systemInitContent = await readFile(
+    path.join(__dirname, 'systemInit.sas')
+  )
+
+  await createFile(
+    path.join(sysInitCompiledPath),
+    await compiledSystemInit(systemInitContent)
+  )
+}
+
+createSysInitFile()

api/scripts/systemInit.sas

@@ -0,0 +1,13 @@
+/**
+  @file
+  @brief The systemInit program
+  @details This program is inserted into every sasjs/server program invocation,
+  _before_ any user-provided content.
+
+  <h4> SAS Macros </h4>
+  @li mcf_stpsrv_header.sas
+
+**/
+
+
+%mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)

api/src/controllers/internal/Execution.ts

@@ -18,10 +18,6 @@ export class ExecutionController {
 
     let program = await readFile(programPath)
 
-    Object.keys(vars).forEach(
-      (key: string) => (program = `%let ${key}=${vars[key]};\n${program}`)
-    )
-
     const sessionController = getSessionController()
 
     const session = await sessionController.getSession()
@@ -38,7 +34,12 @@ export class ExecutionController {
       preProgramVariables?.accessToken ?? 'accessToken'
     )
 
-    program = `
+    const varStatments = Object.keys(vars).reduce(
+      (computed: string, key: string) =>
+        `${computed}%let ${key}=${vars[key]};\n`,
+      ''
+    )
+    const preProgramVarStatments = `
 %let _sasjs_tokenfile=${tokenFile};
 %let _sasjs_username=${preProgramVariables?.username};
 %let _sasjs_userid=${preProgramVariables?.userId};
@@ -47,8 +48,17 @@ export class ExecutionController {
 %let _sasjs_apipath=/SASjsApi/stp/execute;
 %let _metaperson=&_sasjs_displayname;
 %let _metauser=&_sasjs_username;
-%let sasjsprocessmode=Stored Program;
-filename _webout "${weboutPath}";
+%let sasjsprocessmode=Stored Program;`
+
+    program = `
+/* runtime vars */
+${varStatments}
+filename _webout "${weboutPath}" mod;
+
+/* dynamic user-provided vars */
+${preProgramVarStatments}
+
+/* actual job code */
 ${program}`
 
     // if no files are uploaded filesNamesMap will be undefined
@@ -91,6 +101,7 @@ ${program}`
       typeof vars._debug === 'string' ? parseInt(vars._debug) : vars._debug
 
     let debugResponse: string | undefined
+
     if ((debugValue && debugValue >= 131) || session.crashed) {
       debugResponse = `<html><body>${webout}<div style="text-align:left"><hr /><h2>SAS Log</h2><pre>${log}</pre></div></body></html>`
     }
@@ -98,7 +109,16 @@ ${program}`
     session.inUse = false
     sessionController.deleteSession(session)
 
-    if (returnJson) return { result: debugResponse ?? webout, log }
+    if (returnJson) {
+      const response: any = {
+        webout: webout
+      }
+      if ((debugValue && debugValue >= 131) || session.crashed) {
+        response.log = log
+      }
+
+      return response
+    }
     return debugResponse ?? webout
   }
 

api/src/controllers/internal/Session.ts

@@ -2,12 +2,17 @@ import path from 'path'
 import { Session } from '../../types'
 import { promisify } from 'util'
 import { execFile } from 'child_process'
-import { getTmpSessionsFolderPath, generateUniqueFileName } from '../../utils'
+import {
+  getTmpSessionsFolderPath,
+  generateUniqueFileName,
+  sysInitCompiledPath
+} from '../../utils'
 import {
   deleteFolder,
   createFile,
   fileExists,
-  generateTimestamp
+  generateTimestamp,
+  readFile
 } from '@sasjs/utils'
 
 const execFilePromise = promisify(execFile)
@@ -52,9 +57,13 @@ export class SessionController {
     // we clean them up after a predefined period, if unused
     this.scheduleSessionDestroy(session)
 
+    // Place compiled system init code to autoexec
+    const compiledSystemInitContent = await readFile(sysInitCompiledPath)
+
     // the autoexec file is executed on SAS startup
     const autoExecPath = path.join(sessionFolder, 'autoexec.sas')
-    await createFile(autoExecPath, autoExecContent)
+    const contentForAutoExec = `/* compiled systemInit */\n${compiledSystemInitContent}\n/* autoexec */\n${autoExecContent}`
+    await createFile(autoExecPath, contentForAutoExec)
 
     // create empty code.sas as SAS will not start without a SYSIN
     const codePath = path.join(session.path, 'code.sas')

api/src/controllers/stp.ts

@@ -14,7 +14,6 @@ import {
 import { ExecutionController } from './internal'
 import { PreProgramVars } from '../types'
 import { getTmpFilesFolderPath, makeFilesNamesMap } from '../utils'
-import { request } from 'https'
 
 interface ExecuteReturnJsonPayload {
   /**
@@ -26,7 +25,7 @@ interface ExecuteReturnJsonPayload {
 interface ExecuteReturnJsonResponse {
   status: string
   log?: string
-  result?: string
+  _webout?: string
   message?: string
 }
 
@@ -121,7 +120,7 @@ const executeReturnJson = async (
     )
     return {
       status: 'success',
-      result: jsonResult.result,
+      _webout: jsonResult.webout,
       log: jsonResult.log
     }
   } catch (err: any) {

api/src/routes/web/web.ts

@@ -1,4 +1,4 @@
-import { fileExists, readFile } from '@sasjs/utils'
+import { readFile } from '@sasjs/utils'
 import express from 'express'
 import path from 'path'
 import { getWebBuildFolderPath } from '../../utils'
@@ -12,22 +12,23 @@ const codeToInject = `
 </script>`
 
 webRouter.get('/', async (_, res) => {
-  const indexHtmlPath = path.join(getWebBuildFolderPath(), 'index.html')
-  if (!(await fileExists(indexHtmlPath))) {
+  let content: string
+  try {
+    const indexHtmlPath = path.join(getWebBuildFolderPath(), 'index.html')
+    content = await readFile(indexHtmlPath)
+  } catch (_) {
     return res.send('Web Build is not present')
   }
 
   const { MODE } = process.env
   if (MODE?.trim() !== 'server') {
-    const content = await readFile(indexHtmlPath)
-
     const injectedContent = content.replace('</head>', `${codeToInject}</head>`)
 
     res.setHeader('Content-Type', 'text/html')
     return res.send(injectedContent)
   }
 
-  res.sendFile(indexHtmlPath)
+  return res.send(content)
 })
 
 export default webRouter

api/src/utils/connectDB.ts

@@ -3,33 +3,30 @@ import mongoose from 'mongoose'
 import { configuration } from '../../package.json'
 import { getDesktopFields } from '.'
 import { populateClients } from '../routes/api/auth'
-import shelljs from 'shelljs'
 
 export const connectDB = async () => {
-  shelljs.exec(`sasjs v`)
-
-  const { MODE } = process.env
-  if (MODE?.trim() !== 'server') {
-    console.log('Running in Destop Mode, no DB to connect.')
-
-    const { sasLoc, driveLoc } = await getDesktopFields()
-
-    process.sasLoc = sasLoc
-    process.driveLoc = driveLoc
-
-    return
-  } else {
-    const { SAS_PATH } = process.env
-    const sasDir = SAS_PATH ?? configuration.sasPath
-
-    process.sasLoc = path.join(sasDir, 'sas')
-  }
-
-  console.log('sasLoc: ', process.sasLoc)
-
   // NOTE: when exporting app.js as agent for supertest
   // we should exlcude connecting to the real database
   if (process.env.NODE_ENV !== 'test') {
+    const { MODE } = process.env
+    if (MODE?.trim() !== 'server') {
+      console.log('Running in Destop Mode, no DB to connect.')
+
+      const { sasLoc, driveLoc } = await getDesktopFields()
+
+      process.sasLoc = sasLoc
+      process.driveLoc = driveLoc
+
+      return
+    } else {
+      const { SAS_PATH } = process.env
+      const sasDir = SAS_PATH ?? configuration.sasPath
+
+      process.sasLoc = path.join(sasDir, 'sas')
+    }
+
+    console.log('sasLoc: ', process.sasLoc)
+
     mongoose.connect(process.env.DB_CONNECT as string, async (err) => {
       if (err) throw err
 

api/src/utils/file.ts

@@ -1,8 +1,16 @@
 import path from 'path'
 import { getRealPath } from '@sasjs/utils'
 
+export const apiRoot = path.join(__dirname, '..', '..')
+export const codebaseRoot = path.join(apiRoot, '..')
+export const sysInitCompiledPath = path.join(
+  apiRoot,
+  'sasjsbuild',
+  'systemInitCompiled.sas'
+)
+
 export const getWebBuildFolderPath = () =>
-  path.join(__dirname, '..', '..', '..', 'web', 'build')
+  path.join(codebaseRoot, 'web', 'build')
 
 export const getTmpFolderPath = () =>
   process.driveLoc ?? getRealPath(path.join(process.cwd(), 'tmp'))