feat: compile systemInit and inject to autoExec

.gitignore

@@ -6,6 +6,7 @@ node_modules/
 sas/
 tmp/
 build/
+sasjsbuild/
 certificates/
 executables/
 .env

api/package-lock.json

@@ -8,7 +8,8 @@
       "name": "api",
       "version": "0.0.1",
       "dependencies": {
-        "@sasjs/utils": "^2.33.1",
+        "@sasjs/core": "^2.48.6",
+        "@sasjs/utils": "file:../../utils/build/sasjs-utils-5.0.0.tgz",
         "bcryptjs": "^2.4.3",
         "cors": "^2.8.5",
         "express": "^4.17.1",
@@ -1560,11 +1561,17 @@
         "@octokit/openapi-types": "^7.2.3"
       }
     },
+    "node_modules/@sasjs/core": {
+      "version": "2.48.6",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.48.6.tgz",
+      "integrity": "sha512-5kCf4TdCVOYve4wSHVTi+db34hknDwvY2C/JVEPHT6T3CkQ5cnwRVPSFz/1WzXzcVvdUi4ag5xd9SDOsU12oWA=="
+    },
     "node_modules/@sasjs/utils": {
-      "version": "2.33.1",
+      "version": "5.0.0",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.33.1.tgz",
+      "resolved": "file:../../utils/build/sasjs-utils-5.0.0.tgz",
-      "integrity": "sha512-QQqI9+G/riMrbNSjambYwaJwGY3om2f3N9z9tNxBSva+W0g3JaOl4qeOqpRS91KhOmNhrLMCohg6jScMCz3YFQ==",
+      "integrity": "sha512-ZSX1oHLEl3Gaz1ILnmbf1hMt3WLfvAddck65BY1NduOePtHx2ioja9CjzXKxB75CphBjLdjR95tbmb8jjzGDZA==",
       "hasInstallScript": true,
+      "license": "ISC",
       "dependencies": {
         "@types/fs-extra": "^9.0.11",
         "@types/prompts": "^2.0.13",
@@ -1572,8 +1579,11 @@
         "cli-table": "^0.3.6",
         "consola": "^2.15.0",
         "csv-stringify": "^5.6.5",
+        "find": "0.3.0",
         "fs-extra": "^10.0.0",
         "jwt-decode": "^3.1.2",
+        "lodash.groupby": "4.6.0",
+        "lodash.uniqby": "4.7.0",
         "prompts": "^2.4.1",
         "rimraf": "^3.0.2",
         "valid-url": "^1.0.9"
@@ -4933,6 +4943,14 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/find": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/find/-/find-0.3.0.tgz",
+      "integrity": "sha512-iSd+O4OEYV/I36Zl8MdYJO0xD82wH528SaCieTVHhclgiYNe9y+yPKSwK+A7/WsmHL1EZ+pYUJBXWTL5qofksw==",
+      "dependencies": {
+        "traverse-chain": "~0.1.0"
+      }
+    },
     "node_modules/find-cache-dir": {
       "version": "3.3.2",
       "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz",
@@ -7975,6 +7993,11 @@
       "integrity": "sha1-ZHYsSGGAglGKw99Mz11YhtriA0c=",
       "dev": true
     },
+    "node_modules/lodash.groupby": {
+      "version": "4.6.0",
+      "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz",
+      "integrity": "sha1-Cwih3PaDl8OXhVwyOXg4Mt90A9E="
+    },
     "node_modules/lodash.includes": {
       "version": "4.3.0",
       "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
@@ -8025,8 +8048,7 @@
     "node_modules/lodash.uniqby": {
       "version": "4.7.0",
       "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz",
-      "integrity": "sha1-2ZwHpmnp5tJOE2Lf4mbGdhavEwI=",
+      "integrity": "sha1-2ZwHpmnp5tJOE2Lf4mbGdhavEwI="
-      "dev": true
     },
     "node_modules/lowercase-keys": {
       "version": "1.0.1",
@@ -14220,6 +14242,11 @@
       "integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc=",
       "dev": true
     },
+    "node_modules/traverse-chain": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/traverse-chain/-/traverse-chain-0.1.0.tgz",
+      "integrity": "sha1-YdvC1Ttp/2CRoSoWj9fUMxB+QPE="
+    },
     "node_modules/trim-newlines": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-3.0.1.tgz",
@@ -16141,10 +16168,14 @@
         "@octokit/openapi-types": "^7.2.3"
       }
     },
+    "@sasjs/core": {
+      "version": "2.48.6",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.48.6.tgz",
+      "integrity": "sha512-5kCf4TdCVOYve4wSHVTi+db34hknDwvY2C/JVEPHT6T3CkQ5cnwRVPSFz/1WzXzcVvdUi4ag5xd9SDOsU12oWA=="
+    },
     "@sasjs/utils": {
-      "version": "2.33.1",
+      "version": "file:../../utils/build/sasjs-utils-5.0.0.tgz",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.33.1.tgz",
+      "integrity": "sha512-ZSX1oHLEl3Gaz1ILnmbf1hMt3WLfvAddck65BY1NduOePtHx2ioja9CjzXKxB75CphBjLdjR95tbmb8jjzGDZA==",
-      "integrity": "sha512-QQqI9+G/riMrbNSjambYwaJwGY3om2f3N9z9tNxBSva+W0g3JaOl4qeOqpRS91KhOmNhrLMCohg6jScMCz3YFQ==",
       "requires": {
         "@types/fs-extra": "^9.0.11",
         "@types/prompts": "^2.0.13",
@@ -16152,8 +16183,11 @@
         "cli-table": "^0.3.6",
         "consola": "^2.15.0",
         "csv-stringify": "^5.6.5",
+        "find": "0.3.0",
         "fs-extra": "^10.0.0",
         "jwt-decode": "^3.1.2",
+        "lodash.groupby": "4.6.0",
+        "lodash.uniqby": "4.7.0",
         "prompts": "^2.4.1",
         "rimraf": "^3.0.2",
         "valid-url": "^1.0.9"
@@ -18889,6 +18923,14 @@
         "unpipe": "~1.0.0"
       }
     },
+    "find": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/find/-/find-0.3.0.tgz",
+      "integrity": "sha512-iSd+O4OEYV/I36Zl8MdYJO0xD82wH528SaCieTVHhclgiYNe9y+yPKSwK+A7/WsmHL1EZ+pYUJBXWTL5qofksw==",
+      "requires": {
+        "traverse-chain": "~0.1.0"
+      }
+    },
     "find-cache-dir": {
       "version": "3.3.2",
       "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz",
@@ -21170,6 +21212,11 @@
       "integrity": "sha1-ZHYsSGGAglGKw99Mz11YhtriA0c=",
       "dev": true
     },
+    "lodash.groupby": {
+      "version": "4.6.0",
+      "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz",
+      "integrity": "sha1-Cwih3PaDl8OXhVwyOXg4Mt90A9E="
+    },
     "lodash.includes": {
       "version": "4.3.0",
       "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
@@ -21220,8 +21267,7 @@
     "lodash.uniqby": {
       "version": "4.7.0",
       "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz",
-      "integrity": "sha1-2ZwHpmnp5tJOE2Lf4mbGdhavEwI=",
+      "integrity": "sha1-2ZwHpmnp5tJOE2Lf4mbGdhavEwI="
-      "dev": true
     },
     "lowercase-keys": {
       "version": "1.0.1",
@@ -25795,6 +25841,11 @@
       "integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc=",
       "dev": true
     },
+    "traverse-chain": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/traverse-chain/-/traverse-chain-0.1.0.tgz",
+      "integrity": "sha1-YdvC1Ttp/2CRoSoWj9fUMxB+QPE="
+    },
     "trim-newlines": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-3.0.1.tgz",

api/package.json

@@ -4,8 +4,10 @@
   "description": "Api of SASjs server",
   "main": "./src/server.ts",
   "scripts": {
-    "prestart": "npm run swagger",
+    "initial": "npm run swagger && npm run compileSysInit",
-    "prestart:prod": "npm run swagger",
+    "prestart": "npm run initial",
+    "prestart:prod": "npm run initial",
+    "prebuild": "npm run initial",
     "start": "nodemon ./src/server.ts",
     "start:prod": "nodemon ./src/prod-server.ts",
     "build": "rimraf build && tsc",
@@ -16,14 +18,18 @@
     "lint:fix": "npx prettier --write \"src/**/*.{ts,tsx,js,jsx,html,css,sass,less,yml,md,graphql}\"",
     "lint": "npx prettier --check \"src/**/*.{ts,tsx,js,jsx,html,css,sass,less,yml,md,graphql}\"",
     "package:lib": "npm run build && cp ./package.json build && cp README.md build && cd build && npm version \"5.0.0\" && npm pack",
-    "exe": "npm run build && npm run public:copy && npm run web:copy && pkg .",
+    "exe": "npm run build && npm run exe:copy && pkg .",
+    "exe:copy": "npm run public:copy && npm run sasjsbuild:copy && npm run web:copy",
     "public:copy": "cp -r ./public/ ./build/public/",
-    "web:copy": "rimraf web && mkdir web && cp -r ../web/build/ ./web/build/"
+    "sasjsbuild:copy": "cp -r ./sasjsbuild/ ./build/sasjsbuild/",
+    "web:copy": "rimraf web && mkdir web && cp -r ../web/build/ ./web/build/",
+    "compileSysInit": "ts-node ./scripts/compileSysInit.ts"
   },
   "bin": "./build/src/server.js",
   "pkg": {
     "assets": [
       "./build/public/**/*",
+      "./build/sasjsbuild/**/*",
       "./web/build/**/*"
     ],
     "targets": [
@@ -40,7 +46,8 @@
   },
   "author": "Analytium Ltd",
   "dependencies": {
-    "@sasjs/utils": "^2.33.1",
+    "@sasjs/core": "^2.48.6",
+    "@sasjs/utils": "file:../../utils/build/sasjs-utils-5.0.0.tgz",
     "bcryptjs": "^2.4.3",
     "cors": "^2.8.5",
     "express": "^4.17.1",

api/scripts/compileSysInit.ts

@@ -0,0 +1,35 @@
+import path from 'path'
+import {
+  createFile,
+  loadDependenciesFile,
+  readFile,
+  SASJsFileType
+} from '@sasjs/utils'
+import { apiRoot, sysInitCompiledPath } from '../src/utils'
+
+const macroCorePath = path.join(apiRoot, 'node_modules', '@sasjs', 'core')
+
+const compiledSystemInit = async (systemInit: string) =>
+  'options ps=max;\n' +
+  (await loadDependenciesFile({
+    fileContent: systemInit,
+    type: SASJsFileType.job,
+    programFolders: [],
+    macroFolders: [],
+    buildSourceFolder: '',
+    macroCorePath
+  }))
+
+const createSysInitFile = async () => {
+  console.log('macroCorePath', macroCorePath)
+  const systemInitContent = await readFile(
+    path.join(__dirname, 'systemInit.sas')
+  )
+
+  await createFile(
+    path.join(sysInitCompiledPath),
+    await compiledSystemInit(systemInitContent)
+  )
+}
+
+createSysInitFile()

api/scripts/systemInit.sas

@@ -0,0 +1,15 @@
+/**
+  @file
+  @brief The systemInit program
+  @details This program is inserted into every sasjs/server program invocation,
+  _before_ any user-provided content.
+
+  <h4> SAS Macros </h4>
+  @li mcf_stpsrv_header.sas
+
+**/
+
+
+proc fcmp outcat=work.sasjs.utils;
+%mcf_stpsrv_header()
+quit;

api/src/controllers/internal/Execution.ts

@@ -18,10 +18,6 @@ export class ExecutionController {
 
     let program = await readFile(programPath)
 
-    Object.keys(vars).forEach(
-      (key: string) => (program = `%let ${key}=${vars[key]};\n${program}`)
-    )
-
     const sessionController = getSessionController()
 
     const session = await sessionController.getSession()
@@ -38,7 +34,12 @@ export class ExecutionController {
       preProgramVariables?.accessToken ?? 'accessToken'
     )
 
-    program = `
+    const varStatments = Object.keys(vars).reduce(
+      (computed: string, key: string) =>
+        `${computed}%let ${key}=${vars[key]};\n`,
+      ''
+    )
+    const preProgramVarStatments = `
 %let _sasjs_tokenfile=${tokenFile};
 %let _sasjs_username=${preProgramVariables?.username};
 %let _sasjs_userid=${preProgramVariables?.userId};
@@ -47,8 +48,17 @@ export class ExecutionController {
 %let _sasjs_apipath=/SASjsApi/stp/execute;
 %let _metaperson=&_sasjs_displayname;
 %let _metauser=&_sasjs_username;
-%let sasjsprocessmode=Stored Program;
+%let sasjsprocessmode=Stored Program;`
+
+    program = `
+/* runtime vars */
+${varStatments}
 filename _webout "${weboutPath}";
+
+/* dynamic user-provided vars */
+${preProgramVarStatments}
+
+/* actual job code */
 ${program}`
 
     // if no files are uploaded filesNamesMap will be undefined

api/src/controllers/internal/Session.ts

@@ -2,12 +2,17 @@ import path from 'path'
 import { Session } from '../../types'
 import { promisify } from 'util'
 import { execFile } from 'child_process'
-import { getTmpSessionsFolderPath, generateUniqueFileName } from '../../utils'
+import {
+  getTmpSessionsFolderPath,
+  generateUniqueFileName,
+  sysInitCompiledPath
+} from '../../utils'
 import {
   deleteFolder,
   createFile,
   fileExists,
-  generateTimestamp
+  generateTimestamp,
+  readFile
 } from '@sasjs/utils'
 
 const execFilePromise = promisify(execFile)
@@ -52,9 +57,13 @@ export class SessionController {
     // we clean them up after a predefined period, if unused
     this.scheduleSessionDestroy(session)
 
+    // Place compiled system init code to autoexec
+    const compiledSystemInitContent = await readFile(sysInitCompiledPath)
+
     // the autoexec file is executed on SAS startup
     const autoExecPath = path.join(sessionFolder, 'autoexec.sas')
-    await createFile(autoExecPath, autoExecContent)
+    const contentForAutoExec = `/* compiled systemInit */\n${compiledSystemInitContent}\n/* autoexec */\n${autoExecContent}`
+    await createFile(autoExecPath, contentForAutoExec)
 
     // create empty code.sas as SAS will not start without a SYSIN
     const codePath = path.join(session.path, 'code.sas')

api/src/controllers/stp.ts

@@ -14,7 +14,6 @@ import {
 import { ExecutionController } from './internal'
 import { PreProgramVars } from '../types'
 import { getTmpFilesFolderPath, makeFilesNamesMap } from '../utils'
-import { request } from 'https'
 
 interface ExecuteReturnJsonPayload {
   /**

api/src/routes/web/web.ts

@@ -1,4 +1,4 @@
-import { fileExists, readFile } from '@sasjs/utils'
+import { readFile } from '@sasjs/utils'
 import express from 'express'
 import path from 'path'
 import { getWebBuildFolderPath } from '../../utils'
@@ -12,22 +12,23 @@ const codeToInject = `
 </script>`
 
 webRouter.get('/', async (_, res) => {
-  const indexHtmlPath = path.join(getWebBuildFolderPath(), 'index.html')
+  let content: string
-  if (!(await fileExists(indexHtmlPath))) {
+  try {
+    const indexHtmlPath = path.join(getWebBuildFolderPath(), 'index.html')
+    content = await readFile(indexHtmlPath)
+  } catch (_) {
     return res.send('Web Build is not present')
   }
 
   const { MODE } = process.env
   if (MODE?.trim() !== 'server') {
-    const content = await readFile(indexHtmlPath)
-
     const injectedContent = content.replace('</head>', `${codeToInject}</head>`)
 
     res.setHeader('Content-Type', 'text/html')
     return res.send(injectedContent)
   }
 
-  res.sendFile(indexHtmlPath)
+  return res.send(content)
 })
 
 export default webRouter

api/src/utils/file.ts

@@ -1,8 +1,16 @@
 import path from 'path'
 import { getRealPath } from '@sasjs/utils'
 
+export const apiRoot = path.join(__dirname, '..', '..')
+export const codebaseRoot = path.join(apiRoot, '..')
+export const sysInitCompiledPath = path.join(
+  apiRoot,
+  'sasjsbuild',
+  'systemInitCompiled.sas'
+)
+
 export const getWebBuildFolderPath = () =>
-  path.join(__dirname, '..', '..', '..', 'web', 'build')
+  path.join(codebaseRoot, 'web', 'build')
 
 export const getTmpFolderPath = () =>
   process.driveLoc ?? getRealPath(path.join(process.cwd(), 'tmp'))