feat: compile systemInit and inject to autoExec

2025-12-10 11:24:35 +00:00 · 2021-11-18 03:12:05 +05:00
parent 319743a23e
commit b75139dda5
10 changed files with 169 additions and 33 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,7 @@ node_modules/
 sas/
 tmp/
 build/
+sasjsbuild/
 certificates/
 executables/
 .env
--- a/api/package-lock.json
+++ b/api/package-lock.json
@@ -8,7 +8,8 @@
      "name": "api",
      "version": "0.0.1",
      "dependencies": {
-        "@sasjs/utils": "^2.33.1",
+        "@sasjs/core": "^2.48.6",
+        "@sasjs/utils": "file:../../utils/build/sasjs-utils-5.0.0.tgz",
        "bcryptjs": "^2.4.3",
        "cors": "^2.8.5",
        "express": "^4.17.1",
@@ -1560,11 +1561,17 @@
        "@octokit/openapi-types": "^7.2.3"
      }
    },
+    "node_modules/@sasjs/core": {
+      "version": "2.48.6",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.48.6.tgz",
+      "integrity": "sha512-5kCf4TdCVOYve4wSHVTi+db34hknDwvY2C/JVEPHT6T3CkQ5cnwRVPSFz/1WzXzcVvdUi4ag5xd9SDOsU12oWA=="
+    },
    "node_modules/@sasjs/utils": {
-      "version": "2.33.1",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.33.1.tgz",
-      "integrity": "sha512-QQqI9+G/riMrbNSjambYwaJwGY3om2f3N9z9tNxBSva+W0g3JaOl4qeOqpRS91KhOmNhrLMCohg6jScMCz3YFQ==",
+      "version": "5.0.0",
+      "resolved": "file:../../utils/build/sasjs-utils-5.0.0.tgz",
+      "integrity": "sha512-ZSX1oHLEl3Gaz1ILnmbf1hMt3WLfvAddck65BY1NduOePtHx2ioja9CjzXKxB75CphBjLdjR95tbmb8jjzGDZA==",
      "hasInstallScript": true,
+      "license": "ISC",
      "dependencies": {
        "@types/fs-extra": "^9.0.11",
        "@types/prompts": "^2.0.13",
@@ -1572,8 +1579,11 @@
        "cli-table": "^0.3.6",
        "consola": "^2.15.0",
        "csv-stringify": "^5.6.5",
+        "find": "0.3.0",
        "fs-extra": "^10.0.0",
        "jwt-decode": "^3.1.2",
+        "lodash.groupby": "4.6.0",
+        "lodash.uniqby": "4.7.0",
        "prompts": "^2.4.1",
        "rimraf": "^3.0.2",
        "valid-url": "^1.0.9"
@@ -4933,6 +4943,14 @@
        "node": ">= 0.8"
      }
    },
+    "node_modules/find": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/find/-/find-0.3.0.tgz",
+      "integrity": "sha512-iSd+O4OEYV/I36Zl8MdYJO0xD82wH528SaCieTVHhclgiYNe9y+yPKSwK+A7/WsmHL1EZ+pYUJBXWTL5qofksw==",
+      "dependencies": {
+        "traverse-chain": "~0.1.0"
+      }
+    },
    "node_modules/find-cache-dir": {
      "version": "3.3.2",
      "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz",
@@ -7975,6 +7993,11 @@
      "integrity": "sha1-ZHYsSGGAglGKw99Mz11YhtriA0c=",
      "dev": true
    },
+    "node_modules/lodash.groupby": {
+      "version": "4.6.0",
+      "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz",
+      "integrity": "sha1-Cwih3PaDl8OXhVwyOXg4Mt90A9E="
+    },
    "node_modules/lodash.includes": {
      "version": "4.3.0",
      "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
@@ -8025,8 +8048,7 @@
    "node_modules/lodash.uniqby": {
      "version": "4.7.0",
      "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz",
-      "integrity": "sha1-2ZwHpmnp5tJOE2Lf4mbGdhavEwI=",
-      "dev": true
+      "integrity": "sha1-2ZwHpmnp5tJOE2Lf4mbGdhavEwI="
    },
    "node_modules/lowercase-keys": {
      "version": "1.0.1",
@@ -14220,6 +14242,11 @@
      "integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc=",
      "dev": true
    },
+    "node_modules/traverse-chain": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/traverse-chain/-/traverse-chain-0.1.0.tgz",
+      "integrity": "sha1-YdvC1Ttp/2CRoSoWj9fUMxB+QPE="
+    },
    "node_modules/trim-newlines": {
      "version": "3.0.1",
      "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-3.0.1.tgz",
@@ -16141,10 +16168,14 @@
        "@octokit/openapi-types": "^7.2.3"
      }
    },
+    "@sasjs/core": {
+      "version": "2.48.6",
+      "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.48.6.tgz",
+      "integrity": "sha512-5kCf4TdCVOYve4wSHVTi+db34hknDwvY2C/JVEPHT6T3CkQ5cnwRVPSFz/1WzXzcVvdUi4ag5xd9SDOsU12oWA=="
+    },
    "@sasjs/utils": {
-      "version": "2.33.1",
-      "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.33.1.tgz",
-      "integrity": "sha512-QQqI9+G/riMrbNSjambYwaJwGY3om2f3N9z9tNxBSva+W0g3JaOl4qeOqpRS91KhOmNhrLMCohg6jScMCz3YFQ==",
+      "version": "file:../../utils/build/sasjs-utils-5.0.0.tgz",
+      "integrity": "sha512-ZSX1oHLEl3Gaz1ILnmbf1hMt3WLfvAddck65BY1NduOePtHx2ioja9CjzXKxB75CphBjLdjR95tbmb8jjzGDZA==",
      "requires": {
        "@types/fs-extra": "^9.0.11",
        "@types/prompts": "^2.0.13",
@@ -16152,8 +16183,11 @@
        "cli-table": "^0.3.6",
        "consola": "^2.15.0",
        "csv-stringify": "^5.6.5",
+        "find": "0.3.0",
        "fs-extra": "^10.0.0",
        "jwt-decode": "^3.1.2",
+        "lodash.groupby": "4.6.0",
+        "lodash.uniqby": "4.7.0",
        "prompts": "^2.4.1",
        "rimraf": "^3.0.2",
        "valid-url": "^1.0.9"
@@ -18889,6 +18923,14 @@
        "unpipe": "~1.0.0"
      }
    },
+    "find": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/find/-/find-0.3.0.tgz",
+      "integrity": "sha512-iSd+O4OEYV/I36Zl8MdYJO0xD82wH528SaCieTVHhclgiYNe9y+yPKSwK+A7/WsmHL1EZ+pYUJBXWTL5qofksw==",
+      "requires": {
+        "traverse-chain": "~0.1.0"
+      }
+    },
    "find-cache-dir": {
      "version": "3.3.2",
      "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz",
@@ -21170,6 +21212,11 @@
      "integrity": "sha1-ZHYsSGGAglGKw99Mz11YhtriA0c=",
      "dev": true
    },
+    "lodash.groupby": {
+      "version": "4.6.0",
+      "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz",
+      "integrity": "sha1-Cwih3PaDl8OXhVwyOXg4Mt90A9E="
+    },
    "lodash.includes": {
      "version": "4.3.0",
      "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz",
@@ -21220,8 +21267,7 @@
    "lodash.uniqby": {
      "version": "4.7.0",
      "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz",
-      "integrity": "sha1-2ZwHpmnp5tJOE2Lf4mbGdhavEwI=",
-      "dev": true
+      "integrity": "sha1-2ZwHpmnp5tJOE2Lf4mbGdhavEwI="
    },
    "lowercase-keys": {
      "version": "1.0.1",
@@ -25795,6 +25841,11 @@
      "integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc=",
      "dev": true
    },
+    "traverse-chain": {
+      "version": "0.1.0",
+      "resolved": "https://registry.npmjs.org/traverse-chain/-/traverse-chain-0.1.0.tgz",
+      "integrity": "sha1-YdvC1Ttp/2CRoSoWj9fUMxB+QPE="
+    },
    "trim-newlines": {
      "version": "3.0.1",
      "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-3.0.1.tgz",
--- a/api/package.json
+++ b/api/package.json
@@ -4,8 +4,10 @@
  "description": "Api of SASjs server",
  "main": "./src/server.ts",
  "scripts": {
-    "prestart": "npm run swagger",
-    "prestart:prod": "npm run swagger",
+    "initial": "npm run swagger && npm run compileSysInit",
+    "prestart": "npm run initial",
+    "prestart:prod": "npm run initial",
+    "prebuild": "npm run initial",
    "start": "nodemon ./src/server.ts",
    "start:prod": "nodemon ./src/prod-server.ts",
    "build": "rimraf build && tsc",
@@ -16,14 +18,18 @@
    "lint:fix": "npx prettier --write \"src/**/*.{ts,tsx,js,jsx,html,css,sass,less,yml,md,graphql}\"",
    "lint": "npx prettier --check \"src/**/*.{ts,tsx,js,jsx,html,css,sass,less,yml,md,graphql}\"",
    "package:lib": "npm run build && cp ./package.json build && cp README.md build && cd build && npm version \"5.0.0\" && npm pack",
-    "exe": "npm run build && npm run public:copy && npm run web:copy && pkg .",
+    "exe": "npm run build && npm run exe:copy && pkg .",
+    "exe:copy": "npm run public:copy && npm run sasjsbuild:copy && npm run web:copy",
    "public:copy": "cp -r ./public/ ./build/public/",
-    "web:copy": "rimraf web && mkdir web && cp -r ../web/build/ ./web/build/"
+    "sasjsbuild:copy": "cp -r ./sasjsbuild/ ./build/sasjsbuild/",
+    "web:copy": "rimraf web && mkdir web && cp -r ../web/build/ ./web/build/",
+    "compileSysInit": "ts-node ./scripts/compileSysInit.ts"
  },
  "bin": "./build/src/server.js",
  "pkg": {
    "assets": [
      "./build/public/**/*",
+      "./build/sasjsbuild/**/*",
      "./web/build/**/*"
    ],
    "targets": [
@@ -40,7 +46,8 @@
  },
  "author": "Analytium Ltd",
  "dependencies": {
-    "@sasjs/utils": "^2.33.1",
+    "@sasjs/core": "^2.48.6",
+    "@sasjs/utils": "file:../../utils/build/sasjs-utils-5.0.0.tgz",
    "bcryptjs": "^2.4.3",
    "cors": "^2.8.5",
    "express": "^4.17.1",
--- a/api/scripts/compileSysInit.ts
+++ b/api/scripts/compileSysInit.ts
@@ -0,0 +1,35 @@
+import path from 'path'
+import {
+  createFile,
+  loadDependenciesFile,
+  readFile,
+  SASJsFileType
+} from '@sasjs/utils'
+import { apiRoot, sysInitCompiledPath } from '../src/utils'
+
+const macroCorePath = path.join(apiRoot, 'node_modules', '@sasjs', 'core')
+
+const compiledSystemInit = async (systemInit: string) =>
+  'options ps=max;\n' +
+  (await loadDependenciesFile({
+    fileContent: systemInit,
+    type: SASJsFileType.job,
+    programFolders: [],
+    macroFolders: [],
+    buildSourceFolder: '',
+    macroCorePath
+  }))
+
+const createSysInitFile = async () => {
+  console.log('macroCorePath', macroCorePath)
+  const systemInitContent = await readFile(
+    path.join(__dirname, 'systemInit.sas')
+  )
+
+  await createFile(
+    path.join(sysInitCompiledPath),
+    await compiledSystemInit(systemInitContent)
+  )
+}
+
+createSysInitFile()
--- a/api/scripts/systemInit.sas
+++ b/api/scripts/systemInit.sas
@@ -0,0 +1,15 @@
+/**
+  @file
+  @brief The systemInit program
+  @details This program is inserted into every sasjs/server program invocation,
+  _before_ any user-provided content.
+
+  <h4> SAS Macros </h4>
+  @li mcf_stpsrv_header.sas
+
+**/
+
+
+proc fcmp outcat=work.sasjs.utils;
+%mcf_stpsrv_header()
+quit;
--- a/api/src/controllers/internal/Execution.ts
+++ b/api/src/controllers/internal/Execution.ts
@@ -18,10 +18,6 @@ export class ExecutionController {

    let program = await readFile(programPath)

-    Object.keys(vars).forEach(
-      (key: string) => (program = `%let ${key}=${vars[key]};\n${program}`)
-    )
-
    const sessionController = getSessionController()

    const session = await sessionController.getSession()
@@ -38,7 +34,12 @@ export class ExecutionController {
      preProgramVariables?.accessToken ?? 'accessToken'
    )

-    program = `
+    const varStatments = Object.keys(vars).reduce(
+      (computed: string, key: string) =>
+        `${computed}%let ${key}=${vars[key]};\n`,
+      ''
+    )
+    const preProgramVarStatments = `
 %let _sasjs_tokenfile=${tokenFile};
 %let _sasjs_username=${preProgramVariables?.username};
 %let _sasjs_userid=${preProgramVariables?.userId};
@@ -47,8 +48,17 @@ export class ExecutionController {
 %let _sasjs_apipath=/SASjsApi/stp/execute;
 %let _metaperson=&_sasjs_displayname;
 %let _metauser=&_sasjs_username;
-%let sasjsprocessmode=Stored Program;
+%let sasjsprocessmode=Stored Program;`
+
+    program = `
+/* runtime vars */
+${varStatments}
 filename _webout "${weboutPath}";
+
+/* dynamic user-provided vars */
+${preProgramVarStatments}
+
+/* actual job code */
 ${program}`

    // if no files are uploaded filesNamesMap will be undefined
--- a/api/src/controllers/internal/Session.ts
+++ b/api/src/controllers/internal/Session.ts
@@ -2,12 +2,17 @@ import path from 'path'
 import { Session } from '../../types'
 import { promisify } from 'util'
 import { execFile } from 'child_process'
-import { getTmpSessionsFolderPath, generateUniqueFileName } from '../../utils'
+import {
+  getTmpSessionsFolderPath,
+  generateUniqueFileName,
+  sysInitCompiledPath
+} from '../../utils'
 import {
  deleteFolder,
  createFile,
  fileExists,
-  generateTimestamp
+  generateTimestamp,
+  readFile
 } from '@sasjs/utils'

 const execFilePromise = promisify(execFile)
@@ -52,9 +57,13 @@ export class SessionController {
    // we clean them up after a predefined period, if unused
    this.scheduleSessionDestroy(session)

+    // Place compiled system init code to autoexec
+    const compiledSystemInitContent = await readFile(sysInitCompiledPath)
+
    // the autoexec file is executed on SAS startup
    const autoExecPath = path.join(sessionFolder, 'autoexec.sas')
-    await createFile(autoExecPath, autoExecContent)
+    const contentForAutoExec = `/* compiled systemInit */\n${compiledSystemInitContent}\n/* autoexec */\n${autoExecContent}`
+    await createFile(autoExecPath, contentForAutoExec)

    // create empty code.sas as SAS will not start without a SYSIN
    const codePath = path.join(session.path, 'code.sas')
--- a/api/src/controllers/stp.ts
+++ b/api/src/controllers/stp.ts
@@ -14,7 +14,6 @@ import {
 import { ExecutionController } from './internal'
 import { PreProgramVars } from '../types'
 import { getTmpFilesFolderPath, makeFilesNamesMap } from '../utils'
-import { request } from 'https'

 interface ExecuteReturnJsonPayload {
  /**
--- a/api/src/routes/web/web.ts
+++ b/api/src/routes/web/web.ts
@@ -1,4 +1,4 @@
-import { fileExists, readFile } from '@sasjs/utils'
+import { readFile } from '@sasjs/utils'
 import express from 'express'
 import path from 'path'
 import { getWebBuildFolderPath } from '../../utils'
@@ -12,22 +12,23 @@ const codeToInject = `
 </script>`

 webRouter.get('/', async (_, res) => {
-  const indexHtmlPath = path.join(getWebBuildFolderPath(), 'index.html')
-  if (!(await fileExists(indexHtmlPath))) {
+  let content: string
+  try {
+    const indexHtmlPath = path.join(getWebBuildFolderPath(), 'index.html')
+    content = await readFile(indexHtmlPath)
+  } catch (_) {
    return res.send('Web Build is not present')
  }

  const { MODE } = process.env
  if (MODE?.trim() !== 'server') {
-    const content = await readFile(indexHtmlPath)
-
    const injectedContent = content.replace('</head>', `${codeToInject}</head>`)

    res.setHeader('Content-Type', 'text/html')
    return res.send(injectedContent)
  }

-  res.sendFile(indexHtmlPath)
+  return res.send(content)
 })

 export default webRouter
--- a/api/src/utils/file.ts
+++ b/api/src/utils/file.ts
@@ -1,8 +1,16 @@
 import path from 'path'
 import { getRealPath } from '@sasjs/utils'

+export const apiRoot = path.join(__dirname, '..', '..')
+export const codebaseRoot = path.join(apiRoot, '..')
+export const sysInitCompiledPath = path.join(
+  apiRoot,
+  'sasjsbuild',
+  'systemInitCompiled.sas'
+)
+
 export const getWebBuildFolderPath = () =>
-  path.join(__dirname, '..', '..', '..', 'web', 'build')
+  path.join(codebaseRoot, 'web', 'build')

 export const getTmpFolderPath = () =>
  process.driveLoc ?? getRealPath(path.join(process.cwd(), 'tmp'))