diff --git a/.gitignore b/.gitignore index 4fe91fa..3d08c5d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ node_modules/ sas/ tmp/ build/ +sasjsbuild/ certificates/ executables/ .env diff --git a/api/package-lock.json b/api/package-lock.json index e6dd55e..e03345a 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -8,7 +8,8 @@ "name": "api", "version": "0.0.1", "dependencies": { - "@sasjs/utils": "^2.33.1", + "@sasjs/core": "^2.48.6", + "@sasjs/utils": "file:../../utils/build/sasjs-utils-5.0.0.tgz", "bcryptjs": "^2.4.3", "cors": "^2.8.5", "express": "^4.17.1", @@ -1560,11 +1561,17 @@ "@octokit/openapi-types": "^7.2.3" } }, + "node_modules/@sasjs/core": { + "version": "2.48.6", + "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.48.6.tgz", + "integrity": "sha512-5kCf4TdCVOYve4wSHVTi+db34hknDwvY2C/JVEPHT6T3CkQ5cnwRVPSFz/1WzXzcVvdUi4ag5xd9SDOsU12oWA==" + }, "node_modules/@sasjs/utils": { - "version": "2.33.1", - "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.33.1.tgz", - "integrity": "sha512-QQqI9+G/riMrbNSjambYwaJwGY3om2f3N9z9tNxBSva+W0g3JaOl4qeOqpRS91KhOmNhrLMCohg6jScMCz3YFQ==", + "version": "5.0.0", + "resolved": "file:../../utils/build/sasjs-utils-5.0.0.tgz", + "integrity": "sha512-ZSX1oHLEl3Gaz1ILnmbf1hMt3WLfvAddck65BY1NduOePtHx2ioja9CjzXKxB75CphBjLdjR95tbmb8jjzGDZA==", "hasInstallScript": true, + "license": "ISC", "dependencies": { "@types/fs-extra": "^9.0.11", "@types/prompts": "^2.0.13", @@ -1572,8 +1579,11 @@ "cli-table": "^0.3.6", "consola": "^2.15.0", "csv-stringify": "^5.6.5", + "find": "0.3.0", "fs-extra": "^10.0.0", "jwt-decode": "^3.1.2", + "lodash.groupby": "4.6.0", + "lodash.uniqby": "4.7.0", "prompts": "^2.4.1", "rimraf": "^3.0.2", "valid-url": "^1.0.9" @@ -4933,6 +4943,14 @@ "node": ">= 0.8" } }, + "node_modules/find": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/find/-/find-0.3.0.tgz", + "integrity": "sha512-iSd+O4OEYV/I36Zl8MdYJO0xD82wH528SaCieTVHhclgiYNe9y+yPKSwK+A7/WsmHL1EZ+pYUJBXWTL5qofksw==", + "dependencies": { + "traverse-chain": "~0.1.0" + } + }, "node_modules/find-cache-dir": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz", @@ -7975,6 +7993,11 @@ "integrity": "sha1-ZHYsSGGAglGKw99Mz11YhtriA0c=", "dev": true }, + "node_modules/lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha1-Cwih3PaDl8OXhVwyOXg4Mt90A9E=" + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", @@ -8025,8 +8048,7 @@ "node_modules/lodash.uniqby": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz", - "integrity": "sha1-2ZwHpmnp5tJOE2Lf4mbGdhavEwI=", - "dev": true + "integrity": "sha1-2ZwHpmnp5tJOE2Lf4mbGdhavEwI=" }, "node_modules/lowercase-keys": { "version": "1.0.1", @@ -14220,6 +14242,11 @@ "integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc=", "dev": true }, + "node_modules/traverse-chain": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/traverse-chain/-/traverse-chain-0.1.0.tgz", + "integrity": "sha1-YdvC1Ttp/2CRoSoWj9fUMxB+QPE=" + }, "node_modules/trim-newlines": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-3.0.1.tgz", @@ -16141,10 +16168,14 @@ "@octokit/openapi-types": "^7.2.3" } }, + "@sasjs/core": { + "version": "2.48.6", + "resolved": "https://registry.npmjs.org/@sasjs/core/-/core-2.48.6.tgz", + "integrity": "sha512-5kCf4TdCVOYve4wSHVTi+db34hknDwvY2C/JVEPHT6T3CkQ5cnwRVPSFz/1WzXzcVvdUi4ag5xd9SDOsU12oWA==" + }, "@sasjs/utils": { - "version": "2.33.1", - "resolved": "https://registry.npmjs.org/@sasjs/utils/-/utils-2.33.1.tgz", - "integrity": "sha512-QQqI9+G/riMrbNSjambYwaJwGY3om2f3N9z9tNxBSva+W0g3JaOl4qeOqpRS91KhOmNhrLMCohg6jScMCz3YFQ==", + "version": "file:../../utils/build/sasjs-utils-5.0.0.tgz", + "integrity": "sha512-ZSX1oHLEl3Gaz1ILnmbf1hMt3WLfvAddck65BY1NduOePtHx2ioja9CjzXKxB75CphBjLdjR95tbmb8jjzGDZA==", "requires": { "@types/fs-extra": "^9.0.11", "@types/prompts": "^2.0.13", @@ -16152,8 +16183,11 @@ "cli-table": "^0.3.6", "consola": "^2.15.0", "csv-stringify": "^5.6.5", + "find": "0.3.0", "fs-extra": "^10.0.0", "jwt-decode": "^3.1.2", + "lodash.groupby": "4.6.0", + "lodash.uniqby": "4.7.0", "prompts": "^2.4.1", "rimraf": "^3.0.2", "valid-url": "^1.0.9" @@ -18889,6 +18923,14 @@ "unpipe": "~1.0.0" } }, + "find": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/find/-/find-0.3.0.tgz", + "integrity": "sha512-iSd+O4OEYV/I36Zl8MdYJO0xD82wH528SaCieTVHhclgiYNe9y+yPKSwK+A7/WsmHL1EZ+pYUJBXWTL5qofksw==", + "requires": { + "traverse-chain": "~0.1.0" + } + }, "find-cache-dir": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-3.3.2.tgz", @@ -21170,6 +21212,11 @@ "integrity": "sha1-ZHYsSGGAglGKw99Mz11YhtriA0c=", "dev": true }, + "lodash.groupby": { + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/lodash.groupby/-/lodash.groupby-4.6.0.tgz", + "integrity": "sha1-Cwih3PaDl8OXhVwyOXg4Mt90A9E=" + }, "lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", @@ -21220,8 +21267,7 @@ "lodash.uniqby": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/lodash.uniqby/-/lodash.uniqby-4.7.0.tgz", - "integrity": "sha1-2ZwHpmnp5tJOE2Lf4mbGdhavEwI=", - "dev": true + "integrity": "sha1-2ZwHpmnp5tJOE2Lf4mbGdhavEwI=" }, "lowercase-keys": { "version": "1.0.1", @@ -25795,6 +25841,11 @@ "integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc=", "dev": true }, + "traverse-chain": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/traverse-chain/-/traverse-chain-0.1.0.tgz", + "integrity": "sha1-YdvC1Ttp/2CRoSoWj9fUMxB+QPE=" + }, "trim-newlines": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-3.0.1.tgz", diff --git a/api/package.json b/api/package.json index 4742b51..3204c6b 100644 --- a/api/package.json +++ b/api/package.json @@ -4,8 +4,10 @@ "description": "Api of SASjs server", "main": "./src/server.ts", "scripts": { - "prestart": "npm run swagger", - "prestart:prod": "npm run swagger", + "initial": "npm run swagger && npm run compileSysInit", + "prestart": "npm run initial", + "prestart:prod": "npm run initial", + "prebuild": "npm run initial", "start": "nodemon ./src/server.ts", "start:prod": "nodemon ./src/prod-server.ts", "build": "rimraf build && tsc", @@ -16,14 +18,18 @@ "lint:fix": "npx prettier --write \"src/**/*.{ts,tsx,js,jsx,html,css,sass,less,yml,md,graphql}\"", "lint": "npx prettier --check \"src/**/*.{ts,tsx,js,jsx,html,css,sass,less,yml,md,graphql}\"", "package:lib": "npm run build && cp ./package.json build && cp README.md build && cd build && npm version \"5.0.0\" && npm pack", - "exe": "npm run build && npm run public:copy && npm run web:copy && pkg .", + "exe": "npm run build && npm run exe:copy && pkg .", + "exe:copy": "npm run public:copy && npm run sasjsbuild:copy && npm run web:copy", "public:copy": "cp -r ./public/ ./build/public/", - "web:copy": "rimraf web && mkdir web && cp -r ../web/build/ ./web/build/" + "sasjsbuild:copy": "cp -r ./sasjsbuild/ ./build/sasjsbuild/", + "web:copy": "rimraf web && mkdir web && cp -r ../web/build/ ./web/build/", + "compileSysInit": "ts-node ./scripts/compileSysInit.ts" }, "bin": "./build/src/server.js", "pkg": { "assets": [ "./build/public/**/*", + "./build/sasjsbuild/**/*", "./web/build/**/*" ], "targets": [ @@ -40,7 +46,8 @@ }, "author": "Analytium Ltd", "dependencies": { - "@sasjs/utils": "^2.33.1", + "@sasjs/core": "^2.48.6", + "@sasjs/utils": "file:../../utils/build/sasjs-utils-5.0.0.tgz", "bcryptjs": "^2.4.3", "cors": "^2.8.5", "express": "^4.17.1", diff --git a/api/scripts/compileSysInit.ts b/api/scripts/compileSysInit.ts new file mode 100644 index 0000000..2aa5443 --- /dev/null +++ b/api/scripts/compileSysInit.ts @@ -0,0 +1,35 @@ +import path from 'path' +import { + createFile, + loadDependenciesFile, + readFile, + SASJsFileType +} from '@sasjs/utils' +import { apiRoot, sysInitCompiledPath } from '../src/utils' + +const macroCorePath = path.join(apiRoot, 'node_modules', '@sasjs', 'core') + +const compiledSystemInit = async (systemInit: string) => + 'options ps=max;\n' + + (await loadDependenciesFile({ + fileContent: systemInit, + type: SASJsFileType.job, + programFolders: [], + macroFolders: [], + buildSourceFolder: '', + macroCorePath + })) + +const createSysInitFile = async () => { + console.log('macroCorePath', macroCorePath) + const systemInitContent = await readFile( + path.join(__dirname, 'systemInit.sas') + ) + + await createFile( + path.join(sysInitCompiledPath), + await compiledSystemInit(systemInitContent) + ) +} + +createSysInitFile() diff --git a/api/scripts/systemInit.sas b/api/scripts/systemInit.sas new file mode 100644 index 0000000..9a20dda --- /dev/null +++ b/api/scripts/systemInit.sas @@ -0,0 +1,15 @@ +/** + @file + @brief The systemInit program + @details This program is inserted into every sasjs/server program invocation, + _before_ any user-provided content. + +

SAS Macros

+ @li mcf_stpsrv_header.sas + +**/ + + +proc fcmp outcat=work.sasjs.utils; +%mcf_stpsrv_header() +quit; \ No newline at end of file diff --git a/api/src/controllers/internal/Execution.ts b/api/src/controllers/internal/Execution.ts index 29ca4b9..a313c9f 100644 --- a/api/src/controllers/internal/Execution.ts +++ b/api/src/controllers/internal/Execution.ts @@ -18,10 +18,6 @@ export class ExecutionController { let program = await readFile(programPath) - Object.keys(vars).forEach( - (key: string) => (program = `%let ${key}=${vars[key]};\n${program}`) - ) - const sessionController = getSessionController() const session = await sessionController.getSession() @@ -38,7 +34,12 @@ export class ExecutionController { preProgramVariables?.accessToken ?? 'accessToken' ) - program = ` + const varStatments = Object.keys(vars).reduce( + (computed: string, key: string) => + `${computed}%let ${key}=${vars[key]};\n`, + '' + ) + const preProgramVarStatments = ` %let _sasjs_tokenfile=${tokenFile}; %let _sasjs_username=${preProgramVariables?.username}; %let _sasjs_userid=${preProgramVariables?.userId}; @@ -47,8 +48,17 @@ export class ExecutionController { %let _sasjs_apipath=/SASjsApi/stp/execute; %let _metaperson=&_sasjs_displayname; %let _metauser=&_sasjs_username; -%let sasjsprocessmode=Stored Program; +%let sasjsprocessmode=Stored Program;` + + program = ` +/* runtime vars */ +${varStatments} filename _webout "${weboutPath}"; + +/* dynamic user-provided vars */ +${preProgramVarStatments} + +/* actual job code */ ${program}` // if no files are uploaded filesNamesMap will be undefined diff --git a/api/src/controllers/internal/Session.ts b/api/src/controllers/internal/Session.ts index 56a6e58..4b17fd6 100644 --- a/api/src/controllers/internal/Session.ts +++ b/api/src/controllers/internal/Session.ts @@ -2,12 +2,17 @@ import path from 'path' import { Session } from '../../types' import { promisify } from 'util' import { execFile } from 'child_process' -import { getTmpSessionsFolderPath, generateUniqueFileName } from '../../utils' +import { + getTmpSessionsFolderPath, + generateUniqueFileName, + sysInitCompiledPath +} from '../../utils' import { deleteFolder, createFile, fileExists, - generateTimestamp + generateTimestamp, + readFile } from '@sasjs/utils' const execFilePromise = promisify(execFile) @@ -52,9 +57,13 @@ export class SessionController { // we clean them up after a predefined period, if unused this.scheduleSessionDestroy(session) + // Place compiled system init code to autoexec + const compiledSystemInitContent = await readFile(sysInitCompiledPath) + // the autoexec file is executed on SAS startup const autoExecPath = path.join(sessionFolder, 'autoexec.sas') - await createFile(autoExecPath, autoExecContent) + const contentForAutoExec = `/* compiled systemInit */\n${compiledSystemInitContent}\n/* autoexec */\n${autoExecContent}` + await createFile(autoExecPath, contentForAutoExec) // create empty code.sas as SAS will not start without a SYSIN const codePath = path.join(session.path, 'code.sas') diff --git a/api/src/controllers/stp.ts b/api/src/controllers/stp.ts index 460ad0d..ea6e276 100644 --- a/api/src/controllers/stp.ts +++ b/api/src/controllers/stp.ts @@ -14,7 +14,6 @@ import { import { ExecutionController } from './internal' import { PreProgramVars } from '../types' import { getTmpFilesFolderPath, makeFilesNamesMap } from '../utils' -import { request } from 'https' interface ExecuteReturnJsonPayload { /** diff --git a/api/src/routes/web/web.ts b/api/src/routes/web/web.ts index 6a366dc..c9d1dd3 100644 --- a/api/src/routes/web/web.ts +++ b/api/src/routes/web/web.ts @@ -1,4 +1,4 @@ -import { fileExists, readFile } from '@sasjs/utils' +import { readFile } from '@sasjs/utils' import express from 'express' import path from 'path' import { getWebBuildFolderPath } from '../../utils' @@ -12,22 +12,23 @@ const codeToInject = ` ` webRouter.get('/', async (_, res) => { - const indexHtmlPath = path.join(getWebBuildFolderPath(), 'index.html') - if (!(await fileExists(indexHtmlPath))) { + let content: string + try { + const indexHtmlPath = path.join(getWebBuildFolderPath(), 'index.html') + content = await readFile(indexHtmlPath) + } catch (_) { return res.send('Web Build is not present') } const { MODE } = process.env if (MODE?.trim() !== 'server') { - const content = await readFile(indexHtmlPath) - const injectedContent = content.replace('', `${codeToInject}`) res.setHeader('Content-Type', 'text/html') return res.send(injectedContent) } - res.sendFile(indexHtmlPath) + return res.send(content) }) export default webRouter diff --git a/api/src/utils/file.ts b/api/src/utils/file.ts index 14c25bb..e9d7731 100644 --- a/api/src/utils/file.ts +++ b/api/src/utils/file.ts @@ -1,8 +1,16 @@ import path from 'path' import { getRealPath } from '@sasjs/utils' +export const apiRoot = path.join(__dirname, '..', '..') +export const codebaseRoot = path.join(apiRoot, '..') +export const sysInitCompiledPath = path.join( + apiRoot, + 'sasjsbuild', + 'systemInitCompiled.sas' +) + export const getWebBuildFolderPath = () => - path.join(__dirname, '..', '..', '..', 'web', 'build') + path.join(codebaseRoot, 'web', 'build') export const getTmpFolderPath = () => process.driveLoc ?? getRealPath(path.join(process.cwd(), 'tmp'))