diff --git a/api/package-lock.json b/api/package-lock.json index 9a0a131..136a63a 100644 --- a/api/package-lock.json +++ b/api/package-lock.json @@ -48,11 +48,13 @@ "@types/swagger-ui-express": "^4.1.3", "@types/unzipper": "^0.10.5", "adm-zip": "^0.5.9", + "axios": "0.27.2", "csrf": "^3.1.0", "dotenv": "^10.0.0", "http-headers-validation": "^0.0.1", "jest": "^27.0.6", "mongodb-memory-server": "^8.0.0", + "nodejs-file-downloader": "4.10.2", "nodemon": "^2.0.7", "pkg": "5.6.0", "prettier": "^2.3.1", @@ -2507,6 +2509,30 @@ "node": ">= 4.0.0" } }, + "node_modules/axios": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", + "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", + "dev": true, + "dependencies": { + "follow-redirects": "^1.14.9", + "form-data": "^4.0.0" + } + }, + "node_modules/axios/node_modules/form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/babel-jest": { "version": "27.0.6", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-27.0.6.tgz", @@ -4134,6 +4160,26 @@ "node": ">=8" } }, + "node_modules/follow-redirects": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/form-data": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", @@ -7439,6 +7485,18 @@ "integrity": "sha512-uW7fodD6pyW2FZNZnp/Z3hvWKeEW1Y8R1+1CnErE8cXFXzl5blBOoVB41CvMer6P6Q0S5FXDwcHgFd1Wj0U9zg==", "dev": true }, + "node_modules/nodejs-file-downloader": { + "version": "4.10.2", + "resolved": "https://registry.npmjs.org/nodejs-file-downloader/-/nodejs-file-downloader-4.10.2.tgz", + "integrity": "sha512-pTVlytER/4wxcIpEhLXoqhuJ7WH1+xSFNLbI0wPmbwH3pWlJRRebb1Kbu91mz1CyOJmO4sj6YLH1wkF1B6efrQ==", + "dev": true, + "dependencies": { + "follow-redirects": "^1.15.1", + "https-proxy-agent": "^5.0.0", + "mime-types": "^2.1.27", + "sanitize-filename": "^1.6.3" + } + }, "node_modules/nodemon": { "version": "2.0.19", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.19.tgz", @@ -8381,6 +8439,15 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "node_modules/sanitize-filename": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/sanitize-filename/-/sanitize-filename-1.6.3.tgz", + "integrity": "sha512-y/52Mcy7aw3gRm7IrcGDFx/bCk4AhRh2eI9luHOQM86nZsqwiRkkq2GekHXBBD+SmPidc8i2PqtYZl+pWJ8Oeg==", + "dev": true, + "dependencies": { + "truncate-utf8-bytes": "^1.0.0" + } + }, "node_modules/saslprep": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", @@ -9235,6 +9302,15 @@ "resolved": "https://registry.npmjs.org/traverse-chain/-/traverse-chain-0.1.0.tgz", "integrity": "sha1-YdvC1Ttp/2CRoSoWj9fUMxB+QPE=" }, + "node_modules/truncate-utf8-bytes": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz", + "integrity": "sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ==", + "dev": true, + "dependencies": { + "utf8-byte-length": "^1.0.1" + } + }, "node_modules/ts-jest": { "version": "27.0.3", "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-27.0.3.tgz", @@ -9538,6 +9614,12 @@ "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" }, + "node_modules/utf8-byte-length": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/utf8-byte-length/-/utf8-byte-length-1.0.4.tgz", + "integrity": "sha512-4+wkEYLBbWxqTahEsWrhxepcoVOJ+1z5PGIjPZxRkytcdSUaNjIjBM7Xn8E+pdSuV7SzvWovBFA54FO0JSoqhA==", + "dev": true + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -11909,6 +11991,29 @@ "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==", "dev": true }, + "axios": { + "version": "0.27.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz", + "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==", + "dev": true, + "requires": { + "follow-redirects": "^1.14.9", + "form-data": "^4.0.0" + }, + "dependencies": { + "form-data": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dev": true, + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + } + } + }, "babel-jest": { "version": "27.0.6", "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-27.0.6.tgz", @@ -13167,6 +13272,12 @@ "path-exists": "^4.0.0" } }, + "follow-redirects": { + "version": "1.15.2", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", + "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "dev": true + }, "form-data": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", @@ -15654,6 +15765,18 @@ "integrity": "sha512-uW7fodD6pyW2FZNZnp/Z3hvWKeEW1Y8R1+1CnErE8cXFXzl5blBOoVB41CvMer6P6Q0S5FXDwcHgFd1Wj0U9zg==", "dev": true }, + "nodejs-file-downloader": { + "version": "4.10.2", + "resolved": "https://registry.npmjs.org/nodejs-file-downloader/-/nodejs-file-downloader-4.10.2.tgz", + "integrity": "sha512-pTVlytER/4wxcIpEhLXoqhuJ7WH1+xSFNLbI0wPmbwH3pWlJRRebb1Kbu91mz1CyOJmO4sj6YLH1wkF1B6efrQ==", + "dev": true, + "requires": { + "follow-redirects": "^1.15.1", + "https-proxy-agent": "^5.0.0", + "mime-types": "^2.1.27", + "sanitize-filename": "^1.6.3" + } + }, "nodemon": { "version": "2.0.19", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.19.tgz", @@ -16327,6 +16450,15 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "sanitize-filename": { + "version": "1.6.3", + "resolved": "https://registry.npmjs.org/sanitize-filename/-/sanitize-filename-1.6.3.tgz", + "integrity": "sha512-y/52Mcy7aw3gRm7IrcGDFx/bCk4AhRh2eI9luHOQM86nZsqwiRkkq2GekHXBBD+SmPidc8i2PqtYZl+pWJ8Oeg==", + "dev": true, + "requires": { + "truncate-utf8-bytes": "^1.0.0" + } + }, "saslprep": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz", @@ -16990,6 +17122,15 @@ "resolved": "https://registry.npmjs.org/traverse-chain/-/traverse-chain-0.1.0.tgz", "integrity": "sha1-YdvC1Ttp/2CRoSoWj9fUMxB+QPE=" }, + "truncate-utf8-bytes": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz", + "integrity": "sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ==", + "dev": true, + "requires": { + "utf8-byte-length": "^1.0.1" + } + }, "ts-jest": { "version": "27.0.3", "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-27.0.3.tgz", @@ -17212,6 +17353,12 @@ } } }, + "utf8-byte-length": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/utf8-byte-length/-/utf8-byte-length-1.0.4.tgz", + "integrity": "sha512-4+wkEYLBbWxqTahEsWrhxepcoVOJ+1z5PGIjPZxRkytcdSUaNjIjBM7Xn8E+pdSuV7SzvWovBFA54FO0JSoqhA==", + "dev": true + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/api/package.json b/api/package.json index 56165ed..dc0b127 100644 --- a/api/package.json +++ b/api/package.json @@ -4,7 +4,7 @@ "description": "Api of SASjs server", "main": "./src/server.ts", "scripts": { - "initial": "npm run swagger && npm run compileSysInit && npm run copySASjsCore", + "initial": "npm run swagger && npm run compileSysInit && npm run copySASjsCore && npm run downloadMacros", "prestart": "npm run initial", "prebuild": "npm run initial", "start": "NODE_ENV=development nodemon ./src/server.ts", @@ -17,20 +17,21 @@ "lint:fix": "npx prettier --write \"src/**/*.{ts,tsx,js,jsx,html,css,sass,less,yml,md,graphql}\"", "lint": "npx prettier --check \"src/**/*.{ts,tsx,js,jsx,html,css,sass,less,yml,md,graphql}\"", "exe": "npm run build && pkg .", - "copy:files": "npm run public:copy && npm run sasjsbuild:copy && npm run sasjscore:copy && npm run web:copy", + "copy:files": "npm run public:copy && npm run sasjsbuild:copy && npm run sas:copy && npm run web:copy", "public:copy": "cp -r ./public/ ./build/public/", "sasjsbuild:copy": "cp -r ./sasjsbuild/ ./build/sasjsbuild/", - "sasjscore:copy": "cp -r ./sasjscore/ ./build/sasjscore/", + "sas:copy": "cp -r ./sas/ ./build/sas/", "web:copy": "rimraf web && mkdir web && cp -r ../web/build/ ./web/build/", "compileSysInit": "ts-node ./scripts/compileSysInit.ts", - "copySASjsCore": "ts-node ./scripts/copySASjsCore.ts" + "copySASjsCore": "ts-node ./scripts/copySASjsCore.ts", + "downloadMacros": "ts-node ./scripts/downloadMacros.ts" }, "bin": "./build/src/server.js", "pkg": { "assets": [ "./build/public/**/*", "./build/sasjsbuild/**/*", - "./build/sasjscore/**/*", + "./build/sas/**/*", "./web/build/**/*" ], "targets": [ @@ -84,11 +85,13 @@ "@types/swagger-ui-express": "^4.1.3", "@types/unzipper": "^0.10.5", "adm-zip": "^0.5.9", + "axios": "0.27.2", "csrf": "^3.1.0", "dotenv": "^10.0.0", "http-headers-validation": "^0.0.1", "jest": "^27.0.6", "mongodb-memory-server": "^8.0.0", + "nodejs-file-downloader": "4.10.2", "nodemon": "^2.0.7", "pkg": "5.6.0", "prettier": "^2.3.1", diff --git a/api/scripts/downloadMacros.ts b/api/scripts/downloadMacros.ts new file mode 100644 index 0000000..8deeac3 --- /dev/null +++ b/api/scripts/downloadMacros.ts @@ -0,0 +1,39 @@ +import axios from 'axios' +import Downloader from 'nodejs-file-downloader' +import { createFile, listFilesInFolder } from '@sasjs/utils' + +import { sasJSCoreMacros, sasJSCoreMacrosInfo } from '../src/utils/file' + +export const downloadMacros = async () => { + const url = + 'https://api.github.com/repos/yabwon/SAS_PACKAGES/contents/SPF/Macros' + + console.info(`Downloading macros from ${url}`) + + await axios + .get(url) + .then(async (res) => { + await downloadFiles(res.data) + }) + .catch((err) => { + throw new Error(err) + }) +} + +const downloadFiles = async function (fileList: any) { + for (const file of fileList) { + const downloader = new Downloader({ + url: file.download_url, + directory: sasJSCoreMacros, + fileName: file.path.replace(/^SPF\/Macros/, ''), + cloneFiles: false + }) + await downloader.download() + } + + const fileNames = await listFilesInFolder(sasJSCoreMacros) + + await createFile(sasJSCoreMacrosInfo, fileNames.join('\n')) +} + +downloadMacros() diff --git a/api/src/controllers/internal/Session.ts b/api/src/controllers/internal/Session.ts index 3e61e34..edcc6f5 100644 --- a/api/src/controllers/internal/Session.ts +++ b/api/src/controllers/internal/Session.ts @@ -3,6 +3,7 @@ import { Session } from '../../types' import { promisify } from 'util' import { execFile } from 'child_process' import { + getPackagesFolder, getSessionsFolder, generateUniqueFileName, sysInitCompiledPath, @@ -104,7 +105,8 @@ export class SASSessionController extends SessionController { // the autoexec file is executed on SAS startup const autoExecPath = path.join(sessionFolder, 'autoexec.sas') - const contentForAutoExec = `/* compiled systemInit */ + const contentForAutoExec = `filename packages "${getPackagesFolder()}"; +/* compiled systemInit */ ${compiledSystemInitContent} /* autoexec */ ${autoExecContent}` diff --git a/api/src/utils/file.ts b/api/src/utils/file.ts index 985f10b..2d0ac65 100644 --- a/api/src/utils/file.ts +++ b/api/src/utils/file.ts @@ -10,7 +10,7 @@ export const sysInitCompiledPath = path.join( 'systemInitCompiled.sas' ) -export const sasJSCoreMacros = path.join(apiRoot, 'sasjscore') +export const sasJSCoreMacros = path.join(apiRoot, 'sas', 'sasautos') export const sasJSCoreMacrosInfo = path.join(sasJSCoreMacros, '.macrolist') export const getWebBuildFolder = () => path.join(codebaseRoot, 'web', 'build') @@ -28,7 +28,10 @@ export const getAppStreamConfigPath = () => path.join(getSasjsRootFolder(), 'appStreamConfig.json') export const getMacrosFolder = () => - path.join(getSasjsRootFolder(), 'sasjscore') + path.join(getSasjsRootFolder(), 'sas', 'sasautos') + +export const getPackagesFolder = () => + path.join(getSasjsRootFolder(), 'sas', 'sas_packages') export const getUploadsFolder = () => path.join(getSasjsRootFolder(), 'uploads') diff --git a/api/src/utils/setupFolders.ts b/api/src/utils/setupFolders.ts index 4d66455..8f52bbc 100644 --- a/api/src/utils/setupFolders.ts +++ b/api/src/utils/setupFolders.ts @@ -1,10 +1,15 @@ import { createFile, createFolder, fileExists } from '@sasjs/utils' -import { getDesktopUserAutoExecPath, getFilesFolder } from './file' +import { + getDesktopUserAutoExecPath, + getFilesFolder, + getPackagesFolder +} from './file' import { ModeType } from './verifyEnvVariables' export const setupFolders = async () => { const drivePath = getFilesFolder() await createFolder(drivePath) + await createFolder(getPackagesFolder()) if (process.env.MODE === ModeType.Desktop) { if (!(await fileExists(getDesktopUserAutoExecPath()))) {