feat: Enable SAS_PACKAGES in SASjs Server

api/package-lock.json

@@ -48,11 +48,13 @@
         "@types/swagger-ui-express": "^4.1.3",
         "@types/unzipper": "^0.10.5",
         "adm-zip": "^0.5.9",
+        "axios": "0.27.2",
         "csrf": "^3.1.0",
         "dotenv": "^10.0.0",
         "http-headers-validation": "^0.0.1",
         "jest": "^27.0.6",
         "mongodb-memory-server": "^8.0.0",
+        "nodejs-file-downloader": "4.10.2",
         "nodemon": "^2.0.7",
         "pkg": "5.6.0",
         "prettier": "^2.3.1",
@@ -2507,6 +2509,30 @@
         "node": ">= 4.0.0"
       }
     },
+    "node_modules/axios": {
+      "version": "0.27.2",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz",
+      "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==",
+      "dev": true,
+      "dependencies": {
+        "follow-redirects": "^1.14.9",
+        "form-data": "^4.0.0"
+      }
+    },
+    "node_modules/axios/node_modules/form-data": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+      "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+      "dev": true,
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/babel-jest": {
       "version": "27.0.6",
       "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-27.0.6.tgz",
@@ -4134,6 +4160,26 @@
         "node": ">=8"
       }
     },
+    "node_modules/follow-redirects": {
+      "version": "1.15.2",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
+      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
+      "dev": true,
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/RubenVerborgh"
+        }
+      ],
+      "engines": {
+        "node": ">=4.0"
+      },
+      "peerDependenciesMeta": {
+        "debug": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/form-data": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
@@ -7439,6 +7485,18 @@
       "integrity": "sha512-uW7fodD6pyW2FZNZnp/Z3hvWKeEW1Y8R1+1CnErE8cXFXzl5blBOoVB41CvMer6P6Q0S5FXDwcHgFd1Wj0U9zg==",
       "dev": true
     },
+    "node_modules/nodejs-file-downloader": {
+      "version": "4.10.2",
+      "resolved": "https://registry.npmjs.org/nodejs-file-downloader/-/nodejs-file-downloader-4.10.2.tgz",
+      "integrity": "sha512-pTVlytER/4wxcIpEhLXoqhuJ7WH1+xSFNLbI0wPmbwH3pWlJRRebb1Kbu91mz1CyOJmO4sj6YLH1wkF1B6efrQ==",
+      "dev": true,
+      "dependencies": {
+        "follow-redirects": "^1.15.1",
+        "https-proxy-agent": "^5.0.0",
+        "mime-types": "^2.1.27",
+        "sanitize-filename": "^1.6.3"
+      }
+    },
     "node_modules/nodemon": {
       "version": "2.0.19",
       "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.19.tgz",
@@ -8381,6 +8439,15 @@
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
     },
+    "node_modules/sanitize-filename": {
+      "version": "1.6.3",
+      "resolved": "https://registry.npmjs.org/sanitize-filename/-/sanitize-filename-1.6.3.tgz",
+      "integrity": "sha512-y/52Mcy7aw3gRm7IrcGDFx/bCk4AhRh2eI9luHOQM86nZsqwiRkkq2GekHXBBD+SmPidc8i2PqtYZl+pWJ8Oeg==",
+      "dev": true,
+      "dependencies": {
+        "truncate-utf8-bytes": "^1.0.0"
+      }
+    },
     "node_modules/saslprep": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz",
@@ -9235,6 +9302,15 @@
       "resolved": "https://registry.npmjs.org/traverse-chain/-/traverse-chain-0.1.0.tgz",
       "integrity": "sha1-YdvC1Ttp/2CRoSoWj9fUMxB+QPE="
     },
+    "node_modules/truncate-utf8-bytes": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz",
+      "integrity": "sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ==",
+      "dev": true,
+      "dependencies": {
+        "utf8-byte-length": "^1.0.1"
+      }
+    },
     "node_modules/ts-jest": {
       "version": "27.0.3",
       "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-27.0.3.tgz",
@@ -9538,6 +9614,12 @@
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz",
       "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0="
     },
+    "node_modules/utf8-byte-length": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/utf8-byte-length/-/utf8-byte-length-1.0.4.tgz",
+      "integrity": "sha512-4+wkEYLBbWxqTahEsWrhxepcoVOJ+1z5PGIjPZxRkytcdSUaNjIjBM7Xn8E+pdSuV7SzvWovBFA54FO0JSoqhA==",
+      "dev": true
+    },
     "node_modules/util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
@@ -11909,6 +11991,29 @@
       "integrity": "sha512-+q/t7Ekv1EDY2l6Gda6LLiX14rU9TV20Wa3ofeQmwPFZbOMo9DXrLbOjFaaclkXKWidIaopwAObQDqwWtGUjqg==",
       "dev": true
     },
+    "axios": {
+      "version": "0.27.2",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.27.2.tgz",
+      "integrity": "sha512-t+yRIyySRTp/wua5xEr+z1q60QmLq8ABsS5O9Me1AsE5dfKqgnCFzwiCZZ/cGNd1lq4/7akDWMxdhVlucjmnOQ==",
+      "dev": true,
+      "requires": {
+        "follow-redirects": "^1.14.9",
+        "form-data": "^4.0.0"
+      },
+      "dependencies": {
+        "form-data": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz",
+          "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==",
+          "dev": true,
+          "requires": {
+            "asynckit": "^0.4.0",
+            "combined-stream": "^1.0.8",
+            "mime-types": "^2.1.12"
+          }
+        }
+      }
+    },
     "babel-jest": {
       "version": "27.0.6",
       "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-27.0.6.tgz",
@@ -13167,6 +13272,12 @@
         "path-exists": "^4.0.0"
       }
     },
+    "follow-redirects": {
+      "version": "1.15.2",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz",
+      "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==",
+      "dev": true
+    },
     "form-data": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz",
@@ -15654,6 +15765,18 @@
       "integrity": "sha512-uW7fodD6pyW2FZNZnp/Z3hvWKeEW1Y8R1+1CnErE8cXFXzl5blBOoVB41CvMer6P6Q0S5FXDwcHgFd1Wj0U9zg==",
       "dev": true
     },
+    "nodejs-file-downloader": {
+      "version": "4.10.2",
+      "resolved": "https://registry.npmjs.org/nodejs-file-downloader/-/nodejs-file-downloader-4.10.2.tgz",
+      "integrity": "sha512-pTVlytER/4wxcIpEhLXoqhuJ7WH1+xSFNLbI0wPmbwH3pWlJRRebb1Kbu91mz1CyOJmO4sj6YLH1wkF1B6efrQ==",
+      "dev": true,
+      "requires": {
+        "follow-redirects": "^1.15.1",
+        "https-proxy-agent": "^5.0.0",
+        "mime-types": "^2.1.27",
+        "sanitize-filename": "^1.6.3"
+      }
+    },
     "nodemon": {
       "version": "2.0.19",
       "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-2.0.19.tgz",
@@ -16327,6 +16450,15 @@
       "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
     },
+    "sanitize-filename": {
+      "version": "1.6.3",
+      "resolved": "https://registry.npmjs.org/sanitize-filename/-/sanitize-filename-1.6.3.tgz",
+      "integrity": "sha512-y/52Mcy7aw3gRm7IrcGDFx/bCk4AhRh2eI9luHOQM86nZsqwiRkkq2GekHXBBD+SmPidc8i2PqtYZl+pWJ8Oeg==",
+      "dev": true,
+      "requires": {
+        "truncate-utf8-bytes": "^1.0.0"
+      }
+    },
     "saslprep": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/saslprep/-/saslprep-1.0.3.tgz",
@@ -16990,6 +17122,15 @@
       "resolved": "https://registry.npmjs.org/traverse-chain/-/traverse-chain-0.1.0.tgz",
       "integrity": "sha1-YdvC1Ttp/2CRoSoWj9fUMxB+QPE="
     },
+    "truncate-utf8-bytes": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/truncate-utf8-bytes/-/truncate-utf8-bytes-1.0.2.tgz",
+      "integrity": "sha512-95Pu1QXQvruGEhv62XCMO3Mm90GscOCClvrIUwCM0PYOXK3kaF3l3sIHxx71ThJfcbM2O5Au6SO3AWCSEfW4mQ==",
+      "dev": true,
+      "requires": {
+        "utf8-byte-length": "^1.0.1"
+      }
+    },
     "ts-jest": {
       "version": "27.0.3",
       "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-27.0.3.tgz",
@@ -17212,6 +17353,12 @@
         }
       }
     },
+    "utf8-byte-length": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/utf8-byte-length/-/utf8-byte-length-1.0.4.tgz",
+      "integrity": "sha512-4+wkEYLBbWxqTahEsWrhxepcoVOJ+1z5PGIjPZxRkytcdSUaNjIjBM7Xn8E+pdSuV7SzvWovBFA54FO0JSoqhA==",
+      "dev": true
+    },
     "util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",

api/package.json

@@ -4,7 +4,7 @@
   "description": "Api of SASjs server",
   "main": "./src/server.ts",
   "scripts": {
-    "initial": "npm run swagger && npm run compileSysInit && npm run copySASjsCore",
+    "initial": "npm run swagger && npm run compileSysInit && npm run copySASjsCore && npm run downloadMacros",
     "prestart": "npm run initial",
     "prebuild": "npm run initial",
     "start": "NODE_ENV=development nodemon ./src/server.ts",
@@ -17,20 +17,21 @@
     "lint:fix": "npx prettier --write \"src/**/*.{ts,tsx,js,jsx,html,css,sass,less,yml,md,graphql}\"",
     "lint": "npx prettier --check \"src/**/*.{ts,tsx,js,jsx,html,css,sass,less,yml,md,graphql}\"",
     "exe": "npm run build && pkg .",
-    "copy:files": "npm run public:copy && npm run sasjsbuild:copy && npm run sasjscore:copy && npm run web:copy",
+    "copy:files": "npm run public:copy && npm run sasjsbuild:copy && npm run sas:copy && npm run web:copy",
     "public:copy": "cp -r ./public/ ./build/public/",
     "sasjsbuild:copy": "cp -r ./sasjsbuild/ ./build/sasjsbuild/",
-    "sasjscore:copy": "cp -r ./sasjscore/ ./build/sasjscore/",
+    "sas:copy": "cp -r ./sas/ ./build/sas/",
     "web:copy": "rimraf web && mkdir web && cp -r ../web/build/ ./web/build/",
     "compileSysInit": "ts-node ./scripts/compileSysInit.ts",
-    "copySASjsCore": "ts-node ./scripts/copySASjsCore.ts"
+    "copySASjsCore": "ts-node ./scripts/copySASjsCore.ts",
+    "downloadMacros": "ts-node ./scripts/downloadMacros.ts"
   },
   "bin": "./build/src/server.js",
   "pkg": {
     "assets": [
       "./build/public/**/*",
       "./build/sasjsbuild/**/*",
-      "./build/sasjscore/**/*",
+      "./build/sas/**/*",
       "./web/build/**/*"
     ],
     "targets": [
@@ -84,11 +85,13 @@
     "@types/swagger-ui-express": "^4.1.3",
     "@types/unzipper": "^0.10.5",
     "adm-zip": "^0.5.9",
+    "axios": "0.27.2",
     "csrf": "^3.1.0",
     "dotenv": "^10.0.0",
     "http-headers-validation": "^0.0.1",
     "jest": "^27.0.6",
     "mongodb-memory-server": "^8.0.0",
+    "nodejs-file-downloader": "4.10.2",
     "nodemon": "^2.0.7",
     "pkg": "5.6.0",
     "prettier": "^2.3.1",

api/scripts/downloadMacros.ts

@@ -0,0 +1,39 @@
+import axios from 'axios'
+import Downloader from 'nodejs-file-downloader'
+import { createFile, listFilesInFolder } from '@sasjs/utils'
+
+import { sasJSCoreMacros, sasJSCoreMacrosInfo } from '../src/utils/file'
+
+export const downloadMacros = async () => {
+  const url =
+    'https://api.github.com/repos/yabwon/SAS_PACKAGES/contents/SPF/Macros'
+
+  console.info(`Downloading macros from ${url}`)
+
+  await axios
+    .get(url)
+    .then(async (res) => {
+      await downloadFiles(res.data)
+    })
+    .catch((err) => {
+      throw new Error(err)
+    })
+}
+
+const downloadFiles = async function (fileList: any) {
+  for (const file of fileList) {
+    const downloader = new Downloader({
+      url: file.download_url,
+      directory: sasJSCoreMacros,
+      fileName: file.path.replace(/^SPF\/Macros/, ''),
+      cloneFiles: false
+    })
+    await downloader.download()
+  }
+
+  const fileNames = await listFilesInFolder(sasJSCoreMacros)
+
+  await createFile(sasJSCoreMacrosInfo, fileNames.join('\n'))
+}
+
+downloadMacros()

api/src/controllers/internal/Session.ts

@@ -3,6 +3,7 @@ import { Session } from '../../types'
 import { promisify } from 'util'
 import { execFile } from 'child_process'
 import {
+  getPackagesFolder,
   getSessionsFolder,
   generateUniqueFileName,
   sysInitCompiledPath,
@@ -104,7 +105,8 @@ export class SASSessionController extends SessionController {
 
     // the autoexec file is executed on SAS startup
     const autoExecPath = path.join(sessionFolder, 'autoexec.sas')
-    const contentForAutoExec = `/* compiled systemInit */
+    const contentForAutoExec = `filename packages "${getPackagesFolder()}";
+/* compiled systemInit */
 ${compiledSystemInitContent}
 /* autoexec */
 ${autoExecContent}`

api/src/utils/file.ts

@@ -10,7 +10,7 @@ export const sysInitCompiledPath = path.join(
   'systemInitCompiled.sas'
 )
 
-export const sasJSCoreMacros = path.join(apiRoot, 'sasjscore')
+export const sasJSCoreMacros = path.join(apiRoot, 'sas', 'sasautos')
 export const sasJSCoreMacrosInfo = path.join(sasJSCoreMacros, '.macrolist')
 
 export const getWebBuildFolder = () => path.join(codebaseRoot, 'web', 'build')
@@ -28,7 +28,10 @@ export const getAppStreamConfigPath = () =>
   path.join(getSasjsRootFolder(), 'appStreamConfig.json')
 
 export const getMacrosFolder = () =>
-  path.join(getSasjsRootFolder(), 'sasjscore')
+  path.join(getSasjsRootFolder(), 'sas', 'sasautos')
+
+export const getPackagesFolder = () =>
+  path.join(getSasjsRootFolder(), 'sas', 'sas_packages')
 
 export const getUploadsFolder = () => path.join(getSasjsRootFolder(), 'uploads')
 

api/src/utils/setupFolders.ts

@@ -1,10 +1,15 @@
 import { createFile, createFolder, fileExists } from '@sasjs/utils'
-import { getDesktopUserAutoExecPath, getFilesFolder } from './file'
+import {
+  getDesktopUserAutoExecPath,
+  getFilesFolder,
+  getPackagesFolder
+} from './file'
 import { ModeType } from './verifyEnvVariables'
 
 export const setupFolders = async () => {
   const drivePath = getFilesFolder()
   await createFolder(drivePath)
+  await createFolder(getPackagesFolder())
 
   if (process.env.MODE === ModeType.Desktop) {
     if (!(await fileExists(getDesktopUserAutoExecPath()))) {