Merge pull request #12 from sasjs/issue-8

Issue 8

.github/workflows/build.yml

@@ -0,0 +1,30 @@
+name: SASjs Server Build
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [12.x]
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: Install Dependencies
+        run: npm ci
+      - name: Check Code Style
+        run: npm run lint
+      - name: Run Unit Tests
+        run: npm test
+      - name: Build Package
+        run: npm run package:lib
+        env:
+          CI: true

.github/workflows/npmpublish.yml

@@ -1,22 +0,0 @@
-name: SASjs Server Deploy
-
-on:
-  push:
-    branches:
-      - master
-
-jobs:
-  release:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Install dependencies
-        run: npm ci
-      - name: Build Project
-        run: npm run build
-      - name: Semantic Release
-        uses: cycjimmy/semantic-release-action@v2
-        env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}

.gitignore

@@ -4,4 +4,5 @@ node_modules/
 .DS_Store
 .env*
 sas/
-tmp/
+tmp/
+build/

README.md

@@ -0,0 +1 @@
+# @sasjs/server

jest.config.js

@@ -1,13 +1,15 @@
 module.exports = {
-  preset: 'ts-jest',
+  preset: 'ts-jest/presets/js-with-ts',
   testEnvironment: 'node',
-  coverageThreshold: {
+  // FIXME: improve test coverage and uncomment below lines
-    global: {
+  // coverageThreshold: {
-      branches: 80,
+  //   global: {
-      functions: 80,
+  //     branches: 80,
-      lines: 80,
+  //     functions: 80,
-      statements: -10
+  //     lines: 80,
-    }
+  //     statements: -10
-  },
+  //   }
-  collectCoverageFrom: ['src/**/{!(index),}.ts']
+  // },
+  collectCoverageFrom: ['src/**/{!(index),}.ts'],
+  testPathIgnorePatterns: ['/node_modules/', '<rootDir>/build/']
 }

package-lock.json

@@ -2338,11 +2338,6 @@
       "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==",
       "dev": true
     },
-    "child_process": {
-      "version": "1.0.2",
-      "resolved": "https://registry.npmjs.org/child_process/-/child_process-1.0.2.tgz",
-      "integrity": "sha1-sffn/HPSXn/R1FWtyU4UODAYK1o="
-    },
     "chokidar": {
       "version": "3.5.1",
       "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.1.tgz",

package.json

@@ -4,13 +4,14 @@
   "description": "SASjs server",
   "main": "./src/server.ts",
   "scripts": {
-    "start": "nodemon ./src/index.ts",
+    "start": "nodemon ./src/server.ts",
-    "build": "tsc --project ./",
+    "build": "rimraf build && tsc",
     "semantic-release": "semantic-release -d",
     "prepare": "[ -d .git ] && git config core.hooksPath ./.git-hooks || true",
-    "test": "jest --coverage",
+    "test": "mkdir -p tmp && jest --coverage",
     "lint:fix": "npx prettier --write \"src/**/*.{ts,tsx,js,jsx,html,css,sass,less,yml,md,graphql}\"",
-    "lint": "npx prettier --check \"src/**/*.{ts,tsx,js,jsx,html,css,sass,less,yml,md,graphql}\""
+    "lint": "npx prettier --check \"src/**/*.{ts,tsx,js,jsx,html,css,sass,less,yml,md,graphql}\"",
+    "package:lib": "npm run build && cp ./package.json build && cp README.md build && cd build && npm version \"5.0.0\" && npm pack"
   },
   "release": {
     "branches": [
@@ -20,7 +21,6 @@
   "author": "Analytium Ltd",
   "dependencies": {
     "@sasjs/utils": "^2.23.3",
-    "child_process": "^1.0.2",
     "express": "^4.17.1"
   },
   "devDependencies": {
@@ -31,10 +31,14 @@
     "jest": "^27.0.6",
     "nodemon": "^2.0.7",
     "prettier": "^2.3.1",
+    "rimraf": "^3.0.2",
     "semantic-release": "^17.4.3",
     "supertest": "^6.1.3",
     "ts-jest": "^27.0.3",
     "ts-node": "^10.0.0",
     "typescript": "^4.3.2"
+  },
+  "configuration": {
+    "sasPath": "C:\\Program Files\\SASHome\\SASFoundation\\9.4\\sas.exe"
   }
 }

src/app.ts

@@ -3,7 +3,7 @@ import indexRouter from './routes'
 
 const app = express()
 
-app.use(express.json())
+app.use(express.json({ limit: '50mb' }))
 
 app.use('/', indexRouter)
 

src/controllers/deploy.ts

@@ -34,26 +34,23 @@ export const createFileTree = async (
 }
 
 export const getTreeExample = () => ({
-  message: 'Provided not supported data format.',
+  members: [
-  supportedFormat: {
+    {
-    members: [
+      name: 'jobs',
-      {
+      type: 'folder',
-        name: 'jobs',
+      members: [
-        type: 'folder',
+        {
-        members: [
+          name: 'extract',
-          {
+          type: 'folder',
-            name: 'extract',
+          members: [
-            type: 'folder',
+            {
-            members: [
+              name: 'makedata1',
-              {
+              type: 'service',
-                name: 'makedata1',
+              code: '%put Hello World!;'
-                type: 'service',
+            }
-                code: '%put Hello World!;'
+          ]
-              }
+        }
-            ]
+      ]
-          }
+    }
-        ]
+  ]
-      }
-    ]
-  }
 })

src/controllers/sas.ts

@@ -0,0 +1,95 @@
+import { readFile, deleteFile, fileExists, createFile } from '@sasjs/utils'
+import path from 'path'
+import { ExecutionResult, ExecutionQuery } from '../types'
+import {
+  getTmpFilesFolderPath,
+  getTmpLogFolderPath,
+  getTmpWeboutFolderPath,
+  generateUniqueFileName
+} from '../utils'
+import { configuration } from '../../package.json'
+import { promisify } from 'util'
+import { execFile } from 'child_process'
+const execFilePromise = promisify(execFile)
+
+export const processSas = async (query: ExecutionQuery): Promise<any> => {
+  const sasCodePath = path
+    .join(getTmpFilesFolderPath(), query._program)
+    .replace(new RegExp('/', 'g'), path.sep)
+
+  if (!(await fileExists(sasCodePath))) {
+    return Promise.reject({ error: 'SAS file does not exist.' })
+  }
+
+  const sasFile: string = sasCodePath.split(path.sep).pop() || 'default'
+
+  const sasLogPath = path.join(
+    getTmpLogFolderPath(),
+    generateUniqueFileName(sasFile.replace(/\.sas/g, ''), '.log')
+  )
+
+  const sasWeboutPath = path.join(
+    getTmpWeboutFolderPath(),
+    generateUniqueFileName(sasFile.replace(/\.sas/g, ''), '.json')
+  )
+
+  let sasCode = await readFile(sasCodePath)
+
+  const vars: any = query
+  Object.keys(query).forEach(
+    (key: string) => (sasCode = `%let ${key}=${vars[key]};\n${sasCode}`)
+  )
+
+  sasCode = `filename _webout "${sasWeboutPath}";\n${sasCode}`
+
+  const tmpSasCodePath = sasCodePath.replace(
+    sasFile,
+    generateUniqueFileName(sasFile)
+  )
+
+  await createFile(tmpSasCodePath, sasCode)
+
+  const { stdout, stderr } = await execFilePromise(configuration.sasPath, [
+    '-SYSIN',
+    tmpSasCodePath,
+    '-log',
+    sasLogPath,
+    process.platform === 'win32' ? '-nosplash' : ''
+  ]).catch((err) => ({ stderr: err, stdout: '' }))
+
+  let log = ''
+  if (sasLogPath && (await fileExists(sasLogPath))) {
+    log = await readFile(sasLogPath)
+  }
+
+  await deleteFile(sasLogPath)
+  await deleteFile(tmpSasCodePath)
+
+  if (stderr) return Promise.reject({ error: stderr, log: log })
+
+  if (await fileExists(sasWeboutPath)) {
+    let webout = await readFile(sasWeboutPath)
+
+    await deleteFile(sasWeboutPath)
+
+    const debug = Object.keys(query).find(
+      (key: string) => key.toLowerCase() === '_debug'
+    )
+
+    if (debug && (query as any)[debug] >= 131) {
+      webout = `<html><body>
+${webout}
+<div style="text-align:left">
+<hr /><h2>SAS Log</h2>
+<pre>${log}</pre>
+</div>
+</body></html>`
+    }
+
+    return Promise.resolve(webout)
+  } else {
+    return Promise.resolve({
+      log: log
+    })
+  }
+}

src/routes/index.ts

@@ -21,19 +21,61 @@ router.get('/', async (req, res) => {
 })
 
 router.post('/deploy', async (req, res) => {
-  if (!isFileTree(req.body)) {
+  if (!isFileTree({ members: req.body.members })) {
-    res.status(400).send(getTreeExample())
+    res.status(400).send({
+      status: 'failure',
+      message: 'Provided not supported data format.',
+      example: getTreeExample()
+    })
 
     return
   }
 
-  await createFileTree(req.body.members)
+  await createFileTree(
+    req.body.members,
+    req.body.appLoc ? req.body.appLoc.replace(/^\//, '').split('/') : []
+  )
     .then(() => {
-      res.status(200).send('Files deployed successfully to @sasjs/server.')
+      res.status(200).send({
+        status: 'success',
+        message: 'Files deployed successfully to @sasjs/server.'
+      })
     })
     .catch((err) => {
-      res.status(500).send({ message: 'Deployment failed!', ...err })
+      res
+        .status(500)
+        .send({ status: 'failure', message: 'Deployment failed!', ...err })
     })
 })
 
+// TODO: respond with HTML page including file tree
+router.get('/SASjsExecutor', async (req, res) => {
+  res.status(200).send({ status: 'success', tree: {} })
+})
+
+router.get('/SASjsExecutor/do', async (req, res) => {
+  const queryEntries = Object.keys(req.query).map((entry: string) =>
+    entry.toLowerCase()
+  )
+
+  if (isRequestQuery(req.query)) {
+    await processSas({ ...req.query })
+      .then((result) => {
+        res.status(200).send(result)
+      })
+      .catch((err) => {
+        res.status(400).send({
+          status: 'failure',
+          message: 'Job execution failed.',
+          ...err
+        })
+      })
+  } else {
+    res.status(400).send({
+      status: 'failure',
+      message: `Please provide the location of SAS code`
+    })
+  }
+})
+
 export default router

src/routes/spec/routes.spec.ts

@@ -10,7 +10,11 @@ describe('deploy', () => {
     const res = await request(app).post('/deploy').send(payload)
 
     expect(res.statusCode).toEqual(400)
-    expect(res.body).toEqual(getTreeExample())
+    expect(res.body).toEqual({
+      status: 'failure',
+      message: 'Provided not supported data format.',
+      example: getTreeExample()
+    })
   }
 
   it('should respond with payload example if valid payload was not provided', async () => {
@@ -70,12 +74,12 @@ describe('deploy', () => {
   })
 
   it('should respond with payload example if valid payload was not provided', async () => {
-    const res = await request(app)
+    const res = await request(app).post('/deploy').send(getTreeExample())
-      .post('/deploy')
-      .send(getTreeExample().supportedFormat)
 
     expect(res.statusCode).toEqual(200)
-    expect(res.text).toEqual('Files deployed successfully to @sasjs/server.')
+    expect(res.text).toEqual(
+      '{"status":"success","message":"Files deployed successfully to @sasjs/server."}'
+    )
     await expect(folderExists(getTmpFilesFolderPath())).resolves.toEqual(true)
 
     const testJobFolder = path.join(getTmpFilesFolderPath(), 'jobs', 'extract')
@@ -83,13 +87,13 @@ describe('deploy', () => {
 
     const testJobFile = path.join(
       testJobFolder,
-      getTreeExample().supportedFormat.members[0].members[0].members[0].name
+      getTreeExample().members[0].members[0].members[0].name
     )
 
     await expect(fileExists(testJobFile)).resolves.toEqual(true)
 
     await expect(readFile(testJobFile)).resolves.toEqual(
-      getTreeExample().supportedFormat.members[0].members[0].members[0].code
+      getTreeExample().members[0].members[0].members[0].code
     )
 
     await deleteFolder(getTmpFilesFolderPath())

src/server.ts

@@ -1,6 +1,6 @@
 import app from './app'
 
 const port = 5000
-app.listen(port, () => {
+const listener = app.listen(port, () => {
   console.log(`⚡️[server]: Server is running at http://localhost:${port}`)
 })

src/types/request.ts

@@ -1,6 +1,9 @@
-export interface RequestQuery {
+import { MacroVars } from '@sasjs/utils'
+export interface ExecutionQuery {
   _program: string
+  macroVars?: MacroVars
+  _debug?: boolean
 }
 
-export const isRequestQuery = (arg: any): arg is RequestQuery =>
+export const isRequestQuery = (arg: any): arg is ExecutionQuery =>
   arg && !Array.isArray(arg) && typeof arg._program === 'string'

src/types/sas.ts

@@ -1,4 +1,5 @@
 export interface ExecutionResult {
-  log: string
+  webout?: string
-  logPath: string
+  log?: string
+  logPath?: string
 }

src/utils/file.ts

@@ -1,8 +1,23 @@
 import path from 'path'
-import { getRealPath } from '@sasjs/utils'
+import { getRealPath, generateTimestamp } from '@sasjs/utils'
 
 export const getTmpFolderPath = () =>
   getRealPath(path.join(__dirname, '..', '..', 'tmp'))
 
 export const getTmpFilesFolderPath = () =>
   path.join(getTmpFolderPath(), 'files')
+
+export const getTmpLogFolderPath = () => path.join(getTmpFolderPath(), 'logs')
+
+export const getTmpWeboutFolderPath = () =>
+  path.join(getTmpFolderPath(), 'webouts')
+
+export const generateUniqueFileName = (fileName: string, extension = '') =>
+  [
+    fileName,
+    '-',
+    Math.round(Math.random() * 100000),
+    '-',
+    generateTimestamp(),
+    extension
+  ].join('')

src/utils/index.ts

@@ -0,0 +1 @@
+export * from './file'

tsconfig.json

@@ -3,8 +3,9 @@
     "target": "es5",
     "module": "commonjs",
     "rootDir": "./",
-    "outDir": "./dist",
+    "outDir": "./build",
     "esModuleInterop": true,
-    "strict": true
+    "strict": true,
+    "resolveJsonModule": true
   }
-}
+}