chore(release): 0.0.30

fix: macros are available Sessions with SASAUTOS

.gitignore

@@ -7,6 +7,7 @@ sas/
 tmp/
 build/
 sasjsbuild/
+sasjscore/
 certificates/
 executables/
 .env

CHANGELOG.md

@@ -2,6 +2,32 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+### [0.0.30](https://github.com/sasjs/server/compare/v0.0.29...v0.0.30) (2022-03-05)
+
+
+### Features
+
+* parse log to array ([c5ad72c](https://github.com/sasjs/server/commit/c5ad72c931ec8fbd7d5a6475838adcbd380c8aee))
+* set response headers provded by SAS Code execution ([2c4aa42](https://github.com/sasjs/server/commit/2c4aa420b3119890cafde4265ed5dddbc9d6a636))
+
+
+### Bug Fixes
+
+* added http headers to /code api as well ([da899b9](https://github.com/sasjs/server/commit/da899b90e26d5ee393eefc302be985eb7c9055a5))
+* code api is updated return type ([e2a6810](https://github.com/sasjs/server/commit/e2a6810e9531a8102d3c51fd8df2e1f78f0d965f))
+* **file:** fixes response headers ([ef41691](https://github.com/sasjs/server/commit/ef41691e408ef1c1c7a921cc1050bdd533651331))
+* get file instead of it's content ([efaf38d](https://github.com/sasjs/server/commit/efaf38d3039391392ce0e14a3accddd8f34ea7d6))
+* hot fix for web component ([0a4b202](https://github.com/sasjs/server/commit/0a4b202428e14effc8014a6813cecf7761ce3715))
+* improvement in flow of uploading ([8c1941a](https://github.com/sasjs/server/commit/8c1941a87bc184be4e0e09eeff73fc6cb69e3041))
+* macros are available Sessions with SASAUTOS ([95843fa](https://github.com/sasjs/server/commit/95843fa4c711aa695ee63ad265b8def4ba56360d))
+* minor changes ([0b5f958](https://github.com/sasjs/server/commit/0b5f958f456d291ec7a8697236657c7819d5c654))
+* multi-part file upload + validations + specs ([e60f172](https://github.com/sasjs/server/commit/e60f17268d1fa9ab623313026d46bd3f63756f69))
+* organized code for usage of multer ([ce0a5e1](https://github.com/sasjs/server/commit/ce0a5e1229bed69c450061fac2bc19711448da56))
+* return buffer in case of file response ([3e6234e](https://github.com/sasjs/server/commit/3e6234e6019c5f3ae4280fac079ecc9cb0effc07))
+* **stp:** return json for webout ([5005f20](https://github.com/sasjs/server/commit/5005f203b8d6b1d577cdf094b83886bd1fc817a2))
+* updating docs ([7312763](https://github.com/sasjs/server/commit/7312763339d6769826328561e2c8d11bbfc0c9f4))
+* **upload:** added query param as well for filepath ([feeec4e](https://github.com/sasjs/server/commit/feeec4eb149e9a47e5a52320d1fc95243bf5eb15))
+
 ### [0.0.29](https://github.com/sasjs/server/compare/v0.0.28...v0.0.29) (2022-02-16)
 
 

README.md

@@ -10,6 +10,31 @@ One major benefit of using SASjs Server (alongside other components of the SASjs
 
 SASjs Server is available in two modes - Desktop (without authentication) and Server (with authentiation, and a database)
 
+
+## Configuration
+
+When launching the app, it will make use of specific environment variables. These can be set in the following places:
+
+- Configured globally in /etc/environment file
+- Export in terminal or shell script (`export VAR=VALUE`)
+- Prepend in command
+- Enter in the `.env` file alongside the executable
+
+Example variables:
+
+```
+MODE=[desktop|server] default considered as desktop
+CORS=[disable|enable] default considered as disable
+PROTOCOL=[http|https] default considered as http
+PORT=[5000] default value is 5000
+PORT_WEB=[port for sasjs web component(react)] default value is 3000
+SAS_PATH=/path/to/sas/executable.exe
+DRIVE_PATH=./tmp
+PROTOCOL=[http|https] default considered as http. Use pems below if htttps.
+PRIVATE_KEY=privkey.pem
+FULL_CHAIN=fullchain.pem
+```
+
 ## Desktop Version
 
 ### Manual Installation
@@ -18,7 +43,7 @@ Download the relevant package from the [releases](https://github.com/sasjs/serve
 
 Next, trigger by double clicking (windows) or executing from commandline.
 
-You are presented with two prompts:
+You are presented with two prompts (if not set as ENV vars):
 
 - Location of your `sas.exe` / `sas.sh` executable
 - Path to a filesystem location for Stored Programs and temporary files
@@ -32,24 +57,7 @@ curl -L https://github.com/sasjs/server/releases/latest/download/linux.zip > lin
 unzip linux.zip
 ```
 
-The app can then be launched with `./api-linux` and prompts followed.
-
-When launching the app, it will make use of specific environment variables. These can be set in the following places:
-
-- Configured globally in /etc/environment file
-- Export in terminal or shell script (`export VAR=VALUE`)
-- Prepend in command
-- Enter in the `.env` file alongside the executable
-
-Example variables:
-
-```
-PORT=5004
-SAS_PATH=/path/to/sas/executable.exe
-DRIVE_PATH=./tmp
-```
-
-Setting these prompts variables will avoid the need for prompts.
+The app can then be launched with `./api-linux` and prompts followed (if ENV vars not set).
 
 Normally the server process will stop when your terminal dies. To keep it going you can use the npm package [pm2](https://www.npmjs.com/package/pm2) (`npm install pm2@latest -g`) as follows:
 
@@ -84,7 +92,6 @@ Instead of `app_name` you can pass:
 - `id` to act on a specific process id
 
 
-
 ## Server Version
 
 The following credentials can be used for the initial connection to SASjs/server.  It is recommended to change these on first use.

api/.vscode/launch.json

@@ -0,0 +1,13 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Launch via NPM",
+      "request": "launch",
+      "runtimeArgs": ["run-script", "start"],
+      "runtimeExecutable": "npm",
+      "skipFiles": ["<node_internals>/**"],
+      "type": "pwa-node"
+    }
+  ]
+}

api/package-lock.json

@@ -38,6 +38,7 @@
         "@types/supertest": "^2.0.11",
         "@types/swagger-ui-express": "^4.1.3",
         "dotenv": "^10.0.0",
+        "http-headers-validation": "^0.0.1",
         "jest": "^27.0.6",
         "mongodb-memory-server": "^8.0.0",
         "nodemon": "^2.0.7",
@@ -4651,6 +4652,12 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/http-headers-validation": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/http-headers-validation/-/http-headers-validation-0.0.1.tgz",
+      "integrity": "sha1-0xUbTFjQjySTSnbKgYVIzPBa+3g=",
+      "dev": true
+    },
     "node_modules/http-proxy-agent": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz",
@@ -13747,6 +13754,12 @@
         "toidentifier": "1.0.0"
       }
     },
+    "http-headers-validation": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/http-headers-validation/-/http-headers-validation-0.0.1.tgz",
+      "integrity": "sha1-0xUbTFjQjySTSnbKgYVIzPBa+3g=",
+      "dev": true
+    },
     "http-proxy-agent": {
       "version": "4.0.1",
       "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-4.0.1.tgz",

api/package.json

@@ -4,7 +4,7 @@
   "description": "Api of SASjs server",
   "main": "./src/server.ts",
   "scripts": {
-    "initial": "npm run swagger && npm run compileSysInit",
+    "initial": "npm run swagger && npm run compileSysInit && npm run copySASjsCore",
     "prestart": "npm run initial",
     "prebuild": "npm run initial",
     "start": "nodemon ./src/server.ts",
@@ -16,11 +16,13 @@
     "lint": "npx prettier --check \"src/**/*.{ts,tsx,js,jsx,html,css,sass,less,yml,md,graphql}\"",
     "package:lib": "npm run build && cp ./package.json build && cp README.md build && cd build && npm version \"5.0.0\" && npm pack",
     "exe": "npm run build && npm run exe:copy && pkg .",
-    "exe:copy": "npm run public:copy && npm run sasjsbuild:copy && npm run web:copy",
+    "exe:copy": "npm run public:copy && npm run sasjsbuild:copy && npm run sasjscore:copy && npm run web:copy",
     "public:copy": "cp -r ./public/ ./build/public/",
     "sasjsbuild:copy": "cp -r ./sasjsbuild/ ./build/sasjsbuild/",
+    "sasjscore:copy": "cp -r ./sasjscore/ ./build/sasjscore/",
     "web:copy": "rimraf web && mkdir web && cp -r ../web/build/ ./web/build/",
-    "compileSysInit": "ts-node ./scripts/compileSysInit.ts"
+    "compileSysInit": "ts-node ./scripts/compileSysInit.ts",
+    "copySASjsCore": "ts-node ./scripts/copySASjsCore.ts"
   },
   "bin": "./build/src/server.js",
   "pkg": {
@@ -70,6 +72,7 @@
     "@types/supertest": "^2.0.11",
     "@types/swagger-ui-express": "^4.1.3",
     "dotenv": "^10.0.0",
+    "http-headers-validation": "^0.0.1",
     "jest": "^27.0.6",
     "mongodb-memory-server": "^8.0.0",
     "nodemon": "^2.0.7",

api/public/swagger.yaml

@@ -92,6 +92,48 @@ components:
                 - clientSecret
             type: object
             additionalProperties: false
+        IRecordOfAny:
+            properties: {}
+            type: object
+            additionalProperties: {}
+        LogLine:
+            properties:
+                line:
+                    type: string
+            required:
+                - line
+            type: object
+            additionalProperties: false
+        HTTPHeaders:
+            properties: {}
+            type: object
+            additionalProperties:
+                type: string
+        ExecuteReturnJsonResponse:
+            properties:
+                status:
+                    type: string
+                _webout:
+                    anyOf:
+                        -
+                            type: string
+                        -
+                            $ref: '#/components/schemas/IRecordOfAny'
+                log:
+                    items:
+                        $ref: '#/components/schemas/LogLine'
+                    type: array
+                message:
+                    type: string
+                httpHeaders:
+                    $ref: '#/components/schemas/HTTPHeaders'
+            required:
+                - status
+                - _webout
+                - log
+                - httpHeaders
+            type: object
+            additionalProperties: false
         ExecuteSASCodePayload:
             properties:
                 code:
@@ -181,18 +223,6 @@ components:
                 - fileTree
             type: object
             additionalProperties: false
-        GetFileResponse:
-            properties:
-                status:
-                    type: string
-                fileContent:
-                    type: string
-                message:
-                    type: string
-            required:
-                - status
-            type: object
-            additionalProperties: false
         UpdateFileResponse:
             properties:
                 status:
@@ -368,21 +398,6 @@ components:
                 - description
             type: object
             additionalProperties: false
-        ExecuteReturnJsonResponse:
-            properties:
-                status:
-                    type: string
-                _webout:
-                    type: string
-                log:
-                    type: string
-                message:
-                    type: string
-            required:
-                - status
-                - _webout
-            type: object
-            additionalProperties: false
         ExecuteReturnJsonPayload:
             properties:
                 _program:
@@ -520,7 +535,7 @@ paths:
                     content:
                         application/json:
                             schema:
-                                type: string
+                                $ref: '#/components/schemas/ExecuteReturnJsonResponse'
             description: 'Execute SAS code.'
             summary: 'Run SAS Code and returns log'
             tags:
@@ -583,24 +598,8 @@ paths:
         get:
             operationId: GetFile
             responses:
-                '200':
-                    description: Ok
-                    content:
-                        application/json:
-                            schema:
-                                $ref: '#/components/schemas/GetFileResponse'
-                            examples:
-                                'Example 1':
-                                    value: {status: success, fileContent: 'Contents of the File'}
-                '400':
-                    description: 'Unable to get File'
-                    content:
-                        application/json:
-                            schema:
-                                $ref: '#/components/schemas/GetFileResponse'
-                            examples:
-                                'Example 1':
-                                    value: {status: failure, message: 'File request failed.'}
+                '204':
+                    description: 'No content'
             summary: 'Get file from SASjs Drive'
             tags:
                 - Drive
@@ -636,19 +635,36 @@ paths:
                             examples:
                                 'Example 1':
                                     value: {status: failure, message: 'File request failed.'}
+            description: "It's optional to either provide `_filePath` in url as query parameter\nOr provide `filePath` in body as form field.\nBut it's required to provided else API will respond with Bad Request."
             summary: 'Create a file in SASjs Drive'
             tags:
                 - Drive
             security:
                 -
                     bearerAuth: []
-            parameters: []
+            parameters:
+                -
+                    description: 'Location of SAS program'
+                    in: query
+                    name: _filePath
+                    required: false
+                    schema:
+                        type: string
+                    example: /Public/somefolder/some.file.sas
             requestBody:
                 required: true
                 content:
-                    application/json:
+                    multipart/form-data:
                         schema:
-                            $ref: '#/components/schemas/FilePayload'
+                            type: object
+                            properties:
+                                file:
+                                    type: string
+                                    format: binary
+                                filePath:
+                                    type: string
+                            required:
+                                - file
         patch:
             operationId: UpdateFile
             responses:
@@ -1035,9 +1051,11 @@ paths:
                     content:
                         application/json:
                             schema:
-                                type: string
-            description: "Trigger a SAS program using it's location in the _program parameter.\nEnable debugging using the _debug parameter.\nAdditional URL parameters are turned into SAS macro variables.\nAny files provided are placed into the session and\ncorresponding _WEBIN_XXX variables are created."
-            summary: 'Execute Stored Program, return raw content'
+                                anyOf:
+                                    - {type: string}
+                                    - {type: string, format: byte}
+            description: "Trigger a SAS program using it's location in the _program URL parameter.\nEnable debugging using the _debug URL parameter.  Setting _debug=131 will\ncause the log to be streamed in the output.\n\nAdditional URL parameters are turned into SAS macro variables.\n\nAny files provided in the request body are placed into the SAS session with\ncorresponding _WEBIN_XXX variables created.\n\nThe response headers can be adjusted using the mfs_httpheader() macro.  Any\nfile type can be returned, including binary files such as zip or xls.\n\nThis behaviour differs for POST requests, in which case the reponse is\nalways JSON."
+            summary: 'Execute Stored Program, return raw _webout content.'
             tags:
                 - STP
             security:
@@ -1045,6 +1063,7 @@ paths:
                     bearerAuth: []
             parameters:
                 -
+                    description: 'Location of SAS program'
                     in: query
                     name: _program
                     required: true
@@ -1060,7 +1079,10 @@ paths:
                         application/json:
                             schema:
                                 $ref: '#/components/schemas/ExecuteReturnJsonResponse'
-            description: "Trigger a SAS program using it's location in the _program parameter.\nEnable debugging using the _debug parameter.\nAdditional URL parameters are turned into SAS macro variables.\nAny files provided are placed into the session and\ncorresponding _WEBIN_XXX variables are created."
+                            examples:
+                                'Example 1':
+                                    value: {status: success, _webout: 'webout content', log: [], httpHeaders: {Content-type: application/zip, Cache-Control: 'public, max-age=1000'}}
+            description: "Trigger a SAS program using it's location in the _program URL parameter.\nEnable debugging using the _debug URL parameter.  In any case, the log is\nalways returned in the log object.\n\nAdditional URL parameters are turned into SAS macro variables.\n\nAny files provided in the request body are placed into the SAS session with\ncorresponding _WEBIN_XXX variables created.\n\nThe response will be a JSON object with the following root attributes: log,\nwebout, headers.\n\nThe webout will be a nested JSON object ONLY if the response-header\ncontains a content-type of application/json AND it is valid JSON.\nOtherwise it will be a stringified version of the webout content.\n\nResponse headers from the mfs_httpheader macro are simply listed in the\nheaders object, for POST requests they have no effect on the actual\nresponse header."
             summary: 'Execute Stored Program, return JSON'
             tags:
                 - STP
@@ -1069,6 +1091,7 @@ paths:
                     bearerAuth: []
             parameters:
                 -
+                    description: 'Location of SAS program'
                     in: query
                     name: _program
                     required: false

api/scripts/compileSysInit.ts

@@ -21,7 +21,6 @@ const compiledSystemInit = async (systemInit: string) =>
   }))
 
 const createSysInitFile = async () => {
-  console.log('macroCorePath', macroCorePath)
   const systemInitContent = await readFile(
     path.join(__dirname, 'systemInit.sas')
   )

api/scripts/copySASjsCore.ts

@@ -0,0 +1,25 @@
+import path from 'path'
+import { asyncForEach, copy, createFolder, deleteFolder } from '@sasjs/utils'
+
+import { apiRoot, sasJSCoreMacros } from '../src/utils'
+
+const macroCorePath = path.join(apiRoot, 'node_modules', '@sasjs', 'core')
+
+export const copySASjsCore = async () => {
+  await deleteFolder(sasJSCoreMacros)
+  await createFolder(sasJSCoreMacros)
+
+  console.log('Copying SASjs Core Macros...')
+
+  const foldersToCopy = ['base', 'ddl', 'fcmp', 'lua', 'server']
+
+  await asyncForEach(foldersToCopy, async (coreSubFolder) => {
+    const coreSubFolderPath = path.join(macroCorePath, coreSubFolder)
+
+    await copy(coreSubFolderPath, sasJSCoreMacros)
+  })
+
+  console.log('Macros available at: ', sasJSCoreMacros)
+}
+
+copySASjsCore()

api/src/app.ts

@@ -1,12 +1,10 @@
 import path from 'path'
-import express from 'express'
+import express, { ErrorRequestHandler } from 'express'
 import morgan from 'morgan'
 import dotenv from 'dotenv'
 import cors from 'cors'
 
-import webRouter from './routes/web'
-import apiRouter from './routes/api'
-import { connectDB, getWebBuildFolderPath } from './utils'
+import { connectDB, getWebBuildFolderPath, setProcessVariables } from './utils'
 
 dotenv.config()
 
@@ -26,11 +24,21 @@ if (MODE?.trim() !== 'server' || CORS?.trim() === 'enable') {
 app.use(express.json({ limit: '50mb' }))
 app.use(morgan('tiny'))
 app.use(express.static(path.join(__dirname, '../public')))
-
-app.use('/', webRouter)
-app.use('/SASjsApi', apiRouter)
-app.use(express.json({ limit: '50mb' }))
-
 app.use(express.static(getWebBuildFolderPath()))
 
-export default connectDB().then(() => app)
+const onError: ErrorRequestHandler = (err, req, res, next) => {
+  console.error(err.stack)
+  res.status(500).send('Something broke!')
+}
+
+export default setProcessVariables().then(async () => {
+  // loading these modules after setting up variables due to
+  // multer's usage of process var process.driveLoc
+  const { setupRoutes } = await import('./routes/setupRoutes')
+  setupRoutes(app)
+
+  app.use(onError)
+
+  await connectDB()
+  return app
+})

api/src/controllers/code.ts

@@ -1,7 +1,9 @@
 import express from 'express'
 import { Request, Security, Route, Tags, Post, Body } from 'tsoa'
-import { ExecutionController } from './internal'
+import { ExecuteReturnJson, ExecutionController } from './internal'
 import { PreProgramVars } from '../types'
+import { ExecuteReturnJsonResponse } from '.'
+import { parseLogToArray } from '../utils'
 
 interface ExecuteSASCodePayload {
   /**
@@ -23,22 +25,28 @@ export class CodeController {
   public async executeSASCode(
     @Request() request: express.Request,
     @Body() body: ExecuteSASCodePayload
-  ): Promise<string> {
+  ): Promise<ExecuteReturnJsonResponse> {
     return executeSASCode(request, body)
   }
 }
 
 const executeSASCode = async (req: any, { code }: ExecuteSASCodePayload) => {
   try {
-    const result = await new ExecutionController().executeProgram(
-      code,
-      getPreProgramVariables(req),
-      { ...req.query, _debug: 131 },
-      undefined,
-      true
-    )
+    const { webout, log, httpHeaders } =
+      (await new ExecutionController().executeProgram(
+        code,
+        getPreProgramVariables(req),
+        { ...req.query, _debug: 131 },
+        undefined,
+        true
+      )) as ExecuteReturnJson
 
-    return result as string
+    return {
+      status: 'success',
+      _webout: webout as string,
+      log: parseLogToArray(log),
+      httpHeaders
+    }
   } catch (err: any) {
     throw {
       code: 400,

api/src/controllers/drive.ts

@@ -1,5 +1,8 @@
+import path from 'path'
+import express, { Express } from 'express'
 import {
   Security,
+  Request,
   Route,
   Tags,
   Example,
@@ -8,13 +11,14 @@ import {
   Response,
   Query,
   Get,
-  Patch
+  Patch,
+  UploadedFile,
+  FormField
 } from 'tsoa'
-import { fileExists, readFile, createFile } from '@sasjs/utils'
+import { fileExists, createFile, moveFile, createFolder } from '@sasjs/utils'
 import { createFileTree, ExecutionController, getTreeExample } from './internal'
 
 import { FileTree, isFileTree, TreeNode } from '../types'
-import path from 'path'
 import { getTmpFilesFolderPath } from '../utils'
 
 interface DeployPayload {
@@ -93,21 +97,22 @@ export class DriveController {
    * @query filePath Location of SAS program
    * @example filePath "/Public/somefolder/some.file"
    */
-  @Example<GetFileResponse>({
-    status: 'success',
-    fileContent: 'Contents of the File'
-  })
-  @Response<GetFileResponse>(400, 'Unable to get File', {
-    status: 'failure',
-    message: 'File request failed.'
-  })
   @Get('/file')
-  public async getFile(@Query() filePath: string): Promise<GetFileResponse> {
-    return getFile(filePath)
+  public async getFile(
+    @Request() request: express.Request,
+    @Query() filePath: string
+  ) {
+    return getFile(request, filePath)
   }
 
   /**
+   * It's optional to either provide `_filePath` in url as query parameter
+   * Or provide `filePath` in body as form field.
+   * But it's required to provided else API will respond with Bad Request.
+   *
    * @summary Create a file in SASjs Drive
+   * @param _filePath Location of SAS program
+   * @example _filePath "/Public/somefolder/some.file.sas"
    *
    */
   @Example<UpdateFileResponse>({
@@ -119,9 +124,11 @@ export class DriveController {
   })
   @Post('/file')
   public async saveFile(
-    @Body() body: FilePayload
+    @UploadedFile() file: Express.Multer.File,
+    @Query() _filePath?: string,
+    @FormField() filePath?: string
   ): Promise<UpdateFileResponse> {
-    return saveFile(body)
+    return saveFile((_filePath ?? filePath)!, file)
   }
 
   /**
@@ -172,47 +179,47 @@ const deploy = async (data: DeployPayload) => {
   return successDeployResponse
 }
 
-const getFile = async (filePath: string): Promise<GetFileResponse> => {
-  try {
-    const filePathFull = path
-      .join(getTmpFilesFolderPath(), filePath)
-      .replace(new RegExp('/', 'g'), path.sep)
+const getFile = async (req: express.Request, filePath: string) => {
+  const driveFilesPath = getTmpFilesFolderPath()
 
-    await validateFilePath(filePathFull)
-    const fileContent = await readFile(filePathFull)
+  const filePathFull = path
+    .join(getTmpFilesFolderPath(), filePath)
+    .replace(new RegExp('/', 'g'), path.sep)
 
-    return { status: 'success', fileContent: fileContent }
-  } catch (err: any) {
-    throw {
-      code: 400,
-      status: 'failure',
-      message: 'File request failed.',
-      error: typeof err === 'object' ? err.toString() : err
-    }
+  if (!filePathFull.includes(driveFilesPath)) {
+    throw new Error('Cannot get file outside drive.')
   }
+
+  if (!(await fileExists(filePathFull))) {
+    throw new Error('File does not exist.')
+  }
+
+  req.res?.download(filePathFull)
 }
 
-const saveFile = async (body: FilePayload): Promise<GetFileResponse> => {
-  const { filePath, fileContent } = body
-  try {
-    const filePathFull = path
-      .join(getTmpFilesFolderPath(), filePath)
-      .replace(new RegExp('/', 'g'), path.sep)
+const saveFile = async (
+  filePath: string,
+  multerFile: Express.Multer.File
+): Promise<GetFileResponse> => {
+  const driveFilesPath = getTmpFilesFolderPath()
 
-    if (await fileExists(filePathFull)) {
-      throw 'DriveController: File already exists.'
-    }
-    await createFile(filePathFull, fileContent)
+  const filePathFull = path
+    .join(driveFilesPath, filePath)
+    .replace(new RegExp('/', 'g'), path.sep)
 
-    return { status: 'success' }
-  } catch (err: any) {
-    throw {
-      code: 400,
-      status: 'failure',
-      message: 'File request failed.',
-      error: typeof err === 'object' ? err.toString() : err
-    }
+  if (!filePathFull.includes(driveFilesPath)) {
+    throw new Error('Cannot put file outside drive.')
   }
+
+  if (await fileExists(filePathFull)) {
+    throw new Error('File already exists.')
+  }
+
+  const folderPath = path.dirname(filePathFull)
+  await createFolder(folderPath)
+  await moveFile(multerFile.path, filePathFull)
+
+  return { status: 'success' }
 }
 
 const updateFile = async (body: FilePayload): Promise<GetFileResponse> => {

api/src/controllers/internal/Execution.ts

@@ -1,14 +1,37 @@
 import path from 'path'
 import fs from 'fs'
 import { getSessionController } from './'
-import { readFile, fileExists, createFile, moveFile } from '@sasjs/utils'
+import {
+  readFile,
+  fileExists,
+  createFile,
+  moveFile,
+  readFileBinary
+} from '@sasjs/utils'
 import { PreProgramVars, TreeNode } from '../../types'
-import { generateFileUploadSasCode, getTmpFilesFolderPath } from '../../utils'
+import {
+  extractHeaders,
+  generateFileUploadSasCode,
+  getTmpFilesFolderPath,
+  HTTPHeaders,
+  sasJSCoreMacros
+} from '../../utils'
 
 export interface ExecutionVars {
   [key: string]: string | number | undefined
 }
 
+export interface ExecuteReturnRaw {
+  httpHeaders: HTTPHeaders
+  result: string | Buffer
+}
+
+export interface ExecuteReturnJson {
+  httpHeaders: HTTPHeaders
+  webout: string | Buffer
+  log?: string
+}
+
 export class ExecutionController {
   async executeFile(
     programPath: string,
@@ -30,13 +53,14 @@ export class ExecutionController {
       returnJson
     )
   }
+
   async executeProgram(
     program: string,
     preProgramVariables: PreProgramVars,
     vars: ExecutionVars,
     otherArgs?: any,
     returnJson?: boolean
-  ) {
+  ): Promise<ExecuteReturnRaw | ExecuteReturnJson> {
     const sessionController = getSessionController()
 
     const session = await sessionController.getSession()
@@ -44,11 +68,11 @@ export class ExecutionController {
     session.consumed = true
 
     const logPath = path.join(session.path, 'log.log')
-
+    const headersPath = path.join(session.path, 'stpsrv_header.txt')
     const weboutPath = path.join(session.path, 'webout.txt')
-    await createFile(weboutPath, '')
-
     const tokenFile = path.join(session.path, 'accessToken.txt')
+
+    await createFile(weboutPath, '')
     await createFile(
       tokenFile,
       preProgramVariables?.accessToken ?? 'accessToken'
@@ -81,6 +105,8 @@ export class ExecutionController {
 `
 
     program = `
+options insert=(SASAUTOS="${sasJSCoreMacros}");
+
 /* runtime vars */
 ${varStatments}
 filename _webout "${weboutPath}" mod;
@@ -121,8 +147,17 @@ ${program}`
     }
 
     const log = (await fileExists(logPath)) ? await readFile(logPath) : ''
+    const headersContent = (await fileExists(headersPath))
+      ? await readFile(headersPath)
+      : ''
+    const httpHeaders: HTTPHeaders = extractHeaders(headersContent)
+    const fileResponse: boolean =
+      httpHeaders.hasOwnProperty('content-type') && !returnJson
+
     const webout = (await fileExists(weboutPath))
-      ? await readFile(weboutPath)
+      ? fileResponse
+        ? await readFileBinary(weboutPath)
+        : await readFile(weboutPath)
       : ''
 
     const debugValue =
@@ -133,15 +168,21 @@ ${program}`
 
     if (returnJson) {
       return {
+        httpHeaders,
         webout,
         log:
           (debugValue && debugValue >= 131) || session.crashed ? log : undefined
       }
     }
 
-    return (debugValue && debugValue >= 131) || session.crashed
-      ? `<html><body>${webout}<div style="text-align:left"><hr /><h2>SAS Log</h2><pre>${log}</pre></div></body></html>`
-      : webout
+    return {
+      httpHeaders,
+      result: fileResponse
+        ? webout
+        : (debugValue && debugValue >= 131) || session.crashed
+        ? `<html><body>${webout}<div style="text-align:left"><hr /><h2>SAS Log</h2><pre>${log}</pre></div></body></html>`
+        : webout
+    }
   }
 
   buildDirectoryTree() {

api/src/controllers/internal/Session.ts

@@ -67,7 +67,10 @@ export class SessionController {
 
     // the autoexec file is executed on SAS startup
     const autoExecPath = path.join(sessionFolder, 'autoexec.sas')
-    const contentForAutoExec = `/* compiled systemInit */\n${compiledSystemInitContent}\n/* autoexec */\n${autoExecContent}`
+    const contentForAutoExec = `/* compiled systemInit */
+${compiledSystemInitContent}
+/* autoexec */
+${autoExecContent}`
     await createFile(autoExecPath, contentForAutoExec)
 
     // create empty code.sas as SAS will not start without a SYSIN

api/src/controllers/stp.ts

@@ -1,9 +1,30 @@
 import express from 'express'
 import path from 'path'
-import { Request, Security, Route, Tags, Post, Body, Get, Query } from 'tsoa'
-import { ExecutionController, ExecutionVars } from './internal'
+import {
+  Request,
+  Security,
+  Route,
+  Tags,
+  Post,
+  Body,
+  Get,
+  Query,
+  Example
+} from 'tsoa'
+import {
+  ExecuteReturnJson,
+  ExecuteReturnRaw,
+  ExecutionController,
+  ExecutionVars
+} from './internal'
 import { PreProgramVars } from '../types'
-import { getTmpFilesFolderPath, makeFilesNamesMap } from '../utils'
+import {
+  getTmpFilesFolderPath,
+  HTTPHeaders,
+  LogLine,
+  makeFilesNamesMap,
+  parseLogToArray
+} from '../utils'
 
 interface ExecuteReturnJsonPayload {
   /**
@@ -12,11 +33,16 @@ interface ExecuteReturnJsonPayload {
    */
   _program?: string
 }
-interface ExecuteReturnJsonResponse {
+
+interface IRecordOfAny {
+  [key: string]: any
+}
+export interface ExecuteReturnJsonResponse {
   status: string
-  _webout: string
-  log?: string
+  _webout: string | IRecordOfAny
+  log: LogLine[]
   message?: string
+  httpHeaders: HTTPHeaders
 }
 
 @Security('bearerAuth')
@@ -24,33 +50,67 @@ interface ExecuteReturnJsonResponse {
 @Tags('STP')
 export class STPController {
   /**
-   * Trigger a SAS program using it's location in the _program parameter.
-   * Enable debugging using the _debug parameter.
+   * Trigger a SAS program using it's location in the _program URL parameter.
+   * Enable debugging using the _debug URL parameter.  Setting _debug=131 will
+   * cause the log to be streamed in the output.
+   *
    * Additional URL parameters are turned into SAS macro variables.
-   * Any files provided are placed into the session and
-   * corresponding _WEBIN_XXX variables are created.
-   * @summary Execute Stored Program, return raw content
-   * @query _program Location of SAS program
+   *
+   * Any files provided in the request body are placed into the SAS session with
+   * corresponding _WEBIN_XXX variables created.
+   *
+   * The response headers can be adjusted using the mfs_httpheader() macro.  Any
+   * file type can be returned, including binary files such as zip or xls.
+   *
+   * This behaviour differs for POST requests, in which case the reponse is
+   * always JSON.
+   *
+   * @summary Execute Stored Program, return raw _webout content.
+   * @param _program Location of SAS program
    * @example _program "/Public/somefolder/some.file"
    */
   @Get('/execute')
   public async executeReturnRaw(
     @Request() request: express.Request,
     @Query() _program: string
-  ): Promise<string> {
+  ): Promise<string | Buffer> {
     return executeReturnRaw(request, _program)
   }
 
   /**
-   * Trigger a SAS program using it's location in the _program parameter.
-   * Enable debugging using the _debug parameter.
+   * Trigger a SAS program using it's location in the _program URL parameter.
+   * Enable debugging using the _debug URL parameter.  In any case, the log is
+   * always returned in the log object.
+   *
    * Additional URL parameters are turned into SAS macro variables.
-   * Any files provided are placed into the session and
-   * corresponding _WEBIN_XXX variables are created.
+   *
+   * Any files provided in the request body are placed into the SAS session with
+   * corresponding _WEBIN_XXX variables created.
+   *
+   * The response will be a JSON object with the following root attributes: log,
+   * webout, headers.
+   *
+   * The webout will be a nested JSON object ONLY if the response-header
+   * contains a content-type of application/json AND it is valid JSON.
+   * Otherwise it will be a stringified version of the webout content.
+   *
+   * Response headers from the mfs_httpheader macro are simply listed in the
+   * headers object, for POST requests they have no effect on the actual
+   * response header.
+   *
    * @summary Execute Stored Program, return JSON
-   * @query _program Location of SAS program
+   * @param _program Location of SAS program
    * @example _program "/Public/somefolder/some.file"
    */
+  @Example<ExecuteReturnJsonResponse>({
+    status: 'success',
+    _webout: 'webout content',
+    log: [],
+    httpHeaders: {
+      'Content-type': 'application/zip',
+      'Cache-Control': 'public, max-age=1000'
+    }
+  })
   @Post('/execute')
   public async executeReturnJson(
     @Request() request: express.Request,
@@ -65,7 +125,7 @@ export class STPController {
 const executeReturnRaw = async (
   req: express.Request,
   _program: string
-): Promise<string> => {
+): Promise<string | Buffer> => {
   const query = req.query as ExecutionVars
   const sasCodePath =
     path
@@ -73,13 +133,20 @@ const executeReturnRaw = async (
       .replace(new RegExp('/', 'g'), path.sep) + '.sas'
 
   try {
-    const result = await new ExecutionController().executeFile(
-      sasCodePath,
-      getPreProgramVariables(req),
-      query
-    )
+    const { result, httpHeaders } =
+      (await new ExecutionController().executeFile(
+        sasCodePath,
+        getPreProgramVariables(req),
+        query
+      )) as ExecuteReturnRaw
 
-    return result as string
+    req.res?.set(httpHeaders)
+
+    if (result instanceof Buffer) {
+      ;(req as any).sasHeaders = httpHeaders
+    }
+
+    return result
   } catch (err: any) {
     throw {
       code: 400,
@@ -102,17 +169,27 @@ const executeReturnJson = async (
   const filesNamesMap = req.files?.length ? makeFilesNamesMap(req.files) : null
 
   try {
-    const { webout, log } = (await new ExecutionController().executeFile(
-      sasCodePath,
-      getPreProgramVariables(req),
-      { ...req.query, ...req.body },
-      { filesNamesMap: filesNamesMap },
-      true
-    )) as { webout: string; log: string }
+    const { webout, log, httpHeaders } =
+      (await new ExecutionController().executeFile(
+        sasCodePath,
+        getPreProgramVariables(req),
+        { ...req.query, ...req.body },
+        { filesNamesMap: filesNamesMap },
+        true
+      )) as ExecuteReturnJson
+
+    let weboutRes: string | IRecordOfAny = webout
+    if (httpHeaders['content-type']?.toLowerCase() === 'application/json') {
+      try {
+        weboutRes = JSON.parse(webout as string)
+      } catch (_) {}
+    }
+
     return {
       status: 'success',
-      _webout: webout,
-      log
+      _webout: weboutRes,
+      log: parseLogToArray(log),
+      httpHeaders
     }
   } catch (err: any) {
     throw {

api/src/middlewares/multer.ts

@@ -0,0 +1,77 @@
+import path from 'path'
+import { Request } from 'express'
+import multer, { FileFilterCallback, Options } from 'multer'
+import { getTmpUploadsPath } from '../utils'
+
+const acceptableExtensions = ['.sas']
+const fieldNameSize = 300
+const fileSize = 10485760 // 10 MB
+
+const storage = multer.diskStorage({
+  destination: getTmpUploadsPath(),
+  filename: function (
+    _req: Request,
+    file: Express.Multer.File,
+    callback: (error: Error | null, filename: string) => void
+  ) {
+    callback(
+      null,
+      file.fieldname + path.extname(file.originalname) + '-' + Date.now()
+    )
+  }
+})
+
+const limits: Options['limits'] = {
+  fieldNameSize,
+  fileSize
+}
+
+const fileFilter: Options['fileFilter'] = (
+  req: Request,
+  file: Express.Multer.File,
+  callback: FileFilterCallback
+) => {
+  const fileExtension = path.extname(file.originalname).toLocaleLowerCase()
+
+  if (!acceptableExtensions.includes(fileExtension)) {
+    return callback(
+      new Error(
+        `File extension '${fileExtension}' not acceptable. Valid extension(s): ${acceptableExtensions.join(
+          ', '
+        )}`
+      )
+    )
+  }
+
+  const uploadFileSize = parseInt(req.headers['content-length'] ?? '')
+  if (uploadFileSize > fileSize) {
+    return callback(
+      new Error(
+        `File size is over limit. File limit is: ${fileSize / 1024 / 1024} MB`
+      )
+    )
+  }
+
+  callback(null, true)
+}
+
+const options: Options = { storage, limits, fileFilter }
+
+const multerInstance = multer(options)
+
+export const multerSingle = (fileName: string, arg: any) => {
+  const [req, res, next] = arg
+  const upload = multerInstance.single(fileName)
+
+  upload(req, res, function (err) {
+    if (err instanceof multer.MulterError) {
+      return res.status(500).send(err.message)
+    } else if (err) {
+      return res.status(400).send(err.message)
+    }
+    // Everything went fine.
+    next()
+  })
+}
+
+export default multerInstance

api/src/routes/api/code.ts

@@ -12,6 +12,12 @@ runRouter.post('/execute', async (req, res) => {
 
   try {
     const response = await controller.executeSASCode(req, body)
+
+    if (response instanceof Buffer) {
+      res.writeHead(200, (req as any).sasHeaders)
+      return res.end(response)
+    }
+
     res.send(response)
   } catch (err: any) {
     const statusCode = err.code

api/src/routes/api/drive.ts

@@ -1,11 +1,20 @@
 import express from 'express'
+import { deleteFile } from '@sasjs/utils'
+
+import { multerSingle } from '../../middlewares/multer'
 import { DriveController } from '../../controllers/'
-import { getFileDriveValidation, updateFileDriveValidation } from '../../utils'
+import {
+  getFileDriveValidation,
+  updateFileDriveValidation,
+  uploadFileBodyValidation,
+  uploadFileParamValidation
+} from '../../utils'
+
+const controller = new DriveController()
 
 const driveRouter = express.Router()
 
 driveRouter.post('/deploy', async (req, res) => {
-  const controller = new DriveController()
   try {
     const response = await controller.deploy(req.body)
     res.send(response)
@@ -22,41 +31,45 @@ driveRouter.get('/file', async (req, res) => {
   const { error, value: query } = getFileDriveValidation(req.query)
   if (error) return res.status(400).send(error.details[0].message)
 
-  const controller = new DriveController()
   try {
-    const response = await controller.getFile(query.filePath)
-    res.send(response)
+    await controller.getFile(req, query.filePath)
   } catch (err: any) {
-    const statusCode = err.code
-
-    delete err.code
-
-    res.status(statusCode).send(err)
+    res.status(403).send(err.toString())
   }
 })
 
-driveRouter.post('/file', async (req, res) => {
-  const { error, value: body } = updateFileDriveValidation(req.body)
-  if (error) return res.status(400).send(error.details[0].message)
+driveRouter.post(
+  '/file',
+  (...arg) => multerSingle('file', arg),
+  async (req, res) => {
+    const { error: errQ, value: query } = uploadFileParamValidation(req.query)
+    const { error: errB, value: body } = uploadFileBodyValidation(req.body)
 
-  const controller = new DriveController()
-  try {
-    const response = await controller.saveFile(body)
-    res.send(response)
-  } catch (err: any) {
-    const statusCode = err.code
+    if (errQ && errB) {
+      if (req.file) await deleteFile(req.file.path)
+      return res.status(400).send(errB.details[0].message)
+    }
 
-    delete err.code
+    if (!req.file) return res.status(400).send('"file" is not present.')
 
-    res.status(statusCode).send(err)
+    try {
+      const response = await controller.saveFile(
+        req.file,
+        query._filePath,
+        body.filePath
+      )
+      res.send(response)
+    } catch (err: any) {
+      await deleteFile(req.file.path)
+      res.status(403).send(err.toString())
+    }
   }
-})
+)
 
 driveRouter.patch('/file', async (req, res) => {
   const { error, value: body } = updateFileDriveValidation(req.body)
   if (error) return res.status(400).send(error.details[0].message)
 
-  const controller = new DriveController()
   try {
     const response = await controller.updateFile(body)
     res.send(response)
@@ -70,7 +83,6 @@ driveRouter.patch('/file', async (req, res) => {
 })
 
 driveRouter.get('/fileTree', async (req, res) => {
-  const controller = new DriveController()
   try {
     const response = await controller.getFileTree()
     res.send(response)

api/src/routes/api/spec/drive.spec.ts

@@ -1,15 +1,34 @@
+import path from 'path'
 import { Express } from 'express'
 import mongoose, { Mongoose } from 'mongoose'
 import { MongoMemoryServer } from 'mongodb-memory-server'
 import request from 'supertest'
+
+import {
+  folderExists,
+  fileExists,
+  readFile,
+  deleteFolder,
+  generateTimestamp,
+  copy
+} from '@sasjs/utils'
+import * as fileUtilModules from '../../../utils/file'
+
+const timestamp = generateTimestamp()
+const tmpFolder = path.join(process.cwd(), `tmp-${timestamp}`)
+jest
+  .spyOn(fileUtilModules, 'getTmpFolderPath')
+  .mockImplementation(() => tmpFolder)
+jest
+  .spyOn(fileUtilModules, 'getTmpUploadsPath')
+  .mockImplementation(() => path.join(tmpFolder, 'uploads'))
+
 import appPromise from '../../../app'
 import { UserController } from '../../../controllers/'
 import { getTreeExample } from '../../../controllers/internal'
-import { getTmpFilesFolderPath } from '../../../utils/file'
-import { folderExists, fileExists, readFile, deleteFolder } from '@sasjs/utils'
-import path from 'path'
-import { generateAccessToken, saveTokensInDB } from '../../../utils'
 import { FolderMember, ServiceMember } from '../../../types'
+import { generateAccessToken, saveTokensInDB } from '../../../utils/'
+const { getTmpFilesFolderPath } = fileUtilModules
 
 let app: Express
 appPromise.then((_app) => {
@@ -30,28 +49,27 @@ describe('files', () => {
   let mongoServer: MongoMemoryServer
   const controller = new UserController()
 
+  let accessToken: string
+
   beforeAll(async () => {
     mongoServer = await MongoMemoryServer.create()
     con = await mongoose.connect(mongoServer.getUri())
+
+    const dbUser = await controller.createUser(user)
+    accessToken = generateAccessToken({
+      clientId,
+      userId: dbUser.id
+    })
+    await saveTokensInDB(dbUser.id, clientId, accessToken, 'refreshToken')
   })
 
   afterAll(async () => {
     await con.connection.dropDatabase()
     await con.connection.close()
     await mongoServer.stop()
+    await deleteFolder(tmpFolder)
   })
   describe('deploy', () => {
-    let accessToken: string
-    let dbUser: any
-
-    beforeAll(async () => {
-      dbUser = await controller.createUser(user)
-      accessToken = generateAccessToken({
-        clientId,
-        userId: dbUser.id
-      })
-      await saveTokensInDB(dbUser.id, clientId, accessToken, 'refreshToken')
-    })
     const shouldFailAssertion = async (payload: any) => {
       const res = await request(app)
         .post('/SASjsApi/drive/deploy')
@@ -144,8 +162,6 @@ describe('files', () => {
       const exampleService = getExampleService()
       const testJobFile = path.join(testJobFolder, exampleService.name) + '.sas'
 
-      console.log(`[testJobFile]`, testJobFile)
-
       await expect(fileExists(testJobFile)).resolves.toEqual(true)
 
       await expect(readFile(testJobFile)).resolves.toEqual(exampleService.code)
@@ -153,6 +169,159 @@ describe('files', () => {
       await deleteFolder(getTmpFilesFolderPath())
     })
   })
+
+  describe('file', () => {
+    describe('create', () => {
+      it('should create a SAS file on drive having filePath as form field', async () => {
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', '/my/path/code.sas')
+          .attach('file', path.join(__dirname, 'files', 'sample.sas'))
+
+        expect(res.statusCode).toEqual(200)
+        expect(res.body).toEqual({
+          status: 'success'
+        })
+      })
+
+      it('should create a SAS file on drive having _filePath as query param', async () => {
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .query({ _filePath: '/my/path/code1.sas' })
+          .attach('file', path.join(__dirname, 'files', 'sample.sas'))
+
+        expect(res.statusCode).toEqual(200)
+        expect(res.body).toEqual({
+          status: 'success'
+        })
+      })
+
+      it('should respond with Unauthorized if access token is not present', async () => {
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .field('filePath', '/my/path/code.sas')
+          .attach('file', path.join(__dirname, 'files', 'sample.sas'))
+          .expect(401)
+
+        expect(res.text).toEqual('Unauthorized')
+        expect(res.body).toEqual({})
+      })
+
+      it('should respond with Forbidden if file is already present', async () => {
+        const fileToAttachPath = path.join(__dirname, 'files', 'sample.sas')
+        const pathToUpload = '/my/path/code.sas'
+
+        const pathToCopy = path.join(
+          fileUtilModules.getTmpFilesFolderPath(),
+          pathToUpload
+        )
+        await copy(fileToAttachPath, pathToCopy)
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .attach('file', fileToAttachPath)
+          .expect(403)
+
+        expect(res.text).toEqual('Error: File already exists.')
+        expect(res.body).toEqual({})
+      })
+
+      it('should respond with Forbidden if filePath outside Drive', async () => {
+        const fileToAttachPath = path.join(__dirname, 'files', 'sample.sas')
+        const pathToUpload = '/../path/code.sas'
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .attach('file', fileToAttachPath)
+          .expect(403)
+
+        expect(res.text).toEqual('Error: Cannot put file outside drive.')
+        expect(res.body).toEqual({})
+      })
+
+      it('should respond with Bad Request if filePath is missing', async () => {
+        const fileToAttachPath = path.join(__dirname, 'files', 'sample.sas')
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .attach('file', fileToAttachPath)
+          .expect(400)
+
+        expect(res.text).toEqual(`"filePath" is required`)
+        expect(res.body).toEqual({})
+      })
+
+      it("should respond with Bad Request if filePath doesn't has correct extension", async () => {
+        const fileToAttachPath = path.join(__dirname, 'files', 'sample.sas')
+        const pathToUpload = '/my/path/code.oth'
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .attach('file', fileToAttachPath)
+          .expect(400)
+
+        expect(res.text).toEqual('Valid extensions for filePath: .sas')
+        expect(res.body).toEqual({})
+      })
+
+      it('should respond with Bad Request if file is missing', async () => {
+        const pathToUpload = '/my/path/code.sas'
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .expect(400)
+
+        expect(res.text).toEqual('"file" is not present.')
+        expect(res.body).toEqual({})
+      })
+
+      it("should respond with Bad Request if attached file doesn't has correct extension", async () => {
+        const fileToAttachPath = path.join(__dirname, 'files', 'sample.oth')
+        const pathToUpload = '/my/path/code.sas'
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .attach('file', fileToAttachPath)
+          .expect(400)
+
+        expect(res.text).toEqual(
+          `File extension '.oth' not acceptable. Valid extension(s): .sas`
+        )
+        expect(res.body).toEqual({})
+      })
+
+      it('should respond with Bad Request if attached file exceeds file limit', async () => {
+        const pathToUpload = '/my/path/code.sas'
+
+        const attachedFile = Buffer.from('.'.repeat(20 * 1024 * 1024))
+
+        const res = await request(app)
+          .post('/SASjsApi/drive/file')
+          .auth(accessToken, { type: 'bearer' })
+          .field('filePath', pathToUpload)
+          .attach('file', attachedFile, 'another.sas')
+          .expect(400)
+
+        expect(res.text).toEqual(
+          'File size is over limit. File limit is: 10 MB'
+        )
+        expect(res.body).toEqual({})
+      })
+    })
+  })
 })
 
 const getExampleService = (): ServiceMember =>

api/src/routes/api/spec/files/sample.oth

@@ -0,0 +1 @@
+some code of sas

api/src/routes/api/spec/files/sample.sas

@@ -0,0 +1 @@
+some code of sas

api/src/routes/api/stp.ts

@@ -14,6 +14,12 @@ stpRouter.get('/execute', async (req, res) => {
 
   try {
     const response = await controller.executeReturnRaw(req, query._program)
+
+    if (response instanceof Buffer) {
+      res.writeHead(200, (req as any).sasHeaders)
+      return res.end(response)
+    }
+
     res.send(response)
   } catch (err: any) {
     const statusCode = err.code
@@ -40,6 +46,12 @@ stpRouter.post(
         body,
         query?._program
       )
+
+      if (response instanceof Buffer) {
+        res.writeHead(200, (req as any).sasHeaders)
+        return res.end(response)
+      }
+
       res.send(response)
     } catch (err: any) {
       const statusCode = err.code

api/src/routes/setupRoutes.ts

@@ -0,0 +1,9 @@
+import { Express } from 'express'
+
+import webRouter from './web'
+import apiRouter from './api'
+
+export const setupRoutes = (app: Express) => {
+  app.use('/', webRouter)
+  app.use('/SASjsApi', apiRouter)
+}

api/src/server.ts

@@ -7,6 +7,8 @@ appPromise.then(async (app) => {
   const protocol = process.env.PROTOCOL ?? 'http'
   const sasJsPort = process.env.PORT ?? 5000
 
+  console.log('PROTOCOL: ', protocol)
+
   if (protocol !== 'https') {
     app.listen(sasJsPort, () => {
       console.log(

api/src/utils/connectDB.ts

@@ -1,40 +1,19 @@
-import path from 'path'
 import mongoose from 'mongoose'
-import { configuration } from '../../package.json'
-import { getDesktopFields } from '.'
 import { populateClients } from '../routes/api/auth'
-import { getRealPath } from '@sasjs/utils'
 
 export const connectDB = async () => {
   // NOTE: when exporting app.js as agent for supertest
   // we should exclude connecting to the real database
   if (process.env.NODE_ENV === 'test') {
-    process.driveLoc = path.join(process.cwd(), 'tmp')
     return
   } else {
     const { MODE } = process.env
 
     if (MODE?.trim() !== 'server') {
       console.log('Running in Destop Mode, no DB to connect.')
-
-      const { sasLoc, driveLoc } = await getDesktopFields()
-
-      process.sasLoc = sasLoc
-      process.driveLoc = driveLoc
-    } else {
-      const { SAS_PATH, DRIVE_PATH } = process.env
-
-      process.sasLoc = SAS_PATH ?? configuration.sasPath
-      process.driveLoc = getRealPath(
-        path.join(process.cwd(), DRIVE_PATH ?? 'tmp')
-      )
+      return
     }
 
-    console.log('sasLoc: ', process.sasLoc)
-    console.log('sasDrive: ', process.driveLoc)
-
-    if (MODE?.trim() !== 'server') return
-
     mongoose.connect(process.env.DB_CONNECT as string, async (err) => {
       if (err) throw err
 

api/src/utils/extractHeaders.ts

@@ -0,0 +1,25 @@
+const headerUtils = require('http-headers-validation')
+
+export interface HTTPHeaders {
+  [key: string]: string
+}
+
+export const extractHeaders = (content?: string): HTTPHeaders => {
+  const headersObj: HTTPHeaders = {}
+  const headersArr = content
+    ?.split('\n')
+    .map((line) => line.trim())
+    .filter((line) => !!line)
+
+  headersArr?.forEach((headerStr) => {
+    const [key, value] = headerStr.split(':').map((data) => data.trim())
+
+    if (value && headerUtils.validateHeader(key, value)) {
+      headersObj[key.toLowerCase()] = value
+    } else {
+      delete headersObj[key.toLowerCase()]
+    }
+  })
+
+  return headersObj
+}

api/src/utils/file.ts

@@ -8,11 +8,15 @@ export const sysInitCompiledPath = path.join(
   'systemInitCompiled.sas'
 )
 
+export const sasJSCoreMacros = path.join(apiRoot, 'sasjscore')
+
 export const getWebBuildFolderPath = () =>
   path.join(codebaseRoot, 'web', 'build')
 
 export const getTmpFolderPath = () => process.driveLoc
 
+export const getTmpUploadsPath = () => path.join(getTmpFolderPath(), 'uploads')
+
 export const getTmpFilesFolderPath = () =>
   path.join(getTmpFolderPath(), 'files')
 

api/src/utils/getCertificates.ts

@@ -7,6 +7,9 @@ export const getCertificates = async () => {
   const keyPath = PRIVATE_KEY ?? (await getFileInput('Private Key (PEM)'))
   const certPath = FULL_CHAIN ?? (await getFileInput('Full Chain (PEM)'))
 
+  console.log('keyPath: ', keyPath)
+  console.log('certPath: ', certPath)
+
   const key = await readFile(keyPath)
   const cert = await readFile(certPath)
 

api/src/utils/index.ts

@@ -1,12 +1,15 @@
 export * from './connectDB'
+export * from './extractHeaders'
 export * from './file'
 export * from './generateAccessToken'
 export * from './generateAuthCode'
 export * from './generateRefreshToken'
 export * from './getCertificates'
 export * from './getDesktopFields'
+export * from './parseLogToArray'
 export * from './removeTokensInDB'
 export * from './saveTokensInDB'
+export * from './setProcessVariables'
 export * from './sleep'
 export * from './upload'
 export * from './validation'

api/src/utils/parseLogToArray.ts

@@ -0,0 +1,9 @@
+export interface LogLine {
+  line: string
+}
+
+export const parseLogToArray = (content?: string): LogLine[] => {
+  if (!content) return []
+
+  return content.split('\n').map((line) => ({ line: line }))
+}

api/src/utils/setProcessVariables.ts

@@ -0,0 +1,31 @@
+import path from 'path'
+import { getRealPath } from '@sasjs/utils'
+
+import { configuration } from '../../package.json'
+import { getDesktopFields } from '.'
+
+export const setProcessVariables = async () => {
+  if (process.env.NODE_ENV === 'test') {
+    process.driveLoc = path.join(process.cwd(), 'tmp')
+    return
+  }
+
+  const { MODE } = process.env
+
+  if (MODE?.trim() !== 'server') {
+    const { sasLoc, driveLoc } = await getDesktopFields()
+
+    process.sasLoc = sasLoc
+    process.driveLoc = driveLoc
+  } else {
+    const { SAS_PATH, DRIVE_PATH } = process.env
+
+    process.sasLoc = SAS_PATH ?? configuration.sasPath
+    process.driveLoc = getRealPath(
+      path.join(process.cwd(), DRIVE_PATH ?? 'tmp')
+    )
+  }
+
+  console.log('sasLoc: ', process.sasLoc)
+  console.log('sasDrive: ', process.driveLoc)
+}

api/src/utils/specs/extractHeaders.spec.ts

@@ -0,0 +1,52 @@
+import { extractHeaders } from '..'
+
+describe('extractHeaders', () => {
+  it('should return valid http headers', () => {
+    const headers = extractHeaders(`
+      Content-type: application/csv
+      Cache-Control: public, max-age=2000
+      Content-type: application/text
+      Cache-Control: public, max-age=1500
+      Content-type: application/zip
+      Cache-Control: public, max-age=1000
+    `)
+
+    expect(headers).toEqual({
+      'content-type': 'application/zip',
+      'cache-control': 'public, max-age=1000'
+    })
+  })
+
+  it('should not return http headers if last occurrence is blank', () => {
+    const headers = extractHeaders(`
+      Content-type: application/csv
+      Cache-Control: public, max-age=1000
+      Content-type: application/text
+      Content-type:    
+    `)
+
+    expect(headers).toEqual({ 'cache-control': 'public, max-age=1000' })
+  })
+
+  it('should return only valid http headers', () => {
+    const headers = extractHeaders(`
+      Content-type[]: application/csv
+      Content//-type: application/text
+      Content()-type: application/zip
+    `)
+
+    expect(headers).toEqual({})
+  })
+
+  it('should return http headers if empty', () => {
+    const headers = extractHeaders('')
+
+    expect(headers).toEqual({})
+  })
+
+  it('should return http headers if not provided', () => {
+    const headers = extractHeaders()
+
+    expect(headers).toEqual({})
+  })
+})

api/src/utils/specs/parseLogToArray.spec.ts

@@ -0,0 +1,33 @@
+import { parseLogToArray } from '..'
+
+describe('parseLogToArray', () => {
+  it('should parse log to array type', () => {
+    const log = parseLogToArray(`
+line 1 of log content
+line 2 of log content
+line 3 of log content
+line 4 of log content
+    `)
+
+    expect(log).toEqual([
+      { line: '' },
+      { line: 'line 1 of log content' },
+      { line: 'line 2 of log content' },
+      { line: 'line 3 of log content' },
+      { line: 'line 4 of log content' },
+      { line: '    ' }
+    ])
+  })
+
+  it('should parse log to array type if empty', () => {
+    const log = parseLogToArray('')
+
+    expect(log).toEqual([])
+  })
+
+  it('should parse log to array type if not provided', () => {
+    const log = parseLogToArray()
+
+    expect(log).toEqual([])
+  })
+})

api/src/utils/validation.ts

@@ -77,6 +77,20 @@ export const updateFileDriveValidation = (data: any): Joi.ValidationResult =>
     fileContent: Joi.string().required()
   }).validate(data)
 
+export const uploadFileBodyValidation = (data: any): Joi.ValidationResult =>
+  Joi.object({
+    filePath: Joi.string().pattern(/.sas$/).required().messages({
+      'string.pattern.base': `Valid extensions for filePath: .sas`
+    })
+  }).validate(data)
+
+export const uploadFileParamValidation = (data: any): Joi.ValidationResult =>
+  Joi.object({
+    _filePath: Joi.string().pattern(/.sas$/).required().messages({
+      'string.pattern.base': `Valid extensions for filePath: .sas`
+    })
+  }).validate(data)
+
 export const runSASValidation = (data: any): Joi.ValidationResult =>
   Joi.object({
     code: Joi.string().required()

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "server",
-  "version": "0.0.29",
+  "version": "0.0.30",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "server",
-      "version": "0.0.29",
+      "version": "0.0.30",
       "devDependencies": {
         "prettier": "^2.3.1",
         "standard-version": "^9.3.2"

package.json

@@ -1,6 +1,6 @@
 {
   "name": "server",
-  "version": "0.0.29",
+  "version": "0.0.30",
   "description": "NodeJS wrapper for calling the SAS binary executable",
   "repository": "https://github.com/sasjs/server",
   "scripts": {

restClient/auth.rest

@@ -0,0 +1,22 @@
+### Get Auth Code
+POST http://localhost:5000/SASjsApi/auth/authorize
+Content-Type: application/json
+
+{
+  "username": "secretuser",
+  "password": "secretpassword",
+  "client_id": "clientID1"
+}
+
+### Exchange AuthCode with Access/Refresh Tokens
+POST http://localhost:5000/SASjsApi/auth/token
+Content-Type: application/json
+
+{
+  "client_id": "clientID1",
+  "client_secret": "clientID1secret",
+  "code": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjbGllbnRfaWQiOiJjbGllbnRJRDEiLCJ1c2VybmFtZSI6InVzZXJuYW1lMSIsImlzYWRtaW4iOmZhbHNlLCJpc2FjdGl2ZSI6dHJ1ZSwiaWF0IjoxNjM1ODA0MDYxLCJleHAiOjE2MzU4MDQwOTF9.jV7DpBWG7XAGODs22zAW_kWOqVLZvOxmmYJGpSNQ-KM"
+}
+
+### Perform logout to deactivate access token instantly
+DELETE http://localhost:5000/SASjsApi/auth/logout

restClient/drive.rest

@@ -0,0 +1,45 @@
+###
+POST http://localhost:5000/SASjsApi/drive/deploy
+Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjbGllbnRfaWQiOiJjbGllbnRJRDEiLCJ1c2VybmFtZSI6InVzZXJuYW1lMSIsImlzYWRtaW4iOmZhbHNlLCJpc2FjdGl2ZSI6dHJ1ZSwiaWF0IjoxNjM1ODA0MDc2LCJleHAiOjE2MzU4OTA0NzZ9.Cx1F54ILgAUtnkit0Wg1K1YVO2RdNjOnTKdPhUtDm5I
+
+### multipart upload to sas server file
+POST http://localhost:5000/SASjsApi/drive/file
+Content-Type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW
+
+------WebKitFormBoundary7MA4YWxkTrZu0gW
+Content-Disposition: form-data; name="filePath"
+
+/saad/files/new.sas
+------WebKitFormBoundary7MA4YWxkTrZu0gW
+Content-Disposition: form-data; name="file"; filename="sample_new.sas"
+Content-Type: application/octet-stream
+
+< ./sample.sas
+------WebKitFormBoundary7MA4YWxkTrZu0gW--
+
+### multipart upload to sas server file text
+POST http://localhost:5000/SASjsApi/drive/file
+Content-Type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW
+  
+------WebKitFormBoundary7MA4YWxkTrZu0gW \n
+Content-Disposition: form-data; name="filePath"
+
+/saad/files/new2.sas
+------WebKitFormBoundary7MA4YWxkTrZu0gW
+Content-Disposition: form-data; name="file"; filename="sample_new.sas"
+Content-Type: text/plain
+
+SOME CONTENTS OF SAS FILE IN REQUEST
+
+------WebKitFormBoundary7MA4YWxkTrZu0gW--
+
+
+Users
+  "username": "username1",
+  "password": "some password",
+
+  "username": "username2",
+  "password": "some password",
+Admins
+  "username": "secretuser",
+  "password": "secretpassword",

restClient/routes.rest

@@ -23,7 +23,7 @@ Content-Type: application/json
   "client_secret": "newClientSecret"
 }
 ###
-POST https://sas.analytium.co.uk:5002/SASjsApi/auth/authorize
+POST http://localhost:5000/SASjsApi/auth/authorize
 Content-Type: application/json
 
 {
@@ -45,6 +45,41 @@ Content-Type: application/json
 ###
 DELETE http://localhost:5000/SASjsApi/auth/logout
 
+###
+GET http://localhost:5000/SASjsApi/session
+
+
+### multipart upload to sas server file
+POST http://localhost:5000/SASjsApi/drive/file
+Content-Type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW
+
+------WebKitFormBoundary7MA4YWxkTrZu0gW
+Content-Disposition: form-data; name="filePath"
+
+/saad/files/new.sas
+------WebKitFormBoundary7MA4YWxkTrZu0gW
+Content-Disposition: form-data; name="file"; filename="sample_new.sas"
+Content-Type: application/octet-stream
+
+< ./sample.sas
+------WebKitFormBoundary7MA4YWxkTrZu0gW--
+
+### multipart upload to sas server file text
+POST http://localhost:5000/SASjsApi/drive/file
+Content-Type: multipart/form-data; boundary=----WebKitFormBoundary7MA4YWxkTrZu0gW
+  
+------WebKitFormBoundary7MA4YWxkTrZu0gW \n
+Content-Disposition: form-data; name="filePath"
+
+/saad/files/new2.sas
+------WebKitFormBoundary7MA4YWxkTrZu0gW
+Content-Disposition: form-data; name="file"; filename="sample_new.sas"
+Content-Type: text/plain
+
+SOME CONTENTS OF SAS FILE IN REQUEST
+
+------WebKitFormBoundary7MA4YWxkTrZu0gW--
+
 
 Users
   "username": "username1",

restClient/sample.sas

@@ -0,0 +1 @@
+some code of sas

restClient/session.rest

@@ -0,0 +1,2 @@
+### Get current user's info via access token
+GET http://localhost:5000/SASjsApi/session

restClient/users.rest

@@ -0,0 +1,10 @@
+### Create User
+POST http://localhost:5000/SASjsApi/user
+Authorization: Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJjbGllbnRfaWQiOiJjbGllbnRJRDEiLCJ1c2VybmFtZSI6InNlY3JldHVzZXIiLCJpc2FkbWluIjp0cnVlLCJpc2FjdGl2ZSI6dHJ1ZSwiaWF0IjoxNjM1ODAzOTc3LCJleHAiOjE2MzU4OTAzNzd9.f-FLgLwryKvB5XrihdzaGZajO3d5E5OHEEuJI_03GRI
+Content-Type: application/json
+
+{
+  "displayname": "User 2",
+  "username": "username2",
+  "password": "some password"
+}

web/src/containers/Studio/index.tsx

@@ -46,7 +46,11 @@ const Studio = () => {
     axios
       .post(`/SASjsApi/code/execute`, { code })
       .then((res: any) => {
-        setLog(`<div><h2>SAS Log</h2><pre>${res?.data?.log}</pre></div>`)
+        const parsedLog = res?.data?.log
+          .map((logLine: any) => logLine.line)
+          .join('\n')
+
+        setLog(`<div><h2>SAS Log</h2><pre>${parsedLog}</pre></div>`)
 
         let weboutString: string
         try {