From ffcf193b87d811b166d79af74013776a253b50b0 Mon Sep 17 00:00:00 2001 From: Yury Date: Tue, 29 Oct 2024 11:18:04 +0300 Subject: [PATCH 1/4] feat(code): added code/trigger API endpoint --- api/.env.example | 2 +- api/public/swagger.yaml | 60 ++++++++++++++++- api/src/controllers/code.ts | 89 ++++++++++++++++++++++++- api/src/controllers/internal/Session.ts | 32 ++++++--- api/src/routes/api/code.ts | 20 +++++- api/src/types/Session.ts | 1 + api/src/utils/validation.ts | 7 ++ 7 files changed, 194 insertions(+), 17 deletions(-) diff --git a/api/.env.example b/api/.env.example index d59f43e..1804918 100644 --- a/api/.env.example +++ b/api/.env.example @@ -25,7 +25,7 @@ LDAP_USERS_BASE_DN = LDAP_GROUPS_BASE_DN = #default value is 100 -MAX_WRONG_ATTEMPTS_BY_IP_PER_DAY=100 +MAX_WRONG_ATTEMPTS_BY_IP_PER_DAY=100 #default value is 10 MAX_CONSECUTIVE_FAILS_BY_USERNAME_AND_IP=10 diff --git a/api/public/swagger.yaml b/api/public/swagger.yaml index dc73202..04dacd3 100644 --- a/api/public/swagger.yaml +++ b/api/public/swagger.yaml @@ -109,6 +109,36 @@ components: - runTime type: object additionalProperties: false + TriggerCodeResponse: + properties: + sessionId: + type: string + description: "Session ID (SAS WORK folder) used to execute code.\nThis session ID should be used to poll job status." + example: '{ sessionId: ''20241028074744-54132-1730101664824'' }' + required: + - sessionId + type: object + additionalProperties: false + TriggerCodePayload: + properties: + code: + type: string + description: 'Code of program' + example: '* Code HERE;' + runTime: + $ref: '#/components/schemas/RunTimeType' + description: 'runtime for program' + example: sas + expiresAfterMins: + type: number + format: double + description: "Amount of minutes after the completion of the job when the session must be\ndestroyed." + example: 15 + required: + - code + - runTime + type: object + additionalProperties: false MemberType.folder: enum: - folder @@ -805,6 +835,30 @@ paths: application/json: schema: $ref: '#/components/schemas/ExecuteCodePayload' + /SASjsApi/code/trigger: + post: + operationId: TriggerCode + responses: + '200': + description: Ok + content: + application/json: + schema: + $ref: '#/components/schemas/TriggerCodeResponse' + description: 'Trigger Code on the Specified Runtime' + summary: 'Trigger Code and Return Session Id not awaiting for the job completion' + tags: + - Code + security: + - + bearerAuth: [] + parameters: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/TriggerCodePayload' /SASjsApi/drive/deploy: post: operationId: Deploy @@ -1789,7 +1843,7 @@ paths: anyOf: - {type: string} - {type: string, format: byte} - description: "Trigger a Stored Program using the _program URL parameter.\n\nAccepts URL parameters and file uploads. For more details, see docs:\n\nhttps://server.sasjs.io/storedprograms" + description: "Trigger a Stored Program using the _program URL parameter.\n\nAccepts additional URL parameters (converted to session variables)\nand file uploads. For more details, see docs:\n\nhttps://server.sasjs.io/storedprograms" summary: 'Execute a Stored Program, returns _webout and (optionally) log.' tags: - STP @@ -1798,7 +1852,7 @@ paths: bearerAuth: [] parameters: - - description: 'Location of the Stored Program in SASjs Drive' + description: 'Location of code in SASjs Drive' in: query name: _program required: true @@ -1806,7 +1860,7 @@ paths: type: string example: /Projects/myApp/some/program - - description: 'Optional query param for setting debug mode (returns the session log in the response body)' + description: 'Optional query param for setting debug mode, which will return the session log.' in: query name: _debug required: false diff --git a/api/src/controllers/code.ts b/api/src/controllers/code.ts index 72fa376..11912d9 100644 --- a/api/src/controllers/code.ts +++ b/api/src/controllers/code.ts @@ -1,11 +1,10 @@ import express from 'express' import { Request, Security, Route, Tags, Post, Body } from 'tsoa' -import { ExecutionController } from './internal' +import { ExecutionController, getSessionController } from './internal' import { getPreProgramVariables, getUserAutoExec, ModeType, - parseLogToArray, RunTimeType } from '../utils' @@ -22,6 +21,34 @@ interface ExecuteCodePayload { runTime: RunTimeType } +interface TriggerCodePayload { + /** + * Code of program + * @example "* Code HERE;" + */ + code: string + /** + * runtime for program + * @example "sas" + */ + runTime: RunTimeType + /** + * Amount of minutes after the completion of the job when the session must be + * destroyed. + * @example 15 + */ + expiresAfterMins?: number +} + +interface TriggerCodeResponse { + /** + * Session ID (SAS WORK folder) used to execute code. + * This session ID should be used to poll job status. + * @example "{ sessionId: '20241028074744-54132-1730101664824' }" + */ + sessionId: string +} + @Security('bearerAuth') @Route('SASjsApi/code') @Tags('Code') @@ -44,6 +71,18 @@ export class CodeController { ): Promise { return executeCode(request, body) } + + /** + * Trigger Code on the Specified Runtime + * @summary Trigger Code and Return Session Id not awaiting for the job completion + */ + @Post('/trigger') + public async triggerCode( + @Request() request: express.Request, + @Body() body: TriggerCodePayload + ): Promise { + return triggerCode(request, body) + } } const executeCode = async ( @@ -76,3 +115,49 @@ const executeCode = async ( } } } + +const triggerCode = async ( + req: express.Request, + { code, runTime, expiresAfterMins }: TriggerCodePayload +): Promise<{ sessionId: string }> => { + const { user } = req + const userAutoExec = + process.env.MODE === ModeType.Server + ? user?.autoExec + : await getUserAutoExec() + + // get session controller based on runTime + const sessionController = getSessionController(runTime) + + // get session + const session = await sessionController.getSession() + + // add expiresAfterMins to session if provided + if (expiresAfterMins) { + // expiresAfterMins.used is set initially to false + session.expiresAfterMins = { mins: expiresAfterMins, used: false } + } + + try { + // call executeProgram method of ExecutionController without awaiting + new ExecutionController().executeProgram({ + program: code, + preProgramVariables: getPreProgramVariables(req), + vars: { ...req.query, _debug: 131 }, + otherArgs: { userAutoExec }, + runTime: runTime, + includePrintOutput: true, + session // session is provided + }) + + // return session id + return { sessionId: session.id } + } catch (err: any) { + throw { + code: 400, + status: 'failure', + message: 'Job execution failed.', + error: typeof err === 'object' ? err.toString() : err + } + } +} diff --git a/api/src/controllers/internal/Session.ts b/api/src/controllers/internal/Session.ts index 31ee7f3..3573658 100644 --- a/api/src/controllers/internal/Session.ts +++ b/api/src/controllers/internal/Session.ts @@ -14,8 +14,7 @@ import { createFile, fileExists, generateTimestamp, - readFile, - isWindows + readFile } from '@sasjs/utils' const execFilePromise = promisify(execFile) @@ -190,20 +189,33 @@ ${autoExecContent}` } private scheduleSessionDestroy(session: Session) { - setTimeout( - async () => { - if (session.inUse) { - // adding 10 more minutes - const newDeathTimeStamp = parseInt(session.deathTimeStamp) + 10 * 1000 + setTimeout(async () => { + if (session.inUse) { + // adding 10 more minutes + const newDeathTimeStamp = + parseInt(session.deathTimeStamp) + 10 * 60 * 1000 + session.deathTimeStamp = newDeathTimeStamp.toString() + + this.scheduleSessionDestroy(session) + } else { + const { expiresAfterMins } = session + + // delay session destroy if expiresAfterMins present + if (expiresAfterMins && !expiresAfterMins.used) { + // calculate session death time using expiresAfterMins + const newDeathTimeStamp = + parseInt(session.deathTimeStamp) + expiresAfterMins.mins * 60 * 1000 session.deathTimeStamp = newDeathTimeStamp.toString() + // set expiresAfterMins to true to avoid using it again + session.expiresAfterMins!.used = true + this.scheduleSessionDestroy(session) } else { await this.deleteSession(session) } - }, - parseInt(session.deathTimeStamp) - new Date().getTime() - 100 - ) + } + }, parseInt(session.deathTimeStamp) - new Date().getTime() - 100) } } diff --git a/api/src/routes/api/code.ts b/api/src/routes/api/code.ts index 09171c0..c823950 100644 --- a/api/src/routes/api/code.ts +++ b/api/src/routes/api/code.ts @@ -1,5 +1,5 @@ import express from 'express' -import { runCodeValidation } from '../../utils' +import { runCodeValidation, triggerCodeValidation } from '../../utils' import { CodeController } from '../../controllers/' const runRouter = express.Router() @@ -28,4 +28,22 @@ runRouter.post('/execute', async (req, res) => { } }) +runRouter.post('/trigger', async (req, res) => { + const { error, value: body } = triggerCodeValidation(req.body) + if (error) return res.status(400).send(error.details[0].message) + + try { + const response = await controller.triggerCode(req, body) + + res.status(200) + res.send(response) + } catch (err: any) { + const statusCode = err.code + + delete err.code + + res.status(statusCode).send(err) + } +}) + export default runRouter diff --git a/api/src/types/Session.ts b/api/src/types/Session.ts index 0507a6d..4fa1cfb 100644 --- a/api/src/types/Session.ts +++ b/api/src/types/Session.ts @@ -8,4 +8,5 @@ export interface Session { consumed: boolean completed: boolean crashed?: string + expiresAfterMins?: { mins: number; used: boolean } } diff --git a/api/src/utils/validation.ts b/api/src/utils/validation.ts index 66070fb..a1e3310 100644 --- a/api/src/utils/validation.ts +++ b/api/src/utils/validation.ts @@ -178,6 +178,13 @@ export const runCodeValidation = (data: any): Joi.ValidationResult => runTime: Joi.string().valid(...process.runTimes) }).validate(data) +export const triggerCodeValidation = (data: any): Joi.ValidationResult => + Joi.object({ + code: Joi.string().required(), + runTime: Joi.string().valid(...process.runTimes), + expiresAfterMins: Joi.number().greater(0) + }).validate(data) + export const executeProgramRawValidation = (data: any): Joi.ValidationResult => Joi.object({ _program: Joi.string().required(), From 76750e864db839693b1890c436df07e9cabd696b Mon Sep 17 00:00:00 2001 From: Yury Date: Tue, 29 Oct 2024 11:30:05 +0300 Subject: [PATCH 2/4] chore(lint): fixed lint issue --- api/src/utils/upload.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/api/src/utils/upload.ts b/api/src/utils/upload.ts index f85cf57..d4a4119 100644 --- a/api/src/utils/upload.ts +++ b/api/src/utils/upload.ts @@ -51,8 +51,9 @@ export const generateFileUploadSasCode = async ( let fileCount = 0 const uploadedFiles: UploadedFiles[] = [] - const sasSessionFolderList: string[] = - await listFilesInFolder(sasSessionFolder) + const sasSessionFolderList: string[] = await listFilesInFolder( + sasSessionFolder + ) sasSessionFolderList.forEach((fileName) => { let fileCountString = fileCount < 100 ? '0' + fileCount : fileCount fileCountString = fileCount < 10 ? '00' + fileCount : fileCount From 3053c68bdf02e5a45d05388916bcc082d17c8768 Mon Sep 17 00:00:00 2001 From: Yury Date: Tue, 29 Oct 2024 11:40:44 +0300 Subject: [PATCH 3/4] chore(lint): fixed linting issues --- api/src/controllers/internal/Session.ts | 46 ++++++++++++++----------- api/src/controllers/user.ts | 2 +- api/src/utils/upload.ts | 5 ++- 3 files changed, 28 insertions(+), 25 deletions(-) diff --git a/api/src/controllers/internal/Session.ts b/api/src/controllers/internal/Session.ts index 3573658..3000c07 100644 --- a/api/src/controllers/internal/Session.ts +++ b/api/src/controllers/internal/Session.ts @@ -189,33 +189,37 @@ ${autoExecContent}` } private scheduleSessionDestroy(session: Session) { - setTimeout(async () => { - if (session.inUse) { - // adding 10 more minutes - const newDeathTimeStamp = - parseInt(session.deathTimeStamp) + 10 * 60 * 1000 - session.deathTimeStamp = newDeathTimeStamp.toString() - - this.scheduleSessionDestroy(session) - } else { - const { expiresAfterMins } = session - - // delay session destroy if expiresAfterMins present - if (expiresAfterMins && !expiresAfterMins.used) { - // calculate session death time using expiresAfterMins + setTimeout( + async () => { + if (session.inUse) { + // adding 10 more minutes const newDeathTimeStamp = - parseInt(session.deathTimeStamp) + expiresAfterMins.mins * 60 * 1000 + parseInt(session.deathTimeStamp) + 10 * 60 * 1000 session.deathTimeStamp = newDeathTimeStamp.toString() - // set expiresAfterMins to true to avoid using it again - session.expiresAfterMins!.used = true - this.scheduleSessionDestroy(session) } else { - await this.deleteSession(session) + const { expiresAfterMins } = session + + // delay session destroy if expiresAfterMins present + if (expiresAfterMins && !expiresAfterMins.used) { + // calculate session death time using expiresAfterMins + const newDeathTimeStamp = + parseInt(session.deathTimeStamp) + + expiresAfterMins.mins * 60 * 1000 + session.deathTimeStamp = newDeathTimeStamp.toString() + + // set expiresAfterMins to true to avoid using it again + session.expiresAfterMins!.used = true + + this.scheduleSessionDestroy(session) + } else { + await this.deleteSession(session) + } } - } - }, parseInt(session.deathTimeStamp) - new Date().getTime() - 100) + }, + parseInt(session.deathTimeStamp) - new Date().getTime() - 100 + ) } } diff --git a/api/src/controllers/user.ts b/api/src/controllers/user.ts index 424df90..d8790d9 100644 --- a/api/src/controllers/user.ts +++ b/api/src/controllers/user.ts @@ -285,7 +285,7 @@ const getUser = async ( username: user.username, isActive: user.isActive, isAdmin: user.isAdmin, - autoExec: getAutoExec ? user.autoExec ?? '' : undefined, + autoExec: getAutoExec ? (user.autoExec ?? '') : undefined, groups: user.groups } } diff --git a/api/src/utils/upload.ts b/api/src/utils/upload.ts index d4a4119..f85cf57 100644 --- a/api/src/utils/upload.ts +++ b/api/src/utils/upload.ts @@ -51,9 +51,8 @@ export const generateFileUploadSasCode = async ( let fileCount = 0 const uploadedFiles: UploadedFiles[] = [] - const sasSessionFolderList: string[] = await listFilesInFolder( - sasSessionFolder - ) + const sasSessionFolderList: string[] = + await listFilesInFolder(sasSessionFolder) sasSessionFolderList.forEach((fileName) => { let fileCountString = fileCount < 100 ? '0' + fileCount : fileCount fileCountString = fileCount < 10 ? '00' + fileCount : fileCount From 049a7f4b80578392272d6f53b10131ada5fd340d Mon Sep 17 00:00:00 2001 From: Yury Date: Tue, 29 Oct 2024 12:02:26 +0300 Subject: [PATCH 4/4] chore(swagger): improved description --- api/public/swagger.yaml | 20 ++++++++++---------- api/src/controllers/code.ts | 17 +++++++++-------- api/src/controllers/stp.ts | 4 ++-- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/api/public/swagger.yaml b/api/public/swagger.yaml index 04dacd3..922b2f4 100644 --- a/api/public/swagger.yaml +++ b/api/public/swagger.yaml @@ -98,11 +98,11 @@ components: properties: code: type: string - description: 'Code of program' - example: '* Code HERE;' + description: 'The code to be executed' + example: '* Your Code HERE;' runTime: $ref: '#/components/schemas/RunTimeType' - description: 'runtime for program' + description: 'The runtime for the code - eg SAS, JS, PY or R' example: js required: - code @@ -113,7 +113,7 @@ components: properties: sessionId: type: string - description: "Session ID (SAS WORK folder) used to execute code.\nThis session ID should be used to poll job status." + description: "The SessionId is the name of the temporary folder used to store the outputs.\nFor SAS, this would be the SASWORK folder. Can be used to poll job status.\nThis session ID should be used to poll job status." example: '{ sessionId: ''20241028074744-54132-1730101664824'' }' required: - sessionId @@ -123,11 +123,11 @@ components: properties: code: type: string - description: 'Code of program' - example: '* Code HERE;' + description: 'The code to be executed' + example: '* Your Code HERE;' runTime: $ref: '#/components/schemas/RunTimeType' - description: 'runtime for program' + description: 'The runtime for the code - eg SAS, JS, PY or R' example: sas expiresAfterMins: type: number @@ -846,7 +846,7 @@ paths: schema: $ref: '#/components/schemas/TriggerCodeResponse' description: 'Trigger Code on the Specified Runtime' - summary: 'Trigger Code and Return Session Id not awaiting for the job completion' + summary: 'Triggers code and returns SessionId immediately - does not wait for job completion' tags: - Code security: @@ -1852,7 +1852,7 @@ paths: bearerAuth: [] parameters: - - description: 'Location of code in SASjs Drive' + description: 'Location of Stored Program in SASjs Drive.' in: query name: _program required: true @@ -1860,7 +1860,7 @@ paths: type: string example: /Projects/myApp/some/program - - description: 'Optional query param for setting debug mode, which will return the session log.' + description: 'Optional query param for setting debug mode (returns the session log in the response body).' in: query name: _debug required: false diff --git a/api/src/controllers/code.ts b/api/src/controllers/code.ts index 11912d9..4620ba5 100644 --- a/api/src/controllers/code.ts +++ b/api/src/controllers/code.ts @@ -10,12 +10,12 @@ import { interface ExecuteCodePayload { /** - * Code of program - * @example "* Code HERE;" + * The code to be executed + * @example "* Your Code HERE;" */ code: string /** - * runtime for program + * The runtime for the code - eg SAS, JS, PY or R * @example "js" */ runTime: RunTimeType @@ -23,12 +23,12 @@ interface ExecuteCodePayload { interface TriggerCodePayload { /** - * Code of program - * @example "* Code HERE;" + * The code to be executed + * @example "* Your Code HERE;" */ code: string /** - * runtime for program + * The runtime for the code - eg SAS, JS, PY or R * @example "sas" */ runTime: RunTimeType @@ -42,7 +42,8 @@ interface TriggerCodePayload { interface TriggerCodeResponse { /** - * Session ID (SAS WORK folder) used to execute code. + * The SessionId is the name of the temporary folder used to store the outputs. + * For SAS, this would be the SASWORK folder. Can be used to poll job status. * This session ID should be used to poll job status. * @example "{ sessionId: '20241028074744-54132-1730101664824' }" */ @@ -74,7 +75,7 @@ export class CodeController { /** * Trigger Code on the Specified Runtime - * @summary Trigger Code and Return Session Id not awaiting for the job completion + * @summary Triggers code and returns SessionId immediately - does not wait for job completion */ @Post('/trigger') public async triggerCode( diff --git a/api/src/controllers/stp.ts b/api/src/controllers/stp.ts index bd6719e..dbc881d 100644 --- a/api/src/controllers/stp.ts +++ b/api/src/controllers/stp.ts @@ -30,8 +30,8 @@ export class STPController { * https://server.sasjs.io/storedprograms * * @summary Execute a Stored Program, returns _webout and (optionally) log. - * @param _program Location of code in SASjs Drive - * @param _debug Optional query param for setting debug mode, which will return the session log. + * @param _program Location of Stored Program in SASjs Drive. + * @param _debug Optional query param for setting debug mode (returns the session log in the response body). * @example _program "/Projects/myApp/some/program" * @example _debug 131 */