Merge pull request #865 from sasjs/viyaCreateFileAndPatch

Viya create file and patch

.github/workflows/server-tests.yml

@@ -1,107 +1,108 @@
-# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node
+# This workflow will do a clean install of node dependencies, build the source code and run tests across different versions of node
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-nodejs-with-github-actions
-
+
-name: SASjs Build and Server Tests
+name: SASjs Build and Server Tests
-
+
-on:
+on:
-  pull_request:
+  pull_request:
-
+
-jobs:
+jobs:
-  test:
+  test:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-22.04
-
+
-    strategy:
+    strategy:
-      matrix:
+      matrix:
-        node-version: [lts/hydrogen]
+        node-version: [lts/hydrogen]
-
+
-    steps:
+    steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v2
-      - name: Use Node.js ${{ matrix.node-version }}
+      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v2
+        uses: actions/setup-node@v2
-        with:
+        with:
-          node-version: ${{ matrix.node-version }}
+          node-version: ${{ matrix.node-version }}
-
+
-      # 2. Restore npm cache manually
+      # 2. Restore npm cache manually
-      - name: Restore npm cache
+      - name: Restore npm cache
-        uses: actions/cache@v3
+        uses: actions/cache@v3
-        id: npm-cache
+        id: npm-cache
-        with:
+        with:
-          path: ~/.npm
+          path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
+          restore-keys: |
-            ${{ runner.os }}-node-
+            ${{ runner.os }}-node-
-
+
-      - name: Install Dependencies
+      - name: Install Dependencies
-        run: npm ci
+        run: npm ci
-
+
-      - name: Install Rimraf
+      - name: Install Rimraf
-        run: npm i rimraf
+        run: npm i rimraf
-
+
-      - name: Build Package
+      - name: Build Package
-        run: npm run package:lib
+        run: npm run package:lib
-        env:
+        env:
-          CI: true
+          CI: true
-
+
-      - name: Write VPN Files
+      - name: Write VPN Files
-        run: |
+        run: |
-          echo "$CA_CRT" > .github/vpn/ca.crt
+          echo "$CA_CRT" > .github/vpn/ca.crt
-          echo "$USER_CRT" > .github/vpn/user.crt
+          echo "$USER_CRT" > .github/vpn/user.crt
-          echo "$USER_KEY" > .github/vpn/user.key
+          echo "$USER_KEY" > .github/vpn/user.key
-          echo "$TLS_KEY" > .github/vpn/tls.key
+          echo "$TLS_KEY" > .github/vpn/tls.key
-        shell: bash
+        shell: bash
-        env:
+        env:
-          CA_CRT: ${{ secrets.CA_CRT}}
+          CA_CRT: ${{ secrets.CA_CRT}}
-          USER_CRT: ${{ secrets.USER_CRT }}
+          USER_CRT: ${{ secrets.USER_CRT }}
-          USER_KEY: ${{ secrets.USER_KEY }}
+          USER_KEY: ${{ secrets.USER_KEY }}
-          TLS_KEY: ${{ secrets.TLS_KEY }}
+          TLS_KEY: ${{ secrets.TLS_KEY }}
-
+
-      - name: Chmod VPN files
+      - name: Chmod VPN files
-        run: |
+        run: |
-          chmod 600 .github/vpn/ca.crt .github/vpn/user.crt .github/vpn/user.key .github/vpn/tls.key
+          chmod 600 .github/vpn/ca.crt .github/vpn/user.crt .github/vpn/user.key .github/vpn/tls.key
-
+
-      - name: Install Open VPN
+      - name: Install Open VPN
-        run: |
+        run: |
-          sudo apt install apt-transport-https
+          sudo apt install apt-transport-https
-          sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
+          sudo wget https://swupdate.openvpn.net/repos/openvpn-repo-pkg-key.pub
-          sudo apt-key add openvpn-repo-pkg-key.pub
+          sudo apt-key add openvpn-repo-pkg-key.pub
-          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
+          sudo wget -O /etc/apt/sources.list.d/openvpn3.list https://swupdate.openvpn.net/community/openvpn3/repos/openvpn3-jammy.list
-          sudo apt update
+          sudo apt update
-          sudo apt install openvpn3=17~betaUb22042+jammy
+          sudo apt install openvpn3=17~betaUb22042+jammy
-
+
-      - name: Start Open VPN 3
+      - name: Start Open VPN 3
-        run: openvpn3 session-start --config .github/vpn/config.ovpn
+        run: openvpn3 session-start --config .github/vpn/config.ovpn
-
+
-      - name: install pm2
+      - name: install pm2
-        run: npm i -g pm2
+        run: npm i -g pm2
-
+
-      - name: Fetch SASJS server
+      - name: Fetch SASJS server
-        run: curl ${{ secrets.SASJS_SERVER_URL }}/SASjsApi/info
+        run: curl ${{ secrets.SASJS_SERVER_URL }}/SASjsApi/info
-
+
-      - name: Deploy sasjs-tests
+      - name: Deploy sasjs-tests
-        run: |
+        run: |
-          npm install -g replace-in-files-cli
+          sudo apt install jq
-          cd sasjs-tests
+          cd sasjs-tests
-          replace-in-files --regex='"@sasjs/adapter".*' --replacement='"@sasjs/adapter":"latest",' ./package.json
+          jq '.dependencies."@sasjs/adapter" |= "latest"' ./package.json > ./package.temp && mv ./package.temp ./package.json
-          npm i
+          npm i
-          replace-in-files --regex='"serverUrl".*' --replacement='"serverUrl":"${{ secrets.SASJS_SERVER_URL }}",' ./public/config.json
+          jq '.sasJsConfig.serverUrl |= "${{ secrets.SASJS_SERVER_URL }}"' ./public/config.json > ./public/config.temp && mv ./public/config.temp ./public/config.json
-          replace-in-files --regex='"userName".*' --replacement='"userName":"${{ secrets.SASJS_USERNAME }}",' ./public/config.json
+          jq '.sasJsConfig.serverType |= "SASJS"' ./public/config.json > ./public/config.temp && mv ./public/config.temp ./public/config.json
-          replace-in-files --regex='"serverType".*' --replacement='"serverType":"SASJS",' ./public/config.json
+          jq '.userName |= "${{ secrets.SASJS_USERNAME }}"' ./public/config.json > ./public/config.temp && mv ./public/config.temp ./public/config.json
-          replace-in-files --regex='"password".*' --replacement='"password":"${{ secrets.SASJS_PASSWORD }}",' ./public/config.json
+          jq '.password |= "${{ secrets.SASJS_PASSWORD }}"' ./public/config.json > ./public/config.temp && mv ./public/config.temp ./public/config.json
-          cat ./public/config.json
+
-
+          cat ./public/config.json
-          npm run update:adapter
+
-          pm2 start --name sasjs-test npm -- start
+          npm run update:adapter
-
+          pm2 start --name sasjs-test npm -- start
-      - name: Sleep for 10 seconds
+
-        run: sleep 10s
+      - name: Sleep for 10 seconds
-        shell: bash
+        run: sleep 10s
-
+        shell: bash
-      - name: Run cypress on sasjs
+
-        run: |
+      - name: Run cypress on sasjs
-          replace-in-files --regex='"sasjsTestsUrl".*' --replacement='"sasjsTestsUrl":"http://localhost:3000",' ./cypress.json
+        run: |
-          replace-in-files --regex='"username".*' --replacement='"username":"${{ secrets.SASJS_USERNAME }}",' ./cypress.json
+          jq '.env.sasjsTestsUrl |= "http://localhost:3000"' ./cypress.json > ./cypress.temp && mv ./cypress.temp ./cypress.json
-          replace-in-files --regex='"password".*' --replacement='"password":"${{ secrets.SASJS_PASSWORD }}",' ./cypress.json
+          jq '.env.username |= "${{ secrets.SASJS_USERNAME }}"' ./cypress.json > ./cypress.temp && mv ./cypress.temp ./cypress.json
-          cat ./cypress.json
+          jq '.env.password |= "${{ secrets.SASJS_PASSWORD }}"' ./cypress.json > ./cypress.temp && mv ./cypress.temp ./cypress.json
-          echo "SASJS_USERNAME=${{ secrets.SASJS_USERNAME }}"
+          cat ./cypress.json
-
+          echo "SASJS_USERNAME=${{ secrets.SASJS_USERNAME }}"
-          sh ./sasjs-tests/sasjs-cypress-run.sh ${{ secrets.MATRIX_TOKEN }} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+
+          sh ./sasjs-tests/sasjs-cypress-run.sh ${{ secrets.MATRIX_TOKEN }} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}

sasjs-tests/index.css

@@ -12,10 +12,6 @@ body {
   background: #f5f5f5;
 }
 
-#app {
-  min-height: 100vh;
-}
-
 .app__error {
   max-width: 800px;
   margin: 50px auto;
@@ -40,4 +36,4 @@ body {
     border-radius: 4px;
     overflow-x: auto;
   }
-}
+}

src/SASViyaApiClient.ts

@@ -35,6 +35,60 @@ interface JobExecutionResult {
   log?: string
   error?: object
 }
+/* Viya /types/types?limit=999999 response structure */
+interface IViyaTypesResponse {
+  accept: string
+  count: number
+  items: IViyaTypesItem[]
+  limit: number
+  links: IViyaTypesLink[]
+  name: string
+  start: number
+  version: number
+}
+
+/* Item element within the Viya types response */
+interface IViyaTypesItem {
+  description?: string
+  extensions?: string[]
+  iconUri?: string
+  label: string
+  links: IViyaTypesLink[]
+  mappedTypes?: string[]
+  mediaType?: string
+  mediaTypes?: string[]
+  name?: string | undefined
+  pluralLabel?: string
+  properties?: IViyaTypesProperties
+  resourceUri?: string
+  serviceRootUri?: string
+  tags?: string[]
+  version: number
+}
+
+/**
+ * Generic structure for a link
+ * in the links array of a Viya
+ * types/types api response
+ */
+type IViyaTypesLink = Record<string, string>
+
+/**
+ * Generic structure for a type's
+ * 'properties' object from the Viya
+ * types/types api response
+ */
+type IViyaTypesProperties = Record<string, string>
+
+/**
+ * Arbitrary interface for storing
+ * sufficient additional detail to
+ * create and patch a new file.
+ */
+interface IViyaTypesExtensionInfo {
+  typeDefName: string | undefined
+  properties: IViyaTypesProperties | undefined
+}
 
 /**
  * A client for interfacing with the SAS Viya REST API.
@@ -61,6 +115,9 @@ export class SASViyaApiClient {
     this.requestClient
   )
   private folderMap = new Map<string, Job[]>()
+  private fileExtensionMap = new Map<string, IViyaTypesExtensionInfo>()
+  private boolExtensionMap = false // required in case the map has zero entries
+  // after an attempt to populate it.
 
   /**
    * A helper method used to call appendRequest method of RequestClient
@@ -434,15 +491,102 @@ export class SASViyaApiClient {
     const formData = new NodeFormData()
     formData.append('file', contentBuffer, fileName)
 
-    return (
+    /** Query Viya for file metadata based on extension type. */
-      await this.requestClient.post<File>(
+
-        `/files/files?parentFolderUri=${parentFolderUri}&typeDefName=file#rawUpload`,
+    // typeDefName - Viya accepts this property during the file creation
-        formData,
+    let typeDefName: string | undefined = undefined
-        accessToken,
+    // Additional properties are supplied by a patch.
-        'multipart/form-data; boundary=' + (formData as any)._boundary,
+    let filePatch:
-        headers
+      | {
-      )
+          name: string
-    ).result
+          properties: IViyaTypesProperties | undefined
+        }
+      | undefined = undefined
+
+    // The patching process requires properties related to the file-extension
+    const fileExtension: string | undefined = fileName
+      .split('.')
+      .pop()
+      ?.toLowerCase()
+
+    if (fileExtension) {
+      if (!this.boolExtensionMap) {
+        // Populate the file extension map
+        // 1. Get Viya's response to this api call
+        const typesQueryUrl = `/types/types?limit=999999`
+        const response = (
+          await this.requestClient.get(typesQueryUrl, accessToken)
+        ).result as IViyaTypesResponse
+        // 2. Filter the returned items that have file extensions into a map
+        //    using forEach as an item may relate to multiple file extensions.
+        response.items
+          .filter((e) => e.extensions)
+          .forEach((e) => {
+            e.extensions?.forEach((ext) => {
+              this.fileExtensionMap.set(ext, {
+                // `name` becomes the typeDefName value at file creation time.
+                // `name` is ignored here if it is not populated in the map, or
+                // has a blank/empty value.
+                typeDefName: e.name
+                  ? e.name.trim().length
+                    ? e.name.trim()
+                    : undefined
+                  : undefined,
+                properties: e.properties
+              })
+            })
+          })
+        // 3. Toggle the flag to avoid repeating this step
+        this.boolExtensionMap = true
+      }
+
+      // Query the map for the current file extension
+      const fileExtInfo = this.fileExtensionMap.get(fileExtension)
+      if (fileExtInfo) {
+        // If the extension was found in the map, record the typeDefName and
+        // create a patch if a properties object was returned.
+        typeDefName = fileExtInfo.typeDefName
+        if (fileExtInfo.properties)
+          filePatch = { name: fileName, properties: fileExtInfo.properties }
+      }
+    }
+
+    // Create the file
+    const createFileResponse = await this.requestClient.post<File>(
+      `/files/files?parentFolderUri=${parentFolderUri}&typeDefName=${
+        typeDefName ?? 'file'
+      }#rawUpload`,
+      formData,
+      accessToken,
+      'multipart/form-data; boundary=' + (formData as any)._boundary,
+      headers
+    )
+
+    // If a patch was created...
+    if (filePatch) {
+      try {
+        const patchHeaders = {
+          Accept: 'application/json',
+          'If-Match': '*'
+        }
+        // Get the URI of the newly created file
+        const fileUri = createFileResponse.result.links.filter(
+          (e) => e.method == 'PATCH' && e.rel == 'patch'
+        )[0].uri
+        // and apply the patch
+        return (
+          await this.requestClient.patch<File>(
+            `${fileUri}`,
+            filePatch,
+            accessToken,
+            patchHeaders
+          )
+        ).result
+      } catch (e: any) {
+        throw new Error(`Error patching file ${fileName}.\n${e.message}`)
+      }
+    }
+    return createFileResponse.result
   }
 
   /**

src/request/RequestClient.ts

@@ -273,9 +273,13 @@ export class RequestClient implements HttpClient {
   public async patch<T>(
     url: string,
     data: any = {},
-    accessToken?: string
+    accessToken?: string,
+    overrideHeaders: { [key: string]: string | number } = {}
   ): Promise<{ result: T; etag: string }> {
-    const headers = this.getHeaders(accessToken, 'application/json')
+    const headers = {
+      ...this.getHeaders(accessToken, 'application/json'),
+      ...overrideHeaders
+    }
 
     return this.httpClient
       .patch<T>(url, data, { headers, withXSRFToken: true })