fix(*): switch to file upload approach with large datasets and special characters

sasjs-tests/package-lock.json

@@ -1,6 +1,6 @@
 {
-  "name": "sasjs-tests",
-  "version": "0.1.0",
+  "name": "@sasjs/tests",
+  "version": "1.0.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -1356,6 +1356,28 @@
       "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-1.1.3.tgz",
       "integrity": "sha512-shAmDyaQC4H92APFoIaVDHCx5bStIocgvbwQyxPRrbUY20V1EYTbSDchWbuwlMG3V17cprZhA6+78JfB+3DTPw=="
     },
+    "@sasjs/adapter": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/@sasjs/adapter/-/adapter-1.0.4.tgz",
+      "integrity": "sha512-wjABkVqPQ0AaSoizCrLInG3HX1o+4f98UsHRGDHLHvlQzIIOUtUNElIAPYoy5Xxu+CPk030jXkgCls7n0k/3cw==",
+      "requires": {
+        "es6-promise": "^4.2.8",
+        "form-data": "^3.0.0",
+        "isomorphic-fetch": "^2.2.1"
+      },
+      "dependencies": {
+        "form-data": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz",
+          "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==",
+          "requires": {
+            "asynckit": "^0.4.0",
+            "combined-stream": "^1.0.8",
+            "mime-types": "^2.1.12"
+          }
+        }
+      }
+    },
     "@sheerun/mutationobserver-shim": {
       "version": "0.3.3",
       "resolved": "https://registry.npmjs.org/@sheerun/mutationobserver-shim/-/mutationobserver-shim-0.3.3.tgz",
@@ -4912,11 +4934,21 @@
       "integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k="
     },
     "encoding": {
-      "version": "0.1.12",
-      "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.12.tgz",
-      "integrity": "sha1-U4tm8+5izRq1HsMjgp0flIDHS+s=",
+      "version": "0.1.13",
+      "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz",
+      "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==",
       "requires": {
-        "iconv-lite": "~0.4.13"
+        "iconv-lite": "^0.6.2"
+      },
+      "dependencies": {
+        "iconv-lite": {
+          "version": "0.6.2",
+          "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.2.tgz",
+          "integrity": "sha512-2y91h5OpQlolefMPmUlivelittSWy0rP+oYVpn6A7GwVHNE8AWzoYOBNmlwks3LobaJxgHCYZAnyNo2GgpNRNQ==",
+          "requires": {
+            "safer-buffer": ">= 2.1.2 < 3.0.0"
+          }
+        }
       }
     },
     "end-of-stream": {
@@ -11950,27 +11982,6 @@
       "resolved": "https://registry.npmjs.org/sanitize.css/-/sanitize.css-10.0.0.tgz",
       "integrity": "sha512-vTxrZz4dX5W86M6oVWVdOVe72ZiPs41Oi7Z6Km4W5Turyz28mrXSJhhEBZoRtzJWIv3833WKVwLSDWWkEfupMg=="
     },
-    "sasjs": {
-      "version": "file:../build/sasjs-5.0.0.tgz",
-      "integrity": "sha512-8Ez2iS8BKzu2GG1Cwf/pe5PgNvdhowFodQNCTHIxMlDYgLqmg1mcpwRjJjnXF9A73gX0NkR65olYYAesp8cMMA==",
-      "requires": {
-        "es6-promise": "^4.2.8",
-        "form-data": "^3.0.0",
-        "isomorphic-fetch": "^2.2.1"
-      },
-      "dependencies": {
-        "form-data": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.0.tgz",
-          "integrity": "sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg==",
-          "requires": {
-            "asynckit": "^0.4.0",
-            "combined-stream": "^1.0.8",
-            "mime-types": "^2.1.12"
-          }
-        }
-      }
-    },
     "sass-graph": {
       "version": "2.2.5",
       "resolved": "https://registry.npmjs.org/sass-graph/-/sass-graph-2.2.5.tgz",

sasjs-tests/package.json

@@ -1,9 +1,10 @@
 {
-  "name": "sasjs-tests",
-  "version": "0.1.0",
+  "name": "@sasjs/tests",
+  "version": "1.0.0",
   "homepage": ".",
   "private": true,
   "dependencies": {
+    "@sasjs/adapter": "*",
     "@testing-library/jest-dom": "^4.2.4",
     "@testing-library/react": "^9.5.0",
     "@testing-library/user-event": "^7.2.1",
@@ -16,7 +17,6 @@
     "react-dom": "^16.13.1",
     "react-router-dom": "^5.2.0",
     "react-scripts": "3.4.1",
-    "sasjs": "file:../build/sasjs-5.0.0.tgz",
     "typescript": "^3.9.6"
   },
   "scripts": {
@@ -24,7 +24,7 @@
     "build": "react-scripts build",
     "test": "react-scripts test",
     "eject": "react-scripts eject",
-    "deploy": "rsync -avhe ssh ./build/* --delete kriaco@sas.analytium.co.uk:/var/www/html/kriaco/sasjs-tests"
+    "deploy": "npm run build && rsync -avhe ssh ./build/* --delete kriaco@sas.analytium.co.uk:/var/www/html/kriaco/sasjs-tests"
   },
   "eslintConfig": {
     "extends": "react-app"

sasjs-tests/src/TestSuiteRunner.tsx

@@ -4,7 +4,7 @@ import TestSuiteCard from "./components/TestSuiteCard";
 import { TestSuite, Test } from "./types";
 import { basicTests } from "./testSuites/Basic";
 import "./TestSuiteRunner.scss";
-import SASjs from "sasjs";
+import SASjs from "@sasjs/adapter";
 import { AppContext } from "./context/AppContext";
 import { sendArrTests, sendObjTests } from "./testSuites/RequestData";
 import { specialCaseTests } from "./testSuites/SpecialCases";
@@ -38,11 +38,11 @@ const TestSuiteRunner = (
   useEffect(() => {
     if (adapter) {
       setTestSuites([
-        // basicTests(adapter, config.userName, config.password),
-        // sendArrTests(adapter),
-        // sendObjTests(adapter),
+        basicTests(adapter, config.userName, config.password),
+        sendArrTests(adapter),
+        sendObjTests(adapter),
         specialCaseTests(adapter),
-        // sasjsRequestTests(adapter),
+        sasjsRequestTests(adapter),
       ]);
       setCompletedTestSuites([]);
     }

sasjs-tests/src/context/AppContext.tsx

@@ -1,5 +1,5 @@
 import React, { createContext, useState, useEffect, ReactNode } from "react";
-import SASjs from "sasjs";
+import SASjs from "@sasjs/adapter";
 
 export const AppContext = createContext<{
   config: any;

sasjs-tests/src/testSuites/Basic.ts

@@ -1,4 +1,4 @@
-import SASjs, { ServerType, SASjsConfig } from "sasjs";
+import SASjs, { ServerType, SASjsConfig } from "@sasjs/adapter";
 import { TestSuite } from "../types";
 
 const defaultConfig: SASjsConfig = {

sasjs-tests/src/testSuites/RequestData.ts

@@ -1,4 +1,4 @@
-import SASjs from "sasjs";
+import SASjs from "@sasjs/adapter";
 import { TestSuite } from "../types";
 
 const stringData: any = { table1: [{ col1: "first col value" }] };

sasjs-tests/src/testSuites/SasjsRequests.ts

@@ -1,4 +1,4 @@
-import SASjs from "sasjs";
+import SASjs from "@sasjs/adapter";
 import { TestSuite } from "../types";
 
 const data: any = { table1: [{ col1: "first col value" }] };
@@ -12,7 +12,7 @@ export const sasjsRequestTests = (adapter: SASjs): TestSuite => ({
       test: async () => {
         return adapter.request("common/sendArr", data);
       },
-      assertion: (res: any) => {
+      assertion: () => {
         const requests = adapter.getSasRequests();
         if (adapter.getSasjsConfig().debug) {
           return requests[0].SASWORK !== null;

sasjs-tests/src/testSuites/SpecialCases.ts

@@ -1,4 +1,4 @@
-import SASjs from "sasjs";
+import SASjs from "@sasjs/adapter";
 import { TestSuite } from "../types";
 
 const specialCharData: any = {
@@ -102,86 +102,86 @@ export const specialCaseTests = (adapter: SASjs): TestSuite => ({
         );
       },
     },
-    // {
-    //   title: "Other special characters",
-    //   description: "Should handle other special characters",
-    //   test: () => {
-    //     return adapter.request("common/sendArr", moreSpecialCharData);
-    //   },
-    //   assertion: (res: any) => {
-    //     return (
-    //       res.table1[0][0] === moreSpecialCharData.table1[0].speech0 &&
-    //       res.table1[0][1] === moreSpecialCharData.table1[0].pct &&
-    //       res.table1[0][2] === moreSpecialCharData.table1[0].speech &&
-    //       res.table1[0][3] === moreSpecialCharData.table1[0].slash &&
-    //       res.table1[0][4] === moreSpecialCharData.table1[0].slashWithSpecial &&
-    //       res.table1[0][5] === moreSpecialCharData.table1[0].macvar &&
-    //       res.table1[0][6] === moreSpecialCharData.table1[0].chinese &&
-    //       res.table1[0][7] === moreSpecialCharData.table1[0].sigma &&
-    //       res.table1[0][8] === moreSpecialCharData.table1[0].at &&
-    //       res.table1[0][9] === moreSpecialCharData.table1[0].serbian &&
-    //       res.table1[0][10] === moreSpecialCharData.table1[0].dollar
-    //     );
-    //   },
-    // },
-    // {
-    //   title: "Wide table with sendArr",
-    //   description: "Should handle data with 10000 columns",
-    //   test: () => {
-    //     return adapter.request("common/sendArr", getWideData());
-    //   },
-    //   assertion: (res: any) => {
-    //     const data = getWideData();
-    //     let result = true;
-    //     for (let i = 0; i <= 10; i++) {
-    //       result =
-    //         result && res.table1[0][i] === data.table1[0]["col" + (i + 1)];
-    //     }
-    //     return result;
-    //   },
-    // },
-    // {
-    //   title: "Wide table with sendObj",
-    //   description: "Should handle data with 10000 columns",
-    //   test: () => {
-    //     return adapter.request("common/sendObj", getWideData());
-    //   },
-    //   assertion: (res: any) => {
-    //     const data = getWideData();
-    //     let result = true;
-    //     for (let i = 0; i <= 10; i++) {
-    //       result =
-    //         result &&
-    //         res.table1[0]["COL" + (i + 1)] === data.table1[0]["col" + (i + 1)];
-    //     }
-    //     return result;
-    //   },
-    // },
-    // {
-    //   title: "Multiple tables",
-    //   description: "Should handle data with 100 tables",
-    //   test: () => {
-    //     return adapter.request("common/sendArr", getTables());
-    //   },
-    //   assertion: (res: any) => {
-    //     const data = getTables();
-    //     return (
-    //       res.table1[0][0] === data.table1[0].col1 &&
-    //       res.table1[0][1] === data.table1[0].col2 &&
-    //       res.table1[0][2] === data.table1[0].col3 &&
-    //       res.table1[0][3] === data.table1[0].col4 &&
-    //       res.table50[0][0] === data.table50[0].col1 &&
-    //       res.table50[0][1] === data.table50[0].col2 &&
-    //       res.table50[0][2] === data.table50[0].col3 &&
-    //       res.table50[0][3] === data.table50[0].col4
-    //     );
-    //   },
-    // },
     {
-      title: "Large dataset",
+      title: "Other special characters",
+      description: "Should handle other special characters",
+      test: () => {
+        return adapter.request("common/sendArr", moreSpecialCharData);
+      },
+      assertion: (res: any) => {
+        return (
+          res.table1[0][0] === moreSpecialCharData.table1[0].speech0 &&
+          res.table1[0][1] === moreSpecialCharData.table1[0].pct &&
+          res.table1[0][2] === moreSpecialCharData.table1[0].speech &&
+          res.table1[0][3] === moreSpecialCharData.table1[0].slash &&
+          res.table1[0][4] === moreSpecialCharData.table1[0].slashWithSpecial &&
+          res.table1[0][5] === moreSpecialCharData.table1[0].macvar &&
+          res.table1[0][6] === moreSpecialCharData.table1[0].chinese &&
+          res.table1[0][7] === moreSpecialCharData.table1[0].sigma &&
+          res.table1[0][8] === moreSpecialCharData.table1[0].at &&
+          res.table1[0][9] === moreSpecialCharData.table1[0].serbian &&
+          res.table1[0][10] === moreSpecialCharData.table1[0].dollar
+        );
+      },
+    },
+    {
+      title: "Wide table with sendArr",
+      description: "Should handle data with 10000 columns",
+      test: () => {
+        return adapter.request("common/sendArr", getWideData());
+      },
+      assertion: (res: any) => {
+        const data = getWideData();
+        let result = true;
+        for (let i = 0; i <= 10; i++) {
+          result =
+            result && res.table1[0][i] === data.table1[0]["col" + (i + 1)];
+        }
+        return result;
+      },
+    },
+    {
+      title: "Wide table with sendObj",
+      description: "Should handle data with 10000 columns",
+      test: () => {
+        return adapter.request("common/sendObj", getWideData());
+      },
+      assertion: (res: any) => {
+        const data = getWideData();
+        let result = true;
+        for (let i = 0; i <= 10; i++) {
+          result =
+            result &&
+            res.table1[0]["COL" + (i + 1)] === data.table1[0]["col" + (i + 1)];
+        }
+        return result;
+      },
+    },
+    {
+      title: "Multiple tables",
+      description: "Should handle data with 100 tables",
+      test: () => {
+        return adapter.request("common/sendArr", getTables());
+      },
+      assertion: (res: any) => {
+        const data = getTables();
+        return (
+          res.table1[0][0] === data.table1[0].col1 &&
+          res.table1[0][1] === data.table1[0].col2 &&
+          res.table1[0][2] === data.table1[0].col3 &&
+          res.table1[0][3] === data.table1[0].col4 &&
+          res.table50[0][0] === data.table50[0].col1 &&
+          res.table50[0][1] === data.table50[0].col2 &&
+          res.table50[0][2] === data.table50[0].col3 &&
+          res.table50[0][3] === data.table50[0].col4
+        );
+      },
+    },
+    {
+      title: "Large dataset with sendObj",
       description: "Should handle 5mb of data",
       test: () => {
-        return adapter.request("common/sendArr", getLargeDataset());
+        return adapter.request("common/sendObj", getLargeDataset());
       },
       assertion: (res: any) => {
         const data = getLargeDataset();
@@ -192,44 +192,59 @@ export const specialCaseTests = (adapter: SASjs): TestSuite => ({
         return result;
       },
     },
-
-    // {
-    //   title: "Error and _csrf tables with sendArr",
-    //   description: "Should handle error and _csrf tables",
-    //   test: () => {
-    //     return adapter.request("common/sendArr", errorAndCsrfData);
-    //   },
-    //   assertion: (res: any) => {
-    //     return (
-    //       res.error[0][0] === errorAndCsrfData.error[0].col1 &&
-    //       res.error[0][1] === errorAndCsrfData.error[0].col2 &&
-    //       res.error[0][2] === errorAndCsrfData.error[0].col3 &&
-    //       res.error[0][3] === errorAndCsrfData.error[0].col4 &&
-    //       res._csrf[0][0] === errorAndCsrfData._csrf[0].col1 &&
-    //       res._csrf[0][1] === errorAndCsrfData._csrf[0].col2 &&
-    //       res._csrf[0][2] === errorAndCsrfData._csrf[0].col3 &&
-    //       res._csrf[0][3] === errorAndCsrfData._csrf[0].col4
-    //     );
-    //   },
-    // },
-    // {
-    //   title: "Error and _csrf tables with sendObj",
-    //   description: "Should handle error and _csrf tables",
-    //   test: () => {
-    //     return adapter.request("common/sendObj", errorAndCsrfData);
-    //   },
-    //   assertion: (res: any) => {
-    //     return (
-    //       res.error[0].COL1 === errorAndCsrfData.error[0].col1 &&
-    //       res.error[0].COL2 === errorAndCsrfData.error[0].col2 &&
-    //       res.error[0].COL3 === errorAndCsrfData.error[0].col3 &&
-    //       res.error[0].COL4 === errorAndCsrfData.error[0].col4 &&
-    //       res._csrf[0].COL1 === errorAndCsrfData._csrf[0].col1 &&
-    //       res._csrf[0].COL2 === errorAndCsrfData._csrf[0].col2 &&
-    //       res._csrf[0].COL3 === errorAndCsrfData._csrf[0].col3 &&
-    //       res._csrf[0].COL4 === errorAndCsrfData._csrf[0].col4
-    //     );
-    //   },
-    // },
+    {
+      title: "Large dataset with sendArr",
+      description: "Should handle 5mb of data",
+      test: () => {
+        return adapter.request("common/sendArr", getLargeDataset());
+      },
+      assertion: (res: any) => {
+        const data = getLargeDataset();
+        let result = true;
+        for (let i = 0; i <= 10; i++) {
+          result =
+            result && res.table1[i][0] === Object.values(data.table1[i])[0];
+        }
+        return result;
+      },
+    },
+    {
+      title: "Error and _csrf tables with sendArr",
+      description: "Should handle error and _csrf tables",
+      test: () => {
+        return adapter.request("common/sendArr", errorAndCsrfData);
+      },
+      assertion: (res: any) => {
+        return (
+          res.error[0][0] === errorAndCsrfData.error[0].col1 &&
+          res.error[0][1] === errorAndCsrfData.error[0].col2 &&
+          res.error[0][2] === errorAndCsrfData.error[0].col3 &&
+          res.error[0][3] === errorAndCsrfData.error[0].col4 &&
+          res._csrf[0][0] === errorAndCsrfData._csrf[0].col1 &&
+          res._csrf[0][1] === errorAndCsrfData._csrf[0].col2 &&
+          res._csrf[0][2] === errorAndCsrfData._csrf[0].col3 &&
+          res._csrf[0][3] === errorAndCsrfData._csrf[0].col4
+        );
+      },
+    },
+    {
+      title: "Error and _csrf tables with sendObj",
+      description: "Should handle error and _csrf tables",
+      test: () => {
+        return adapter.request("common/sendObj", errorAndCsrfData);
+      },
+      assertion: (res: any) => {
+        return (
+          res.error[0].COL1 === errorAndCsrfData.error[0].col1 &&
+          res.error[0].COL2 === errorAndCsrfData.error[0].col2 &&
+          res.error[0].COL3 === errorAndCsrfData.error[0].col3 &&
+          res.error[0].COL4 === errorAndCsrfData.error[0].col4 &&
+          res._csrf[0].COL1 === errorAndCsrfData._csrf[0].col1 &&
+          res._csrf[0].COL2 === errorAndCsrfData._csrf[0].col2 &&
+          res._csrf[0].COL3 === errorAndCsrfData._csrf[0].col3 &&
+          res._csrf[0].COL4 === errorAndCsrfData._csrf[0].col4
+        );
+      },
+    },
   ],
 });

src/SASjs.ts

@@ -479,38 +479,37 @@ export default class SASjs {
 
           formData.append(name, file, `${name}.csv`);
         }
+      } else {
+        // param based approach
+        const sasjsTables = [];
+        let tableCounter = 0;
+        for (const tableName in data) {
+          if (isError) {
+            return;
+          }
+          tableCounter++;
+          sasjsTables.push(tableName);
+          const csv = convertToCSV(data[tableName]);
+          if (csv === "ERROR: LARGE STRING LENGTH") {
+            isError = true;
+            errorMsg =
+              "The max length of a string value in SASjs is 32765 characters.";
+          }
+          // if csv has length more then 16k, send in chunks
+          if (csv.length > 16000) {
+            const csvChunks = splitChunks(csv);
+            // append chunks to form data with same key
+            csvChunks.map((chunk) => {
+              formData.append(`sasjs${tableCounter}data`, chunk);
+            });
+          } else {
+            requestParams[`sasjs${tableCounter}data`] = csv;
+          }
+        }
+        requestParams["sasjs_tables"] = sasjsTables.join(" ");
       }
-      // param based approach
-      const sasjsTables = [];
-      let tableCounter = 0;
-      for (const tableName in data) {
-        if (isError) {
-          return;
-        }
-        tableCounter++;
-        sasjsTables.push(tableName);
-        const csv = convertToCSV(data[tableName]);
-        if (csv === "ERROR: LARGE STRING LENGTH") {
-          isError = true;
-          errorMsg =
-            "The max length of a string value in SASjs is 32765 characters.";
-        }
-        // if csv has length more then 16k, send in chunks
-        if (csv.length > 16000) {
-          const csvChunks = splitChunks(csv);
-          // append chunks to form data with same key
-          csvChunks.map((chunk) => {
-            formData.append(`sasjs${tableCounter}data`, chunk);
-          });
-        } else {
-          requestParams[`sasjs${tableCounter}data`] = csv;
-        }
-      }
-      requestParams["sasjs_tables"] = sasjsTables.join(" ");
     }
 
-    console.log("Request params", requestParams);
-
     for (const key in requestParams) {
       if (requestParams.hasOwnProperty(key)) {
         formData.append(key, requestParams[key]);
@@ -526,9 +525,7 @@ export default class SASjs {
         if (isError) {
           reject({ MESSAGE: errorMsg });
         }
-        const headers: any = {
-          "Content-Type": "application/x-www-form-urlencoded",
-        };
+        const headers: any = {};
         if (this._csrfHeader && this._csrf) {
           headers[this._csrfHeader] = this._csrf;
         }
@@ -720,9 +717,9 @@ export default class SASjs {
   private getRequestParams(): any {
     const requestParams: any = {};
 
-    // if (this._csrf) {
-    //   requestParams["_csrf"] = this._csrf;
-    // }
+    if (this._csrf) {
+      requestParams["_csrf"] = this._csrf;
+    }
 
     if (this.sasjsConfig.debug) {
       requestParams["_omittextlog"] = "false";