fix: adding mod statement to _webout to enable sas-side sasjs testing

fix: switching to data step for JSON generation in mp_jsonout and the…

.gitpod.dockerfile

@@ -1,10 +1,6 @@
-
 FROM gitpod/workspace-full
 
 RUN sudo apt-get update \
  && sudo apt-get install -y \
     doxygen \
- && npm i -g npm@latest \
- && npm i -g @sasjs/cli \
- && npm i \
- && sudo rm -rf /var/lib/apt/lists/*
+ && sudo rm -rf /var/lib/apt/lists/*

.gitpod.yml

@@ -1,8 +1,8 @@
 tasks:
-  - init: npm i && clear
+  - init: npm i -g @sasjs/cli
 
 image:
   file: .gitpod.dockerfile
 vscode:
   extensions:
-    - sasjs.sasjs-for-vscode@1.7.2:R6y1nzpFh2P99BZg5FgH5g==
+    - sasjs.sasjs-for-vscode

.sasjslint

@@ -2,8 +2,11 @@
     "noTrailingSpaces": true,
     "noEncodedPasswords": true,
     "hasDoxygenHeader": true,
+    "hasMacroNameInMend": false,
+    "hasMacroParentheses": true,
+    "noNestedMacros": false,
     "noSpacesInFileNames": true,
-    "maxLineLength": 140,
+    "maxLineLength": 135,
     "lowerCaseFileNames": true,
     "noTabIndentation": true,
     "indentationMultiple": 2

.vscode/extensions.json

@@ -0,0 +1,5 @@
+{
+  "recommendations": [
+    "sasjs.sasjs-for-vscode"
+  ]
+}

all.sas

@@ -573,21 +573,23 @@ options noquotelenmax;
     or "&sysprocessmode"= "SAS Compute Server" %then %do;
         SASVIYA
     %end;
-    %else %if "&sysprocessmode"="SAS Stored Process Server" %then %do;
+    %else %if "&sysprocessmode"="SAS Stored Process Server"
+      or "&sysprocessmode"="SAS Workspace Server"
+    %then %do;
       SASMETA
       %return;
     %end;
     %else %do;
-      SAS
+      BASESAS
       %return;
     %end;
   %end;
-  %else %if %symexist(_metaport) %then %do;
+  %else %if %symexist(_metaport) or %symexist(_metauser) %then %do;
     SASMETA
     %return;
   %end;
   %else %do;
-    SAS
+    BASESAS
     %return;
   %end;
 %end;
@@ -3917,8 +3919,11 @@ create table &outds (rename=(
   @details PROC JSON is faster but will produce errs like the ones below if
   special chars are encountered.
 
-  >An object or array close is not valid at this point in the JSON text.
-  >Date value out of range
+  > ERROR: Some code points did not transcode.
+
+  > An object or array close is not valid at this point in the JSON text.
+
+  > Date value out of range
 
   If this happens, try running with ENGINE=DATASTEP.
 
@@ -3927,7 +3932,9 @@ create table &outds (rename=(
         filename tmp temp;
         data class; set sashelp.class;run;
 
+        %mp_jsonout(OPEN,jref=tmp)
         %mp_jsonout(OBJ,class,jref=tmp)
+        %mp_jsonout(CLOSE,jref=tmp)
 
         data _null_;
         infile tmp;
@@ -3940,18 +3947,18 @@ create table &outds (rename=(
   For more information see https://sasjs.io
 
   @param action Valid values:
-    * OPEN - opens the JSON
-    * OBJ - sends a table with each row as an object
-    * ARR - sends a table with each row in an array
-    * CLOSE - closes the JSON
+    @li OPEN - opens the JSON
+    @li OBJ - sends a table with each row as an object
+    @li ARR - sends a table with each row in an array
+    @li CLOSE - closes the JSON
 
   @param ds the dataset to send.  Must be a work table.
   @param jref= the fileref to which to send the JSON
   @param dslabel= the name to give the table in the exported JSON
   @param fmt= Whether to keep or strip formats from the table
-  @param engine= Which engine to use to send the JSON, options are:
-  * PROCJSON (default)
-  * DATASTEP
+  @param engine= Which engine to use to send the JSON, valid options are:
+    @li PROCJSON (default)
+    @li DATASTEP (more reliable when data has non standard characters)
 
   @param dbg= DEPRECATED - was used to conditionally add PRETTY to
     proc json but this can cause line truncation in large files.
@@ -4073,11 +4080,12 @@ create table &outds (rename=(
 %end;
 
 %else %if &action=CLOSE %then %do;
-  data _null_;file &jref encoding='utf-8';
+  data _null_;file &jref encoding='utf-8' mod;
     put "}";
   run;
 %end;
-%mend;/**
+%mend;
+/**
   @file
   @brief Convert all library members to CARDS files
   @details Gets list of members then calls the <code>%mp_ds2cards()</code> macro.
@@ -4472,7 +4480,8 @@ insert into &outds select distinct * from &append_ds;
 /**
   @file
   @brief Reset an option to original value
-  @details Inspired by the SAS Jedi - https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options/
+  @details Inspired by the SAS Jedi -
+https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options
     Called as follows:
 
     options obs=30;
@@ -7298,8 +7307,6 @@ run;
     */
   %mm_updatestpsourcecode(stp=&tree/&stpname
     ,stpcode="&directory/&filename"
-    ,frefin=&frefin.
-    ,frefout=&frefout.
     ,mdebug=&mdebug
     ,minify=&minify)
 
@@ -7507,7 +7514,7 @@ data _null_;
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8''; ';
+  put '  data _null_;file &jref encoding=''utf-8'' mod; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
@@ -7588,14 +7595,14 @@ data _null_;
   put '      i+1; ';
   put '      call symputx(''wt''!!left(i),name,''l''); ';
   put '      call symputx(''wtcnt'',i,''l''); ';
-  put '    data _null_; file &fref encoding=''utf-8''; ';
+  put '    data _null_; file &fref mod encoding=''utf-8''; ';
   put '      put ",""WORK"":{"; ';
   put '    %do i=1 %to &wtcnt; ';
   put '      %let wt=&&wt&i; ';
   put '      proc contents noprint data=&wt ';
   put '        out=_data_ (keep=name type length format:); ';
   put '      run;%let tempds=%scan(&syslast,2,.); ';
-  put '      data _null_; file &fref encoding=''utf-8''; ';
+  put '      data _null_; file &fref mod encoding=''utf-8''; ';
   put '        dsid=open("WORK.&wt",''is''); ';
   put '        nlobs=attrn(dsid,''NLOBS''); ';
   put '        nvars=attrn(dsid,''NVARS''); ';
@@ -7606,10 +7613,10 @@ data _null_;
   put '        put '',"nvars":'' nvars; ';
   put '      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP) ';
   put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP) ';
-  put '      data _null_; file &fref encoding=''utf-8''; ';
+  put '      data _null_; file &fref mod encoding=''utf-8''; ';
   put '        put "}"; ';
   put '    %end; ';
-  put '    data _null_; file &fref encoding=''utf-8''; ';
+  put '    data _null_; file &fref mod encoding=''utf-8''; ';
   put '      put "}"; ';
   put '    run; ';
   put '  %end; ';
@@ -11057,14 +11064,14 @@ run;
       i+1;
       call symputx('wt'!!left(i),name,'l');
       call symputx('wtcnt',i,'l');
-    data _null_; file &fref encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8';
       put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
       proc contents noprint data=&wt
         out=_data_ (keep=name type length format:);
       run;%let tempds=%scan(&syslast,2,.);
-      data _null_; file &fref encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8';
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
         nvars=attrn(dsid,'NVARS');
@@ -11075,10 +11082,10 @@ run;
         put ',"nvars":' nvars;
       %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
       %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
-      data _null_; file &fref encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8';
         put "}";
     %end;
-    data _null_; file &fref encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8';
       put "}";
     run;
   %end;
@@ -11430,13 +11437,13 @@ options noquotelenmax;
     needs to be attached to the beginning of the job
   @param code= Fileref(s) of the actual code to be added
   @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values are "password" or "authorization_code" (unquoted).
-    The default is authorization_code.
+  @param grant_type= valid values are "password" or "authorization_code"
+    (unquoted). The default is authorization_code.
   @param replace= select NO to avoid replacing any existing job in that location
   @param contextname= Choose a specific context on which to run the Job.  Leave
     blank to use the default context.  From Viya 3.5 it is possible to configure
     a shared context - see
-    https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5pyhn1wg3ja0drdl6h.htm&docsetVersion=3.5&locale=en
+https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5pyhn1wg3ja0drdl6h.htm&docsetVersion=3.5&locale=en
 
   @version VIYA V.03.04
   @author [Allan Bowe](https://www.linkedin.com/in/allanbowe)
@@ -12034,7 +12041,7 @@ data _null_;
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8''; ';
+  put '  data _null_;file &jref encoding=''utf-8'' mod; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
@@ -12166,7 +12173,7 @@ data _null_;
   put '%end; ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
   put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
-  put '      ,jref=&fref,engine=PROCJSON,dbg=%str(&_debug) ';
+  put '      ,jref=&fref,engine=DATASTEP,dbg=%str(&_debug) ';
   put '    ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';
@@ -15373,7 +15380,7 @@ libname &libref clear;
 filename &fref1 clear;
 
 %mend;/**
-  @file mv_webout.sas
+  @file
   @brief Send data to/from the SAS Viya Job Execution Service
   @details This macro should be added to the start of each Job Execution
   Service, **immediately** followed by a call to:
@@ -15385,7 +15392,7 @@ filename &fref1 clear;
     following syntax:
 
         data some datasets; * make some data ;
-        retain some columns;
+          retain some columns;
         run;
 
         %mv_webout(OPEN)
@@ -15536,7 +15543,7 @@ filename &fref1 clear;
 %end;
 %else %if &action=ARR or &action=OBJ %then %do;
     %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt
-      ,jref=&fref,engine=PROCJSON,dbg=%str(&_debug)
+      ,jref=&fref,engine=DATASTEP,dbg=%str(&_debug)
     )
 %end;
 %else %if &action=CLOSE %then %do;

base/mf_getplatform.sas

@@ -27,21 +27,23 @@
     or "&sysprocessmode"= "SAS Compute Server" %then %do;
         SASVIYA
     %end;
-    %else %if "&sysprocessmode"="SAS Stored Process Server" %then %do;
+    %else %if "&sysprocessmode"="SAS Stored Process Server"
+      or "&sysprocessmode"="SAS Workspace Server"
+    %then %do;
       SASMETA
       %return;
     %end;
     %else %do;
-      SAS
+      BASESAS
       %return;
     %end;
   %end;
-  %else %if %symexist(_metaport) %then %do;
+  %else %if %symexist(_metaport) or %symexist(_metauser) %then %do;
     SASMETA
     %return;
   %end;
   %else %do;
-    SAS
+    BASESAS
     %return;
   %end;
 %end;

base/mp_jsonout.sas

@@ -4,8 +4,11 @@
   @details PROC JSON is faster but will produce errs like the ones below if
   special chars are encountered.
 
-  >An object or array close is not valid at this point in the JSON text.
-  >Date value out of range
+  > ERROR: Some code points did not transcode.
+
+  > An object or array close is not valid at this point in the JSON text.
+
+  > Date value out of range
 
   If this happens, try running with ENGINE=DATASTEP.
 
@@ -14,7 +17,9 @@
         filename tmp temp;
         data class; set sashelp.class;run;
 
+        %mp_jsonout(OPEN,jref=tmp)
         %mp_jsonout(OBJ,class,jref=tmp)
+        %mp_jsonout(CLOSE,jref=tmp)
 
         data _null_;
         infile tmp;
@@ -27,18 +32,18 @@
   For more information see https://sasjs.io
 
   @param action Valid values:
-    * OPEN - opens the JSON
-    * OBJ - sends a table with each row as an object
-    * ARR - sends a table with each row in an array
-    * CLOSE - closes the JSON
+    @li OPEN - opens the JSON
+    @li OBJ - sends a table with each row as an object
+    @li ARR - sends a table with each row in an array
+    @li CLOSE - closes the JSON
 
   @param ds the dataset to send.  Must be a work table.
   @param jref= the fileref to which to send the JSON
   @param dslabel= the name to give the table in the exported JSON
   @param fmt= Whether to keep or strip formats from the table
-  @param engine= Which engine to use to send the JSON, options are:
-  * PROCJSON (default)
-  * DATASTEP
+  @param engine= Which engine to use to send the JSON, valid options are:
+    @li PROCJSON (default)
+    @li DATASTEP (more reliable when data has non standard characters)
 
   @param dbg= DEPRECATED - was used to conditionally add PRETTY to
     proc json but this can cause line truncation in large files.
@@ -160,8 +165,8 @@
 %end;
 
 %else %if &action=CLOSE %then %do;
-  data _null_;file &jref encoding='utf-8';
+  data _null_;file &jref encoding='utf-8' mod;
     put "}";
   run;
 %end;
-%mend;
+%mend;

base/mp_resetoption.sas

@@ -1,7 +1,8 @@
 /**
   @file
   @brief Reset an option to original value
-  @details Inspired by the SAS Jedi - https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options/
+  @details Inspired by the SAS Jedi -
+https://blogs.sas.com/content/sastraining/2012/08/14/jedi-sas-tricks-reset-sas-system-options
     Called as follows:
 
     options obs=30;

meta/mm_createstp.sas

@@ -376,8 +376,6 @@ run;
     */
   %mm_updatestpsourcecode(stp=&tree/&stpname
     ,stpcode="&directory/&filename"
-    ,frefin=&frefin.
-    ,frefout=&frefout.
     ,mdebug=&mdebug
     ,minify=&minify)
 

meta/mm_createwebservice.sas

@@ -196,7 +196,7 @@ data _null_;
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8''; ';
+  put '  data _null_;file &jref encoding=''utf-8'' mod; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
@@ -277,14 +277,14 @@ data _null_;
   put '      i+1; ';
   put '      call symputx(''wt''!!left(i),name,''l''); ';
   put '      call symputx(''wtcnt'',i,''l''); ';
-  put '    data _null_; file &fref encoding=''utf-8''; ';
+  put '    data _null_; file &fref mod encoding=''utf-8''; ';
   put '      put ",""WORK"":{"; ';
   put '    %do i=1 %to &wtcnt; ';
   put '      %let wt=&&wt&i; ';
   put '      proc contents noprint data=&wt ';
   put '        out=_data_ (keep=name type length format:); ';
   put '      run;%let tempds=%scan(&syslast,2,.); ';
-  put '      data _null_; file &fref encoding=''utf-8''; ';
+  put '      data _null_; file &fref mod encoding=''utf-8''; ';
   put '        dsid=open("WORK.&wt",''is''); ';
   put '        nlobs=attrn(dsid,''NLOBS''); ';
   put '        nvars=attrn(dsid,''NVARS''); ';
@@ -295,10 +295,10 @@ data _null_;
   put '        put '',"nvars":'' nvars; ';
   put '      %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP) ';
   put '      %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP) ';
-  put '      data _null_; file &fref encoding=''utf-8''; ';
+  put '      data _null_; file &fref mod encoding=''utf-8''; ';
   put '        put "}"; ';
   put '    %end; ';
-  put '    data _null_; file &fref encoding=''utf-8''; ';
+  put '    data _null_; file &fref mod encoding=''utf-8''; ';
   put '      put "}"; ';
   put '    run; ';
   put '  %end; ';

meta/mm_webout.sas

@@ -108,14 +108,14 @@
       i+1;
       call symputx('wt'!!left(i),name,'l');
       call symputx('wtcnt',i,'l');
-    data _null_; file &fref encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8';
       put ",""WORK"":{";
     %do i=1 %to &wtcnt;
       %let wt=&&wt&i;
       proc contents noprint data=&wt
         out=_data_ (keep=name type length format:);
       run;%let tempds=%scan(&syslast,2,.);
-      data _null_; file &fref encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8';
         dsid=open("WORK.&wt",'is');
         nlobs=attrn(dsid,'NLOBS');
         nvars=attrn(dsid,'NVARS');
@@ -126,10 +126,10 @@
         put ',"nvars":' nvars;
       %mp_jsonout(OBJ,&tempds,jref=&fref,dslabel=colattrs,engine=DATASTEP)
       %mp_jsonout(OBJ,&wt,jref=&fref,dslabel=first10rows,engine=DATASTEP)
-      data _null_; file &fref encoding='utf-8';
+      data _null_; file &fref mod encoding='utf-8';
         put "}";
     %end;
-    data _null_; file &fref encoding='utf-8';
+    data _null_; file &fref mod encoding='utf-8';
       put "}";
     run;
   %end;

viya/mv_createjob.sas

@@ -39,13 +39,13 @@
     needs to be attached to the beginning of the job
   @param code= Fileref(s) of the actual code to be added
   @param access_token_var= The global macro variable to contain the access token
-  @param grant_type= valid values are "password" or "authorization_code" (unquoted).
-    The default is authorization_code.
+  @param grant_type= valid values are "password" or "authorization_code"
+    (unquoted). The default is authorization_code.
   @param replace= select NO to avoid replacing any existing job in that location
   @param contextname= Choose a specific context on which to run the Job.  Leave
     blank to use the default context.  From Viya 3.5 it is possible to configure
     a shared context - see
-    https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5pyhn1wg3ja0drdl6h.htm&docsetVersion=3.5&locale=en
+https://go.documentation.sas.com/?docsetId=calcontexts&docsetTarget=n1hjn8eobk5pyhn1wg3ja0drdl6h.htm&docsetVersion=3.5&locale=en
 
   @version VIYA V.03.04
   @author [Allan Bowe](https://www.linkedin.com/in/allanbowe)

viya/mv_createwebservice.sas

@@ -335,7 +335,7 @@ data _null_;
   put '%end; ';
   put ' ';
   put '%else %if &action=CLOSE %then %do; ';
-  put '  data _null_;file &jref encoding=''utf-8''; ';
+  put '  data _null_;file &jref encoding=''utf-8'' mod; ';
   put '    put "}"; ';
   put '  run; ';
   put '%end; ';
@@ -467,7 +467,7 @@ data _null_;
   put '%end; ';
   put '%else %if &action=ARR or &action=OBJ %then %do; ';
   put '    %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt ';
-  put '      ,jref=&fref,engine=PROCJSON,dbg=%str(&_debug) ';
+  put '      ,jref=&fref,engine=DATASTEP,dbg=%str(&_debug) ';
   put '    ) ';
   put '%end; ';
   put '%else %if &action=CLOSE %then %do; ';

viya/mv_webout.sas

@@ -1,5 +1,5 @@
 /**
-  @file mv_webout.sas
+  @file
   @brief Send data to/from the SAS Viya Job Execution Service
   @details This macro should be added to the start of each Job Execution
   Service, **immediately** followed by a call to:
@@ -11,7 +11,7 @@
     following syntax:
 
         data some datasets; * make some data ;
-        retain some columns;
+          retain some columns;
         run;
 
         %mv_webout(OPEN)
@@ -162,7 +162,7 @@
 %end;
 %else %if &action=ARR or &action=OBJ %then %do;
     %mp_jsonout(&action,&ds,dslabel=&dslabel,fmt=&fmt
-      ,jref=&fref,engine=PROCJSON,dbg=%str(&_debug)
+      ,jref=&fref,engine=DATASTEP,dbg=%str(&_debug)
     )
 %end;
 %else %if &action=CLOSE %then %do;