Merge pull request #191 from sasjs/allanbowe/mp-loadformat-not-appending-190

fix: ensuring audit table gets loaded in mp_loadformat.

all.sas

@@ -7573,6 +7573,11 @@ create table &outds as
   outfile=0
 )/*/STORE SOURCE*/;
 
+  %if "%substr(&sysver,1,4)"="V.04" %then %do;
+    %put %str(ERR)OR: Viya 4 does not support the IO library in lua;
+    %return;
+  %end;
+
   %ml_gsubfile()
 
 %mend mp_gsubfile;
@@ -7932,7 +7937,7 @@ run;
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
   @param [in] iftrue= A condition under which the macro should be executed.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
-  will contain one column (hashkey) with one observation (a hex32.
+  will contain one column (hashkey) with one observation (a $hex32.
   representation of the input hash)
   |hashkey:$32.|
   |---|
@@ -8867,6 +8872,9 @@ options ibufsize=&ibufsize;
       ,mdebug=&mdebug
     )
 
+    proc append base=&auditlibds data=&storediffs;
+    run;
+
     %if &locklibds ne 0 %then %do;
       %mp_lockanytable(UNLOCK
         ,lib=%scan(&auditlibds,1,.)
@@ -9396,7 +9404,7 @@ put(md5(
     &sep put(md5(trim(put(ifn(missing(&var),&var,&var*1),binary64.))),$hex32.)
     %let sep=!!;
   %end;
-),hex32.)
+),$hex32.)
 %mend mp_md5;
 /**
   @file
@@ -9688,7 +9696,7 @@ filename &inref &infile lrecl=1 recfm=n;
 
 data &ds1;
   infile &inref;
-  input sourcechar $ 1. @@;
+  input sourcechar $char1. @@;
   format sourcechar hex2.;
 run;
 
@@ -18854,6 +18862,38 @@ run;
 )
 
 %mend ms_createfile;
+/**
+  @file
+  @brief Gets a file from SASjs Drive
+  @details Fetches a file on SASjs Drive and stores it in the output fileref.
+
+  Example:
+
+      %ms_getfile(/some/stored/file.ext, outref=myfile)
+
+  @param [in] driveloc The full path to the file in SASjs Drive
+  @param [out] outref= (msgetfil) The fileref to contain the file.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+
+**/
+
+%macro ms_getfile(driveloc
+    ,outref=msgetfil
+    ,mdebug=0
+  );
+
+filename &outref temp;
+
+proc http method='GET' out=&outref
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file?filePath=&driveloc";
+%if &mdebug=1 %then %do;
+  debug level=2;
+%end;
+run;
+
+
+%mend ms_getfile;
 /**
   @file
   @brief Executes a SASjs Server Stored Program
@@ -20969,10 +21009,16 @@ libname &libref1a clear;
       %mv_deleteviyafolder(path=/Public/test)
 
 
-  @param path= The full path of the folder to be deleted
+  @param [in] path= The full path of the folder to be deleted
-  @param access_token_var= The global macro variable to contain the access token
+  @param [in] access_token_var= (ACCESS_TOKEN) The global macro variable to
-  @param grant_type= valid values are "password" or "authorization_code" (unquoted).
+    contain the access token
-    The default is authorization_code.
+  @param [in] grant_type= (sas_services) Valid values are:
+    @li password
+    @li authorization_code
+    @li detect - will check if access_token exists, if not will use sas_services
+      if a SASStudioV session else authorization_code.  Default option.
+    @li sas_services - will use oauth_bearer=sas_services.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
 
 
   @version VIYA V.03.04
@@ -20990,6 +21036,7 @@ libname &libref1a clear;
 %macro mv_deleteviyafolder(path=
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
+    ,mdebug=0
   );
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
@@ -21066,14 +21113,17 @@ run;
 %let libref1a=%mf_getuniquelibref();
 libname &libref1a JSON fileref=&fname1a;
 
-data _null_;
+%if %mf_existds(&libref1a..items_links) %then %do;
+  data _null_;
     set &libref1a..items_links;
     if href=:'/folders/folders' then return;
     if rel='deleteResource' then
-    call execute('proc http method="DELETE" url='!!quote("&base_uri"!!trim(href))
+      call execute('proc http method="DELETE" url='
+      !!quote("&base_uri"!!trim(href))
       !!'; headers "Authorization"="Bearer &&&access_token_var" '
       !!' "Accept"="*/*";run; /**/');
-run;
+  run;
+%end;
 
 %put &sysmacroname: perform the delete operation ;
 %local fname2;
@@ -21094,12 +21144,15 @@ run;
 %end;
 %else %put &sysmacroname: &path successfully deleted;
 
-/* clear refs */
+%if &mdebug=0 %then %do;
-filename &fname1 clear;
+  /* clear refs */
-filename &fname2 clear;
+  filename &fname1 clear;
-libname &libref1 clear;
+  filename &fname2 clear;
+  libname &libref1 clear;
+%end;
 
-%mend mv_deleteviyafolder;/**
+%mend mv_deleteviyafolder;
+/**
   @file mv_getclients.sas
   @brief Get a list of Viya Clients
   @details First, be sure you have an access token (which requires an app token).
@@ -21477,7 +21530,7 @@ filename &fname1 clear;
     ,grant_type=sas_services
     ,outds=work.viyagroups
   );
-%local oauth_bearer;
+%local oauth_bearer base_uri fname1 libref1;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
   %else %let grant_type=sas_services;
@@ -21495,11 +21548,10 @@ filename &fname1 clear;
 )
 
 options noquotelenmax;
-%local base_uri; /* location of rest apis */
+/* location of rest apis */
 %let base_uri=%mf_getplatform(VIYARESTAPI);
 
 /* fetching folder details for provided path */
-%local fname1;
 %let fname1=%mf_getuniquefileref();
 %let libref1=%mf_getuniquelibref();
 
@@ -21523,12 +21575,12 @@ data &outds;
 run;
 
 
-
 /* clear refs */
 filename &fname1 clear;
 libname &libref1 clear;
 
-%mend mv_getgroups;/**
+%mend mv_getgroups;
+/**
   @file
   @brief Extract the source code from a SAS Viya Job
   @details Extracts the SAS code from a Job into a fileref or physical file.
@@ -21563,7 +21615,6 @@ libname &libref1 clear;
   @li mf_getplatform.sas
   @li mf_getuniquefileref.sas
   @li mv_getfoldermembers.sas
-  @li ml_json.sas
 
 **/
 
@@ -21574,9 +21625,9 @@ libname &libref1 clear;
     ,grant_type=sas_services
     ,mdebug=0
   );
-%local dbg;
+%local dbg bufsize varcnt fname1 fname2 errmsg;
 %if &mdebug=1 %then %do;
-  %put &sysmacroname entry vars:;
+  %put &sysmacroname local entry vars:;
   %put _local_;
 %end;
 %else %let dbg=*;
@@ -21634,7 +21685,6 @@ run;
 )
 
 /* prepare request*/
-%local  fname1;
 %let fname1=%mf_getuniquefileref();
 proc http method='GET' out=&fname1 &oauth_bearer
   url="&base_uri&joburi";
@@ -21644,37 +21694,95 @@ proc http method='GET' out=&fname1 &oauth_bearer
   %end;
   ;
 run;
-%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
-%do;
-  data _null_;infile &fname1;input;putlog _infile_;run;
-  %mp_abort(mac=&sysmacroname
-    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
-  )
-%end;
-%local  fname2 fname3 fpath1 fpath2 fpath3;
-%let fname2=%mf_getuniquefileref();
-%let fname3=%mf_getuniquefileref();
-%let fpath1=%sysfunc(pathname(&fname1));
-%let fpath2=%sysfunc(pathname(&fname2));
-%let fpath3=%sysfunc(pathname(&fname3));
 
-/* compile the lua JSON module */
+%if &mdebug=1 %then %do;
-%ml_json()
+  data _null_;
-/* read using LUA - this allows the code to be of any length */
+    infile &fname1;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+%let fname2=%mf_getuniquefileref();
+filename &fname2 temp ;
+
+/* cannot use lua IO package as not available in Viya 4 */
+/* so use data step to read the JSON until the string `"code":"` is found */
 data _null_;
-  file "&fpath3..lua";
+  file &fname2 recfm=n;
-  put '
+  infile &fname1 lrecl=1 recfm=n;
-    infile = io.open (sas.symget("fpath1"), "r")
+  input sourcechar $char1. @@;
-    outfile = io.open (sas.symget("fpath2"), "w")
+  format sourcechar hex2.;
-    io.input(infile)
+  retain startwrite 0;
-    local resp=json.decode(io.read())
+  if startwrite=0 and sourcechar='"' then do;
-    local job=resp["code"]
+    reentry:
-    outfile:write(job)
+    input sourcechar $ 1. @@;
-    io.close(infile)
+    if sourcechar='c' then do;
-    io.close(outfile)
+      reentry2:
-  ';
+      input sourcechar $ 1. @@;
+      if sourcechar='o' then do;
+        input sourcechar $ 1. @@;
+        if sourcechar='d' then do;
+          input sourcechar $ 1. @@;
+          if sourcechar='e' then do;
+            input sourcechar $ 1. @@;
+            if sourcechar='"' then do;
+              input sourcechar $ 1. @@;
+              if sourcechar=':' then do;
+                input sourcechar $ 1. @@;
+                if sourcechar='"' then do;
+                  putlog 'code found';
+                  startwrite=1;
+                  input sourcechar $ 1. @@;
+                end;
+              end;
+              else if sourcechar='c' then goto reentry2;
+            end;
+          end;
+          else if sourcechar='"' then goto reentry;
+        end;
+        else if sourcechar='"' then goto reentry;
+      end;
+      else if sourcechar='"' then goto reentry;
+    end;
+    else if sourcechar='"' then goto reentry;
+  end;
+  /* once the `"code":"` string is found, write until unescaped `"` is found */
+  if startwrite=1 then do;
+    if sourcechar='\' then do;
+      input sourcechar $ 1. @@;
+      if sourcechar in ('"','\') then put sourcechar char1.;
+      else if sourcechar='n' then put '0A'x;
+      else if sourcechar='r' then put '0D'x;
+      else if sourcechar='t' then put '09'x;
+      else if sourcechar='u' then do;
+        length uni $4;
+        input uni $ 4. @@;
+        sourcechar=unicode('\u'!!uni);
+        put sourcechar char1.;
+      end;
+      else do;
+        call symputx('errmsg',"Uncaught escape char: "!!sourcechar,'l');
+        call symputx('syscc',99);
+        stop;
+      end;
+    end;
+    else if sourcechar='"' then stop;
+    else put sourcechar char1.;
+  end;
 run;
-%inc "&fpath3..lua";
+
+%mp_abort(iftrue=("&syscc"="99")
+  ,mac=mv_getjobcode
+  ,msg=%str(&errmsg)
+)
+
 /* export to desired destination */
 %if "&outref"="0" %then %do;
   data _null_;
@@ -21699,7 +21807,6 @@ run;
   /* clear refs */
   filename &fname1 clear;
   filename &fname2 clear;
-  filename &fname3 clear;
 %end;
 
 %mend mv_getjobcode;
@@ -21791,7 +21898,8 @@ run;
   @li mp_abort.sas
   @li mf_getplatform.sas
   @li mf_existfileref.sas
-  @li ml_json.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
 
 **/
 
@@ -21800,7 +21908,7 @@ run;
     ,grant_type=sas_services
     ,mdebug=0
   );
-%local dbg;
+%local dbg libref1 libref2 loglocation fname1 fname2;
 %if &mdebug=1 %then %do;
   %put &sysmacroname entry vars:;
   %put _local_;
@@ -21859,8 +21967,8 @@ options noquotelenmax;
 %let base_uri=%mf_getplatform(VIYARESTAPI);
 
 /* prepare request*/
-%local  fname1;
 %let fname1=%mf_getuniquefileref();
+%let fname2=%mf_getuniquefileref();
 proc http method='GET' out=&fname1 &oauth_bearer
   url="&base_uri&uri";
   headers
@@ -21880,37 +21988,19 @@ run;
     ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
   )
 %end;
-%local  fname2 fname3 fpath1 fpath2 fpath3;
-%let fname2=%mf_getuniquefileref();
-%let fname3=%mf_getuniquefileref();
-%let fpath1=%sysfunc(pathname(&fname1));
-%let fpath2=%sysfunc(pathname(&fname2));
-%let fpath3=%sysfunc(pathname(&fname3));
 
-/* compile the lua JSON module */
+%let libref1=%mf_getuniquelibref();
-%ml_json()
+libname &libref1 JSON fileref=&fname1;
-/* read using LUA - this allows the code to be of any length */
 data _null_;
-  file "&fpath3..lua";
+  set &libref1..root;
-  put '
+  call symputx('loglocation',loglocation,'l');
-    infile = io.open (sas.symget("fpath1"), "r")
-    outfile = io.open (sas.symget("fpath2"), "w")
-    io.input(infile)
-    local resp=json.decode(io.read())
-    local logloc=resp["logLocation"]
-    outfile:write(logloc)
-    io.close(infile)
-    io.close(outfile)
-  ';
 run;
-%inc "&fpath3..lua";
+
-/* get log path*/
+/* validate log path*/
 %let errflg=1;
-%let errmsg=No entry in &fname2 fileref;
+%let errmsg=No loglocation entry in &fname1 fileref;
 data _null_;
-  infile &fname2;
+  uri=symget('loglocation');
-  input;
-  uri=cats(_infile_);
   if length(uri)<12 then do;
     call symputx('errflg',1);
     call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
@@ -21937,7 +22027,7 @@ run;
 
 /* we have a log uri - now fetch the log */
 %&dbg.put &sysmacroname: querying &base_uri&logloc/content;
-proc http method='GET' out=&fname1 &oauth_bearer
+proc http method='GET' out=&fname2 &oauth_bearer
   url="&base_uri&logloc/content?limit=10000";
   headers
   %if &grant_type=authorization_code %then %do;
@@ -21948,14 +22038,14 @@ run;
 
 %if &mdebug=1 %then %do;
   %put &sysmacroname: fetching log content from &base_uri&logloc/content;
-  data _null_;infile &fname1;input;putlog _infile_;run;
+  data _null_;infile &fname2;input;putlog _infile_;run;
 %end;
 
 %if &SYS_PROCHTTP_STATUS_CODE=400 %then %do;
   /* fetch log from parent session */
   %let logloc=%substr(&logloc,1,%index(&logloc,%str(/jobs/))-1);
   %&dbg.put &sysmacroname: Now querying &base_uri&logloc/log/content;
-  proc http method='GET' out=&fname1 &oauth_bearer
+  proc http method='GET' out=&fname2 &oauth_bearer
     url="&base_uri&logloc/log/content?limit=10000";
     headers
     %if &grant_type=authorization_code %then %do;
@@ -21965,47 +22055,32 @@ run;
   run;
   %if &mdebug=1 %then %do;
     %put &sysmacroname: fetching log content from &base_uri&logloc/log/content;
-    data _null_;infile &fname1;input;putlog _infile_;run;
+    data _null_;infile &fname2;input;putlog _infile_;run;
   %end;
 %end;
 
 %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201
 %then %do;
   %if &mdebug ne 1 %then %do; /* have already output above */
-    data _null_;infile &fname1;input;putlog _infile_;run;
+    data _null_;infile &fname2;input;putlog _infile_;run;
   %end;
   %mp_abort(mac=&sysmacroname
     ,msg=%str(logfetch: &SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
   )
 %end;
-data _null_;
-  file "&fpath3..lua";
-  put '
-    infile = io.open (sas.symget("fpath1"), "r")
-    outfile = io.open (sas.symget("fpath2"), "w")
-    io.input(infile)
-    local resp=json.decode(io.read())
-    for i, v in pairs(resp["items"]) do
-      outfile:write(v.line,"\n")
-    end
-    io.close(infile)
-    io.close(outfile)
-  ';
-run;
-%inc "&fpath3..lua";
 
-/* write log out to the specified fileref */
+%let libref2=%mf_getuniquelibref();
+libname &libref2 JSON fileref=&fname2;
 data _null_;
-  infile &fname2 end=last;
   file &outref mod;
   if _n_=1 then do;
     put "/** SASJS Viya Job Log Extract start: &uri **/";
   end;
-  input;
+  set &libref2..items end=last;
-  put _infile_;
   %if &mdebug=1 %then %do;
-    putlog _infile_;
+    putlog line;
   %end;
+  put line;
   if last then do;
     put "/** SASJS Viya Job Log Extract end: &uri **/";
   end;
@@ -22014,7 +22089,8 @@ run;
 %if &mdebug=0 %then %do;
   filename &fname1 clear;
   filename &fname2 clear;
-  filename &fname3 clear;
+  libname &libref1 clear;
+  libname &libref2 clear;
 %end;
 %else %do;
   %put &sysmacroname exit vars:;
@@ -24971,115 +25047,6 @@ endsub;
 %end;
 
 %mend mcf_length;/**
-  @file
-  @brief Provides a replacement for the stpsrv_header function
-  @details The stpsrv_header is normally a built-in function, used to set the
-  headers for SAS 9 Stored Processes as documented here:
-  https://go.documentation.sas.com/doc/en/itechcdc/9.4/stpug/srvhead.htm
-
-  The purpose of this custom function is to provide a replacement when running
-  similar code as a web service against
-  [sasjs/server](https://github.com/sasjs/server).  It operates by creating a
-  text file with the headers.  The location of this text file is determined by
-  a macro variable (`sasjs_stpsrv_header_loc`) which needs to be injected into
-  each service by the calling process, eg:
-
-      %let sasjs_stpsrv_header_loc = C:/temp/some_uuid/stpsrv_header.txt;
-
-  Note - the function works by appending headers to the file.  If multiple same-
-  named headers are provided, they will all be appended - the calling process
-  needs to pick up the last one.  This will mean removing the attribute if the
-  final record has an empty value.
-
-  The function takes the following (positional) parameters:
-
-  | PARAMETER | DESCRIPTION |
-  |------------|-------------|
-  | name $  | name of the header attribute to create|
-  | value  $  | value of the header attribute|
-
-  It returns 0 if successful, or -1 if an error occured.
-
-  Usage:
-
-      %let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/stpsrv_header.txt;
-
-      %mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)
-
-      data _null_;
-        rc=stpsrv_header('Content-type','application/text');
-        rc=stpsrv_header('Content-disposition',"attachment; filename=file.txt");
-      run;
-
-      data _null_;
-        infile "&sasjs_stpsrv_header_loc";
-        input;
-        putlog _infile_;
-      run;
-
-
-  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
-  @param [out] lib= (work) The output library in which to create the catalog.
-  @param [out] cat= (sasjs) The output catalog in which to create the package.
-  @param [out] pkg= (utils) The output package in which to create the function.
-    Uses a 3 part format:  libref.catalog.package
-  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
-    values inserted only if needed.
-
-  <h4> SAS Macros </h4>
-  @li mcf_init.sas
-
-  <h4> Related Programs </h4>
-  @li mcf_stpsrv_header.test.sas
-  @li mp_init.sas
-
-**/
-
-%macro mcf_stpsrv_header(wrap=NO
-  ,insert_cmplib=DEPRECATED
-  ,lib=WORK
-  ,cat=SASJS
-  ,pkg=UTILS
-)/*/STORE SOURCE*/;
-%local i var cmpval found;
-%if %mcf_init(stpsrv_header)=1 %then %return;
-
-%if &wrap=YES  %then %do;
-  proc fcmp outlib=&lib..&cat..&pkg;
-%end;
-
-function stpsrv_header(name $, value $);
-  length loc $128 val $512;
-  loc=symget('sasjs_stpsrv_header_loc');
-  val=trim(name)!!': '!!value;
-  length fref $8;
-  rc=filename(fref,loc);
-  if (rc ne 0) then return( -1 );
-  fid = fopen(fref,'a');
-  if (fid = 0) then return( -1 );
-  rc=fput(fid, val);
-  rc=fwrite(fid);
-  rc=fclose(fid);
-  rc=filename(fref);
-  return(0);
-endsub;
-
-%if &wrap=YES %then %do;
-  quit;
-%end;
-
-/* insert the CMPLIB if not already there */
-%let cmpval=%sysfunc(getoption(cmplib));
-%let found=0;
-%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
-  %let var=%scan(&cmpval,&i,%str( %(%)));
-  %if &var=&lib..&cat %then %let found=1;
-%end;
-%if &found=0 %then %do;
-  options insert=(CMPLIB=(&lib..&cat));
-%end;
-
-%mend mcf_stpsrv_header;/**
   @file
   @brief Adds a string to a file
   @details Creates an fcmp function for appending a string to an external file.

base/mf_getuser.sas

@@ -33,7 +33,8 @@
   %else %if %symexist(&metavar) %then %do;
     %if %length(&&&metavar)=0 %then %let user=&sysuserid;
     /* sometimes SAS will add @domain extension - remove for consistency */
-    %else %let user=%scan(&&&metavar,1,@);
+    /* but be sure to quote in case of usernames with commas */
+    %else %let user=%unquote(%scan(%quote(&&&metavar),1,@));
   %end;
   %else %let user=&sysuserid;
 

base/mp_gsubfile.sas

@@ -48,6 +48,11 @@
   outfile=0
 )/*/STORE SOURCE*/;
 
+  %if "%substr(&sysver,1,4)"="V.04" %then %do;
+    %put %str(ERR)OR: Viya 4 does not support the IO library in lua;
+    %return;
+  %end;
+
   %ml_gsubfile()
 
 %mend mp_gsubfile;

base/mp_hashdataset.sas

@@ -23,7 +23,7 @@
   @param [in] salt= Provide a salt (could be, for instance, the dataset name)
   @param [in] iftrue= A condition under which the macro should be executed.
   @param [out] outds= (work.mf_hashdataset) The output dataset to create. This
-  will contain one column (hashkey) with one observation (a hex32.
+  will contain one column (hashkey) with one observation (a $hex32.
   representation of the input hash)
   |hashkey:$32.|
   |---|

base/mp_loadformat.sas

@@ -40,13 +40,13 @@
   @li mp_abort.sas
   @li mp_cntlout.sas
   @li mp_lockanytable.sas
+  @li mp_storediffs.sas
 
   <h4> Related Macros </h4>
   @li mddl_dc_difftable.sas
   @li mddl_dc_locktable.sas
   @li mp_loadformat.test.sas
   @li mp_lockanytable.sas
-  @li mp_storediffs.sas
   @li mp_stackdiffs.sas
 
 
@@ -272,6 +272,9 @@ options ibufsize=&ibufsize;
       ,mdebug=&mdebug
     )
 
+    proc append base=&auditlibds data=&storediffs;
+    run;
+
     %if &locklibds ne 0 %then %do;
       %mp_lockanytable(UNLOCK
         ,lib=%scan(&auditlibds,1,.)

base/mp_md5.sas

@@ -54,5 +54,5 @@ put(md5(
     &sep put(md5(trim(put(ifn(missing(&var),&var,&var*1),binary64.))),$hex32.)
     %let sep=!!;
   %end;
-),hex32.)
+),$hex32.)
 %mend mp_md5;

base/mp_replace.sas

@@ -70,7 +70,7 @@ filename &inref &infile lrecl=1 recfm=n;
 
 data &ds1;
   infile &inref;
-  input sourcechar $ 1. @@;
+  input sourcechar $char1. @@;
   format sourcechar hex2.;
 run;
 

fcmp/mcf_stpsrv_header.sas

@@ -1,110 +0,0 @@
-/**
-  @file
-  @brief Provides a replacement for the stpsrv_header function
-  @details The stpsrv_header is normally a built-in function, used to set the
-  headers for SAS 9 Stored Processes as documented here:
-  https://go.documentation.sas.com/doc/en/itechcdc/9.4/stpug/srvhead.htm
-
-  The purpose of this custom function is to provide a replacement when running
-  similar code as a web service against
-  [sasjs/server](https://github.com/sasjs/server).  It operates by creating a
-  text file with the headers.  The location of this text file is determined by
-  a macro variable (`sasjs_stpsrv_header_loc`) which needs to be injected into
-  each service by the calling process, eg:
-
-      %let sasjs_stpsrv_header_loc = C:/temp/some_uuid/stpsrv_header.txt;
-
-  Note - the function works by appending headers to the file.  If multiple same-
-  named headers are provided, they will all be appended - the calling process
-  needs to pick up the last one.  This will mean removing the attribute if the
-  final record has an empty value.
-
-  The function takes the following (positional) parameters:
-
-  | PARAMETER | DESCRIPTION |
-  |------------|-------------|
-  | name $  | name of the header attribute to create|
-  | value  $  | value of the header attribute|
-
-  It returns 0 if successful, or -1 if an error occured.
-
-  Usage:
-
-      %let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/stpsrv_header.txt;
-
-      %mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)
-
-      data _null_;
-        rc=stpsrv_header('Content-type','application/text');
-        rc=stpsrv_header('Content-disposition',"attachment; filename=file.txt");
-      run;
-
-      data _null_;
-        infile "&sasjs_stpsrv_header_loc";
-        input;
-        putlog _infile_;
-      run;
-
-
-  @param [out] wrap= (NO) Choose YES to add the proc fcmp wrapper.
-  @param [out] lib= (work) The output library in which to create the catalog.
-  @param [out] cat= (sasjs) The output catalog in which to create the package.
-  @param [out] pkg= (utils) The output package in which to create the function.
-    Uses a 3 part format:  libref.catalog.package
-  @param [out] insert_cmplib= DEPRECATED - The CMPLIB option is checked and
-    values inserted only if needed.
-
-  <h4> SAS Macros </h4>
-  @li mcf_init.sas
-
-  <h4> Related Programs </h4>
-  @li mcf_stpsrv_header.test.sas
-  @li mp_init.sas
-
-**/
-
-%macro mcf_stpsrv_header(wrap=NO
-  ,insert_cmplib=DEPRECATED
-  ,lib=WORK
-  ,cat=SASJS
-  ,pkg=UTILS
-)/*/STORE SOURCE*/;
-%local i var cmpval found;
-%if %mcf_init(stpsrv_header)=1 %then %return;
-
-%if &wrap=YES  %then %do;
-  proc fcmp outlib=&lib..&cat..&pkg;
-%end;
-
-function stpsrv_header(name $, value $);
-  length loc $128 val $512;
-  loc=symget('sasjs_stpsrv_header_loc');
-  val=trim(name)!!': '!!value;
-  length fref $8;
-  rc=filename(fref,loc);
-  if (rc ne 0) then return( -1 );
-  fid = fopen(fref,'a');
-  if (fid = 0) then return( -1 );
-  rc=fput(fid, val);
-  rc=fwrite(fid);
-  rc=fclose(fid);
-  rc=filename(fref);
-  return(0);
-endsub;
-
-%if &wrap=YES %then %do;
-  quit;
-%end;
-
-/* insert the CMPLIB if not already there */
-%let cmpval=%sysfunc(getoption(cmplib));
-%let found=0;
-%do i=1 %to %sysfunc(countw(&cmpval,%str( %(%))));
-  %let var=%scan(&cmpval,&i,%str( %(%)));
-  %if &var=&lib..&cat %then %let found=1;
-%end;
-%if &found=0 %then %do;
-  options insert=(CMPLIB=(&lib..&cat));
-%end;
-
-%mend mcf_stpsrv_header;

sasjs/sasjsconfig.json

@@ -95,6 +95,19 @@
         "tests/sas9only",
         "tests/viyaonly"
       ]
+    },
+    {
+      "name": "viya4",
+      "serverUrl": "https://azureuse011059.my-trials.sas.com",
+      "serverType": "SASVIYA",
+      "appLoc": "/Public/temp/macrocore",
+      "macroFolders": [
+        "tests/viyaonly"
+      ],
+      "deployConfig": {
+        "deployServicePack": true
+      },
+      "contextName": "SAS Job Execution compute context"
     }
   ]
 }

server/ms_getfile.sas

@@ -0,0 +1,32 @@
+/**
+  @file
+  @brief Gets a file from SASjs Drive
+  @details Fetches a file on SASjs Drive and stores it in the output fileref.
+
+  Example:
+
+      %ms_getfile(/some/stored/file.ext, outref=myfile)
+
+  @param [in] driveloc The full path to the file in SASjs Drive
+  @param [out] outref= (msgetfil) The fileref to contain the file.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+
+**/
+
+%macro ms_getfile(driveloc
+    ,outref=msgetfil
+    ,mdebug=0
+  );
+
+filename &outref temp;
+
+proc http method='GET' out=&outref
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file?filePath=&driveloc";
+%if &mdebug=1 %then %do;
+  debug level=2;
+%end;
+run;
+
+
+%mend ms_getfile;

tests/crossplatform/mcf_stpsrv_header.test.sas

@@ -1,39 +0,0 @@
-/**
-  @file
-  @brief Testing mcf_stpsrv_header macro
-
-  <h4> SAS Macros </h4>
-  @li mcf_stpsrv_header.sas
-  @li mp_assert.sas
-
-**/
-
-%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/stpsrv_header.txt;
-
-%mcf_stpsrv_header(wrap=YES, insert_cmplib=YES)
-
-data _null_;
-  rc=stpsrv_header('Content-type','application/text');
-  rc=stpsrv_header('Content-disposition',"attachment; filename=file.txt");
-run;
-
-%let test1=FAIL;
-%let test2=FAIL;
-
-data _null_;
-  infile "&sasjs_stpsrv_header_loc";
-  input;
-  if _n_=1 and _infile_='Content-type: application/text'
-  then call symputx('test1','PASS');
-  else if _n_=2 & _infile_='Content-disposition: attachment; filename=file.txt'
-  then call symputx('test2','PASS');
-run;
-
-%mp_assert(
-  iftrue=(%str(&test1)=%str(PASS)),
-  desc=Check first header line
-)
-%mp_assert(
-  iftrue=(%str(&test2)=%str(PASS)),
-  desc=Check second header line
-)

tests/crossplatform/mp_gsubfile.test.sas

@@ -8,6 +8,13 @@
 
 **/
 
+
+%macro gsubtest();
+%if "%substr(&sysver,1,4)"="V.04" %then %do;
+  %put %str(ERR)OR: Viya 4 does not support the IO library in lua;
+  %return;
+%end;
+
 /**
   * test 1 - simple replace
   */
@@ -64,3 +71,7 @@ run;
   desc=Check that multi line replacement was successful (line3),
   outds=work.test_results
 )
+
+%mend gsubtest;
+
+%gsubtest()

tests/crossplatform/mp_loadformat.test.sas

@@ -3,6 +3,7 @@
   @brief Testing mp_loadformat.sas macro
 
   <h4> SAS Macros </h4>
+  @li mddl_dc_difftable.sas
   @li mp_loadformat.sas
   @li mp_assert.sas
   @li mp_assertscope.sas
@@ -12,6 +13,8 @@
 /* prep format catalog */
 libname perm (work);
 
+%mddl_dc_difftable(libds=perm.audit)
+
 data work.loadfmts;
   length fmtname $32;
   eexcl='Y';
@@ -49,7 +52,7 @@ run;
 %mp_loadformat(perm.testcat
   ,work.stagedata
   ,loadtarget=YES
-  ,auditlibds=0
+  ,auditlibds=perm.audit
   ,locklibds=0
   ,delete_col=deleteme
   ,outds_add=add_test1
@@ -74,3 +77,8 @@ run;
   desc=Test 1 - mod obs,
   outds=work.test_results
 )
+%mp_assert(
+  iftrue=(%mf_nobs(perm.audit)>7329),
+  desc=Test 1 - audit table updated,
+  outds=work.test_results
+)

tests/crossplatform/mp_replace.test.sas

@@ -63,3 +63,33 @@ run;
   desc=Checking second replace 3rd row,
   outds=work.test_results
 )
+
+
+%let test3="&sasjswork/file3.txt";
+%let str=%str(replace.string.with.dots   );
+%let rep=%str( more.dots);
+data _null_;
+  file &test3;
+  put 'blahblah';
+  put "blahblah&str.blah&str. replace &str.X";
+  put "blahbreplacewith&str.spacesahblah";
+run;
+%mp_replace(&test3, findvar=str, replacevar=rep)
+
+data _null_;
+  infile &test3;
+  input;
+  if _n_=2 then call symputx('test3resulta',_infile_);
+  if _n_=3 then call symputx('test3resultb',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test3resulta" = "blahblah&rep.blah&rep. replace &rep.X"),
+  desc=Checking third replace 2nd row (dots),
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test3resultb" = "blahbreplacewith&rep.spacesahblah"),
+  desc=Checking third replace 3rd row (dots),
+  outds=work.test_results
+)

tests/serveronly/ms_getfile.test.sas

@@ -0,0 +1,45 @@
+/**
+  @file
+  @brief Testing ms_getfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li ms_createfile.sas
+  @li ms_getfile.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+
+/* first make a remote file */
+filename stpcode temp;
+%let fname=%mf_getuniquename();
+data _null_;
+  file stpcode;
+  put "data &fname;run;";
+run;
+%ms_createfile(/sasjs/tests/&fname..sas
+  ,inref=stpcode
+  ,mdebug=1
+)
+
+%mp_assertscope(SNAPSHOT)
+%ms_getfile(/sasjs/tests/&fname..sas,outref=testref)
+%mp_assertscope(COMPARE)
+
+%let test1=0;
+data _null_;
+  infile testref;
+  input;
+  call symputx('test1',_infile_);
+run;
+
+%mp_assert(
+  iftrue=("&test1"="data &fname;run;"),
+  desc=Checking file was created with the same content,
+  outds=work.test_results
+)
+
+
+
+

tests/viyaonly/mv_deleteviyafolder.test.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Testing mv_deleteviyafolder macro function
+
+  <h4> SAS Macros </h4>
+  @li mf_uid.sas
+  @li mfv_existfolder.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+  @li mv_createfolder.sas
+  @li mv_deleteviyafolder.sas
+
+**/
+
+options mprint sgen;
+
+%let folder=%mf_uid();
+%let tgtfolder=&mcTestAppLoc/temp/&folder;
+
+/* create a folder */
+%mv_createfolder(path=&tgtfolder)
+
+
+%mp_assert(
+  iftrue=(%mfv_existfolder(&tgtfolder)=1),
+  desc=Check if created folder exists
+)
+
+%mp_assertscope(SNAPSHOT)
+%mv_deleteviyafolder(path=&tgtfolder)
+/* ignore proc json vars */
+%mp_assertscope(COMPARE
+  ,ignorelist=MCLIB0_JADP1LEN MCLIB0_JADP2LEN MCLIB0_JADVLEN MCLIB2_JADP1LEN
+    MCLIB2_JADVLEN
+)
+
+%mp_assert(
+  iftrue=(%mfv_existfolder(&tgtfolder)=0),
+  desc=Check if deleted folder is gone
+)
+
+/* delete folder with content */
+%mv_createfolder(path=&tgtfolder/content/and/stuff)
+%mp_assert(
+  iftrue=(%mfv_existfolder(&tgtfolder/content/and/stuff)=1),
+  desc=Check if folder with content exists
+)
+%mv_deleteviyafolder(path=&tgtfolder)
+%mp_assert(
+  iftrue=(%mfv_existfolder(&tgtfolder)=0),
+  desc=Check if deleted folder with subfolders is gone
+)

tests/viyaonly/mv_getjobcode.test.sas

@@ -4,6 +4,7 @@
 
   <h4> SAS Macros </h4>
   @li mp_assert.sas
+  @li mp_assertscope.sas
   @li mv_createjob.sas
   @li mv_getjobcode.sas
 
@@ -27,11 +28,17 @@ run;
 )
 
 /* now get the code back */
+%mp_assertscope(SNAPSHOT)
 %mv_getjobcode(
   path=&mcTestAppLoc/services/temp,
   name=some_job,
   outref=mycode
 )
+/* exclude automatic proc json macro variables from scope check */
+%mp_assertscope(COMPARE,
+  ignorelist=MCLIB2_JADP1LEN MCLIB2_JADP2LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
+    MCLIB2_JADP3LEN
+)
 
 %let diditexist=NO;
 data work.test1;

tests/viyaonly/mv_getjoblog.test.sas

@@ -4,6 +4,7 @@
 
   <h4> SAS Macros </h4>
   @li mp_assert.sas
+  @li mp_assertscope.sas
   @li mv_createjob.sas
   @li mv_jobexecute.sas
   @li mv_jobwaitfor.sas
@@ -49,8 +50,12 @@ data _null_;
 run;
 
 %* Finally, fetch the log;
-%mv_getjoblog(uri=%str(&uri),outref=mylog)
+%mp_assertscope(SNAPSHOT)
-
+%mv_getjoblog(uri=%str(&uri),outref=mylog,mdebug=1)
+/* ignore auto proc json vars */
+%mp_assertscope(COMPARE
+  ,ignorelist=MCLIB2_JADP2LEN MCLIB2_JADPNUM MCLIB2_JADVLEN
+)
 
 data _null_;
   infile mylog end=eof;

viya/mv_deleteviyafolder.sas

@@ -8,10 +8,16 @@
       %mv_deleteviyafolder(path=/Public/test)
 
 
-  @param path= The full path of the folder to be deleted
+  @param [in] path= The full path of the folder to be deleted
-  @param access_token_var= The global macro variable to contain the access token
+  @param [in] access_token_var= (ACCESS_TOKEN) The global macro variable to
-  @param grant_type= valid values are "password" or "authorization_code" (unquoted).
+    contain the access token
-    The default is authorization_code.
+  @param [in] grant_type= (sas_services) Valid values are:
+    @li password
+    @li authorization_code
+    @li detect - will check if access_token exists, if not will use sas_services
+      if a SASStudioV session else authorization_code.  Default option.
+    @li sas_services - will use oauth_bearer=sas_services.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
 
 
   @version VIYA V.03.04
@@ -29,6 +35,7 @@
 %macro mv_deleteviyafolder(path=
     ,access_token_var=ACCESS_TOKEN
     ,grant_type=sas_services
+    ,mdebug=0
   );
 %local oauth_bearer;
 %if &grant_type=detect %then %do;
@@ -105,14 +112,17 @@ run;
 %let libref1a=%mf_getuniquelibref();
 libname &libref1a JSON fileref=&fname1a;
 
-data _null_;
+%if %mf_existds(&libref1a..items_links) %then %do;
+  data _null_;
     set &libref1a..items_links;
     if href=:'/folders/folders' then return;
     if rel='deleteResource' then
-    call execute('proc http method="DELETE" url='!!quote("&base_uri"!!trim(href))
+      call execute('proc http method="DELETE" url='
+      !!quote("&base_uri"!!trim(href))
       !!'; headers "Authorization"="Bearer &&&access_token_var" '
       !!' "Accept"="*/*";run; /**/');
-run;
+  run;
+%end;
 
 %put &sysmacroname: perform the delete operation ;
 %local fname2;
@@ -133,9 +143,11 @@ run;
 %end;
 %else %put &sysmacroname: &path successfully deleted;
 
-/* clear refs */
+%if &mdebug=0 %then %do;
-filename &fname1 clear;
+  /* clear refs */
-filename &fname2 clear;
+  filename &fname1 clear;
-libname &libref1 clear;
+  filename &fname2 clear;
+  libname &libref1 clear;
+%end;
 
 %mend mv_deleteviyafolder;

viya/mv_getgroups.sas

@@ -32,7 +32,7 @@
     ,grant_type=sas_services
     ,outds=work.viyagroups
   );
-%local oauth_bearer;
+%local oauth_bearer base_uri fname1 libref1;
 %if &grant_type=detect %then %do;
   %if %symexist(&access_token_var) %then %let grant_type=authorization_code;
   %else %let grant_type=sas_services;
@@ -50,11 +50,10 @@
 )
 
 options noquotelenmax;
-%local base_uri; /* location of rest apis */
+/* location of rest apis */
 %let base_uri=%mf_getplatform(VIYARESTAPI);
 
 /* fetching folder details for provided path */
-%local fname1;
 %let fname1=%mf_getuniquefileref();
 %let libref1=%mf_getuniquelibref();
 
@@ -78,7 +77,6 @@ data &outds;
 run;
 
 
-
 /* clear refs */
 filename &fname1 clear;
 libname &libref1 clear;

viya/mv_getjobcode.sas

@@ -33,7 +33,6 @@
   @li mf_getplatform.sas
   @li mf_getuniquefileref.sas
   @li mv_getfoldermembers.sas
-  @li ml_json.sas
 
 **/
 
@@ -44,9 +43,9 @@
     ,grant_type=sas_services
     ,mdebug=0
   );
-%local dbg;
+%local dbg bufsize varcnt fname1 fname2 errmsg;
 %if &mdebug=1 %then %do;
-  %put &sysmacroname entry vars:;
+  %put &sysmacroname local entry vars:;
   %put _local_;
 %end;
 %else %let dbg=*;
@@ -104,7 +103,6 @@ run;
 )
 
 /* prepare request*/
-%local  fname1;
 %let fname1=%mf_getuniquefileref();
 proc http method='GET' out=&fname1 &oauth_bearer
   url="&base_uri&joburi";
@@ -114,37 +112,95 @@ proc http method='GET' out=&fname1 &oauth_bearer
   %end;
   ;
 run;
-%if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201 %then
-%do;
-  data _null_;infile &fname1;input;putlog _infile_;run;
-  %mp_abort(mac=&sysmacroname
-    ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
-  )
-%end;
-%local  fname2 fname3 fpath1 fpath2 fpath3;
-%let fname2=%mf_getuniquefileref();
-%let fname3=%mf_getuniquefileref();
-%let fpath1=%sysfunc(pathname(&fname1));
-%let fpath2=%sysfunc(pathname(&fname2));
-%let fpath3=%sysfunc(pathname(&fname3));
 
-/* compile the lua JSON module */
+%if &mdebug=1 %then %do;
-%ml_json()
+  data _null_;
-/* read using LUA - this allows the code to be of any length */
+    infile &fname1;
+    input;
+    putlog _infile_;
+  run;
+%end;
+
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+%let fname2=%mf_getuniquefileref();
+filename &fname2 temp ;
+
+/* cannot use lua IO package as not available in Viya 4 */
+/* so use data step to read the JSON until the string `"code":"` is found */
 data _null_;
-  file "&fpath3..lua";
+  file &fname2 recfm=n;
-  put '
+  infile &fname1 lrecl=1 recfm=n;
-    infile = io.open (sas.symget("fpath1"), "r")
+  input sourcechar $char1. @@;
-    outfile = io.open (sas.symget("fpath2"), "w")
+  format sourcechar hex2.;
-    io.input(infile)
+  retain startwrite 0;
-    local resp=json.decode(io.read())
+  if startwrite=0 and sourcechar='"' then do;
-    local job=resp["code"]
+    reentry:
-    outfile:write(job)
+    input sourcechar $ 1. @@;
-    io.close(infile)
+    if sourcechar='c' then do;
-    io.close(outfile)
+      reentry2:
-  ';
+      input sourcechar $ 1. @@;
+      if sourcechar='o' then do;
+        input sourcechar $ 1. @@;
+        if sourcechar='d' then do;
+          input sourcechar $ 1. @@;
+          if sourcechar='e' then do;
+            input sourcechar $ 1. @@;
+            if sourcechar='"' then do;
+              input sourcechar $ 1. @@;
+              if sourcechar=':' then do;
+                input sourcechar $ 1. @@;
+                if sourcechar='"' then do;
+                  putlog 'code found';
+                  startwrite=1;
+                  input sourcechar $ 1. @@;
+                end;
+              end;
+              else if sourcechar='c' then goto reentry2;
+            end;
+          end;
+          else if sourcechar='"' then goto reentry;
+        end;
+        else if sourcechar='"' then goto reentry;
+      end;
+      else if sourcechar='"' then goto reentry;
+    end;
+    else if sourcechar='"' then goto reentry;
+  end;
+  /* once the `"code":"` string is found, write until unescaped `"` is found */
+  if startwrite=1 then do;
+    if sourcechar='\' then do;
+      input sourcechar $ 1. @@;
+      if sourcechar in ('"','\') then put sourcechar char1.;
+      else if sourcechar='n' then put '0A'x;
+      else if sourcechar='r' then put '0D'x;
+      else if sourcechar='t' then put '09'x;
+      else if sourcechar='u' then do;
+        length uni $4;
+        input uni $ 4. @@;
+        sourcechar=unicode('\u'!!uni);
+        put sourcechar char1.;
+      end;
+      else do;
+        call symputx('errmsg',"Uncaught escape char: "!!sourcechar,'l');
+        call symputx('syscc',99);
+        stop;
+      end;
+    end;
+    else if sourcechar='"' then stop;
+    else put sourcechar char1.;
+  end;
 run;
-%inc "&fpath3..lua";
+
+%mp_abort(iftrue=("&syscc"="99")
+  ,mac=mv_getjobcode
+  ,msg=%str(&errmsg)
+)
+
 /* export to desired destination */
 %if "&outref"="0" %then %do;
   data _null_;
@@ -169,7 +225,6 @@ run;
   /* clear refs */
   filename &fname1 clear;
   filename &fname2 clear;
-  filename &fname3 clear;
 %end;
 
 %mend mv_getjobcode;

viya/mv_getjoblog.sas

@@ -86,7 +86,8 @@
   @li mp_abort.sas
   @li mf_getplatform.sas
   @li mf_existfileref.sas
-  @li ml_json.sas
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquelibref.sas
 
 **/
 
@@ -95,7 +96,7 @@
     ,grant_type=sas_services
     ,mdebug=0
   );
-%local dbg;
+%local dbg libref1 libref2 loglocation fname1 fname2;
 %if &mdebug=1 %then %do;
   %put &sysmacroname entry vars:;
   %put _local_;
@@ -154,8 +155,8 @@ options noquotelenmax;
 %let base_uri=%mf_getplatform(VIYARESTAPI);
 
 /* prepare request*/
-%local  fname1;
 %let fname1=%mf_getuniquefileref();
+%let fname2=%mf_getuniquefileref();
 proc http method='GET' out=&fname1 &oauth_bearer
   url="&base_uri&uri";
   headers
@@ -175,37 +176,19 @@ run;
     ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
   )
 %end;
-%local  fname2 fname3 fpath1 fpath2 fpath3;
-%let fname2=%mf_getuniquefileref();
-%let fname3=%mf_getuniquefileref();
-%let fpath1=%sysfunc(pathname(&fname1));
-%let fpath2=%sysfunc(pathname(&fname2));
-%let fpath3=%sysfunc(pathname(&fname3));
 
-/* compile the lua JSON module */
+%let libref1=%mf_getuniquelibref();
-%ml_json()
+libname &libref1 JSON fileref=&fname1;
-/* read using LUA - this allows the code to be of any length */
 data _null_;
-  file "&fpath3..lua";
+  set &libref1..root;
-  put '
+  call symputx('loglocation',loglocation,'l');
-    infile = io.open (sas.symget("fpath1"), "r")
-    outfile = io.open (sas.symget("fpath2"), "w")
-    io.input(infile)
-    local resp=json.decode(io.read())
-    local logloc=resp["logLocation"]
-    outfile:write(logloc)
-    io.close(infile)
-    io.close(outfile)
-  ';
 run;
-%inc "&fpath3..lua";
+
-/* get log path*/
+/* validate log path*/
 %let errflg=1;
-%let errmsg=No entry in &fname2 fileref;
+%let errmsg=No loglocation entry in &fname1 fileref;
 data _null_;
-  infile &fname2;
+  uri=symget('loglocation');
-  input;
-  uri=cats(_infile_);
   if length(uri)<12 then do;
     call symputx('errflg',1);
     call symputx('errmsg',"URI is invalid (too short) - '&uri'",'l');
@@ -232,7 +215,7 @@ run;
 
 /* we have a log uri - now fetch the log */
 %&dbg.put &sysmacroname: querying &base_uri&logloc/content;
-proc http method='GET' out=&fname1 &oauth_bearer
+proc http method='GET' out=&fname2 &oauth_bearer
   url="&base_uri&logloc/content?limit=10000";
   headers
   %if &grant_type=authorization_code %then %do;
@@ -243,14 +226,14 @@ run;
 
 %if &mdebug=1 %then %do;
   %put &sysmacroname: fetching log content from &base_uri&logloc/content;
-  data _null_;infile &fname1;input;putlog _infile_;run;
+  data _null_;infile &fname2;input;putlog _infile_;run;
 %end;
 
 %if &SYS_PROCHTTP_STATUS_CODE=400 %then %do;
   /* fetch log from parent session */
   %let logloc=%substr(&logloc,1,%index(&logloc,%str(/jobs/))-1);
   %&dbg.put &sysmacroname: Now querying &base_uri&logloc/log/content;
-  proc http method='GET' out=&fname1 &oauth_bearer
+  proc http method='GET' out=&fname2 &oauth_bearer
     url="&base_uri&logloc/log/content?limit=10000";
     headers
     %if &grant_type=authorization_code %then %do;
@@ -260,47 +243,32 @@ run;
   run;
   %if &mdebug=1 %then %do;
     %put &sysmacroname: fetching log content from &base_uri&logloc/log/content;
-    data _null_;infile &fname1;input;putlog _infile_;run;
+    data _null_;infile &fname2;input;putlog _infile_;run;
   %end;
 %end;
 
 %if &SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201
 %then %do;
   %if &mdebug ne 1 %then %do; /* have already output above */
-    data _null_;infile &fname1;input;putlog _infile_;run;
+    data _null_;infile &fname2;input;putlog _infile_;run;
   %end;
   %mp_abort(mac=&sysmacroname
     ,msg=%str(logfetch: &SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
   )
 %end;
-data _null_;
-  file "&fpath3..lua";
-  put '
-    infile = io.open (sas.symget("fpath1"), "r")
-    outfile = io.open (sas.symget("fpath2"), "w")
-    io.input(infile)
-    local resp=json.decode(io.read())
-    for i, v in pairs(resp["items"]) do
-      outfile:write(v.line,"\n")
-    end
-    io.close(infile)
-    io.close(outfile)
-  ';
-run;
-%inc "&fpath3..lua";
 
-/* write log out to the specified fileref */
+%let libref2=%mf_getuniquelibref();
+libname &libref2 JSON fileref=&fname2;
 data _null_;
-  infile &fname2 end=last;
   file &outref mod;
   if _n_=1 then do;
     put "/** SASJS Viya Job Log Extract start: &uri **/";
   end;
-  input;
+  set &libref2..items end=last;
-  put _infile_;
   %if &mdebug=1 %then %do;
-    putlog _infile_;
+    putlog line;
   %end;
+  put line;
   if last then do;
     put "/** SASJS Viya Job Log Extract end: &uri **/";
   end;
@@ -309,7 +277,8 @@ run;
 %if &mdebug=0 %then %do;
   filename &fname1 clear;
   filename &fname2 clear;
-  filename &fname3 clear;
+  libname &libref1 clear;
+  libname &libref2 clear;
 %end;
 %else %do;
   %put &sysmacroname exit vars:;