Merge pull request #172 from sasjs/serverupdates

fix: updating mp_streamfile for sasjs/server compatibility

README.md

@@ -47,7 +47,7 @@ Documentation: https://core.sasjs.io
 
 This library will not be used for storing data entries (such as formats or datalines).  Where this becomes necessary in the future, a new repo will be created, in order to keep the NPM bundle size down (for the benefit of those looking to embed purely macros in their applications).
 
-#### FCMP library (All Platforms)
+### FCMP library (All Platforms)
 
 - Function and macro names are identical, except for special cases
 - Prefixes: _mcf_
@@ -217,7 +217,6 @@ If you find this library useful, please leave a [star](https://github.com/sasjs/
 ![](https://starchart.cc/sasjs/core.svg)
 
 
-
 ## Contributors ✨
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
 [![All Contributors](https://img.shields.io/badge/all_contributors-10-orange.svg?style=flat-square)](#contributors-)

base/mf_getplatform.sas

@@ -22,6 +22,12 @@
 )/*/STORE SOURCE*/;
 %local a b c;
 %if &switch.NONE=NONE %then %do;
+  %if %symexist(sasjsprocessmode) %then %do;
+    %if &sasjsprocessmode=Stored Program %then %do;
+      SASJS
+      %return;
+    %end;
+  %end;
   %if %symexist(sysprocessmode) %then %do;
     %if "&sysprocessmode"="SAS Object Server"
     or "&sysprocessmode"= "SAS Compute Server" %then %do;

base/mp_assertscope.sas

@@ -74,7 +74,8 @@
   outds=work.test_results
 )/*/STORE SOURCE*/;
 %local ds test_result test_comments del add mod ilist;
-%let ilist=%upcase(&sasjs_prefix._FUNCTIONS &ignorelist);
+%let ilist=%upcase(&sasjs_prefix._FUNCTIONS SYS_PROCHTTP_STATUS_CODE
+  SYS_PROCHTTP_STATUS_CODE SYS_PROCHTTP_STATUS_PHRASE &ignorelist);
 
 /**
   * this sets up the global vars, it will also enter STRICT mode.  If this
@@ -89,7 +90,7 @@
   create table &scopeds as
     select name,offset,value
     from dictionary.macros
-    where scope="&scope" and name not in (%mf_getquotedstr(&ilist))
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
     order by name,offset;
 %end;
 %else %if &action=COMPARE %then %do;
@@ -98,7 +99,7 @@
   create table _data_ as
     select name,offset,value
     from dictionary.macros
-    where scope="&scope" and name not in (%mf_getquotedstr(&ilist))
+    where scope="&scope" and upcase(name) not in (%mf_getquotedstr(&ilist))
     order by name,offset;
 
   %let ds=&syslast;
@@ -141,4 +142,4 @@
   drop table &ds;
 %end;
 
-%mend mp_assertscope;
+%mend mp_assertscope;

base/mp_cntlout.sas

@@ -0,0 +1,85 @@
+/**
+  @file mp_cntlout.sas
+  @brief Creates a cntlout dataset in a consistent format
+  @details The dataset produced by proc format in the cntlout option will vary
+  according to its contents.
+
+  When dealing with formats from an ETL perspective (eg in [Data Controller for
+  SAS](https://datacontroller.io)), it is important that the output dataset
+  has a consistent model (and compariable values).
+
+  This macro makes use of mddl_sas_cntlout.sas to provide the consistent model,
+  and will left-align the start and end values when dealing with numeric ranges
+  to enable consistency when checking for differences.
+
+  usage:
+
+      %mp_cntlout(libcat=yourlib.cat,cntlout=work.formatexport)
+
+  @param [in] libcat The library.catalog reference
+  @param [in] fmtlist= (0) provide a space separated list of specific formats to
+    extract
+  @param [in] iftrue= (1=1) A condition under which the macro should be executed
+  @param [out] cntlout= (work.fmtextract) Libds reference for the output dataset
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+
+  <h4> Related Macros </h4>
+  @li mf_getvarformat.sas
+  @li mp_getformats.sas
+  @li mp_loadformat.sas
+  @li mp_ds2fmtds.sas
+
+  @version 9.2
+  @author Allan Bowe
+  @cond
+**/
+
+%macro mp_cntlout(
+  iftrue=(1=1)
+  ,libcat=
+  ,cntlout=work.fmtextract
+  ,fmtlist=0
+)/*/STORE SOURCE*/;
+%local ddlds cntlds i;
+
+%if not(%eval(%unquote(&iftrue))) %then %return;
+
+%let ddlds=%mf_getuniquename();
+%let cntlds=%mf_getuniquename();
+
+%mddl_sas_cntlout(libds=&ddlds)
+
+%if %index(&libcat,-)>0 and %scan(&libcat,2,-)=FC %then %do;
+  %let libcat=%scan(&libcat,1,-);
+%end;
+
+proc format lib=&libcat cntlout=&cntlds;
+%if "&fmtlist" ne "0" %then %do;
+  select
+  %do i=1 %to %sysfunc(countw(&fmtlist));
+    %scan(&fmtlist,&i,%str( ))
+  %end;
+  ;
+%end;
+run;
+
+data &cntlout;
+  if 0 then set &ddlds;
+  set &cntlds;
+  if type="N" then do;
+    start=cats(start);
+    end=cats(end);
+  end;
+run;
+proc sort;
+  by fmtname start;
+run;
+
+proc sql;
+drop table &ddlds,&cntlds;
+
+%mend mp_cntlout;
+/** @endcond */

base/mp_getformats.sas

@@ -113,10 +113,10 @@ create table &outsummary as
       select &&fmtname&i;
     run;
     data &tempds;
-      length label $256;
+      if 0 then set &outdetail;
       set &tempds;
     run;
-    proc append base=&outdetail data=&tempds;
+    proc append base=&outdetail data=&tempds ;
     run;
   %end;
 %end;

base/mp_loadformat.sas

@@ -0,0 +1,298 @@
+/**
+  @file
+  @brief Loads a format catalog from a staging dataset
+  @details When loading staged data, it is common to receive only the records
+  that have actually changed.  However, when loading a format catalog, if
+  records are missing they are presumed to be no longer required.
+
+  This macro will augment a staging dataset with other records from the same
+  format, to prevent loss of data - UNLESS the input dataset contains a marker
+  column, specifying that a particular row needs to be deleted (`delete_col=`).
+
+  This macro can also be used to identify which records would be (or were)
+  considered new, modified or deleted (`loadtarget=`) by creating the following
+  tables:
+
+  @li work.outds_add
+  @li work.outds_del
+  @li work.outds_mod
+
+  For example usage, see mp_loadformat.test.sas
+
+  @param [in] libcat The format catalog to be loaded
+  @param [in] libds The staging table to load
+  @param [in] loadtarget= (NO) Set to YES to actually load the target catalog
+  @param [in] delete_col= (_____DELETE__THIS__RECORD_____) The column used to
+    mark a record for deletion.  Values should be "Yes" or "No".
+  @param [out] auditlibds= (0) For change tracking, set to the libds of an audit
+    table as defined in mddl_dc_difftable.sas
+  @param [in] locklibds= (0) For multi-user (parallel) situations, set to the
+    libds of the DC lock table as defined in the mddl_dc_locktable.sas macro.
+  @param [out] outds_add= (0) Set a libds here to see the new records added
+  @param [out] outds_del= (0) Set a libds here to see the records deleted
+  @param [out] outds_mod= (0) Set a libds here to see the modified records
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
+
+  <h4> SAS Macros </h4>
+  @li mddl_sas_cntlout.sas
+  @li mf_getuniquename.sas
+  @li mf_nobs.sas
+  @li mp_abort.sas
+  @li mp_cntlout.sas
+  @li mp_lockanytable.sas
+
+  <h4> Related Macros </h4>
+  @li mddl_dc_difftable.sas
+  @li mddl_dc_locktable.sas
+  @li mp_loadformat.test.sas
+  @li mp_lockanytable.sas
+  @li mp_storediffs.sas
+  @li mp_stackdiffs.sas
+
+
+  @version 9.2
+  @author Allan Bowe
+
+**/
+
+%macro mp_loadformat(libcat,libds
+  ,loadtarget=NO
+  ,auditlibds=0
+  ,locklibds=0
+  ,delete_col=_____DELETE__THIS__RECORD_____
+  ,outds_add=0
+  ,outds_del=0
+  ,outds_mod=0
+  ,mdebug=0
+);
+/* set up local macro variables and temporary tables (with a prefix) */
+%local err msg prefix dslist i var fmtlist ibufsize;
+%let dslist=base_fmts template inlibds ds1 stagedata storediffs;
+%if &outds_add=0 %then %let dslist=&dslist outds_add;
+%if &outds_del=0 %then %let dslist=&dslist outds_del;
+%if &outds_mod=0 %then %let dslist=&dslist outds_mod;
+%let prefix=%substr(%mf_getuniquename(),1,21);
+%do i=1 %to %sysfunc(countw(&dslist));
+  %let var=%scan(&dslist,&i);
+  %local &var;
+  %let &var=%upcase(&prefix._&var);
+%end;
+
+/*
+format values can be up to 32767 wide.  SQL joins on such a wide column can
+cause buffer issues.  Update ibufsize and reset at the end.
+*/
+%let ibufsize=%sysfunc(getoption(ibufsize));
+options ibufsize=32767 ;
+
+/* in DC, format catalogs maybe specified in the libds with a -FC extension */
+%let libcat=%scan(&libcat,1,-);
+
+/* perform input validations */
+%let err=0;
+%let msg=0;
+data _null_;
+  if _n_=1 then putlog "&sysmacroname entry vars:";
+  set sashelp.vmacro;
+  where scope="&sysmacroname";
+  value=upcase(value);
+  if &mdebug=0 then put name '=' value;
+  if name=:'LOAD' and value not in ('YES','NO') then do;
+    call symputx('msg',"invalid value for "!!name!!":"!!value);
+    call symputx('err',1);
+    stop;
+  end;
+  else if name='LIBCAT' then do;
+    if exist(value,'CATALOG') le 0 then do;
+      call symputx('msg',"Unable to open catalog: "!!value);
+      call symputx('err',1);
+      stop;
+    end;
+  end;
+  else if name='LIBDS' then do;
+    if exist(value) le 0 then do;
+      call symputx('msg',"Unable to open staging table: "!!value);
+      call symputx('err',1);
+      stop;
+    end;
+  end;
+  else if (name=:'OUTDS' or name in ('DELETE_COL','LOCKLIBDS','AUDITLIBDS'))
+  and missing(value) then do;
+    call symputx('msg',"missing value in var: "!!name);
+    call symputx('err',1);
+    stop;
+  end;
+run;
+
+%mp_abort(
+  iftrue=(&err ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(&msg)
+)
+
+/**
+  * First, extract only relevant formats from the catalog
+  */
+proc sql noprint;
+select distinct fmtname into: fmtlist separated by ' ' from &libds;
+
+%mp_cntlout(libcat=&libcat,fmtlist=&fmtlist,cntlout=&base_fmts)
+
+
+/**
+  * Ensure input table and base_formats have consistent lengths and types
+  */
+%mddl_sas_cntlout(libds=&template)
+data &inlibds;
+  if 0 then set &template;
+  set &libds;
+  if missing(type) then do;
+    if substr(fmtname,1,1)='$' then type='C';
+    else type='N';
+  end;
+  if type='N' then do;
+    start=cats(start);
+    end=cats(end);
+  end;
+run;
+
+
+/**
+  * Identify new records
+  */
+proc sql;
+create table &outds_add(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  left join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where b.fmtname is null
+    and upcase(a.&delete_col) ne "YES"
+  order by fmtname, start;;
+
+/**
+  * Identify deleted records
+  */
+create table &outds_del(drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where upcase(a.&delete_col)="YES"
+  order by fmtname, start;
+
+/**
+  * Identify modified records
+  */
+create table &outds_mod (drop=&delete_col) as
+  select a.*
+  from &inlibds a
+  inner join &base_fmts b
+  on a.fmtname=b.fmtname
+    and a.start=b.start
+  where upcase(a.&delete_col) ne "YES"
+  order by fmtname, start;
+
+options ibufsize=&ibufsize;
+
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc prior to load prep)
+)
+
+%if &loadtarget=YES %then %do;
+  data &ds1;
+    merge &base_fmts(in=base)
+      &outds_mod(in=mod)
+      &outds_add(in=add)
+      &outds_del(in=del);
+    if not del and not mod;
+    by fmtname start;
+  run;
+  data &stagedata;
+    set &ds1 &outds_mod;
+  run;
+  proc sort;
+    by fmtname start;
+  run;
+%end;
+/* mp abort needs to run outside of conditional blocks */
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc prior to actual load)
+)
+%if &loadtarget=YES %then %do;
+  %if %mf_nobs(&stagedata)=0 %then %do;
+    %put There are no changes to load in &libcat!;
+    %return;
+  %end;
+  %if &locklibds ne 0 %then %do;
+    /* prevent parallel updates */
+    %mp_lockanytable(LOCK
+      ,lib=%scan(&libcat,1,.)
+      ,ds=%scan(&libcat,2,.)-FC
+      ,ref=MP_LOADFORMAT commencing format load
+      ,ctl_ds=&locklibds
+    )
+  %end;
+  /* do the actual load */
+  proc format lib=&libcat cntlin=&stagedata;
+  run;
+  %if &locklibds ne 0 %then %do;
+    /* unlock the table */
+    %mp_lockanytable(UNLOCK
+      ,lib=%scan(&libcat,1,.)
+      ,ds=%scan(&libcat,2,.)-FC
+      ,ref=MP_LOADFORMAT completed format load
+      ,ctl_ds=&locklibds
+    )
+  %end;
+  /* track the changes */
+  %if &auditlibds ne 0 %then %do;
+    %if &locklibds ne 0 %then %do;
+      %mp_lockanytable(LOCK
+        ,lib=%scan(&auditlibds,1,.)
+        ,ds=%scan(&auditlibds,2,.)
+        ,ref=MP_LOADFORMAT commencing audit table load
+        ,ctl_ds=&locklibds
+      )
+    %end;
+
+    %mp_storediffs(&libcat-FC
+      ,&inlibds
+      ,FMTNAME START
+      ,delds=&outds_del
+      ,modds=&outds_mod
+      ,appds=&outds_add
+      ,outds=&storediffs
+      ,mdebug=&mdebug
+    )
+
+    %if &locklibds ne 0 %then %do;
+      %mp_lockanytable(UNLOCK
+        ,lib=%scan(&auditlibds,1,.)
+        ,ds=%scan(&auditlibds,2,.)
+        ,ref=MP_LOADFORMAT commencing audit table load
+        ,ctl_ds=&locklibds
+      )
+    %end;
+  %end;
+%end;
+%mp_abort(
+  iftrue=(&syscc ne 0)
+  ,mac=&sysmacroname
+  ,msg=%str(SYSCC=&syscc after load)
+)
+
+%if &mdebug=0 %then %do;
+  proc datasets lib=work;
+    delete &prefix:;
+  run;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%mend mp_loadformat;

base/mp_lockanytable.sas

@@ -1,7 +1,8 @@
 /**
   @file
   @brief Mechanism for locking tables to prevent parallel modifications
-  @details Uses a control table to enable ANY table to be locked for updates.
+  @details Uses a control table to enable ANY table to be locked for updates
+  (not just SAS datasets).
   Only useful if every update uses the macro!   Used heavily within
   [Data Controller for SAS](https://datacontroller.io).
 
@@ -15,7 +16,7 @@
     length is 200 characters.
   @param [out] ctl_ds= (0) The control table which controls the actual locking.
     Should already be assigned and available.  The definition is available by
-    running mp_coretable.sas as follows:  `%mp_coretable(LOCKTABLE)`.
+    running the mddl_dc_locktable.sas macro.
 
   @param [in] loops= (25) Number of times to check for a lock.
   @param [in] loop_secs= (1) Seconds to wait between each lock attempt
@@ -48,7 +49,7 @@ data _null_;
   put name '=' value;
 run;
 
-%mp_abort(iftrue= (&ds=0 and &action ne MAKETABLE)
+%mp_abort(iftrue= ("&ds"="0" and &action ne MAKETABLE)
   ,mac=&sysmacroname
   ,msg=%str(dataset was not provided)
 )

base/mp_lockfilecheck.sas

@@ -37,7 +37,7 @@ run;
   ,mac=checklock.sas
   ,msg=Aborting with syscc=&syscc on entry.
 )
-%mp_abort(iftrue= (&libds=0)
+%mp_abort(iftrue= ("&libds"="0")
   ,mac=&sysmacroname
   ,msg=%str(libds not provided)
 )
@@ -46,6 +46,12 @@ run;
 %let lib=%upcase(%scan(&libds,1,.));
 %let ds=%upcase(%scan(&libds,2,.));
 
+/* in DC, format catalogs are passed with a -FC suffix. No saslock here! */
+%if %scan(&libds,2,-)=FC %then %do;
+  %put &sysmacroname: Format Catalog detected, no lockfile applied to &libds;
+  %return;
+%end;
+
 /* do not proceed if no observations can be processed */
 %let msg=options obs = 0. syserrortext=%superq(syserrortext);
 %mp_abort(iftrue= (%sysfunc(getoption(OBS))=0)

base/mp_md5.sas

@@ -0,0 +1,58 @@
+/**
+  @file
+  @brief Generates an md5 expression for hashing a set of variables
+  @details This is the same algorithm used to hash records in
+  [Data Controller for SAS](https://datacontroller.io) (free for up
+  to 5 users).
+
+  It is not designed to be efficient - it is designed to be effective,
+  given the range of edge cases (large floating points, special missing
+  numerics, thousands of columns, very wide columns).
+
+  It can be used only in data step, eg as follows:
+
+      data _null_;
+        set sashelp.class;
+        hashvar=%mp_md5(cvars=name sex, nvars=age height weight);
+        put hashvar=;
+      run;
+
+  Unfortunately it will not run in SQL - it fails with the following message:
+
+  > The width value for HEX is out of bounds. It should be between 1 and 16
+
+  The macro will also cause errors if the data contains (non-special) missings
+  and the (undocumented) `options dsoptions=nonote2err;` is in effect.
+
+  This can be avoided in two ways:
+
+  @li Global option:  `options dsoptions=nonote2err;`
+  @li Data step option: `data YOURLIB.YOURDATASET /nonote2err;`
+
+  @param cvars= Space seperated list of character variables
+  @param nvars= Space seperated list of numeric variables
+
+  <h4> Related Programs </h4>
+  @li mp_init.sas
+
+  @version 9.2
+  @author Allan Bowe
+**/
+
+%macro mp_md5(cvars=,nvars=);
+%local i var sep;
+put(md5(
+  %do i=1 %to %sysfunc(countw(&cvars));
+    %let var=%scan(&cvars,&i,%str( ));
+    &sep put(md5(trim(&var)),$hex32.)
+    %let sep=!!;
+  %end;
+  %do i=1 %to %sysfunc(countw(&nvars));
+    %let var=%scan(&nvars,&i,%str( ));
+    /* multiply by 1 to strip precision errors (eg 0 != 0) */
+    /* but ONLY if not missing, else will lose any special missing values */
+    &sep put(md5(trim(put(ifn(missing(&var),&var,&var*1),binary64.))),$hex32.)
+    %let sep=!!;
+  %end;
+),hex32.)
+%mend mp_md5;

base/mp_storediffs.sas

@@ -90,7 +90,7 @@
 %else %let dbg=*;
 
 /* set up unique and temporary vars */
-%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist;
+%local ds1 ds2 ds3 ds4 hashkey inds_auto inds_keep dslist vlist;
 %let ds1=%upcase(work.%mf_getuniquename(prefix=mpsd_ds1));
 %let ds2=%upcase(work.%mf_getuniquename(prefix=mpsd_ds2));
 %let ds3=%upcase(work.%mf_getuniquename(prefix=mpsd_ds3));
@@ -144,12 +144,21 @@ proc transpose data=&ds1
   by &inds_keep &hashkey;
   var _character_;
 run;
+
+%if %index(&libds,-)>0 and %scan(&libds,2,-)=FC %then %do;
+  /* this is a format catalog - cannot query cols directly */
+  %let vlist="FMTNAME","START","END","LABEL","MIN","MAX","DEFAULT","LENGTH"
+    ,"FUZZ","PREFIX","MULT","FILL","NOEDIT","TYPE","SEXCL","EEXCL","HLO"
+    ,"DECSEP","DIG3SEP","DATATYPE","LANGUAGE";
+%end;
+%else %let vlist=%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE);
+
 data &ds4;
   length &inds_keep $41 tgtvar_nm $32;
   set &ds2 &ds3 indsname=&inds_auto;
 
   tgtvar_nm=upcase(tgtvar_nm);
-  if tgtvar_nm in (%upcase(%mf_getvarlist(&libds,dlm=%str(,),quote=DOUBLE)));
+  if tgtvar_nm in (%upcase(&vlist));
 
   if &inds_auto="&ds2" then tgtvar_type='N';
   else if &inds_auto="&ds3" then tgtvar_type='C';

base/mp_streamfile.sas

@@ -12,7 +12,7 @@
 
       %mp_streamfile(contenttype=csv,inloc=/some/where.txt,outname=myfile.txt)
 
-  @param [in] contenttype= (TEXTS) Either TEXT, ZIP, CSV, EXCEL
+  @param [in] contenttype= (TEXT) Either TEXT, ZIP, CSV, EXCEL
   @param [in] inloc= /path/to/file.ext to be sent
   @param [in] inref= fileref of file to be sent (if provided, overrides `inloc`)
   @param [in] iftrue= (1=1) Provide a condition under which to execute.
@@ -22,6 +22,7 @@
 
   <h4> SAS Macros </h4>
   @li mf_getplatform.sas
+  @li mfs_httpheader.sas
   @li mp_binarycopy.sas
 
   @author Allan Bowe
@@ -55,7 +56,7 @@ data _null_;
 run;
 
 %if &contentype=CSV %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/csv');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -66,10 +67,14 @@ run;
       contenttype='application/csv'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,application/csv)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
 %else %if &contentype=EXCEL %then %do;
   /* suitable for XLS format */
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/vnd.ms-excel');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -80,9 +85,13 @@ run;
       contenttype='application/vnd.ms-excel'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,application/vnd.ms-excel)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
 %else %if &contentype=GIF or &contentype=JPEG or &contentype=PNG %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type',"image/%lowcase(&contenttype)");
     run;
@@ -91,15 +100,21 @@ run;
     filename &outref filesrvc parenturi="&SYS_JES_JOB_URI"
       contenttype="image/%lowcase(&contenttype)";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,image/%lowcase(&contenttype))
+  %end;
 %end;
 %else %if &contentype=HTML %then %do;
   %if &platform=SASVIYA %then %do;
     filename &outref filesrvc parenturi="&SYS_JES_JOB_URI" name="_webout.json"
       contenttype="text/html";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,text/html)
+  %end;
 %end;
 %else %if &contentype=TEXT %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/text');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -110,9 +125,13 @@ run;
       contenttype='application/text'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,application/text)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
 %else %if &contentype=WOFF or &contentype=WOFF2 or &contentype=TTF %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type',"font/%lowcase(&contenttype)");
     run;
@@ -121,9 +140,12 @@ run;
     filename &outref filesrvc parenturi="&SYS_JES_JOB_URI"
       contenttype="font/%lowcase(&contenttype)";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,font/%lowcase(&contenttype))
+  %end;
 %end;
 %else %if &contentype=XLSX %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type',
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet');
@@ -136,9 +158,15 @@ run;
         'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type
+      ,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+    )
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
 %else %if &contentype=ZIP %then %do;
-  %if &platform=SASMETA and &streamweb=1 %then %do;
+  %if (&platform=SASMETA and &streamweb=1) %then %do;
     data _null_;
       rc=stpsrv_header('Content-type','application/zip');
       rc=stpsrv_header('Content-disposition',"attachment; filename=&outname");
@@ -149,6 +177,10 @@ run;
       contenttype='application/zip'
       contentdisp="attachment; filename=&outname";
   %end;
+  %else %if &platform=SASJS %then %do;
+    %mfs_httpheader(Content-type,application/zip)
+    %mfs_httpheader(Content-disposition,%str(attachment; filename=&outname))
+  %end;
 %end;
 %else %do;
   %put %str(ERR)OR: Content Type &contenttype NOT SUPPORTED by &sysmacroname!;

base/mp_wait4file.sas

@@ -4,9 +4,7 @@
   @details Loops with a `sleep()` command until a file arrives or the max wait
   period expires.
 
-  @example
+  Example: Wait 3 minutes OR for /tmp/flag.txt to appear
-
-  Wait 3 minutes OR for /tmp/flag.txt to appear
 
     %mp_wait4file(/tmp/flag.txt , maxwait=60*3)
 

ddl/mddl_sas_cntlout.sas

@@ -12,25 +12,30 @@
 
 proc sql;
 create table &libds(
-    FMTNAME char(32)     label='Format name'
+    FMTNAME char(32)      label='Format name'
-    ,START char(16)     label='Starting value for format'
+    /*
-    ,END char(16)     label='Ending value for format'
+      to accomodate larger START values, mp_loadformat.sas will need the
-    ,LABEL char(256)     label='Format value label'
+      SQL dependency removed (proc sql needs to accomodate 3 index values in
+      a 32767 ibufsize limit)
+    */
+    ,START char(10000)    label='Starting value for format'
+    ,END char(32767)      label='Ending value for format'
+    ,LABEL char(32767)    label='Format value label'
     ,MIN num length=3     label='Minimum length'
     ,MAX num length=3     label='Maximum length'
-    ,DEFAULT num length=3     label='Default length'
+    ,DEFAULT num length=3 label='Default length'
-    ,LENGTH num length=3     label='Format length'
+    ,LENGTH num length=3  label='Format length'
-    ,FUZZ num     label='Fuzz value'
+    ,FUZZ num             label='Fuzz value'
-    ,PREFIX char(2)     label='Prefix characters'
+    ,PREFIX char(2)       label='Prefix characters'
-    ,MULT num     label='Multiplier'
+    ,MULT num             label='Multiplier'
-    ,FILL char(1)     label='Fill character'
+    ,FILL char(1)         label='Fill character'
-    ,NOEDIT num length=3     label='Is picture string noedit?'
+    ,NOEDIT num length=3  label='Is picture string noedit?'
-    ,TYPE char(1)     label='Type of format'
+    ,TYPE char(1)         label='Type of format'
-    ,SEXCL char(1)     label='Start exclusion'
+    ,SEXCL char(1)        label='Start exclusion'
-    ,EEXCL char(1)     label='End exclusion'
+    ,EEXCL char(1)        label='End exclusion'
-    ,HLO char(13)     label='Additional information'
+    ,HLO char(13)         label='Additional information'
-    ,DECSEP char(1)     label='Decimal separator'
+    ,DECSEP char(1)       label='Decimal separator'
-    ,DIG3SEP char(1)     label='Three-digit separator'
+    ,DIG3SEP char(1)      label='Three-digit separator'
     ,DATATYPE char(8)     label='Date/time/datetime?'
     ,LANGUAGE char(8)     label='Language for date strings'
 );

meta/mm_assignlib.sas

@@ -40,6 +40,12 @@
       /* now try and assign it */
       if libname("&libref",,'meta',cats('liburi="',liburi,'";')) ne 0 then do;
         putlog "&libref could not be assigned";
+        putlog liburi=;
+        /**
+          * Fetch the system message for display in the abort modal.  This is
+          * not always helpful though.  One example, previously received:
+          * NOTE: Libref XX refers to the same library metadata as libref XX.
+          */
         call symputx('msg',sysmsg(),'l');
         if "&mabort"='HARD' then call symputx('mp_abort',1,'l');
       end;
@@ -61,7 +67,7 @@
 
   %if &mp_abort=1 %then %do;
     %mp_abort(iftrue= (&mp_abort=1)
-      ,mac=&sysmacroname
+      ,mac=mm_assignlib.sas
       ,msg=&msg
     )
     %return;

meta/mm_createfolder.sas

@@ -12,13 +12,14 @@
     The macro is idempotent - if you run it twice, it will only create a folder
     once.
 
-  usage:
+  Usage:
 
       %mm_createfolder(path=/some/meta/folder)
 
   @param [in] path= Name of the folder to create.
   @param [in] mdebug= set DBG to 1 to disable DEBUG messages
 
+
   @version 9.4
   @author Allan Bowe
 

meta/mm_createstp.sas

@@ -12,7 +12,7 @@
     This macro is idempotent - if you run it twice, it will only create an STP
     once.
 
-  usage (type 1 STP):
+  Usage (type 1 STP):
 
       %mm_createstp(stpname=MyNewSTP
         ,filename=mySpecialProgram.sas
@@ -31,7 +31,8 @@
         putlog (_all_)(=);
       run;
 
-  usage (type 2 STP):
+  Usage (type 2 STP):
+
       %mm_createstp(stpname=MyNewType2STP
         ,filename=mySpecialProgram.sas
         ,directory=SASEnvironment/SASCode/STPs
@@ -74,8 +75,9 @@
   @li mf_verifymacvars.sas
   @li mm_getdirectories.sas
   @li mm_updatestpsourcecode.sas
-  @li mp_dropmembers.sas
   @li mm_getservercontexts.sas
+  @li mp_abort.sas
+  @li mp_dropmembers.sas
 
   <h4> Related Macros </h4>
   @li mm_createwebservice.sas

meta/mm_getauthinfo.sas

@@ -1,16 +1,21 @@
 /**
   @file mm_getauthinfo.sas
-  @brief extracts authentication info
+  @brief Extracts authentication info for each user in metadata
-  @details usage:
+  @details
+  Usage:
 
-    %mm_getauthinfo(outds=auths)
+      %mm_getauthinfo(outds=auths)
 
-  @param outds= the ONE LEVEL work dataset to create
+
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages and preserve outputs
+  @param [out] outds= (mm_getauthinfo) The output dataset to create
 
   <h4> SAS Macros </h4>
-  @li mm_getobjects.sas
   @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
   @li mm_getdetails.sas
+  @li mm_getobjects.sas
+
 
   @version 9.4
   @author Allan Bowe
@@ -18,67 +23,69 @@
 **/
 
 %macro mm_getauthinfo(outds=mm_getauthinfo
+  ,mdebug=0
 )/*/STORE SOURCE*/;
+%local prefix fileref;
+%let prefix=%substr(%mf_getuniquename(),1,25);
 
-%if %length(&outds)>30 %then %do;
+%mm_getobjects(type=Login,outds=&prefix.0)
-  %put %str(ERR)OR: Temp tables are created with the &outds prefix, which
-    therefore needs to be 30 characters or less;
-  %return;
-%end;
-%if %index(&outds,'.')>0 %then %do;
-  %put %str(ERR)OR: Table &outds should be ONE LEVEL (no library);
-  %return;
-%end;
-
-%mm_getobjects(type=Login,outds=&outds.0)
 
 %local fileref;
 %let fileref=%mf_getuniquefileref();
 
 data _null_;
   file &fileref;
-  set &outds.0 end=last;
+  set &prefix.0 end=last;
   /* run macro */
-  str=cats('%mm_getdetails(uri=',id,",outattrs=&outds.d",_n_
+  str=cats('%mm_getdetails(uri=',id,",outattrs=&prefix.d",_n_
-    ,",outassocs=&outds.a",_n_,")");
+    ,",outassocs=&prefix.a",_n_,")");
   put str;
   /* transpose attributes */
-  str=cats("proc transpose data=&outds.d",_n_,"(drop=type) out=&outds.da"
+  str=cats("proc transpose data=&prefix.d",_n_,"(drop=type) out=&prefix.da"
     ,_n_,"(drop=_name_);var value;id name;run;");
   put str;
   /* add extra info to attributes */
-  str=cats("data &outds.da",_n_,";length login_id login_name $256; login_id="
+  str=cats("data &prefix.da",_n_,";length login_id login_name $256; login_id="
-    ,quote(trim(id)),";set &outds.da",_n_
+    ,quote(trim(id)),";set &prefix.da",_n_
     ,";login_name=trim(subpad(name,1,256));drop name;run;");
   put str;
   /* add extra info to associations */
-  str=cats("data &outds.a",_n_,";length login_id login_name $256; login_id="
+  str=cats("data &prefix.a",_n_,";length login_id login_name $256; login_id="
     ,quote(trim(id)),";login_name=",quote(trim(name))
-    ,";set &outds.a",_n_,";run;");
+    ,";set &prefix.a",_n_,";run;");
   put str;
   if last then do;
     /* collate attributes */
-    str=cats("data &outds._logat; set &outds.da1-&outds.da",_n_,";run;");
+    str=cats("data &prefix._logat; set &prefix.da1-&prefix.da",_n_,";run;");
     put str;
     /* collate associations */
-    str=cats("data &outds._logas; set &outds.a1-&outds.a",_n_,";run;");
+    str=cats("data &prefix._logas; set &prefix.a1-&prefix.a",_n_,";run;");
     put str;
     /* tidy up */
-    str=cats("proc delete data=&outds.da1-&outds.da",_n_,";run;");
+    str=cats("proc delete data=&prefix.da1-&prefix.da",_n_,";run;");
     put str;
-    str=cats("proc delete data=&outds.d1-&outds.d",_n_,";run;");
+    str=cats("proc delete data=&prefix.d1-&prefix.d",_n_,";run;");
     put str;
-    str=cats("proc delete data=&outds.a1-&outds.a",_n_,";run;");
+    str=cats("proc delete data=&prefix.a1-&prefix.a",_n_,";run;");
     put str;
   end;
 run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fileref;
+    if _n_=1 then putlog // "Now executing the following code:" //;
+    input; putlog _infile_;
+  run;
+%end;
 %inc &fileref;
+filename &fileref clear;
 
 /* get libraries */
-proc sort data=&outds._logas(where=(assoc='Libraries')) out=&outds._temp;
+proc sort data=&prefix._logas(where=(assoc='Libraries')) out=&prefix._temp;
   by login_id;
-data &outds._temp;
+data &prefix._temp;
-  set &outds._temp;
+  set &prefix._temp;
   by login_id;
   length library_list $32767;
   retain library_list;
@@ -86,31 +93,27 @@ data &outds._temp;
   else library_list=catx(' !! ',library_list,name);
 proc sql;
 /* get auth domain */
-create table &outds._dom as
+create table &prefix._dom as
   select login_id,name as domain
-  from &outds._logas
+  from &prefix._logas
   where assoc='Domain';
-create unique index login_id on &outds._dom(login_id);
+create unique index login_id on &prefix._dom(login_id);
 /* join it all together */
-create table &outds._logins as
+create table &outds as
   select a.*
     ,c.domain
     ,b.library_list
-  from &outds._logat (drop=ishidden lockedby usageversion publictype) a
+  from &prefix._logat (drop=ishidden lockedby usageversion publictype) a
-  left join &outds._temp b
+  left join &prefix._temp b
   on a.login_id=b.login_id
-  left join &outds._dom c
+  left join &prefix._dom c
   on a.login_id=c.login_id;
-drop table &outds._temp;
-drop table &outds._logat;
-drop table &outds._logas;
 
-data _null_;
+%if &mdebug=0 %then %do;
-  infile &fileref;
+  proc datasets lib=work;
-  if _n_=1 then putlog // "Now executing the following code:" //;
+    delete &prefix:;
-  input; putlog _infile_;
+  run;
-run;
+%end;
 
-filename &fileref clear;
 
 %mend mm_getauthinfo;

package-lock.json

@@ -1150,9 +1150,9 @@
       }
     },
     "node_modules/follow-redirects": {
-      "version": "1.14.7",
+      "version": "1.14.8",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.8.tgz",
-      "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==",
+      "integrity": "sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA==",
       "dev": true,
       "funding": [
         {
@@ -3678,9 +3678,9 @@
       }
     },
     "follow-redirects": {
-      "version": "1.14.7",
+      "version": "1.14.8",
-      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.7.tgz",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.14.8.tgz",
-      "integrity": "sha512-+hbxoLbFMbRKDwohX8GkTataGqO6Jb7jGwpAlwgy2bIz25XtRm7KEzJM76R1WiNT5SwZkX4Y75SwBolkpmE7iQ==",
+      "integrity": "sha512-1x0S9UVJHsQprFcEC/qnNzBLcIxsjAV905f/UkQxbclCsoTWlacCNOpQa/anodLl2uaEKFhfWOvM2Qg77+15zA==",
       "dev": true
     },
     "form-data": {

sasjs/sasjsconfig.json

@@ -73,17 +73,24 @@
       "name": "server",
       "serverUrl": "https://sas.analytium.co.uk:5000",
       "serverType": "SASJS",
-      "appLoc": "/Shared Data/temp/macrocore",
+      "httpsAgentOptions": {
+        "allowInsecureRequests": false
+      },
+      "appLoc": "/sasjs/core",
       "macroFolders": [
         "tests/serveronly"
       ],
+      "programFolders": [],
+      "binaryFolders": [],
       "deployConfig": {
-        "deployServicePack": true
+        "deployServicePack": true,
+        "deployScripts": []
       }
     },
     {
       "name": "docsonly",
       "serverType": "SAS9",
+      "appLoc": "dummy",
       "macroFolders": [
         "tests/sas9only",
         "tests/viyaonly"

server/mfs_httpheader.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Sets the http headers in the SASjs/server response
+  @details For GET requests, SASjs server will use the file generated by this
+  macro for setting the appropriate http headers in the response.
+
+  It works by writing a file to the session directory, that is then ingested by
+  the server.
+
+  The location of this file is driven by the global variable
+  `sasjs_stpsrv_header_loc` which is made available in the autoexec.
+
+  Usage:
+
+      %mfs_httpheader(Content-type,application/csv)
+
+  @param [in] header_name Name of the http header to set
+  @param [in] header_value Value of the http header to set
+
+  <h4> Related Macros </h4>
+  @li mcf_stpsrv_header.sas
+
+  @version 9.3
+  @author Allan Bowe
+
+**/
+
+%macro mfs_httpheader(header_name
+  ,header_value
+)/*/STORE SOURCE*/;
+%local fref fid i;
+
+%if %sysfunc(filename(fref,&sasjs_stpsrv_header_loc)) ne 0 %then %do;
+  %put &=fref &=sasjs_stpsrv_header_loc;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let fid=%sysfunc(fopen(&fref,A));
+
+%if &fid=0 %then %do;
+  %put %str(ERR)OR: %sysfunc(sysmsg());
+  %return;
+%end;
+
+%let rc=%sysfunc(fput(&fid,%str(&header_name): %str(&header_value)));
+%let rc=%sysfunc(fwrite(&fid));
+
+%let rc=%sysfunc(fclose(&fid));
+%let rc=%sysfunc(filename(&fref));
+
+%mend mfs_httpheader;

server/ms_createfile.sas

@@ -0,0 +1,89 @@
+/**
+  @file
+  @brief Creates a file on SASjs Drive
+  @details Creates a file on SASjs Drive. To use the file as a Stored Program,
+  it must have a ".sas" extension.
+
+  Example:
+
+      filename stpcode temp;
+      data _null_;
+        file stpcode;
+        put '%put hello world;';
+      run;
+      %ms_createfile(/some/stored/program.sas, inref=stpcode)
+
+  @param [in] driveloc The full path to the file in SASjs Drive
+  @param [in] inref= (0) The fileref containing the file to create.
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mf_getuniquename.sas
+  @li mp_abort.sas
+
+**/
+
+%macro ms_createfile(driveloc
+    ,inref=0
+    ,mdebug=0
+  );
+
+%local fname0 fname1 boundary fname statcd msg;
+%let fname0=%mf_getuniquefileref();
+%let fname1=%mf_getuniquefileref();
+%let boundary=%mf_getuniquename();
+
+data _null_;
+  file &fname0 termstr=crlf;
+  infile &inref end=eof;
+  if _n_ = 1 then do;
+    put "--&boundary.";
+    put 'Content-Disposition: form-data; name="filePath"';
+    put ;
+    put "&driveloc";
+    put "--&boundary";
+    put 'Content-Disposition: form-data; name="file"; filename="ignore.sas"';
+    put "Content-Type: text/plain";
+    put ;
+  end;
+  input;
+  put _infile_; /* add the actual file to be sent */
+  if eof then do;
+    put ;
+    put "--&boundary--";
+  end;
+run;
+
+%if &mdebug=1 %then %do;
+  data _null_;
+    infile &fname0;
+    input;
+    put _infile_;
+  run;
+%end;
+
+proc http method='POST' in=&fname0 out=&fname1
+  url="&_sasjs_apiserverurl/SASjsApi/drive/file";
+  headers "Content-Type"="multipart/form-data; boundary=&boundary";
+%if &mdebug=1 %then %do;
+  debug level=1;
+%end;
+run;
+
+%let statcd=0;
+data _null_;
+  infile &fname1;
+  input;
+  putlog _infile_;
+  if _infile_='{"status":"success"}' then call symputx('statcd',1,'l');
+  else call symputx('msg',_infile_,'l');
+run;
+
+%mp_abort(
+  iftrue=(&statcd=0)
+  ,mac=ms_createfile.sas
+  ,msg=%superq(msg)
+)
+
+%mend ms_createfile;

server/ms_runstp.sas

@@ -0,0 +1,77 @@
+/**
+  @file
+  @brief Executes a SASjs Server Stored Program
+  @details Runs a Stored Program (using POST method) and extracts the webout and
+  log from the response JSON.
+
+  Example:
+
+      %ms_runstp(/some/stored/program
+        ,debug=131
+        ,outref=weboot
+      )
+
+  @param [in] pgm The full path to the Stored Program in SASjs Drive (_program
+    parameter)
+  @param [in] debug= (131) The value to supply to the _debug URL parameter
+  @param [in] mdebug= (0) Set to 1 to enable DEBUG messages
+  @param [out] outref= (outweb) The output fileref to contain the response JSON
+    (will be created using temp engine)
+
+  <h4> SAS Macros </h4>
+  @li mf_getuniquefileref.sas
+  @li mp_abort.sas
+
+**/
+
+%macro ms_runstp(pgm
+    ,debug=131
+    ,outref=outweb
+    ,mdebug=0
+  );
+%local dbg fname1;
+%if &mdebug=1 %then %do;
+  %put &sysmacroname entry vars:;
+  %put _local_;
+%end;
+%else %let dbg=*;
+%let fname1=%mf_getuniquefileref();
+
+%mp_abort(iftrue=("&pgm"="")
+  ,mac=&sysmacroname
+  ,msg=%str(Program not provided)
+)
+
+data _null_;
+  file &fname1;
+  infile "&_sasjs_tokenfile";
+  input;
+  put 'Authorization: Bearer' _infile_;
+run;
+
+filename &outref temp;
+
+/* prepare request*/
+proc http method='POST' headerin=&fname1 out=&outref
+  url="&_sasjs_apiserverurl.&_sasjs_apipath?_program=&pgm%str(&)_debug=131";
+run;
+%if (&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  or &mdebug=1 %then %do;
+  data _null_;infile &outref;input;putlog _infile_;run;
+%end;
+%mp_abort(
+  iftrue=(&SYS_PROCHTTP_STATUS_CODE ne 200 and &SYS_PROCHTTP_STATUS_CODE ne 201)
+  ,mac=&sysmacroname
+  ,msg=%str(&SYS_PROCHTTP_STATUS_CODE &SYS_PROCHTTP_STATUS_PHRASE)
+)
+
+
+%if &mdebug=1 %then %do;
+  %put &sysmacroname exit vars:;
+  %put _local_;
+%end;
+%else %do;
+  /* clear refs */
+  filename &fname1 clear;
+%end;
+%mend ms_runstp;

server/ms_webout.sas

@@ -33,8 +33,9 @@
     `,"$tablename":{"formats":{"col1":"$CHAR1"},"types":{"COL1":"C"}}`
 
   <h4> SAS Macros </h4>
-  @li mp_jsonout.sas
   @li mf_getuser.sas
+  @li mp_jsonout.sas
+  @li mfs_httpheader.sas
 
   <h4> Related Macros </h4>
   @li mv_webout.sas
@@ -86,11 +87,11 @@
   /* fix encoding */
   OPTIONS NOBOMFILE;
 
+  /* set the header */
+  %mfs_httpheader(Content-type,application/json)
+
   /* setup json */
   data _null_;file &fref encoding='utf-8' termstr=lf;
-  %if %str(&_debug) ge 131 %then %do;
-    put '>>weboutBEGIN<<';
-  %end;
     put '{"SYSDATE" : "' "&SYSDATE" '"';
     put ',"SYSTIME" : "' "&SYSTIME" '"';
   run;
@@ -170,9 +171,6 @@
     memsize=quote(cats(memsize));
     put ',"MEMSIZE" : ' memsize;
     put "}" @;
-  %if %str(&_debug) ge 131 %then %do;
-    put '>>weboutEND<<';
-  %end;
   run;
 %end;
 

tests/crossplatform/mp_cntlout.test.sas

@@ -0,0 +1,38 @@
+/**
+  @file
+  @brief Testing mp_cntlout.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_cntlout.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+libname perm (work);
+data work.loadfmts;
+  length fmtname $32;
+  eexcl='Y';
+  type='N';
+  do i=1 to 100;
+    fmtname=cats('SASJS_',i,'X');
+    do j=1 to 100;
+      start=cats(j);
+      end=cats(j+1);
+      label= cats('Dummy ',start);
+      output;
+    end;
+  end;
+run;
+proc format cntlin=work.loadfmts library=perm.testcat;
+run;
+
+%mp_assertscope(SNAPSHOT)
+%mp_cntlout(libcat=perm.testcat,cntlout=work.cntlout)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(%mf_nobs(work.cntlout)=10000),
+  desc=Checking first hash diff,
+  outds=work.test_results
+)

tests/crossplatform/mp_loadformat.test.sas

@@ -0,0 +1,76 @@
+/**
+  @file
+  @brief Testing mp_loadformat.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_loadformat.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+/* prep format catalog */
+libname perm (work);
+
+data work.loadfmts;
+  length fmtname $32;
+  eexcl='Y';
+  type='N';
+  do i=1 to 100;
+    fmtname=cats('SASJS_',i,'X');
+    do j=1 to 100;
+      start=cats(j);
+      end=cats(j+1);
+      label= cats('Dummy ',start);
+      output;
+    end;
+  end;
+run;
+proc format cntlin=work.loadfmts library=perm.testcat;
+run;
+
+/* make some test data */
+data work.stagedata;
+  set work.loadfmts;
+  type='N';
+  eexcl='Y';
+  if _n_<150 then deleteme='Yes';
+  else if _n_<250 then label='mod'!!cats(_n_);
+  else if _n_<350 then do;
+    start=cats(_n_);
+    end=cats(_n_+1);
+    label='newval'!!cats(_N_);
+  end;
+  else stop;
+run;
+
+/* load the above */
+%mp_assertscope(SNAPSHOT)
+%mp_loadformat(perm.testcat
+  ,work.stagedata
+  ,loadtarget=YES
+  ,auditlibds=0
+  ,locklibds=0
+  ,delete_col=deleteme
+  ,outds_add=add_test1
+  ,outds_del=del_test1
+  ,outds_mod=mod_test1
+  ,mdebug=1
+)
+%mp_assertscope(COMPARE)
+
+%mp_assert(
+  iftrue=(%mf_nobs(del_test1)=149),
+  desc=Test 1 - delete obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(add_test1)=100),
+  desc=Test 1 - add obs,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=(%mf_nobs(mod_test1)=100),
+  desc=Test 1 - mod obs,
+  outds=work.test_results
+)

tests/crossplatform/mp_md5.test.sas

@@ -0,0 +1,41 @@
+/**
+  @file
+  @brief Testing mp_md5.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mp_md5.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+%global hash1 hash2 hash3;
+
+%mp_assertscope(SNAPSHOT)
+data work.test1 /nonote2err;
+  c1='';
+  c2=repeat('x',32767);
+  c3='  f';
+  n1=.a;
+  n2=.;
+  n3=1.0000000001;
+  hash=%mp_md5(cvars=c1 c2 c3,nvars=n1 n2 n3);
+  call symputx('hash1',hash);
+  n1=.b;
+  hash=%mp_md5(cvars=c1 c2 c3,nvars=n1 n2 n3);
+  call symputx('hash2',hash);
+  c3='f';
+  hash=%mp_md5(cvars=c1 c2 c3,nvars=n1 n2 n3);
+  call symputx('hash3',hash);
+run;
+%mp_assertscope(COMPARE,ignorelist=HASH1 HASH2 HASH3)
+
+%mp_assert(
+  iftrue=("&hash1" ne "&hash2"),
+  desc=Checking first hash diff,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&hash2" ne "&hash3"),
+  desc=Checking first hash diff,
+  outds=work.test_results
+)

tests/crossplatform/mp_stackdiffs.test.sas

@@ -42,6 +42,7 @@ run;
   ,mdebug=1
 )
 
+
 %mp_assertscope(SNAPSHOT)
 
 /**

tests/sas9only/mm_getauthinfo.test.sas

@@ -0,0 +1,21 @@
+/**
+  @file
+  @brief Testing mm_getauthinfo macro
+
+  <h4> SAS Macros </h4>
+  @li mf_existds.sas
+  @li mm_getauthinfo.sas
+  @li mp_assertscope.sas
+
+**/
+
+
+%mp_assertscope(SNAPSHOT)
+%mm_getauthinfo(outds=auths)
+%mp_assertscope(COMPARE)
+
+
+%mp_assert(
+  iftrue=(%mf_existds(work.auths)=1),
+  desc=Check if the auths dataset was created
+)

tests/serveronly/mfs_httpheader.test.sas

@@ -0,0 +1,52 @@
+/**
+  @file
+  @brief Testing mfs_httpheader.sas macro
+
+  <h4> SAS Macros </h4>
+  @li mfs_httpheader.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%let sasjs_stpsrv_header_loc=%sysfunc(pathname(work))/header.txt;
+
+%mp_assertscope(SNAPSHOT)
+%mfs_httpheader(Content-type,application/csv)
+%mp_assertscope(COMPARE)
+
+data _null_;
+  infile "&sasjs_stpsrv_header_loc";
+  input;
+  if _n_=1 then call symputx('test1',_infile_);
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check code ran without errors,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test1"="Content-type: application/csv"),
+  desc=Checking line was created,
+  outds=work.test_results
+)
+
+%mfs_httpheader(Content-type,application/text)
+%let test2=0;
+data _null_;
+  infile "&sasjs_stpsrv_header_loc";
+  input;
+  if _n_=2 then call symputx('test2',_infile_);
+run;
+
+%mp_assert(
+  iftrue=(&syscc=0),
+  desc=Check code ran without errors for test2,
+  outds=work.test_results
+)
+%mp_assert(
+  iftrue=("&test2"="Content-type: application/text"),
+  desc=Checking line was created,
+  outds=work.test_results
+)

tests/serveronly/ms_createfile.test.sas

@@ -0,0 +1,30 @@
+/**
+  @file
+  @brief Testing ms_createfile.sas macro
+
+  <h4> SAS Macros </h4>
+  @li ms_createfile.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+
+filename stpcode temp;
+data _null_;
+  file stpcode;
+  put '%put hello world;';
+run;
+
+options mprint;
+%let fname=%mf_getuniquename();
+
+%mp_assertscope(SNAPSHOT)
+%ms_createfile(/sasjs/tests/&fname..sas
+  ,inref=stpcode
+  ,mdebug=1
+)
+%mp_assertscope(COMPARE)
+
+
+

tests/serveronly/ms_runstp.test.sas

@@ -0,0 +1,44 @@
+/**
+  @file
+  @brief Testing ms_runstp.sas macro
+
+  <h4> SAS Macros </h4>
+  @li ms_runstp.sas
+  @li mp_assert.sas
+  @li mp_assertscope.sas
+
+**/
+
+%mp_assertscope(SNAPSHOT)
+%ms_runstp(/Public/app/frs/allan/services/common/appinit
+  ,debug=131
+  ,outref=weboot
+)
+%mp_assertscope(COMPARE)
+
+libname webeen json (weboot);
+
+data _null_;
+  infile weboot;
+  input;
+  putlog _infile_;
+run;
+
+data work.httpheaders;
+  set webeen.httpheaders;
+  call symputx('test1',content_type);
+run;
+
+data work.log;
+  set webeen.log;
+  put (_all_)(=);
+  if _n_>10 then stop;
+run;
+
+%mp_assert(
+  iftrue=("&test1"="application/json"),
+  desc=Checking line was created,
+  outds=work.test_results
+)
+
+

tests/testinit.sas

@@ -19,7 +19,7 @@
 %macro loglevel();
   %if "&_debug"="2477" or "&_debug"="fields,log,trace" %then %do;
     %put debug mode activated;
-    options mprint;
+    options mprint mprintnest;
   %end;
 %mend loglevel;
 

viya/mv_registerclient.sas

@@ -1,10 +1,10 @@
 /**
   @file mv_registerclient.sas
   @brief Register Client and Secret (admin task)
-  @details When building apps on SAS Viya, an client id and secret are sometimes
+  @details When building apps on SAS Viya, a client id and secret are usually
-  required.  In order to generate them, filesystem access to the Consul Token
+  required.  In order to generate them, the Consul Token is required.  To access
-  is needed (it is not enough to be in the SASAdministrator group in SAS
+  this token, you need to be a system administrator (it is not enough to be in
-  Environment Manager).
+  the SASAdministrator group in SAS Environment Manager).
 
   If you are registering a lot of clients / secrets, you may find it more
   convenient to use the [Viya Token Generator]
@@ -25,51 +25,56 @@
         "https://raw.githubusercontent.com/sasjs/core/main/all.sas";
       %inc mc;
 
+      %* generate random client using consul token as input parameter;
+      %mv_registerclient(consul_token=12x34sa43v2345n234lasd)
+
+      %* generate random client details with all scopes;
+      %mv_registerclient(scopes=openid *)
+
       %* specific client with just openid scope;
       %mv_registerclient(client_id=YourClient
         ,client_secret=YourSecret
         ,scopes=openid
       )
 
-      %* generate random client details with all scopes;
-      %mv_registerclient(scopes=openid *)
-
       %* generate random client with 90/180 second access/refresh token expiry;
       %mv_registerclient(scopes=openid *
         ,access_token_validity=90
         ,refresh_token_validity=180
       )
 
-  @param client_id= The client name.  Auto generated if blank.
+  @param [in,out] client_id= The client name.  Auto generated if blank.
-  @param client_secret= Client secret.  Auto generated if client is blank.
+  @param [in,out] client_secret= Client secret.  Auto generated if client is
-  @param scopes=(openid) List of space-seperated unquoted scopes
+    blank.
-  @param grant_type=(authorization_code|refresh_token) Valid values are
+  @param [in] consul_token= (0) Provide the actual consul token value here if
-    "password" or "authorization_code" (unquoted)
+    using Viya 4 or above.
-  @param outds=(mv_registerclient) The dataset to contain the registered client
+  @param [in] scopes= (openid) List of space-seperated unquoted scopes
-    id and secret
+  @param [in] grant_type= (authorization_code|refresh_token) Valid values are
-  @param access_token_validity=(DEFAULT) The duration of validity of the access
+    "password" or "authorization_code" (unquoted).  Pipe seperated.
-    token in seconds.  A value of DEFAULT will omit the entry (and use system
+  @param [out] outds=(mv_registerclient) The dataset to contain the registered
-    default)
+    client id and secret
-  @param refresh_token_validity=(DEFAULT)  The duration of validity of the
+  @param [in] access_token_validity= (DEFAULT) The access token duration in
+    seconds.  A value of DEFAULT will omit the entry (and use system default)
+  @param [in] refresh_token_validity= (DEFAULT)  The duration of validity of the
     refresh token in seconds.  A value of DEFAULT will omit the entry (and use
     system default)
-  @param name= An optional, human readable name for the client
+  @param [in] client_name= (DEFAULT) An optional, human readable name for the
-  @param required_user_groups= A list of group names. If a user does not belong
+    client.
-    to all the required groups, the user will not be authenticated and no tokens
+  @param [in] required_user_groups= A list of group names. If a user does not
-    are issued to this client for that user. If this field is not specified,
+    belong to all the required groups, the user will not be authenticated and no
-    authentication and token issuance proceeds normally.
+    tokens are issued to this client for that user. If this field is not
-  @param autoapprove= During the auth step the user can choose which scope to
+    specified, authentication and token issuance proceeds normally.
-    apply.  Setting this to true will autoapprove all the client scopes.
+  @param [in] autoapprove= During the auth step the user can choose which scope
-  @param use_session= If true, access tokens issued to this client will be
+    to apply.  Setting this to true will autoapprove all the client scopes.
+  @param [in] use_session= If true, access tokens issued to this client will be
     associated with an HTTP session and revoked upon logout or time-out.
-  @param outjson= (_null_) A dataset containing the lines of JSON submitted.
+  @param [out] outjson= (_null_) A dataset containing the lines of JSON
-    Useful for debugging.
+    submitted. Useful for debugging.
 
   @version VIYA V.03.04
   @author Allan Bowe, source: https://github.com/sasjs/core
 
   <h4> SAS Macros </h4>
-  @li mp_abort.sas
   @li mf_getplatform.sas
   @li mf_getuniquefileref.sas
   @li mf_getuniquelibref.sas
@@ -81,6 +86,7 @@
 
 %macro mv_registerclient(client_id=
     ,client_secret=
+    ,consul_token=0
     ,client_name=DEFAULT
     ,scopes=openid
     ,grant_type=authorization_code|refresh_token
@@ -92,33 +98,40 @@
     ,refresh_token_validity=DEFAULT
     ,outjson=_null_
   );
-%local consul_token fname1 fname2 fname3 libref access_token url tokloc;
+%local fname1 fname2 fname3 libref access_token url tokloc;
 
 %if client_name=DEFAULT %then %let client_name=
-  Generated by %mf_getuser() on %sysfunc(datetime(),datetime19.) using SASjs;
+  Generated by %mf_getuser() (&sysuserid) on %sysfunc(datetime(),datetime19.
+  ) using SASjs;
 
 options noquotelenmax;
-/* first, get consul token needed to get client id / secret */
-%let tokloc=/etc/SASSecurityCertificateFramework/tokens/consul/default;
-%let tokloc=%mf_loc(VIYACONFIG)&tokloc/client.token;
 
+%if "&consul_token"="0" %then %do;
+  /* first, get consul token needed to get client id / secret */
+  %let tokloc=/etc/SASSecurityCertificateFramework/tokens/consul/default;
+  %let tokloc=%mf_loc(VIYACONFIG)&tokloc/client.token;
 
-%mp_abort(iftrue=(%sysfunc(fileexist(&tokloc))=0)
+  %if %sysfunc(fileexist(&tokloc))=0 %then %do;
-  ,mac=&sysmacroname
+    %put &sysmacroname: unable to access the consul token at &tokloc;
-  ,msg=%str(Unable to access the consul token at &tokloc)
+    %put Try passing the value in the consul= macro parameter;
-)
+    %put See docs:  https://core.sasjs.io/mv__registerclient_8sas.html;
+    %abort;
+  %end;
 
-%let consul_token=0;
+  data _null_;
-data _null_;
+    infile "&tokloc";
-  infile "&tokloc";
+    input token:$64.;
-  input token:$64.;
+    call symputx('consul_token',token);
-  call symputx('consul_token',token);
+  run;
-run;
 
-%mp_abort(iftrue=("&consul_token"="0")
+  %if "&consul_token"="0" %then %do;
-  ,mac=&sysmacroname
+    %put &sysmacroname: Unable to source the consul token from &tokloc;
-  ,msg=%str(Unable to source the consul token from &tokloc)
+    %put It seems your account (&sysuserid) does not have admin rights;
-)
+    %put Please speak with your platform adminstrator;
+    %put Docs:  https://core.sasjs.io/mv__registerclient_8sas.html;
+    %abort;
+  %end;
+%end;
 
 %local base_uri; /* location of rest apis */
 %let base_uri=%mf_getplatform(VIYARESTAPI);
@@ -131,6 +144,9 @@ proc http method='POST' out=&fname1
   headers "X-Consul-Token"="&consul_token";
 run;
 
+%put &=SYS_PROCHTTP_STATUS_CODE;
+%put &=SYS_PROCHTTP_STATUS_PHRASE;
+
 %let libref=%mf_getuniquelibref();
 libname &libref JSON fileref=&fname1;