Implement temporary streams.

A temporary stream functions as a memory stream until its size reaches
a preconfigured threshold value.  Once it is reached, the stream storage
is automatically converted to a temporary file, and all data written so far
are transferred to the new storage.  If the temporary file cannot be
created, the stream continues to operate in memory-based mode.

This stream type is intended to decrease the number of used file
descriptors.

* include/mailutils/sys/Makefile.am: Add temp_stream.h
* include/mailutils/sys/temp_stream.h: New file.
* include/mailutils/stream.h (mu_temp_file_threshold_size): New extern.
(mu_temp_stream_create): New proto.
* libmailutils/stream/Makefile.am: Add temp_stream.c.
* libmailutils/stream/temp_stream.c: New file.
* libmailutils/string/Makefile.am: Add strtosize.c

* libmailutils/tests/.gitignore: Update.
* libmailutils/tests/Makefile.am
* libmailutils/tests/strtoc.c: Add tests for mu_c_hsize.
* libmailutils/tests/temp_stream.c: New test program.
* libmailutils/tests/testsuite.at: Test the temp_stream functionality.
* testsuite/testsuite.inc (MU_CHECK): New macro.

* libmailutils/mailbox/body.c (body_get_transport): Use mu_temp_stream_create.

7 files changed, 260 insertions, 1 deletions

diff --git a/libmailutils/mailbox/body.c b/libmailutils/mailbox/body.c
index 7619d4dd7..1ba21b804 100644
--- a/libmailutils/mailbox/body.c
+++ b/libmailutils/mailbox/body.c
@@ -87,7 +87,7 @@ body_get_transport (mu_body_t body, int mode, mu_stream_t *pstr)
 	{
 	  int rc;
 	  
-	  rc = mu_temp_file_stream_create (&body->temp_stream, NULL, 0);
+	  rc = mu_temp_stream_create (&body->temp_stream, 0);
 	  if (rc)
 	    return rc;
 	  mu_stream_set_buffer (body->temp_stream, mu_buffer_full, 0);
diff --git a/libmailutils/stream/Makefile.am b/libmailutils/stream/Makefile.am
index f4e7f1c30..c4b65ad61 100644
--- a/libmailutils/stream/Makefile.am
+++ b/libmailutils/stream/Makefile.am
@@ -40,6 +40,7 @@ libstream_la_SOURCES = \
  syslogstream.c\
  tcp.c\
  temp_file_stream.c\
+ temp_stream.c\
  wordwrap.c\
  xscript-stream.c
 
diff --git a/libmailutils/stream/temp_stream.c b/libmailutils/stream/temp_stream.c
new file mode 100644
index 000000000..c4bb51b31
--- /dev/null
+++ b/libmailutils/stream/temp_stream.c
@@ -0,0 +1,178 @@
+/* GNU Mailutils -- a suite of utilities for electronic mail
+   Copyright (C) 2020 Free Software Foundation, Inc.
+
+   This library is free software; you can redistribute it and/or modify
+   it under the terms of the GNU Lesser General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public License
+   along with GNU Mailutils.  If not, see <http://www.gnu.org/licenses/>. */
+
+/* Implementation of temp_stream.
+
+   Temp_stream combines the functionality of memory and temp_file streams.
+   Streams of that type function as memory streams until their size reaches
+   a preconfigured threshold value.  Once it is reached, the stream storage
+   is automatically converted to temporary file, and all data written so far
+   are transferred to the new storage.  If the temporary file cannot be
+   created, the stream continues to operate in memory-based mode.
+   
+   The stream is created using the following call:
+   
+      int mu_temp_stream_create (mu_stream_t *pstream, size_t threshold)
+
+   If threshold is 0, the threshold value is first looked up in the
+   environment variable MU_TEMP_FILE_THRESHOLD (which should contain
+   a string suitable for input to mu_strtosize).  If it is not set or
+   unparsable, the mu_temp_file_threshold_size global is used instead.
+
+   Two special values of MU_TEMP_FILE_THRESHOLD alter the behavior of
+   mu_temp_stream_create:
+
+     "0"    -  the function creates a pure tempfile-based stream
+               (equivalent to mu_temp_file_stream_create).
+     "inf"  -  the function returns a pure memory-based stream
+               (equivalent to mu_memory_stream_create).
+ */
+#include <stdlib.h>
+#include <errno.h>
+#include <mailutils/stream.h>
+#include <mailutils/sys/temp_stream.h>
+#include <mailutils/cstr.h>
+#include <mailutils/diag.h>
+#include <mailutils/debug.h>
+#include <mailutils/errno.h>
+
+static int
+temp_stream_write (struct _mu_stream *str, const char *buf, size_t size,
+		   size_t *ret_size)
+{
+  struct _mu_temp_stream *ts = (struct _mu_temp_stream *)str;
+  
+  if (ts->s.mem.offset + size > ts->max_size)
+    {
+      int rc;
+      mu_stream_t temp_file;
+      rc = mu_temp_file_stream_create (&temp_file, NULL, 0);
+      if (rc == 0)
+	{
+	  if (ts->s.mem.ptr == NULL)
+	    rc = 0;
+	  else
+	    {
+	      size_t s = 0;
+
+	      while (s < ts->s.mem.size)
+		{
+		  size_t n = ts->s.mem.size - s;
+		  size_t wrn;
+		  
+		  rc = temp_file->write (temp_file, ts->s.mem.ptr + s, n, &wrn);
+		  if (rc)
+		    break;
+		  s += wrn;
+		}
+	      
+	      if (rc == 0)
+		{
+		  mu_off_t res;
+		  rc = temp_file->seek (temp_file, str->offset, &res);
+		}
+	    }
+	  
+	  if (rc == 0)
+	    {
+	      /* Preserve the necessary stream data */
+	      temp_file->ref_count = str->ref_count;
+	      temp_file->buftype = str->buftype;
+	      temp_file->buffer = str->buffer;
+	      temp_file->level = str->level;
+	      temp_file->pos = str->pos;
+
+	      temp_file->statmask = str->statmask;
+	      temp_file->statbuf = str->statbuf;
+
+	      /* Deinitialize previous stream backend */
+	      ts->s.stream.done (str);
+
+	      /* Replace it with the newly created one. */
+	      memcpy (&ts->s.file, temp_file, sizeof (ts->s.file));
+
+	      /* Reclaim the memory used by the stream object */
+	      free (temp_file);
+
+	      /* Write data to the new stream. */
+	      return ts->s.stream.write (str, buf, size, ret_size);
+	    }
+	}
+      else
+	{
+	  mu_diag_funcall (MU_DIAG_WARNING, "mu_temp_file_stream_create",
+			   NULL, rc);
+	  /* Switch to plain memory stream mode */
+	  ts->s.stream.write = ts->saved_write;
+	}
+    }
+
+  return ts->saved_write (str, buf, size, ret_size);
+}
+
+size_t mu_temp_file_threshold_size = 4096;
+
+int
+mu_temp_stream_create (mu_stream_t *pstream, size_t max_size)
+{
+  int rc;
+  mu_stream_t stream;
+  struct _mu_temp_stream *str;
+
+  if (max_size == 0)
+    {
+      char *s;
+      if ((s = getenv ("MU_TEMP_FILE_THRESHOLD")) != NULL)
+	{
+	  char *p;
+
+	  if (strcmp(p, "inf") == 0)
+	    return mu_memory_stream_create (&stream, MU_STREAM_RDWR);
+	  
+	  rc = mu_strtosize (s, &p, &max_size);
+	  if (rc == 0)
+	    {
+	      if (max_size == 0)
+		return mu_temp_file_stream_create (pstream, NULL, 0);
+	    }
+	  else
+	    mu_debug (MU_DEBCAT_STREAM, MU_DEBUG_ERROR,
+		      ("failed parsing MU_TEMP_FILE_THRESHOLD value: %s near %s",
+		       mu_strerror (rc), p));
+	}
+      if (max_size == 0)
+	max_size = mu_temp_file_threshold_size;
+    }
+  
+  rc = mu_memory_stream_create (&stream, MU_STREAM_RDWR);
+  if (rc)
+    return rc;
+
+  str = realloc (stream, sizeof (*str));
+  if (!str)
+    {
+      mu_stream_destroy (&stream);
+      return ENOMEM;
+    }
+
+  str->max_size = max_size;
+  str->saved_write = str->s.stream.write;
+  str->s.stream.write = temp_stream_write;
+
+  *pstream = (mu_stream_t) str;
+  return rc;
+}
+
diff --git a/libmailutils/tests/.gitignore b/libmailutils/tests/.gitignore
index 591f9befd..4a215ccb8 100644
--- a/libmailutils/tests/.gitignore
+++ b/libmailutils/tests/.gitignore
@@ -56,3 +56,4 @@ t-streamshift
 recenv
 readmesg
 stream-getdelim
+temp_stream
diff --git a/libmailutils/tests/Makefile.am b/libmailutils/tests/Makefile.am
index cdaec679f..b29e24cd7 100644
--- a/libmailutils/tests/Makefile.am
+++ b/libmailutils/tests/Makefile.am
@@ -68,6 +68,7 @@ noinst_PROGRAMS = \
  t1-stream\
  t-streamshift\
  tempfile\
+ temp_stream\
  tcli\
  tocrlf\
  url-comp\
diff --git a/libmailutils/tests/temp_stream.c b/libmailutils/tests/temp_stream.c
new file mode 100644
index 000000000..e509e30a6
--- /dev/null
+++ b/libmailutils/tests/temp_stream.c
@@ -0,0 +1,75 @@
+/*
+ */
+#include <mailutils/mailutils.h>
+#include <mailutils/sys/stream.h>
+#include <mailutils/sys/temp_stream.h>
+
+#define MAXMEM 32
+
+extern int mu_temp_stream_create (mu_stream_t *pstream, size_t max_size);
+
+static void
+verify (mu_stream_t str, int len)
+{
+  char buf[2*MAXMEM];
+  int i;
+  
+  MU_ASSERT (mu_stream_seek (str, 0, MU_SEEK_SET, NULL));
+  MU_ASSERT (mu_stream_read (str, buf, len, NULL));
+  for (i = 0; i < len; i++)
+    {
+      if (buf[i] != i)
+	{
+	  mu_error ("bad octet %d: %d", i, buf[i]);
+	  exit (1);
+	}
+    }
+}
+
+static int
+is_file_backed_stream (mu_stream_t str)
+{
+  int state;
+  return mu_stream_ioctl (str, MU_IOCTL_FD, MU_IOCTL_FD_GET_BORROW, &state)
+         == 0;
+}
+
+int
+main (int argc, char **argv)
+{
+  mu_stream_t str;
+  char i;
+  
+  MU_ASSERT (mu_temp_stream_create (&str, MAXMEM));
+  for (i = 0; i < MAXMEM; i++)
+    {
+      MU_ASSERT (mu_stream_write (str, &i, 1, NULL));
+    }
+
+  verify (str, MAXMEM);
+
+  if (is_file_backed_stream (str))
+    {
+      mu_error ("stream switched to file backend too early");
+      return 1;
+    }
+
+  MU_ASSERT (mu_stream_write (str, &i, 1, NULL));
+  ++i;
+  if (!is_file_backed_stream (str))
+    {
+      mu_error ("stream failed to switch to file backend");
+      return 1;
+    }
+      
+  for (; i < 2*MAXMEM; i++)
+    {
+      MU_ASSERT (mu_stream_write (str, &i, 1, NULL));
+    }
+
+  verify (str, 2*MAXMEM);
+
+  mu_stream_destroy (&str);
+
+  return 0;
+}
diff --git a/libmailutils/tests/testsuite.at b/libmailutils/tests/testsuite.at
index 2ca2f7386..d1c6bf358 100644
--- a/libmailutils/tests/testsuite.at
+++ b/libmailutils/tests/testsuite.at
@@ -104,6 +104,9 @@ m4_include([t1-stream.at])
 m4_include([streamshift.at])
 m4_include([getdelim.at])
 
+AT_BANNER([Stream implementations])
+MU_CHECK([temp_stream],[stream],[temp_stream])
+
 AT_BANNER([Conversions])
 m4_include([strtoc.at])