Revamp UID support in dotmail

* include/mailutils/stream.h (mu_stream_header_copy): New proto.
* libmailutils/stream/hdrcpy.c: New file.
* libmailutils/stream/Makefile.am: Add hdrcpy.c
* libmailutils/tests/hdrcpy.c: New test program.
* libmailutils/tests/hdrcpy.at: New testcase.
* libmailutils/tests/Makefile.am: Add new test program and
testcase.
* libmailutils/tests/testsuite.at: Add new testcase.
* libproto/dotmail/tests/append.at: Add new tests.
* libmailutils/tests/.gitignore: Update.

* include/mailutils/sys/dotmail.h (mu_dotmail_message): Remove headers_scanned.
New field: uid_modified.
(mu_dotmail_mailbox): Change type for uidnext to unsigned long.
Remove scanned_uids_count.
(mu_dotmail_message_headers_prescan)
(mu_dotmail_mailbox_scan_uids): Remove.
(mu_dotmail_mailbox_uid_setup): New proto.
* libproto/dotmail/dotmail.c: Revamp UID support. See "Notes" on
line 251 for description.
* libproto/dotmail/message.c (mu_dotmail_message_reconstruct): Optimize
for the case where the message remains unchanged except for the
UID information.
* libproto/dotmail/tests/dm_mesg.c: Add mailbox inspection commands
from dm_mbox. Perhaps the latter can be removed.

13 files changed, 850 insertions, 426 deletions

diff --git a/include/mailutils/stream.h b/include/mailutils/stream.h
index 848feaee9..bd4c3e7ae 100644
--- a/include/mailutils/stream.h
+++ b/include/mailutils/stream.h
@@ -362,2 +362,3 @@ int mu_stream_copy (mu_stream_t dst, mu_stream_t src, mu_off_t size,
 		    mu_off_t *pcsz);
+int mu_stream_header_copy (mu_stream_t dst, mu_stream_t src, char **exclude_names);
 
diff --git a/include/mailutils/sys/dotmail.h b/include/mailutils/sys/dotmail.h
index 7fa65795c..800348bd6 100644
--- a/include/mailutils/sys/dotmail.h
+++ b/include/mailutils/sys/dotmail.h
@@ -41,5 +41,5 @@ struct mu_dotmail_message
   unsigned body_dot_stuffed:1;   /* True if body is dot-stuffed */
-  unsigned headers_scanned:1;    /* True if hdr is filled */
   unsigned attr_scanned:1;       /* True if attr_flags is initialized */
   unsigned body_lines_scanned:1; /* True if body_lines is initialized */
+  unsigned uid_modified:1;/* UID|uidvalidity|uidnext has been modified */
   int attr_flags;         /* Packed "Status:" attribute flags */
@@ -67,6 +67,4 @@ struct mu_dotmail_mailbox
   unsigned long uidvalidity; /* Uidvalidity value */
-  unsigned uidnext;          /* Expected next UID value */
+  unsigned long uidnext;     /* Expected next UID value */
   unsigned uidvalidity_scanned:1; /* True if uidvalidity is initialized */
-  size_t scanned_uids_count; /* Number of messages for which UIDs have been
-				scanned */
   
@@ -80,5 +78,4 @@ void mu_dotmail_message_free (struct mu_dotmail_message *dmsg);
 int mu_dotmail_message_get (struct mu_dotmail_message *dmsg, mu_message_t *mptr);
-int mu_dotmail_message_headers_prescan (struct mu_dotmail_message *dmsg);
 int mu_dotmail_message_attr_load (struct mu_dotmail_message *dmsg);
-int mu_dotmail_mailbox_scan_uids (mu_mailbox_t mailbox, size_t msgno);
+int mu_dotmail_mailbox_uid_setup (struct mu_dotmail_mailbox *dmp);
 int mu_dotmail_message_reconstruct (mu_stream_t dest,
diff --git a/libmailutils/stream/Makefile.am b/libmailutils/stream/Makefile.am
index 9d92920be..2c2c396f9 100644
--- a/libmailutils/stream/Makefile.am
+++ b/libmailutils/stream/Makefile.am
@@ -22,2 +22,3 @@ libstream_la_SOURCES = \
  fltstream.c\
+ hdrcpy.c\
  iostream.c\
diff --git a/libmailutils/stream/hdrcpy.c b/libmailutils/stream/hdrcpy.c
new file mode 100644
index 000000000..46d023f1e
--- /dev/null
+++ b/libmailutils/stream/hdrcpy.c
@@ -0,0 +1,227 @@
+/* GNU Mailutils -- a suite of utilities for electronic mail
+   Copyright (C) 2010-2019 Free Software Foundation, Inc.
+
+   This library is free software; you can redistribute it and/or modify
+   it under the terms of the GNU Lesser General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public License
+   along with GNU Mailutils.  If not, see <http://www.gnu.org/licenses/>. */
+
+#include <config.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <mailutils/stream.h>
+#include <mailutils/cctype.h>
+
+static int
+excmp (const void *a, const void *b)
+{
+  return strcmp (*(const char **)a, *(const char **)b);
+}
+
+/* Given a NULL-terminated list of header names in NAMES, create a
+   corresponding expect table for use in mu_stream_header_copy. The
+   expect table is an array of strings of equal length. Each element
+   contains a header name from the input array, converted to lowercase,
+   with a ':' appended to it and stuffed with 0 bytes to the maximum
+   length. The maximum length is selected as the length of the longest
+   string from NAMES plus one.
+   The resulting array is sorted in lexicographical order.
+   On success, return the created table. Store the number of entries in
+   PCOUNT and the maximum length in PMAX.
+   On error (ENOMEM) return NULL.
+ */
+static char **
+make_exclusion_list (char **names, size_t *pcount, size_t *pmax)
+{
+  size_t i, j;
+  size_t count = 0;
+  size_t max_len = 0;
+  char **exlist;
+  char *p;
+
+  for (i = 0; names[i]; i++)
+    {
+      size_t len = strlen (names[i]) + 1;
+      if (len > max_len)
+	max_len = len;
+    }
+  count = i;
+
+  exlist = calloc (count, sizeof (exlist[0]) + max_len + 1);
+  if (!exlist)
+    return NULL;
+  p = (char*)(exlist + count);
+  for (i = 0; names[i]; i++, p += max_len + 1)
+    {
+      exlist[i] = p;
+      for (j = 0; names[i][j]; j++)
+	p[j] = mu_tolower (names[i][j]);
+      p[j++] = ':';
+      memset (p + j, 0, max_len - j + 1);
+    }
+  qsort (exlist, count, sizeof (exlist[0]), excmp);
+
+  *pcount = count;
+  *pmax = max_len;
+  return exlist;
+}
+
+/* Assuming SRC is a stream of RFC 822 headers, copy it to DST, omitting
+   headers from EXCLUDE_NAMES. Stop copying at the empty line ("\n\n")
+   or end of input, whichever occurs first. The terminating newline is
+   read from SRC, but not written to DST. This allows the caller to append
+   to DST any additional headers.
+   Returns mailutils error code.
+   FIXME: Bail out early with MU_ERR_PARSE if the input is not a well-
+   formed header stream. This can be checked in save_state_init and
+   save_state_expect by ensuring that the character read is in the
+   MU_CTYPE_HEADR class.
+*/
+int
+mu_stream_header_copy (mu_stream_t dst, mu_stream_t src, char **exclude_names)
+{
+  int rc;
+  size_t la_max;
+  char *lookahead;
+  size_t la_idx = 0;
+  enum
+  {
+    save_state_init,
+    save_state_expect,
+    save_state_skip,
+    save_state_copy,
+    save_state_stop
+  } state = save_state_init;
+  int i = 0;
+  int j = 0;
+  char **exclude;
+  size_t excount;
+
+  exclude = make_exclusion_list (exclude_names, &excount, &la_max);
+  if (!exclude)
+    return ENOMEM;
+  lookahead = malloc (la_max);
+  if (!lookahead)
+    {
+      free (exclude);
+      return ENOMEM;
+    }
+
+  while (state != save_state_stop)
+    {
+      char c;
+      size_t n;
+
+      rc = mu_stream_read (src, &c, 1, &n);
+      if (rc || n == 0)
+	break;
+
+      if (state == save_state_init || state == save_state_expect)
+	{
+	  if (la_idx == la_max)
+	    state = save_state_copy;
+	  else
+	    {
+	      lookahead[la_idx++] = c;
+	      c = mu_tolower (c);
+	    }
+	}
+
+      switch (state)
+	{
+	case save_state_init:
+	  if (c == '\n')
+	    {
+	      /* End of headers. */
+	      state = save_state_stop;
+	      break;
+	    }
+
+	  j = 0;
+	  state = save_state_copy;
+	  for (i = 0; i < excount; i++)
+	    {
+	      if (exclude[i][j] == c)
+		{
+		  j++;
+		  state = save_state_expect;
+		  break;
+		}
+	    }
+	  break;
+
+	case save_state_expect:
+	  if (exclude[i][j] != c)
+	    {
+	      while (++i < excount)
+		{
+		  if (memcmp (exclude[i-1], exclude[i], j))
+		    {
+		      state = save_state_copy;
+		      break;
+		    }
+		  if (exclude[i][j] == c)
+		    break;
+		}
+	      if (i == excount)
+		state = save_state_copy;
+	      if (state == save_state_copy)
+		break;
+	    }
+
+	  if (c == ':')
+	    {
+	      la_idx = 0;
+	      state = save_state_skip;
+	    }
+	  else
+	    {
+	      j++;
+	      if (exclude[i][j] == 0)
+		state = save_state_copy;
+	    }
+	  break;
+
+	case save_state_copy:
+	  if (la_idx > 0)
+	    {
+	      rc = mu_stream_write (dst, lookahead, la_idx, NULL);
+	      if (rc)
+		break;
+	      la_idx = 0;
+	    }
+	  rc = mu_stream_write (dst, &c, 1, NULL);
+	  if (c == '\n')
+	    state = save_state_init;
+	  break;
+
+	case save_state_skip:
+	  if (c == '\n')
+	    state = save_state_init;
+	  break;
+
+	default:
+	  abort (); /* Should not happen */
+	}
+    }
+
+  if (rc == 0)
+    {
+      if (la_idx > 1)
+	rc = mu_stream_write (dst, lookahead, la_idx - 1, NULL);
+    }
+
+  free (lookahead);
+  free (exclude);
+
+  return rc;
+}
diff --git a/libmailutils/tests/.gitignore b/libmailutils/tests/.gitignore
index b93c70aaa..d72fd7863 100644
--- a/libmailutils/tests/.gitignore
+++ b/libmailutils/tests/.gitignore
@@ -21,2 +21,3 @@ fsfolder
 globtest
+hdrcpy
 imapio
diff --git a/libmailutils/tests/Makefile.am b/libmailutils/tests/Makefile.am
index 26137df97..b0f3b9295 100644
--- a/libmailutils/tests/Makefile.am
+++ b/libmailutils/tests/Makefile.am
@@ -56,2 +56,3 @@ noinst_PROGRAMS = \
  globtest\
+ hdrcpy\
  imapio\
@@ -110,2 +111,3 @@ TESTSUITE_AT = \
  fsfolder02.at\
+ hdrcpy.at\
  hdrflt.at\
diff --git a/libmailutils/tests/hdrcpy.at b/libmailutils/tests/hdrcpy.at
new file mode 100644
index 000000000..f760ee019
--- /dev/null
+++ b/libmailutils/tests/hdrcpy.at
@@ -0,0 +1,39 @@
+# This file is part of GNU Mailutils. -*- Autotest -*-
+# Copyright (C) 2010-2019 Free Software Foundation, Inc.
+#
+# GNU Mailutils is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 3, or (at
+# your option) any later version.
+#
+# GNU Mailutils is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNU Mailutils.  If not, see <http://www.gnu.org/licenses/>.
+
+AT_SETUP([header exclusion copy])
+AT_KEYWORDS([header-filter hdrcpy])
+AT_CHECK([
+AT_DATA([input],[From: gray@example.com
+Subject: Header exclusion test
+X-UID: 1000
+To: root@example.com
+X-IMAPbase: 1 5
+Message-Id: 0123456778
+X-IMAPtest: foo
+X-Mailer: hidden
+])
+
+hdrcpy X-uid X-imapbase x-Mailer < input
+],
+[0],
+[From: gray@example.com
+Subject: Header exclusion test
+To: root@example.com
+Message-Id: 0123456778
+X-IMAPtest: foo
+])
+AT_CLEANUP
diff --git a/libmailutils/tests/hdrcpy.c b/libmailutils/tests/hdrcpy.c
new file mode 100644
index 000000000..6fc129d45
--- /dev/null
+++ b/libmailutils/tests/hdrcpy.c
@@ -0,0 +1,27 @@
+/* GNU Mailutils -- a suite of utilities for electronic mail
+   Copyright (C) 2011-2019 Free Software Foundation, Inc.
+
+   GNU Mailutils is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3, or (at your option)
+   any later version.
+
+   GNU Mailutils is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with GNU Mailutils.  If not, see <http://www.gnu.org/licenses/>. */
+
+#include <config.h>
+#include <mailutils/mailutils.h>
+
+int
+main (int argc, char **argv)
+{
+  mu_set_program_name (argv[0]);
+  mu_stdstream_setup (MU_STDSTREAM_RESET_NONE);
+  MU_ASSERT (mu_stream_header_copy (mu_strout, mu_strin, argv + 1));
+  return 0;
+}
diff --git a/libmailutils/tests/testsuite.at b/libmailutils/tests/testsuite.at
index d155d0116..db284fd86 100644
--- a/libmailutils/tests/testsuite.at
+++ b/libmailutils/tests/testsuite.at
@@ -213,2 +213,3 @@ m4_include([inline-comment.at])
 m4_include([hdrflt.at])
+m4_include([hdrcpy.at])
 m4_include([linecon.at])
diff --git a/libproto/dotmail/dotmail.c b/libproto/dotmail/dotmail.c
index 8e2898530..41d7e9278 100644
--- a/libproto/dotmail/dotmail.c
+++ b/libproto/dotmail/dotmail.c
@@ -40,2 +40,3 @@
 #include <mailutils/util.h>
+#include <mailutils/cctype.h>
 
@@ -69,3 +70,3 @@ dotmail_mailbox_init_stream (struct mu_dotmail_mailbox *dmp)
   mu_mailbox_t mailbox = dmp->mailbox;
-  
+
   rc = mu_mapfile_stream_create (&mailbox->stream, dmp->name, mailbox->flags);
@@ -249,2 +250,55 @@ dotmail_dispatch (mu_mailbox_t mailbox, int evt, void *data)
 
+/* Notes on the UID subsystem
+
+   1. The values of uidvalidity and uidnext are stored in the
+      X-IMAPbase header in the first message.
+   2. Message UID is stored in the X-UID header in that message.
+   3. To minimize unwanted modifications to the mailbox, the
+      UID subsystem is initialized only in the following cases:
+
+      3a. Upon mailbox scanning, if the first message contains a
+	  valid X-IMAPbase header. In this case, the
+	  dotmail_rescan_unlocked function initializes each
+	  message's uid value from the X-UID header. The first
+	  message that lacks X-UID or with an X-UID that cannot
+	  be parsed, gets assigned new UID. The subsequent
+	  messages are assigned new UIDs no matterwhether they
+	  have X-UID headers.
+
+      3b. When any of the following functions are called for
+	  the first time: dotmail_uidvalidity, dotmail_uidnext,
+	  dotmail_message_uid. This means that the caller used
+	  mu_mailbox_uidvalidity, mu_mailbox_uidnext, or
+	  mu_message_get_uid.
+	  In this case, each message is assigned a UID equal to
+	  its ordinal number (1-based) in the mailbox.
+	  This is done by the mu_dotmail_mailbox_uid_setup function.
+
+   4. When a message is appended to the mailbox, any existing
+      X-IMAPbase and X-UID headers are removed from it. If the
+      UID subsystem is initialized, the message is assigned a new
+      UID.
+   5. Assigning new UID to a message does not change its attributes.
+      Instead, its uid_modified flag is set.
+*/
+
+/* Allocate next available UID for the mailbox.
+   The caller must ensure that the UID subsystem is initialized.
+*/
+static unsigned long
+dotmail_alloc_next_uid (struct mu_dotmail_mailbox *mbox)
+{
+  mbox->mesg[0]->uid_modified = 1;
+  return mbox->uidnext++;
+}
+
+static void
+dotmail_message_alloc_uid (struct mu_dotmail_message *dmsg)
+{
+  free (dmsg->hdr[mu_dotmail_hdr_x_uid]);
+  dmsg->hdr[mu_dotmail_hdr_x_uid] = NULL;
+  dmsg->uid = dotmail_alloc_next_uid (dmsg->mbox);
+  dmsg->uid_modified = 1;
+}
+
 static int
@@ -261,2 +315,3 @@ dotmail_rescan_unlocked (mu_mailbox_t mailbox, mu_off_t offset)
     dotmail_scan_header_newline,
+    dotmail_scan_header_expect,
     dotmail_scan_body,
@@ -268,2 +323,9 @@ dotmail_rescan_unlocked (mu_mailbox_t mailbox, mu_off_t offset)
   int rc;
+  static char *expect[] = {
+    "status:    ",
+    "x-imapbase:",
+    "x-uid:     ",
+  };
+  int i, j;
+  int force_init_uids = 0;
 
@@ -335,3 +397,4 @@ dotmail_rescan_unlocked (mu_mailbox_t mailbox, mu_off_t offset)
 	  --dmsg->message_start;
-	  state = dotmail_scan_header;
+	  state = dotmail_scan_header_newline;
+	  i = j = 0;
 	  break;
@@ -360,3 +423,68 @@ dotmail_rescan_unlocked (mu_mailbox_t mailbox, mu_off_t offset)
 	  else
-	    state = dotmail_scan_header;
+	    {
+	      state = dotmail_scan_header;
+	      j = 0;
+	      for (i = 0; i < MU_DOTMAIL_HDR_MAX; i++)
+		{
+		  if (expect[i][j] == mu_tolower (cur))
+		    {
+		      j++;
+		      state = dotmail_scan_header_expect;
+		      break;
+		    }
+		}
+	    }
+	  break;
+
+	case dotmail_scan_header_expect:
+	  if (cur == '\n')
+	    {
+	      state = dotmail_scan_header_newline;
+	    }
+	  else
+	    {
+	      int c = mu_tolower (cur);
+	      if (expect[i][j] != c)
+		{
+		  if (++i == MU_DOTMAIL_HDR_MAX
+		      || memcmp (expect[i-1], expect[i], j)
+		      || expect[i][j] != c)
+		    {
+		      state = dotmail_scan_header;
+		      break;
+		    }
+		}
+
+	      if (c == ':')
+		{
+		  char *buf = NULL;
+		  size_t size = 0;
+		  size_t n;
+
+		  rc = mu_stream_getline (stream, &buf, &size, &n);
+		  if (rc)
+		    {
+		      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+				("%s:%s (%s): %s",
+				 __func__, "mu_stream_getline",
+				 dmsg->mbox->name,
+				 mu_strerror (rc)));
+		      return rc;
+		    }
+		  if (n > 0)
+		    {
+		      buf[n-1] = 0;
+		      dmsg->hdr[i] = buf;
+		    }
+		  else
+		    free (buf);
+		  state = dotmail_scan_header_newline;
+		}
+	      else
+		{
+		  j++;
+		  if (expect[i][j] == 0)
+		    state = dotmail_scan_header_newline;
+		}
+	    }
 	  break;
@@ -401,2 +529,22 @@ dotmail_rescan_unlocked (mu_mailbox_t mailbox, mu_off_t offset)
 
+	      if (dmsg->num == 0)
+		{
+		  if (dmsg->hdr[mu_dotmail_hdr_x_imapbase]
+		      && sscanf (dmsg->hdr[mu_dotmail_hdr_x_imapbase],
+				 "%lu %lu",
+				 &dmp->uidvalidity, &dmp->uidnext) == 2)
+		    dmp->uidvalidity_scanned = 1;
+		}
+
+	      if (dmp->uidvalidity_scanned)
+		{
+		  if (!(!force_init_uids
+			&& dmsg->hdr[mu_dotmail_hdr_x_uid]
+			&& sscanf (dmsg->hdr[mu_dotmail_hdr_x_uid],
+				   "%lu", &dmsg->uid) == 1))
+		    force_init_uids = 1;
+		  if (force_init_uids)
+		    dotmail_message_alloc_uid (dmsg);
+		}
+
 	      /* Every 100 mesgs update the lock, it should be every minute.  */
@@ -533,18 +681,2 @@ dotmail_scan (mu_mailbox_t mailbox, size_t i, size_t *pcount)
 static int
-dotmail_prescan_headers (mu_mailbox_t mailbox)
-{
-  struct mu_dotmail_mailbox *dmp = mailbox->data;
-  size_t i = 0;
-
-  int rc = dotmail_refresh (mailbox);
-  if (rc)
-    return rc;
-
-  for (; i < dmp->mesg_count; i++)
-    mu_dotmail_message_headers_prescan (dmp->mesg[i]);
-
-  return 0;
-}
-
-static int
 dotmail_messages_recent (mu_mailbox_t mailbox, size_t *pcount)
@@ -553,6 +685,5 @@ dotmail_messages_recent (mu_mailbox_t mailbox, size_t *pcount)
   size_t count = 0;
-  int rc;
   struct mu_dotmail_mailbox *dmp = mailbox->data;
 
-  rc = dotmail_prescan_headers (mailbox);
+  int rc = dotmail_refresh (mailbox);
   if (rc)
@@ -576,6 +707,5 @@ dotmail_message_unseen (mu_mailbox_t mailbox, size_t *pmsgno)
   size_t i;
-  int rc;
   struct mu_dotmail_mailbox *dmp = mailbox->data;
 
-  rc = dotmail_prescan_headers (mailbox);
+  int rc = dotmail_refresh (mailbox);
   if (rc)
@@ -596,39 +726,31 @@ dotmail_message_unseen (mu_mailbox_t mailbox, size_t *pmsgno)
 
-static int
-dotmail_uidvalidity (mu_mailbox_t mailbox, unsigned long *puidvalidity)
+/* Initialize the mailbox UID subsystem. See the Notes above. */
+int
+mu_dotmail_mailbox_uid_setup (struct mu_dotmail_mailbox *dmp)
 {
-  struct mu_dotmail_mailbox *dmp = mailbox->data;
-
   if (!dmp->uidvalidity_scanned)
     {
-      int rc = dotmail_refresh (mailbox);
-      if (rc)
+      size_t i;
+      int rc = dotmail_refresh (dmp->mailbox);
+      if (rc || dmp->uidvalidity_scanned)
 	return rc;
 
-      if (dmp->mesg_count)
-	{
-	  mu_dotmail_message_attr_load (dmp->mesg[0]);
-	  if (dmp->mesg[0]->hdr[mu_dotmail_hdr_x_imapbase])
-	    {
-	      if (sscanf (dmp->mesg[0]->hdr[mu_dotmail_hdr_x_imapbase],
-			  "%lu %u",
-			  &dmp->uidvalidity, &dmp->uidnext) != 2)
-		{
-		  dmp->uidvalidity = (unsigned long)time (NULL);
-		  dmp->uidnext = dmp->mesg_count + 1;
-		}
-	    }
-	}
-      else
-	{
-	  dmp->uidvalidity = (unsigned long)time (NULL);
-	  dmp->uidnext = dmp->mesg_count + 1;
-	}
+      dmp->uidvalidity = (unsigned long)time (NULL);
+      dmp->uidnext = 1;
       dmp->uidvalidity_scanned = 1;
+
+      for (i = 0; i < dmp->mesg_count; i++)
+	dotmail_message_alloc_uid (dmp->mesg[i]);
     }
+  return 0;
+}
 
-  if (puidvalidity)
+static int
+dotmail_uidvalidity (mu_mailbox_t mailbox, unsigned long *puidvalidity)
+{
+  struct mu_dotmail_mailbox *dmp = mailbox->data;
+  int rc = mu_dotmail_mailbox_uid_setup (dmp);
+  if (rc == 0)
     *puidvalidity = dmp->uidvalidity;
-
-  return 0;
+  return rc;
 }
@@ -639,3 +761,3 @@ dotmail_uidnext (mu_mailbox_t mailbox, size_t *puidnext)
   struct mu_dotmail_mailbox *dmp = mailbox->data;
-  int rc = dotmail_uidvalidity (mailbox, NULL);
+  int rc = mu_dotmail_mailbox_uid_setup (dmp);
   if (rc == 0)
@@ -723,2 +845,8 @@ mailbox_append_message (mu_mailbox_t mailbox, mu_message_t msg)
   mu_stream_t istr, flt;
+  static char *exclude_headers[] = {
+    MU_HEADER_X_IMAPBASE,
+    MU_HEADER_X_UID,
+    NULL
+  };
+  struct mu_dotmail_mailbox *dmp = mailbox->data;
 
@@ -732,10 +860,36 @@ mailbox_append_message (mu_mailbox_t mailbox, mu_message_t msg)
 
-  rc = mu_filter_create (&flt, istr, "DOT",
-			 MU_FILTER_ENCODE, MU_STREAM_READ);
-  mu_stream_unref (istr);
+  do
+    {
+      rc = mu_stream_header_copy (mailbox->stream, istr, exclude_headers);
+      if (rc)
+	break;
+
+      /* Write UID-related data */
+      if (dmp->uidvalidity_scanned)
+	{
+	  if (dmp->mesg_count == 0)
+	    mu_stream_printf (mailbox->stream, "%s: %lu %lu\n",
+			      MU_HEADER_X_IMAPBASE,
+			      dmp->uidvalidity,
+			      dmp->uidnext);
+	  mu_stream_printf (mailbox->stream, "%s: %lu\n",
+			    MU_HEADER_X_UID,
+			    dotmail_alloc_next_uid (dmp));
+	}
+
+      rc = mu_stream_write (mailbox->stream, "\n", 1, NULL);
+      if (rc)
+	break;
+
+      rc = mu_filter_create (&flt, istr, "DOT",
+			     MU_FILTER_ENCODE, MU_STREAM_READ);
+      mu_stream_destroy (&istr);
+      rc = mu_stream_copy (mailbox->stream, flt, 0, NULL);
+      mu_stream_unref (flt);
+    }
+  while (0);
 
-  rc = mu_stream_copy (mailbox->stream, flt, 0, NULL);
-  mu_stream_unref (flt);
   if (rc)
     {
+      mu_stream_destroy (&istr);
       rc = mu_stream_truncate (mailbox->stream, size);
@@ -790,38 +944,2 @@ dotmail_append_message (mu_mailbox_t mailbox, mu_message_t msg)
 
-int
-mu_dotmail_mailbox_scan_uids (mu_mailbox_t mailbox, size_t msgno)
-{
-  struct mu_dotmail_mailbox *dmp = mailbox->data;
-  size_t i;
-
-  if (dmp->scanned_uids_count < msgno)
-    {
-      int rc = dotmail_refresh (mailbox);
-      if (rc)
-	return rc;
-
-      for (i = dmp->scanned_uids_count; i < msgno; i++)
-	{
-	  struct mu_dotmail_message *dmsg = dmp->mesg[i];
-	  int rc = mu_dotmail_message_headers_prescan (dmsg);
-	  if (rc)
-	    return rc;
-	  if (!(dmsg->hdr[mu_dotmail_hdr_x_uid]
-		&& sscanf (dmsg->hdr[mu_dotmail_hdr_x_uid], "%lu", &dmsg->uid) == 1))
-	    break;
-	}
-
-      if (i < msgno)
-	{
-	  int rc = dotmail_uidvalidity (mailbox, NULL);
-	  if (rc)
-	    return rc;
-	  for (; i < msgno; i++)
-	    dmp->mesg[i]->uid = dmp->uidnext++;
-	}
-      dmp->scanned_uids_count = msgno;
-    }
-    return 0;
-}
-
 static int
@@ -918,3 +1036,2 @@ dotmail_tracker_sync (struct mu_dotmail_flush_tracker *trk)
       dmp->uidvalidity_scanned = 0;
-      dmp->scanned_uids_count = 0;
     }
@@ -922,4 +1039,2 @@ dotmail_tracker_sync (struct mu_dotmail_flush_tracker *trk)
     {
-      int reset_scanned_uids = 1;
-      
       for (i = 0; i < trk->mesg_count; i++)
@@ -938,8 +1053,4 @@ dotmail_tracker_sync (struct mu_dotmail_flush_tracker *trk)
 	    dmp->mesg[i]->body_lines_scanned = 0;
-	  if (trk->ref[i].orig_num + 1 == dmp->scanned_uids_count)
-	    reset_scanned_uids = 0;
 	}
       dmp->mesg_count = trk->mesg_count;
-      if (reset_scanned_uids)
-	dmp->scanned_uids_count = 0;
       dmp->size = trk->ref[trk->mesg_count - 1].message_end + 2;
@@ -949,2 +1060,5 @@ dotmail_tracker_sync (struct mu_dotmail_flush_tracker *trk)
 
+/* Write to the output stream DEST messages in the range [from,to).
+   Update TRK accordingly.
+*/
 static int
@@ -994,2 +1108,6 @@ dotmail_mailbox_copy_unchanged (struct mu_dotmail_flush_tracker *trk,
 
+/* Flush the mailbox described by the tracker TRK to the stream TEMPSTR.
+   First modified message is I (0-based). EXPUNGE is 1 if the
+   MU_ATTRIBUTE_DELETED attribute is to be honored.
+*/
 static int
@@ -1038,9 +1156,4 @@ dotmail_flush_temp (struct mu_dotmail_flush_tracker *trk,
 
-      if (!dmsg->message)
-	{
-	  i++;
-	  continue;
-	}
-
-      if ((dmsg->attr_flags & MU_ATTRIBUTE_MODIFIED)
+      if (dmsg->uid_modified
+	  || (dmsg->attr_flags & MU_ATTRIBUTE_MODIFIED)
 	  || mu_message_is_modified (dmsg->message))
@@ -1050,8 +1163,11 @@ dotmail_flush_temp (struct mu_dotmail_flush_tracker *trk,
 	    return rc;
+
+	  free (dmsg->hdr[mu_dotmail_hdr_x_imapbase]);
+	  dmsg->hdr[mu_dotmail_hdr_x_imapbase] = NULL;
 	  if (save_imapbase == i)
 	    {
-	      free (dmsg->hdr[mu_dotmail_hdr_x_imapbase]);
-	      mu_asprintf (&dmsg->hdr[mu_dotmail_hdr_x_imapbase], "%lu %u",
+	      mu_asprintf (&dmsg->hdr[mu_dotmail_hdr_x_imapbase], "%lu %lu",
 			   dmp->uidvalidity, dmp->uidnext);
 	    }
+
 	  rc = mu_dotmail_message_reconstruct (tempstr, dmsg,
@@ -1073,2 +1189,6 @@ dotmail_flush_temp (struct mu_dotmail_flush_tracker *trk,
 
+/* Flush the mailbox described by the tracker TRK to the stream TEMPSTR.
+   EXPUNGE is 1 if the MU_ATTRIBUTE_DELETED attribute is to be honored.
+   Assumes that simultaneous access to the mailbox has been blocked.
+*/
 static int
@@ -1100,2 +1220,4 @@ dotmail_flush_unlocked (struct mu_dotmail_flush_tracker *trk, int expunge)
       struct mu_dotmail_message *dmsg = dmp->mesg[dirty];
+      if (dmsg->uid_modified)
+	break;
       mu_dotmail_message_attr_load (dmsg);
@@ -1199,2 +1321,12 @@ dotmail_flush_unlocked (struct mu_dotmail_flush_tracker *trk, int expunge)
 
+/* Flush the changes in the mailbox DMP to disk storage.
+   EXPUNGE is 1 if the MU_ATTRIBUTE_DELETED attribute is to be honored.
+   Block simultaneous access for the duration of the process.
+
+   This is done by creating a temporary mailbox on the same device as
+   DMP and by transferring all messages (whether changed or not) to
+   it. If the process succeeds, old mailbox is removed and the temporary
+   one is renamed to it. In case of failure, the temporary is removed and
+   the original mailbox remains unchanged.
+*/
 static int
diff --git a/libproto/dotmail/message.c b/libproto/dotmail/message.c
index 76c1a97b1..1a8761648 100644
--- a/libproto/dotmail/message.c
+++ b/libproto/dotmail/message.c
@@ -31,148 +31,2 @@
 
-/* Status, UID, uidnext and uidvalidity */
-static char *expect[] = {
-  "status:    ",
-  "x-imapbase:",
-  "x-uid:     ",
-};
-
-static char *canon_name[] = {
-  MU_HEADER_STATUS,
-  MU_HEADER_X_IMAPBASE,
-  MU_HEADER_X_UID
-};
-
-int
-mu_dotmail_message_headers_prescan (struct mu_dotmail_message *dmsg)
-{
-  mu_stream_t stream;
-  int rc;
-  enum
-  {
-    prescan_state_init,
-    prescan_state_expect,
-    prescan_state_skip,
-    prescan_state_stop
-  } state = prescan_state_init;
-
-  int i = 0;
-  int j = 0;
-
-  char cur;
-  size_t n;
-
-  if (dmsg->headers_scanned)
-    return 0;
-
-  rc = mu_streamref_create_abridged (&stream,
-				     dmsg->mbox->mailbox->stream,
-				     dmsg->message_start,
-				     dmsg->body_start - 1);
-  if (rc)
-    return rc;
-
-  rc = mu_stream_seek (stream, 0, MU_SEEK_SET, NULL);
-  if (rc)
-    {
-      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
-		("%s:%s (%s): %s",
-		 __func__, "mu_stream_seek", dmsg->mbox->name,
-		 mu_strerror (rc)));
-      return rc;
-    }
-
-  while (state != prescan_state_stop
-	 && (rc = mu_stream_read (stream, &cur, 1, &n)) == 0
-	 && n == 1)
-    {
-      switch (state)
-	{
-	case prescan_state_init:
-	  j = 0;
-	  state = prescan_state_stop;
-	  for (i = 0; i < MU_DOTMAIL_HDR_MAX; i++)
-	    {
-	      if (dmsg->hdr[i] == NULL)
-		{
-		  state = prescan_state_skip;
-		  if (expect[i][j] == mu_tolower (cur))
-		    {
-		      j++;
-		      state = prescan_state_expect;
-		      break;
-		    }
-		}
-	    }
-	  break;
-
-	case prescan_state_expect:
-	  {
-	    int c = mu_tolower (cur);
-	    if (expect[i][j] != c)
-	      {
-		if (++i == MU_DOTMAIL_HDR_MAX
-		    || memcmp (expect[i-1], expect[i], j)
-		    || expect[i][j] != c)
-		  {
-		    state = prescan_state_skip;
-		    break;
-		  }
-	      }
-
-	    if (c == ':')
-	      {
-		char *buf = NULL;
-		size_t size = 0;
-		size_t n;
-
-		rc = mu_stream_getline (stream, &buf, &size, &n);
-		if (rc)
-		  {
-		    mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
-			      ("%s:%s (%s): %s",
-			       __func__, "mu_stream_getline",
-			       dmsg->mbox->name,
-			       mu_strerror (rc)));
-		    return rc;
-		  }
-		if (n > 0)
-		  {
-		    buf[n-1] = 0;
-		    dmsg->hdr[i] = buf;
-		  }
-		else
-		  free (buf);
-		state = prescan_state_init;
-	      }
-	    else
-	      {
-		j++;
-		if (expect[i][j] == 0)
-		  state = prescan_state_skip;
-	      }
-	  }
-	  break;
-
-	case prescan_state_skip:
-	  if (cur == '\n')
-	    state = prescan_state_init;
-	  break;
-
-	default:
-	  break; /* Should not happen */
-	}
-    }
-
-  if (rc)
-    {
-      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
-		("%s:%s (%s): %s",
-		 __func__, "mu_stream_read", dmsg->mbox->name,
-		 mu_strerror (rc)));
-      /* Try to get on with what we've got this far */
-    }
-  dmsg->headers_scanned = 1;
-  return 0;
-}
-
 void
@@ -303,5 +157,2 @@ mu_dotmail_message_attr_load (struct mu_dotmail_message *dmsg)
     {<