Fixes in imap client: handle EXPUNGE response, fix incorporation of new messages

Certain servers do not issue EXISTS unsolicited response after expunging.
Use EXPUNGE responses to keep track of the deleted messages.

Internal structure of imap mailbox object did not interact well with
incorporating new messages. It is fixed.

* include/mailutils/imap.h (MU_IMAP_CB_EXPUNGE): New callback code.
* include/mailutils/sys/imap.h (_mu_imap_message): New member: msgno.
(_mu_imap_mailbox): Change type of msgs to struct _mu_imap_message **.
Each _mu_imap_message * is registered as owner for various MU objects,
so it is important it does not change during reallocations.
* libproto/imap/mbox.c (_imap_msg_free): Free the message itself.
(_imap_attr_clr_flags): Fix bitwise op.
(_imap_mbx_get_message,_imap_realloc_messages)
(aggregate_attributes,_imap_mbx_gensync)
(_imap_fetch_callback): Reflect change in the type of msgs array.
(_imap_expunge_callback): New callback.
(_imap_mbx_open): Register _imap_expunge_callback.
* libproto/imap/resproc.c (_process_unsolicited_response): handle
the EXPUNGE response.

4 files changed, 207 insertions, 139 deletions

diff --git a/include/mailutils/imap.h b/include/mailutils/imap.h
index 0b2b4ef98..2bfc1a0ed 100644
--- a/include/mailutils/imap.h
+++ b/include/mailutils/imap.h
@@ -30,7 +30,7 @@ extern "C" {
 
 #define MU_IMAP_DEFAULT_PORT 143
 #define MU_IMAP_DEFAULT_SSL_PORT 993
-  
+
 typedef struct _mu_imap *mu_imap_t;
 
 enum mu_imap_session_state
@@ -40,7 +40,7 @@ enum mu_imap_session_state
     MU_IMAP_SESSION_AUTH,     /* Authenticated State */
     MU_IMAP_SESSION_SELECTED  /* Selected State */
   };
-  
+
 int mu_imap_create (mu_imap_t *pimap);
 void mu_imap_destroy (mu_imap_t *pimap);
 
@@ -49,13 +49,13 @@ int mu_imap_disconnect (mu_imap_t imap);
 
 int mu_imap_iserror (mu_imap_t imap);
 void mu_imap_clearerr (mu_imap_t imap);
-  
+
 int mu_imap_capability (mu_imap_t imap, int reread, mu_iterator_t *piter);
 int mu_imap_capability_test (mu_imap_t imap, const char *name,
 			     const char **pret);
 
 int mu_imap_starttls (mu_imap_t imap);
-  
+
 int mu_imap_login (mu_imap_t imap, const char *user, const char *pass);
 int mu_imap_login_secret (mu_imap_t imap, const char *user,
 			  mu_secret_t secret);
@@ -80,7 +80,7 @@ int mu_imap_store (mu_imap_t imap, int uid, mu_msgset_t msgset,
 
 int mu_imap_store_flags (mu_imap_t imap, int uid, mu_msgset_t msgset,
 			 int op, int flags);
-  
+
 int mu_imap_delete (mu_imap_t imap, const char *mailbox);
 int mu_imap_rename (mu_imap_t imap, const char *mailbox,
 		    const char *new_mailbox);
@@ -103,14 +103,14 @@ int mu_imap_append_stream (mu_imap_t imap, const char *mailbox, int flags,
 int mu_imap_append_message (mu_imap_t imap, const char *mailbox, int flags,
 			    struct tm *tm, struct mu_timezone *tz,
 			    mu_message_t msg);
-  
+
 int mu_imap_genlist (mu_imap_t imap, int lsub,
 		     const char *refname, const char *mboxname,
 		     mu_list_t retlist);
 int mu_imap_genlist_new (mu_imap_t imap, int lsub,
 			 const char *refname, const char *mboxname,
 			 mu_list_t *plist);
-  
+
 int mu_imap_list (mu_imap_t imap, const char *refname, const char *mboxname,
 		  mu_list_t retlist);
 int mu_imap_list_new (mu_imap_t imap, const char *refname,
@@ -123,7 +123,7 @@ int mu_imap_lsub_new (mu_imap_t imap, const char *refname,
 
 int mu_imap_subscribe (mu_imap_t imap, const char *mailbox);
 int mu_imap_unsubscribe (mu_imap_t imap, const char *mailbox);
-  
+
 int mu_imap_set_carrier (mu_imap_t imap, mu_stream_t carrier);
 int mu_imap_get_carrier (mu_imap_t imap, mu_stream_t *pcarrier);
 
@@ -135,7 +135,7 @@ int mu_imap_trace_mask (mu_imap_t imap, int op, int lev);
 
 enum mu_imap_response mu_imap_response (mu_imap_t imap);
 int mu_imap_response_code (mu_imap_t imap);
-  
+
 int mu_imap_strerror (mu_imap_t imap, const char **pstr);
 
 int mu_imap_session_state (mu_imap_t imap);
@@ -145,7 +145,7 @@ int mu_imap_tag (mu_imap_t imap, const char **pseq);
 
 int mu_imap_search (mu_imap_t imap, int uid, const char *expr,
 		    mu_msgset_t *msgset);
-  
+
 
 #define MU_IMAP_STAT_DEFINED_FLAGS   0x01
 #define MU_IMAP_STAT_PERMANENT_FLAGS 0x02
@@ -181,7 +181,7 @@ extern struct mu_kwd _mu_imap_status_name_table[];
 #define MU_IMAP_CB_MESSAGE_COUNT    1
 #define MU_IMAP_CB_RECENT_COUNT     2
 #define MU_IMAP_CB_FIRST_UNSEEN     3
-#define MU_IMAP_CB_UIDNEXT          4 
+#define MU_IMAP_CB_UIDNEXT          4
 #define MU_IMAP_CB_UIDVALIDITY      5
 
   /* The following callbacks correspond to unsolicited server responses and
@@ -194,40 +194,44 @@ extern struct mu_kwd _mu_imap_status_name_table[];
 #define MU_IMAP_CB_BYE              9
 #define MU_IMAP_CB_PREAUTH         10
 
-  /* These corresponde to the tagged server responses.  The calling convention
+  /* These correspond to the tagged server responses.  The calling convention
      is the same as above. */
-#define MU_IMAP_CB_TAGGED_OK       11  
+#define MU_IMAP_CB_TAGGED_OK       11
 #define MU_IMAP_CB_TAGGED_NO       12
 #define MU_IMAP_CB_TAGGED_BAD      13
-  
+
   /* FETCH callback.  Arguments: SDAT - message sequence number, PDAT - a
      list (mu_list_t) of union mu_imap_fetch_response (see below). */
 #define MU_IMAP_CB_FETCH           14
 
-#define _MU_IMAP_CB_MAX            15
+  /* This callback is called upon EXPUNGE unsolicited response.
+     Arguments: SDAT - message number, PDAT - address of struct mu_imap_stat */
+#define MU_IMAP_CB_EXPUNGE	   15
+
+#define _MU_IMAP_CB_MAX            16
 
 typedef void (*mu_imap_callback_t) (void *, int code, size_t sdat, void *pdat);
-  
+
 void mu_imap_callback (mu_imap_t imap, int code, size_t sdat, void *pdat);
 
 void mu_imap_register_callback_function (mu_imap_t imap, int code,
 					 mu_imap_callback_t callback,
 					 void *data);
 
-#define MU_IMAP_RESPONSE_UNKNOWN           0 
+#define MU_IMAP_RESPONSE_UNKNOWN           0
 #define MU_IMAP_RESPONSE_ALERT             1
 #define MU_IMAP_RESPONSE_BADCHARSET        2
 #define MU_IMAP_RESPONSE_CAPABILITY        3
 #define MU_IMAP_RESPONSE_PARSE             4
 #define MU_IMAP_RESPONSE_PERMANENTFLAGS    5
-#define MU_IMAP_RESPONSE_READ_ONLY         6   
+#define MU_IMAP_RESPONSE_READ_ONLY         6
 #define MU_IMAP_RESPONSE_READ_WRITE        7
 #define MU_IMAP_RESPONSE_TRYCREATE         8
 #define MU_IMAP_RESPONSE_UIDNEXT           9
 #define MU_IMAP_RESPONSE_UIDVALIDITY      10
 #define MU_IMAP_RESPONSE_UNSEEN           11
 
-extern struct mu_kwd mu_imap_response_codes[];  
+extern struct mu_kwd mu_imap_response_codes[];
 
   /* FETCH Response Codes */
 
@@ -236,7 +240,7 @@ extern struct mu_kwd mu_imap_response_codes[];
   /* BODY & BODYSTRUCTURE */
 #define MU_IMAP_FETCH_BODYSTRUCTURE        1
   /* ENVELOPE */
-#define MU_IMAP_FETCH_ENVELOPE             2 
+#define MU_IMAP_FETCH_ENVELOPE             2
   /* FLAGS */
 #define MU_IMAP_FETCH_FLAGS                3
   /* INTERNALDATE */
@@ -261,7 +265,7 @@ struct mu_imap_fetch_bodystructure
   int type;
   struct mu_bodystructure *bs;
 };
-  
+
 struct mu_imap_fetch_envelope
 {
   int type;
@@ -280,13 +284,13 @@ struct mu_imap_fetch_internaldate
   struct tm tm;
   struct mu_timezone tz;
 };
-  
+
 struct mu_imap_fetch_rfc822_size
 {
   int type;
   size_t size;
 };
-  
+
 struct mu_imap_fetch_uid
 {
   int type;
@@ -310,5 +314,3 @@ union mu_imap_fetch_response
 #endif
 
 #endif /* _MAILUTILS_IMAP_H */
-  
-  
diff --git a/include/mailutils/sys/imap.h b/include/mailutils/sys/imap.h
index a1b743fea..8e34a4b10 100644
--- a/include/mailutils/sys/imap.h
+++ b/include/mailutils/sys/imap.h
@@ -42,7 +42,7 @@ extern "C" {
   while (0)
 #define MU_IMAP_IS_XSCRIPT_MASK(imap,n)         \
   ((imap)->flags & MU_IMAP_XSCRIPT_MASK (n))
-  
+
 enum mu_imap_client_state
   {
     MU_IMAP_CLIENT_READY,
@@ -81,7 +81,7 @@ enum mu_imap_response
     MU_IMAP_NO,
     MU_IMAP_BAD
   };
-  
+
 struct _mu_imap
   {
     int flags;
@@ -90,11 +90,11 @@ struct _mu_imap
     enum mu_imap_response response;
     /* The recent response code */
     int response_code;
-    
+
     /* Error string (if any) */
     char *errstr;
     size_t errsize;
-    
+
     enum mu_imap_client_state client_state;
     enum mu_imap_session_state session_state;
 
@@ -102,7 +102,7 @@ struct _mu_imap
     size_t tag_len;  /* Length of the command tag */
     int *tag_buf;    /* Tag number (BCD) */
     char *tag_str;   /* String representation (tag_len + 1 bytes, asciiz) */
-    
+
     mu_list_t capa;
     mu_imapio_t io;
 
@@ -135,7 +135,7 @@ struct imap_list_element
 };
 
 #define MU_IMAP_FSET(p,f) ((p)->flags |= (f))
-#define MU_IMAP_FISSET(p,f) ((p)->flags & (f))  
+#define MU_IMAP_FISSET(p,f) ((p)->flags & (f))
 #define MU_IMAP_FCLR(p,f) ((p)->flags &= ~(f))
 
 int _mu_imap_init (mu_imap_t imap);
@@ -161,7 +161,7 @@ struct imap_command
 };
 
 int mu_imap_gencom (mu_imap_t imap, struct imap_command *cmd);
-  
+
 /* If status indicates an error, return.
   */
 #define MU_IMAP_CHECK_ERROR(imap, status)			\
@@ -169,8 +169,8 @@ int mu_imap_gencom (mu_imap_t imap, struct imap_command *cmd);
     {								\
       if (status != 0)						\
 	{							\
-          imap->client_state = MU_IMAP_CLIENT_ERROR;		\
-          return status;					\
+	  imap->client_state = MU_IMAP_CLIENT_ERROR;		\
+	  return status;					\
 	}							\
     }								\
   while (0)
@@ -184,18 +184,18 @@ int mu_imap_gencom (mu_imap_t imap, struct imap_command *cmd);
     {							\
       switch (status)					\
 	{						\
-        case 0:						\
+	case 0:						\
 	  break;					\
-        case EAGAIN:					\
-        case EINPROGRESS:				\
-        case EINTR:					\
+	case EAGAIN:					\
+	case EINPROGRESS:				\
+	case EINTR:					\
 	  return status;				\
-        case MU_ERR_REPLY:				\
-        case MU_ERR_BADREPLY:				\
+	case MU_ERR_REPLY:				\
+	case MU_ERR_BADREPLY:				\
 	  imap->client_state = MU_IMAP_CLIENT_READY;	\
 	  return status;				\
-        default:					\
-          imap->client_state = MU_IMAP_CLIENT_ERROR;	\
+	default:					\
+	  imap->client_state = MU_IMAP_CLIENT_ERROR;	\
 	  return status;				\
 	}						\
     }							\
@@ -214,21 +214,21 @@ int _mu_imap_process_untagged_response (mu_imap_t imap, mu_list_t list,
 					mu_imap_response_action_t fun,
 					void *data);
 int _mu_imap_process_tagged_response (mu_imap_t imap, mu_list_t resp);
- 
+
 int _mu_imap_response (mu_imap_t imap, mu_imap_response_action_t fun,
 		       void *data);
-  
+
 int _mu_imap_list_element_is_string (struct imap_list_element *elt,
 				     const char *str);
 int _mu_imap_list_element_is_nil (struct imap_list_element *elt);
 
 int _mu_imap_list_nth_element_is_string (mu_list_t list, size_t n,
 					 const char *str);
-  
+
 int _mu_imap_collect_flags (struct imap_list_element *arg, int *res);
 
 struct imap_list_element *_mu_imap_list_at (mu_list_t list, int idx);
-  
+
 int _mu_imap_parse_fetch_response (mu_list_t resp, mu_list_t *result_list);
 
 void _mu_close_handler (mu_imap_t imap);
@@ -253,18 +253,19 @@ int _mu_imap_mailbox_init (mu_mailbox_t mailbox);
 struct _mu_imap_message
 {
   int flags;
-  size_t uid;               /* Message UID */   
+  size_t msgno;             /* Message sequence number */
+  size_t uid;               /* Message UID */
   int attr_flags;           /* Attributes */
   mu_off_t offset;          /* Offset in the message cache stream */
   mu_off_t body_start;      /* Start of message, relative to offset */
-  mu_off_t body_end;        /* End of message, relative to offset */  
+  mu_off_t body_end;        /* End of message, relative to offset */
   size_t header_lines;      /* Number of lines in the header */
   size_t body_lines;        /* Number of lines in the body */
   size_t message_size;      /* Message size */
   size_t message_lines;     /* Number of lines in the message */
   struct mu_imapenvelope *env; /* IMAP envelope */
   mu_stream_t header_stream;   /* Memory stream with message headers */
-  mu_message_t message;     /* Pointer to the message structure */ 
+  mu_message_t message;     /* Pointer to the message structure */
   struct _mu_imap_mailbox *imbx; /* Back pointer.  */
 };
 
@@ -273,8 +274,8 @@ struct _mu_imap_message
 struct _mu_imap_mailbox
 {
   int flags;
-  struct mu_imap_stat stats;     /* Mailbox statistics */ 
-  struct _mu_imap_message *msgs; /* Array of messages */
+  struct mu_imap_stat stats;     /* Mailbox statistics */
+  struct _mu_imap_message **msgs; /* Array of messages */
   size_t msgs_cnt;               /* Number of used slots in msgs */
   size_t msgs_max;               /* Number of slots in msgs */
   mu_stream_t cache;          /* Message cache stream */
diff --git a/libproto/imap/mbox.c b/libproto/imap/mbox.c
index fde568a69..b0cae48c7 100644
--- a/libproto/imap/mbox.c
+++ b/libproto/imap/mbox.c
@@ -52,10 +52,10 @@ static int _imap_mbx_is_updated (mu_mailbox_t mbox);
 /* Auxiliary message functions     */
 /* ------------------------------- */
 
-static size_t
+static inline size_t
 _imap_msg_no (struct _mu_imap_message *imsg)
 {
-  return imsg - imsg->imbx->msgs + 1;
+  return imsg->msgno;
 }
 
 static int
@@ -63,7 +63,7 @@ _imap_fetch_with_callback (mu_imap_t imap, mu_msgset_t msgset, char *items,
 			   mu_imap_callback_t cb, void *data)
 {
   int rc;
-  
+
   mu_imap_register_callback_function (imap, MU_IMAP_CB_FETCH, cb, data);
   rc = mu_imap_fetch (imap, 0, msgset, items);
   mu_imap_register_callback_function (imap, MU_IMAP_CB_FETCH, NULL, NULL);
@@ -76,6 +76,7 @@ _imap_msg_free (struct _mu_imap_message *msg)
   mu_message_imapenvelope_free (msg->env);
   mu_stream_destroy (&msg->header_stream);
   mu_message_destroy (&msg->message, msg);
+  free (msg);
 }
 
 struct save_closure
@@ -98,10 +99,10 @@ _save_message_parser (void *item, void *data)
       int rc;
       mu_stream_t istr, flt;
       mu_off_t size;
-  
+
       rc = mu_static_memory_stream_create (&istr, resp->body.text,
 					   strlen (resp->body.text));
-      
+
       if (rc)
 	{
 	  mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
@@ -121,7 +122,7 @@ _save_message_parser (void *item, void *data)
 	  imbx->last_error = rc;
 	  return 0;
 	}
-      
+
       rc = mu_stream_copy (clos->save_stream, flt, 0, &size);
       mu_stream_destroy (&flt);
       if (rc)
@@ -155,11 +156,11 @@ __imap_msg_get_stream (struct _mu_imap_message *imsg, size_t msgno,
   struct _mu_imap_mailbox *imbx = imsg->imbx;
   mu_folder_t folder = imbx->mbox->folder;
   mu_imap_t imap = folder->data;
-  
+
   if (!(imsg->flags & _MU_IMAP_MSG_CACHED))
     {
       mu_msgset_t msgset;
-      
+
       mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_TRACE1,
 		(_("caching message %lu"), (unsigned long) msgno));
       if (!imbx->cache)
@@ -184,7 +185,7 @@ __imap_msg_get_stream (struct _mu_imap_message *imsg, size_t msgno,
 	  clos.imsg = imsg;
 	  clos.save_stream = imbx->cache;
 	  clos.size = 0;
-  
+
 	  rc = mu_msgset_add_range (msgset, msgno, msgno, MU_MSGSET_NUM);
 	  if (rc == 0)
 	    {
@@ -221,14 +222,14 @@ _imap_msg_scan (struct _mu_imap_message *imsg)
   mu_stream_t stream;
   struct mu_message_scan scan;
   size_t msgno = _imap_msg_no (imsg);
-  
+
   if (imsg->flags & _MU_IMAP_MSG_SCANNED)
     return 0;
-  
+
   rc = __imap_msg_get_stream (imsg, msgno, &stream);
   if (rc)
     return rc;
-  
+
   scan.flags = MU_SCAN_SEEK | MU_SCAN_SIZE;
   scan.message_start = 0;
   scan.message_size = imsg->message_size;
@@ -244,7 +245,7 @@ _imap_msg_scan (struct _mu_imap_message *imsg)
       imsg->message_lines = imsg->header_lines + 1 + imsg->body_lines;
       imsg->flags |= _MU_IMAP_MSG_SCANNED;
     }
-  
+
   return rc;
 }
 
@@ -259,7 +260,7 @@ _imap_env_date (mu_envelope_t env, char *buf, size_t len,
   struct _mu_imap_message *imsg = mu_envelope_get_owner (env);
   mu_stream_t str;
   int rc;
-  
+
   if (!buf)
     rc = mu_nullstream_create (&str, MU_STREAM_WRITE);
   else
@@ -292,7 +293,7 @@ _imap_env_sender (mu_envelope_t env, char *buf, size_t len,
     return MU_ERR_NOENT;
   return mu_address_get_email (addr, 1, buf, len, pnwrite);
 }
-  
+
 static int
 _imap_msg_env_setup (struct _mu_imap_message *imsg, mu_message_t message)
 {
@@ -341,7 +342,7 @@ _imap_attr_clr_flags (mu_attribute_t attr, int flags)
 
   if (!imsg)
     return EINVAL;
-  imsg->attr_flags |= flags;
+  imsg->attr_flags &= ~flags;
   imsg->flags |= _MU_IMAP_MSG_ATTRCHG;
   return 0;
 }
@@ -372,7 +373,7 @@ _imap_hdr_fill (void *data, char **pbuf, size_t *plen)
   char *buf;
   mu_off_t size;
   int rc = 0;
-  
+
   mu_stream_size (str, &size);
   buf = malloc (size + 1);
   if (!buf)
@@ -500,11 +501,11 @@ _imap_msg_lines (mu_message_t msg, size_t *plines, int quick)
   struct _mu_imap_message *imsg = mu_message_get_owner (msg);
   struct _mu_imap_mailbox *imbx = imsg->imbx;
   mu_mailbox_t mbox = imbx->mbox;
-  
+
   if (!(imsg->flags & _MU_IMAP_MSG_LINES))
     {
       int rc;
-      
+
       if (quick && !(imsg->flags & _MU_IMAP_MSG_CACHED))
 	return MU_ERR_INFO_UNAVAILABLE;
       if (!_imap_mbx_uptodate (imbx))
@@ -520,10 +521,10 @@ _imap_msg_lines (mu_message_t msg, size_t *plines, int quick)
 static int
 _copy_imapenvelope (struct mu_imapenvelope *env,
 		    struct mu_imapenvelope const *src)
-{  
+{
   env->date = src->date;
   env->tz   = src->tz;
-  
+
   if (src->subject && (env->subject = strdup (src->subject)) == NULL)
     return ENOMEM;
   if (src->from && (env->from = mu_address_dup (src->from)) == NULL)
@@ -546,7 +547,7 @@ _copy_imapenvelope (struct mu_imapenvelope *env,
     return ENOMEM;
   return 0;
 }
-		   
+
 static int
 _imap_msg_imapenvelope (mu_message_t msg, struct mu_imapenvelope **penv)
 {
@@ -588,7 +589,7 @@ _imap_bodystructure_callback (void *data, int code, size_t sdat, void *pdat)
   mu_list_t list = pdat;
   mu_list_foreach (list, fetch_bodystructure_parser, data);
 }
-  
+
 static int
 _imap_msg_bodystructure (mu_message_t msg, struct mu_bodystructure **pbs)
 {
@@ -617,7 +618,7 @@ _imap_mbx_get_message (mu_mailbox_t mailbox, size_t msgno, mu_message_t *pmsg)
   struct _mu_imap_mailbox *imbx = mailbox->data;
   struct _mu_imap_message *imsg;
   int rc;
-  
+
   /* If we did not start a scanning yet do it now.  */
   if (!_imap_mbx_uptodate (imbx))
     _imap_mbx_scan (mailbox, 1, NULL);
@@ -625,11 +626,11 @@ _imap_mbx_get_message (mu_mailbox_t mailbox, size_t msgno, mu_message_t *pmsg)
   if (msgno > imbx->msgs_cnt)
     return MU_ERR_NOENT;
 
-  imsg = imbx->msgs + msgno - 1;
+  imsg = imbx->msgs[msgno - 1];
   if (!imsg->message)
     {
       mu_message_t msg;
-  
+
       rc = mu_message_create (&msg, imsg);
       if (rc)
 	return rc;
@@ -638,7 +639,7 @@ _imap_mbx_get_message (mu_mailbox_t mailbox, size_t msgno, mu_message_t *pmsg)
       mu_message_set_lines (msg, _imap_msg_lines, imsg);
       mu_message_set_imapenvelope (msg, _imap_msg_imapenvelope, imsg);
       mu_message_set_bodystructure (msg, _imap_msg_bodystructure, imsg);
-      
+
       do
 	{
 	  rc = _imap_msg_env_setup (imsg, msg);
@@ -653,7 +654,7 @@ _imap_mbx_get_message (mu_mailbox_t mailbox, size_t msgno, mu_message_t *pmsg)
 	  rc = _imap_mbx_body_setup (imsg, msg);
 	}
       while (0);
-      
+
       if (rc)
 	{
 	  mu_message_destroy (&msg, imsg);
@@ -674,8 +675,8 @@ _imap_realloc_messages (struct _mu_imap_mailbox *imbx, size_t count)
 {
   if (count > imbx->msgs_max)
     {
-      struct _mu_imap_message *newmsgs = realloc (imbx->msgs,
-						  count * sizeof (*newmsgs));
+      struct _mu_imap_message **newmsgs = realloc (imbx->msgs,
+						   count * sizeof (*newmsgs));
       if (!newmsgs)
 	return ENOMEM;
       memset (newmsgs + imbx->msgs_max, 0,
@@ -691,14 +692,14 @@ _imap_mbx_destroy (mu_mailbox_t mailbox)
 {
   size_t i;
   struct _mu_imap_mailbox *imbx = mailbox->data;
-  
+
   if (!imbx)
     return;
 
   if (imbx->msgs)
     {
       for (i = 0; i < imbx->msgs_cnt; i++)
-	_imap_msg_free (imbx->msgs + i);
+	_imap_msg_free (imbx->msgs[i]);
       free (imbx->msgs);
     }
   mu_stream_unref (imbx->cache);
@@ -714,6 +715,33 @@ _imap_update_callback (void *data, int code, size_t sdat, void *pdat)
   imbx->flags &= ~_MU_IMAP_MBX_UPTODATE;
 }
 
+static void
+_imap_expunge_callback (void *data, int code, size_t msgno, void *pdat)
+{
+  struct _mu_imap_mailbox *imbx = data;
+  size_t n;
+
+  if (msgno > imbx->msgs_cnt)
+    {
+      mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+		("_imap_expunge_callback called with invalid message number: %lu",
+		 (unsigned long)msgno));
+      return;
+    }
+  _imap_msg_free (imbx->msgs[msgno - 1]);
+  n = imbx->msgs_cnt - msgno;
+  if (n)
+    {
+      size_t i;
+
+      memmove (imbx->msgs + msgno - 1, imbx->msgs + msgno,
+	       n * sizeof (imbx->msgs[0]));
+      for (i = msgno - 1; i < imbx->msgs_cnt; i++)
+	imbx->msgs[i]->msgno = i + 1;
+    }
+  imbx->msgs_cnt--;
+}
+
 static int
 _imap_mbx_open (mu_mailbox_t mbox, int flags)
 {
@@ -723,9 +751,9 @@ _imap_mbx_open (mu_mailbox_t mbox, int flags)
   const char *mbox_name;
   mu_url_t url;
   mu_imap_t imap;
-  
+
   mbox->flags = flags;
-  
+
   rc = mu_mailbox_get_url (mbox, &url);
   if (rc)
     return rc;
@@ -736,7 +764,7 @@ _imap_mbx_open (mu_mailbox_t mbox, int flags)
     mbox_name = "INBOX";
   else if (rc)
     return rc;
-      
+
   rc = mu_folder_open (folder, flags);
   if (rc)
     return rc;
@@ -749,7 +777,10 @@ _imap_mbx_open (mu_mailbox_t mbox, int flags)
   mu_imap_register_callback_function (imap, MU_IMAP_CB_MESSAGE_COUNT,
 				      _imap_update_callback,
 				      imbx);
-  
+  mu_imap_register_callback_function (imap, MU_IMAP_CB_EXPUNGE,
+				      _imap_expunge_callback,
+				      imbx);
+
   rc = mu_imap_select (imap, mbox_name,
 		       flags & (MU_STREAM_WRITE|MU_STREAM_APPEND),
 		       &imbx->stats);
@@ -803,7 +834,7 @@ _imap_messages_recent (mu_mailbox_t mbox, size_t *pcount)
       mu_folder_t folder = mbox->folder;
       mu_imap_t imap = folder->data;
       mu_msgset_t msgset;
-  
+
       rc = mu_imap_search (imap, 0, "RECENT", &msgset);
       if (rc)
 	return rc;
@@ -839,7 +870,7 @@ _imap_message_unseen (mu_mailbox_t mbox, size_t *pn)
       mu_imap_t imap = folder->data;
       mu_msgset_t msgset;
       mu_list_t list;
-  
+
       rc = mu_imap_search (imap, 0, "UNSEEN", &msgset);
       if (rc)
 	return rc;
@@ -898,7 +929,7 @@ attr_tab_cmp (void const *a, void const *b)
   else if (ta->start > tb->start)
     return 1;
   return 0;
-} 
+}
 
 static int
 aggregate_attributes (struct _mu_imap_mailbox *imbx,
@@ -907,12 +938,12 @@ aggregate_attributes (struct _mu_imap_mailbox *imbx,
   size_t i, j;
   size_t count;
   struct attr_tab *tab;
-  
+
   /* Pass 1: Count modified attributes */
   count = 0;
   for (i = 0; i < imbx->msgs_cnt; i++)
     {
-      if (imbx->msgs[i].flags & _MU_IMAP_MSG_ATTRCHG)
+      if (imbx->msgs[i]->flags & _MU_IMAP_MSG_ATTRCHG)
 	count++;
     }
 
@@ -922,21 +953,21 @@ aggregate_attributes (struct _mu_imap_mailbox *imbx,
       *pcnt = 0;
       return 0;
     }
-  
+
   /* Pass 2: Create and populate expanded array */
   tab = calloc (count, sizeof (*tab));
   if (!tab)
     return ENOMEM;
   for (i = j = 0; i < imbx->msgs_cnt; i++)
     {
-      if (imbx->msgs[i].flags & _MU_IMAP_MSG_ATTRCHG)
+      if (imbx->msgs[i]->flags & _MU_IMAP_MSG_ATTRCHG)
 	{
 	  tab[j].start = tab[j].end = i;
-	  tab[j].attr_flags = imbx->msgs[i].attr_flags;
+	  tab[j].attr_flags = imbx->msgs[i]->attr_flags;
 	  j++;
 	}
     }
-  
+
   /* Sort the array */
   qsort (tab, count, sizeof (tab[0]), attr_tab_cmp);
 
@@ -980,7 +1011,7 @@ _imap_mbx_gensync (mu_mailbox_t mbox, int *pdel)
       /* Too bad, but try to use naive approach */
       for (i = 0; i < imbx->msgs_cnt; i++)
 	{
-	  if (imbx->msgs[i].flags & _MU_IMAP_MSG_ATTRCHG)
+	  if (imbx->msgs[i]->flags & _MU_IMAP_MSG_ATTRCHG)
 	    {
 	      mu_msgset_clear (msgset);
 	      mu_msgset_add_range (msgset, i + 1, i + 1, MU_MSGSET_NUM);
@@ -988,8 +1019,8 @@ _imap_mbx_gensync (mu_mailbox_t mbox, int *pdel)
 		break;
 	      rc = mu_imap_store_flags (imap, 0, msgset,
 					MU_IMAP_STORE_SET|MU_IMAP_STORE_SILENT,
-					imbx->msgs[i].attr_flags);
-	      delflg |= imbx->msgs[i].attr_flags & MU_ATTRIBUTE_DELETED;
+					imbx->msgs[i]->attr_flags);
+	      delflg |= imbx->msgs[i]->attr_flags & MU_ATTRIBUTE_DELETED;
 	      if (rc)
 		break;
 	    }
@@ -1036,13 +1067,13 @@ _imap_mbx_gensync (mu_mailbox_t mbox, int *pdel)
       free (tab);
     }
   mu_msgset_free (msgset);
-  
+
   if (rc)
     return rc;
 
   if (pdel)
     *pdel = delflg;
-  
+
   return 0;
 }
 
@@ -1050,7 +1081,7 @@ static int
 _imap_mbx_expunge (mu_mailbox_t mbox)
 {
   int rc, del = 0;
-  
+
   mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_TRACE1,
 	    (_("expunging mailbox %s"), mu_url_to_string (mbox->url)));
   rc = _imap_mbx_gensync (mbox, &del);
@@ -1079,7 +1110,7 @@ _imap_mbx_append_message (mu_mailbox_t mbox, mu_message_t msg)
   mu_imap_t imap = folder->data;
   mu_url_t url;
   const char *mbox_name;
-  
+
   rc = mu_mailbox_get_url (mbox, &url);
   if (rc)
     return rc;
@@ -1098,7 +1129,7 @@ sum_lines (void *item, void *data)
   size_t *pn = data;
   size_t n;
   int rc;
-  
+
   rc = _compute_lines (bs, &n);
   if (rc)
     return rc;
@@ -1121,7 +1152,7 @@ _compute_lines (struct mu_bodystructure *bs, size_t *pcount)
     case mu_message_rfc822:
       *pcount = bs->v.rfc822.body_lines;
       return 0;
-      
+
     case mu_message_multipart:
       *pcount = 0;
       return mu_list_foreach (bs->v.multipart.body_parts, sum_lines, pcount);
@@ -1140,20 +1171,20 @@ fetch_response_parser (void *item, void *data)
     case MU_IMAP_FETCH_UID:
       imsg->uid = resp->uid.uid;
       break;
-      
+
     case MU_IMAP_FETCH_FLAGS:
       imsg->attr_flags = resp->flags.flags;
       break;
-      
+
     case MU_IMAP_FETCH_ENVELOPE:
       imsg->env = resp->envelope.imapenvelope;
       resp->envelope.imapenvelope = NULL; /* Steal the envelope */
       break;
-      
+
     case MU_IMAP_FETCH_RFC822_SIZE:
       imsg->message_size = resp->rfc822_size.size;
       break;
-      
+
     case MU_IMAP_FETCH_BODYSTRUCTURE:
       {
 	size_t n;
@@ -1185,7 +1216,7 @@ fetch_response_parser (void *item, void *data)
 		     mu_strerror (rc)));
       }
       break;
-	    
+
     default:
       mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_TRACE0,
 		(_("fetch returned a not requested item %d"),
@@ -1194,7 +1225,7 @@ fetch_response_parser (void *item, void *data)
     }
   return 0;
 }
-  
+
 static void
 _imap_fetch_callback (void *data, int code, size_t sdat, void *pdat)
 {
@@ -1203,7 +1234,7 @@ _imap_fetch_callback (void *data, int code, size_t sdat, void *pdat)
   mu_list_t list = pdat;
   int rc;
   struct _mu_imap_message *imsg;
-  
+
   rc = _imap_realloc_messages (imbx, sdat);
   if (rc)
     {
@@ -1215,7 +1246,22 @@ _imap_fetch_callback (void *data, int code, size_t sdat, void *pdat)
     }
   if (imbx->msgs_cnt < sdat)
     imbx->msgs_cnt = sdat;
-  imsg = imbx->msgs + sdat - 1;
+
+  if (!imbx->msgs[sdat - 1])
+    {
+      imbx->msgs[sdat - 1] = calloc (1, sizeof (imbx->msgs[sdat - 1][0]));
+      if (!imbx->msgs[sdat - 1])
+	{
+	  mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_ERROR,
+		    (_("cannot allocate imap message: %s"),
+		     mu_strerror (errno)));
+	  imbx->last_error = errno;
+	  return;
+	}
+      imbx->msgs[sdat - 1]->msgno = sdat;
+    }
+
+  imsg = imbx->msgs[sdat - 1];
   imsg->imbx = imbx;
   mu_list_foreach (list, fetch_response_parser, imsg);
 
@@ -1236,7 +1282,7 @@ _imap_mbx_scan (mu_mailbox_t mbox, size_t msgno, size_t *pcount)
   mu_msgset_t msgset;
   int rc;
   static char _imap_scan_items[] = "(UID FLAGS ENVELOPE RFC822.SIZE BODY BODY.PEEK[HEADER])";
-  
+
   mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_TRACE1,
 	    (_("scanning mailbox %s"), mu_url_to_string (mbox->url)));
   if (imbx->stats.message_count > 0)
@@ -1250,7 +1296,7 @@ _imap_mbx_scan (mu_mailbox_t mbox, size_t msgno, size_t *pcount)
 	  mu_msgset_free (msgset);
 	  return rc;
 	}
-  
+
       _imap_mbx_clrerr (imbx);
       rc = _imap_fetch_with_callback (imap, msgset, _imap_scan_items,
 				      _imap_fetch_callback, imbx);
@@ -1261,12 +1307,12 @@ _imap_mbx_scan (mu_mailbox_t mbox, size_t msgno, size_t *pcount)
 	{
 	  size_t i;
 	  mu_off_t total = 0;
-	  
+
 	  imbx->flags |= _MU_IMAP_MBX_UPTODATE;
 
 	  for (i = 1; i <= imbx->msgs_cnt; i++)
 	    {
-	      total += imbx->msgs[i-1].message_size;
+	      total += imbx->msgs[i-1]->message_size;
 	      /* MU_EVT_MESSAGE_ADD must be delivered only when it is already
 		 possible to retrieve the message in question.  It could not be
 		 done in the fetch handler for obvious reasons.  Hence the extra
@@ -1295,7 +1341,7 @@ _imap_mbx_is_updated (mu_mailbox_t mbox)
   mu_folder_t folder = mbox->folder;
   mu_imap_t imap = folder->data;
   int rc;
-  
+
   rc = mu_imap_noop (imap);
   if (rc)
     {
@@ -1314,7 +1360,7 @@ _imap_copy_to_mailbox (mu_mailbox_t mbox, mu_msgset_t msgset,
   mu_folder_t folder = mbox->folder;
   mu_imap_t imap = folder->data;
   int rc;
-  
+
   mu_debug (MU_DEBCAT_MAILBOX, MU_DEBUG_TRACE1,
 	    (_("copying messages to mailbox %s"), mailbox));
   _imap_mbx_clrerr (imbx);
@@ -1361,14 +1407,14 @@ _mu_imap_mailbox_init (mu_mailbox_t mailbox)
   mailbox->_message_unseen = _imap_message_unseen;
   mailbox->_uidvalidity = _imap_uidvalidity;
   mailbox->_uidnext = _imap_uidnext;
-  
+
   mailbox->_scan = _imap_mbx_scan;
   mailbox->_is_updated = _imap_mbx_is_updated;
   mailbox->_get_message = _imap_mbx_get_message;
   mailbox->_sync = _imap_mbx_sync;
-  
+
   mailbox->_append_message = _imap_mbx_append_message;
   mailbox->_copy = _imap_copy_to_mailbox;
-  
+
   return 0;
 }
diff --git a/libproto/imap/resproc.c b/libproto/imap/resproc.c
index 0beff4d5d..64230f455 100644
--- a/libproto/imap/resproc.c
+++ b/libproto/imap/resproc.c
@@ -35,7 +35,7 @@ struct mu_kwd mu_imap_response_codes[] = {
   /* [CAPABILITY (list)] */
   { "CAPABILITY",     MU_IMAP_RESPONSE_CAPABILITY },
   /* [PARSE] text */
-  { "PARSE",          MU_IMAP_RESPONSE_PARSE }, 
+  { "PARSE",          MU_IMAP_RESPONSE_PARSE },
   /* [PERMANENTFLAGS (list)] */
   { "PERMANENTFLAGS", MU_IMAP_RESPONSE_PERMANENTFLAGS },
   /* [READ-ONLY] */
@@ -43,9 +43,9 @@ struct mu_kwd mu_imap_response_codes[] = {
   /* [READ-WRITE] */
   { "READ-WRITE",     MU_IMAP_RESPONSE_READ_WRITE },
   /* [TRYCREATE] */
-  { "TRYCREATE",      MU_IMAP_RESPONSE_TRYCREATE },  
+  { "TRYCREATE",      MU_IMAP_RESPONSE_TRYCREATE },
   /* [UIDNEXT N] */
-  { "UIDNEXT",        MU_IMAP_RESPONSE_UIDNEXT }, 
+  { "UIDNEXT",        MU_IMAP_RESPONSE_UIDNEXT },
   /* [UIDVALIDITY N] */
   { "UIDVALIDITY",    MU_IMAP_RESPONSE_UIDVALIDITY },
   /* [UNSEEN N] */
@@ -62,23 +62,23 @@ parse_response_code (mu_imap_t imap, mu_list_t resp)
   arg = _mu_imap_list_at (resp, 1);
   if (!arg)
     return -1;
-  
+
   if (_mu_imap_list_element_is_string (arg, "["))
     {
       arg = _mu_imap_list_at (resp, 2);
       if (!arg || arg->type != imap_eltype_string)
 	return -1;
-      
+
       if (mu_kwd_xlat_name (mu_imap_response_codes, arg->v.string, &rcode))
 	return -1;
-      
+
       arg = _mu_imap_list_at (resp, 4);
       if (!arg || !_mu_imap_list_element_is_string (arg, "]"))
 	return -1;
     }
   return rcode;
 }
-  
+
 static void
 ok_response (mu_imap_t imap, mu_list_t resp, void *data)
 {
@@ -98,7 +98,7 @@ ok_response (mu_imap_t imap, mu_list_t resp, void *data)
       imap->mbox_stat.flags |= MU_IMAP_STAT_PERMANENT_FLAGS;
       mu_imap_callback (imap, MU_IMAP_CB_PERMANENT_FLAGS, 0, &imap->mbox_stat);
       return;
-	  
+
     case MU_IMAP_RESPONSE_UIDNEXT:
       arg = _mu_imap_list_at (resp, 3);
       if (!arg || arg->type != imap_eltype_string)
@@ -111,7 +111,7 @@ ok_response (mu_imap_t imap, mu_list_t resp, void *data)
 	  mu_imap_callback (imap, MU_IMAP_CB_UIDNEXT, 0, &imap->mbox_stat);
 	}
       return;
-			    
+
     case MU_IMAP_RESPONSE_UIDVALIDITY:
       arg = _mu_imap_list_at (resp, 3);
       if (!arg || arg->type != imap_eltype_string)
@@ -124,7 +124,7 @@ ok_response (mu_imap_t imap, mu_list_t resp, void *data)
 	  mu_imap_callback (imap, MU_IMAP_CB_UIDVALIDITY, 0, &imap->mbox_stat);
 	}
       return;
-      
+
     case MU_IMAP_RESPONSE_UNSEEN:
       arg = _mu_imap_list_at (resp, 3);
       if (!arg || arg->type != imap_eltype_string)
@@ -260,7 +260,7 @@ _process_unsolicited_response (mu_imap_t imap, mu_list_t resp)
 {
   size_t count;
   struct imap_list_element *arg;
-  
+
   if (mu_list_count (resp, &count))
     return 1;
 
@@ -274,13 +274,13 @@ _process_unsolicited_response (mu_imap_t imap, mu_list_t resp)
       arg = _mu_imap_list_at (resp, 1);
       if (!arg)
 	return 1;