Index: imap/sync_client.c =================================================================== RCS file: /cvs/src/cyrus/imap/sync_client.c,v retrieving revision 1.26 diff -u -d -r1.26 sync_client.c --- imap/sync_client.c 10 Oct 2007 21:23:57 -0000 1.26 +++ imap/sync_client.c 11 Oct 2007 18:18:41 -0000 @@ -1018,6 +1018,85 @@ /* ====================================================================== */ +static int check_modseq(struct mailbox *mailbox, struct sync_msg_list *list) +{ + struct sync_msg *msg; + unsigned long msgno; + struct index_record record; + + /* Updates on the client should always cause a highestmodseq */ + if (mailbox->highestmodseq != list->highestmodseq) + return(1); + + msg = list->head; + for (msgno = 1; msg && (msgno <= mailbox->exists) ; msgno++) { + mailbox_read_index_record(mailbox, msgno, &record); + + /* Skip msgs on client missing on server (will upload later) */ + if (record.uid < msg->uid) + continue; + + /* Skip over messages recorded on server which are missing on client + * (either will be expunged or have been expunged already) */ + while (msg && (msg->uid < record.uid)) + msg = msg->next; + + if (!(msg && (record.uid == msg->uid))) + continue; + + /* Got a message on client which has same UID as message on server */ + /* If the modseqs don't match, then we need to make an update */ + if (record.modseq != msg->modseq) + return(1); + + msg = msg->next; + } + return(0); +} + +static int update_modseq(struct mailbox *mailbox, struct sync_msg_list *list) +{ + struct sync_msg *msg; + unsigned long msgno; + struct index_record record; + + prot_printf(toserver, "SETMODSEQ " MODSEQ_FMT, mailbox->highestmodseq); + + msg = list->head; + for (msgno = 1; msg && (msgno <= mailbox->exists) ; msgno++) { + mailbox_read_index_record(mailbox, msgno, &record); + + /* Skip msgs on client missing on server (will upload later) */ + if (record.uid < msg->uid) + continue; + + /* Skip over messages recorded on server which are missing on client + * (either will be expunged or have been expunged already) */ + while (msg && (msg->uid < record.uid)) + msg = msg->next; + + if (!(msg && (record.uid == msg->uid))) + continue; + + /* Got a message on client which has same UID as message on server */ + if (record.modseq == msg->modseq) { + msg = msg->next; + continue; + } + + /* ... but a different modseq */ + prot_printf(toserver, " %lu " MODSEQ_FMT, record.uid, record.modseq); + msg = msg->next; + } + + prot_printf(toserver, "\r\n"); + prot_flush(toserver); + + return(sync_parse_code("SETMODSEQ",fromserver,SYNC_PARSE_EAT_OKLINE,NULL)); +} + +/* ====================================================================== */ + static int check_expunged(struct mailbox *mailbox, struct sync_msg_list *list) { struct sync_msg *msg = list->head; @@ -1677,6 +1756,16 @@ goto bail; } + if (check_modseq(mailbox, list)) { + if (!selected && + (r=folder_select(mailbox->name, mailbox->uniqueid, NULL))) + return(r); + + selected = 1; + if ((r=update_modseq(mailbox, list))) + goto bail; + } + if (check_expunged(mailbox, list)) { if (!selected && (r=folder_select(mailbox->name, mailbox->uniqueid, NULL))) @@ -1856,8 +1945,8 @@ if (!r && do_contents) { struct sync_msg_list *folder_msglist; - /* 0L forces lastuid push */ - folder_msglist = sync_msg_list_create(m.flagname, 0L); + /* 0L forces lastuid, highestmodseq push */ + folder_msglist = sync_msg_list_create(m.flagname, 0L, 0L); r = do_mailbox_work(&m, folder_msglist, 1); sync_msg_list_free(&folder_msglist); } @@ -1903,8 +1992,8 @@ if (!r && do_contents) { struct sync_msg_list *folder_msglist; - /* 0L forces uidlast push */ - folder_msglist = sync_msg_list_create(m.flagname, 0L); + /* 0L forces uidlast, highestmodseq push */ + folder_msglist = sync_msg_list_create(m.flagname, 0L, 0L); r = do_mailbox_work(&m, folder_msglist, 1); sync_msg_list_free(&folder_msglist); } @@ -1946,6 +2035,7 @@ static struct buf name; static struct buf uidvalidity; static struct buf lastuid; + static struct buf highestmodseq; static struct buf options; static struct buf arg; struct quota quota, *quotap; @@ -1984,6 +2074,9 @@ if ((c = getastring(fromserver, toserver, &lastuid)) != ' ') goto parse_err; + if ((c = getastring(fromserver, toserver, &highestmodseq)) != ' ') + goto parse_err; + c = getastring(fromserver, toserver, &options); quotap = NULL; @@ -2002,7 +2095,13 @@ sync_atoul(uidvalidity.s), sync_atoul(options.s), quotap); - folder->msglist = sync_msg_list_create(NULL, sync_atoul(lastuid.s)); +#ifdef HAVE_LONG_LONG_INT + folder->msglist = sync_msg_list_create + (NULL, sync_atoul(lastuid.s), sync_atoull(highestmodseq.s)); +#else + folder->msglist = sync_msg_list_create + (NULL, sync_atoul(lastuid.s), sync_atoul(highestmodseq.s)); +#endif break; case 1: /* New message in current folder */ @@ -2011,7 +2110,13 @@ if (((c = getword(fromserver, &arg)) != ' ') || ((msg->uid = sync_atoul(arg.s)) == 0)) goto parse_err; - +#ifdef HAVE_LONG_LONG_INT + if (((c = getword(fromserver, &arg)) != ' ') || + ((msg->modseq = sync_atoull(arg.s)) == 0)) goto parse_err; +#else + if (((c = getword(fromserver, &arg)) != ' ') || + ((msg->modseq = sync_atoul(arg.s)) == 0)) goto parse_err; +#endif if (((c = getword(fromserver, &arg)) != ' ')) goto parse_err; if (!message_guid_decode(&msg->guid, arg.s)) @@ -2384,6 +2489,7 @@ static struct buf acl; static struct buf uidvalidity; static struct buf lastuid; + static struct buf highestmodseq; static struct buf options; static struct buf arg; struct sync_folder *folder = NULL; @@ -2441,6 +2547,9 @@ if ((c = getastring(fromserver, toserver, &lastuid)) != ' ') goto parse_err; + if ((c = getastring(fromserver, toserver, &highestmodseq)) != ' ') + goto parse_err; + c = getastring(fromserver, toserver, &options); quotap = NULL; @@ -2452,14 +2561,21 @@ if (c == '\r') c = prot_getc(fromserver); if (c != '\n') goto parse_err; - if (!imparse_isnumber(uidvalidity.s)) goto parse_err; - if (!imparse_isnumber(lastuid.s)) goto parse_err; + if (!imparse_isnumber(uidvalidity.s)) goto parse_err; + if (!imparse_isnumber(lastuid.s)) goto parse_err; + if (!imparse_isnumber(highestmodseq.s)) goto parse_err; folder = sync_folder_list_add(server_list, id.s, name.s, acl.s, sync_atoul(uidvalidity.s), sync_atoul(options.s), quotap); - folder->msglist = sync_msg_list_create(NULL, sync_atoul(lastuid.s)); +#ifdef HAVE_LONG_LONG_INT + folder->msglist = sync_msg_list_create + (NULL, sync_atoul(lastuid.s), sync_atoull(highestmodseq.s)); +#else + folder->msglist = sync_msg_list_create + (NULL, sync_atoul(lastuid.s), sync_atoul(highestmodseq.s)); +#endif break; case 1: /* New message in current folder */ @@ -2468,7 +2584,13 @@ if (((c = getword(fromserver, &arg)) != ' ') || ((msg->uid = sync_atoul(arg.s)) == 0)) goto parse_err; - +#ifdef HAVE_LONG_LONG_INT + if (((c = getword(fromserver, &arg)) != ' ') || + ((msg->modseq = sync_atoull(arg.s)) == 0)) goto parse_err; +#else + if (((c = getword(fromserver, &arg)) != ' ') || + ((msg->modseq = sync_atoul(arg.s)) == 0)) goto parse_err; +#endif if (((c = getword(fromserver, &arg)) != ' ')) goto parse_err; if (!message_guid_decode(&msg->guid, arg.s)) Index: imap/sync_commit.c =================================================================== RCS file: /cvs/src/cyrus/imap/sync_commit.c,v retrieving revision 1.13 diff -u -d -r1.13 sync_commit.c --- imap/sync_commit.c 5 Oct 2007 18:18:25 -0000 1.13 +++ imap/sync_commit.c 11 Oct 2007 18:18:41 -0000 @@ -1127,6 +1127,97 @@ /* ====================================================================== */ +int sync_highestmodseq_commit(struct mailbox *mailbox, + modseq_t newhighestmodseq) +{ + unsigned char *hbuf = xmalloc(mailbox->start_offset); + int n; + + /* Fix up information in index header */ + lseek(mailbox->index_fd, 0L, SEEK_SET); + + n = read(mailbox->index_fd, hbuf, mailbox->start_offset); + if ((unsigned long)n != mailbox->start_offset) { + free(hbuf); + syslog(LOG_ERR, + "IOERROR: reading index header for %s: got %d of %lu", + mailbox->name, n, mailbox->start_offset); + return(IMAP_IOERROR); + } + +#ifdef HAVE_LONG_LONG_INT + align_htonll(hbuf+OFFSET_HIGHESTMODSEQ_64, newhighestmodseq); +#else + *((bit32 *)(hbuf+OFFSET_HIGHESTMODSEQ_64)) = htonl(0); + *((bit32 *)(hbuf+OFFSET_HIGHESTMODSEQ)) = htonl(newhighestmodseq); +#endif + + /* And write it back out */ + lseek(mailbox->index_fd, 0L, SEEK_SET); + + n = retry_write(mailbox->index_fd, hbuf, mailbox->start_offset); + + free(hbuf); + if ((unsigned long)n != mailbox->start_offset) { + syslog(LOG_ERR, "IOERROR: writing out new index header for %s", + mailbox->name); + return(IMAP_IOERROR); + } + + /* Ensure everything made it to disk */ + if (fsync(mailbox->index_fd)) { + syslog(LOG_ERR, "IOERROR: writing index for %s: %m", + mailbox->name); + return(IMAP_IOERROR); + } + return(0); +} + +int sync_modseq_commit(struct mailbox *mailbox, + struct sync_modseq_list *modseq_list) +{ + struct index_record record; + struct sync_modseq_item *item = modseq_list->head; + unsigned long msgno = 1; + int r = 0; + time_t now = time(NULL); + + if (!r) r = mailbox_lock_header(mailbox); + if (!r) r = mailbox_lock_index(mailbox); + + if (r) return(r); + + while (item && (msgno <= mailbox->exists)) { + r = mailbox_read_index_record(mailbox, msgno, &record); + + if (r) return(r); + + if (record.uid == item->uid) { + record.modseq = item->modseq; + record.last_updated = ((record.last_updated >= now) ? + record.last_updated + 1 : now); + mailbox_write_index_record(mailbox, msgno, &record, 0); + item = item->next; + } + msgno++; + } + + if (!r) r = mailbox_write_index_header(mailbox); + if (!r) mailbox_unlock_index(mailbox); + if (!r) mailbox_unlock_header(mailbox); + + if (fsync(mailbox->index_fd)) { + syslog(LOG_ERR, "IOERROR: writing index record %lu for %s: %m", + msgno, mailbox->name); + return IMAP_IOERROR; + } + + r = mailbox_open_index(mailbox); /* Update internal index */ + return(r); +} + +/* ====================================================================== */ + #define DB config_mboxlist_db int sync_create_commit(char *name, char *partition, char *uniqueid, char *acl, Index: imap/sync_commit.h =================================================================== RCS file: /cvs/src/cyrus/imap/sync_commit.h,v retrieving revision 1.3 diff -u -d -r1.3 sync_commit.h --- imap/sync_commit.h 12 Sep 2007 15:51:04 -0000 1.3 +++ imap/sync_commit.h 11 Oct 2007 18:18:41 -0000 @@ -61,6 +61,14 @@ int sync_setflags_commit(struct mailbox *mailbox, struct sync_flag_list *flag_list); +int sync_highestmodseq_commit(struct mailbox *mailbox, + modseq_t newhighestmodseq); + + +int sync_modseq_commit(struct mailbox *mailbox, + struct sync_modseq_list *modseq_list); + + int sync_create_commit(char *name, char *partition, char *uniqueid, char *acl, int mbtype, unsigned long options, unsigned long uidvalidity, int isadmin, char *userid, struct auth_state *auth_state); Index: imap/sync_server.c =================================================================== RCS file: /cvs/src/cyrus/imap/sync_server.c,v retrieving revision 1.18 diff -u -d -r1.18 sync_server.c --- imap/sync_server.c 10 Oct 2007 15:14:39 -0000 1.18 +++ imap/sync_server.c 11 Oct 2007 18:18:41 -0000 @@ -168,6 +168,7 @@ time_t lastread, unsigned int last_recent_uid, time_t lastchange, char *seenuid); static void cmd_setseen_all(char *user, struct buf *data); +static void cmd_setmodseq(struct mailbox *mailbox); static void cmd_setacl(char *name, char *acl); static void cmd_setuidvalidity(char *name, unsigned long uidvalidity); static void cmd_expunge(struct mailbox *mailbox); @@ -910,6 +911,9 @@ if (c != ' ') goto missingargs; cmd_setflags(mailbox); continue; + } else if (!strcmp(cmd.s, "Setmodseq")) { + cmd_setmodseq(mailbox); + continue; } else if (!strcmp(cmd.s, "Setseen")) { if (c != ' ') goto missingargs; c = getastring(sync_in, sync_out, &arg1); @@ -1676,8 +1680,8 @@ for (msgno = 1 ; msgno <= mailbox->exists; msgno++) { mailbox_read_index_record(mailbox, msgno, &record); - prot_printf(sync_out, "* %lu %s (", - record.uid, message_guid_encode(&record.guid)); + prot_printf(sync_out, "* %lu " MODSEQ_FMT " %s (", + record.uid, record.modseq, message_guid_encode(&record.guid)); flags_printed = 0; @@ -1708,7 +1712,8 @@ return; } cmd_status_work(mailbox); - prot_printf(sync_out, "OK %lu\r\n", mailbox->last_uid); + prot_printf(sync_out, "OK %lu " MODSEQ_FMT "\r\n", + mailbox->last_uid, mailbox->highestmodseq); } /* ====================================================================== */ @@ -2233,6 +2238,94 @@ /* ====================================================================== */ +static void cmd_setmodseq(struct mailbox *mailbox) +{ + struct sync_modseq_list *modseq_list = sync_modseq_list_create(); + static struct buf arg; + char *err = NULL; + int c; + int r = 0; + unsigned long uid; + modseq_t modseq, highestmodseq; + + if (!mailbox) { + eatline(sync_in, ' '); + prot_printf(sync_out, "NO Mailbox not open\r\n"); + return; + } + + if ((c = getastring(sync_in, sync_out, &arg)) == EOF) + goto bail; + +#ifdef HAVE_LONG_LONG_INT + highestmodseq = sync_atoull(arg.s); +#else + highestmodseq = sync_atoul(arg.s); +#endif + + while (c == ' ') { + err = NULL; + + /* Parse UID */ + if ((c = getastring(sync_in, sync_out, &arg)) == EOF) + goto bail; + + if ((c != ' ') || ((uid = sync_atoul(arg.s)) == 0)) + err = "Invalid UID"; + else if (uid > mailbox->last_uid) + err = "UID out of range"; + + if ((c = getastring(sync_in, sync_out, &arg)) == EOF) + goto bail; + + if ((c != ' ') && (c != '\r') && (c != '\n')) { + if (!err) err = "Invalid modseq"; + } else { +#ifdef HAVE_LONG_LONG_INT + modseq = sync_atoull(arg.s); +#else + modseq = sync_atoul(arg.s); +#endif + if (modseq == 0) + err = "Invalid modseq"; + } + + if (err != NULL) { + eatline(sync_in, c); + prot_printf(sync_out, "BAD Syntax error in Setflags: %s\r\n", err); + goto bail; + } + sync_modseq_list_add(modseq_list, uid, modseq); + + /* if we see a SP, we're trying to set more than one flag */ + } + + if (c == '\r') c = prot_getc(sync_in); + if (c != '\n') { + eatline(sync_in, c); + prot_printf(sync_out, "BAD Garbage at end of Setmodseq sequence\r\n"); + goto bail; + } + + r = sync_highestmodseq_commit(mailbox, highestmodseq); + if (!r) r = sync_modseq_commit(mailbox, modseq_list); + + if (r) { + prot_printf(sync_out, + "NO Failed to commit modseq update for %s: %s\r\n", + mailbox->name, error_message(r)); + } else { + prot_printf(sync_out, "OK Updated modseqs on %lu messages okay\r\n", + modseq_list->count); + } + + bail: + sync_modseq_list_free(&modseq_list); +} + + +/* ====================================================================== */ + struct uid_list { unsigned long *array; unsigned long alloc; @@ -2369,6 +2462,7 @@ sync_printastring(sync_out, m.acl); prot_printf(sync_out, " %lu", m.uidvalidity); prot_printf(sync_out, " %lu", m.last_uid); + prot_printf(sync_out, " " MODSEQ_FMT, m.highestmodseq); prot_printf(sync_out, " %lu", m.options); if (m.quota.root && !strcmp(name, m.quota.root) && !quota_read(&m.quota, NULL, 0)) { Index: imap/sync_support.c =================================================================== RCS file: /cvs/src/cyrus/imap/sync_support.c,v retrieving revision 1.16 diff -u -d -r1.16 sync_support.c --- imap/sync_support.c 5 Oct 2007 16:28:41 -0000 1.16 +++ imap/sync_support.c 11 Oct 2007 18:18:41 -0000 @@ -470,7 +470,8 @@ /* sync_msg stuff */ struct sync_msg_list *sync_msg_list_create(char **flagname, - unsigned long last_uid) + unsigned long last_uid, + modseq_t highestmodseq) { struct sync_msg_list *l = xzmalloc(sizeof (struct sync_msg_list)); @@ -478,6 +479,7 @@ l->tail = NULL; l->count = 0; l->last_uid = last_uid; + l->highestmodseq = highestmodseq; sync_flags_meta_clear(&l->meta); if (flagname) @@ -1537,6 +1539,52 @@ /* ====================================================================== */ +struct sync_modseq_list *sync_modseq_list_create() +{ + struct sync_modseq_list *l = xzmalloc(sizeof (struct sync_modseq_list)); + + l->head = NULL; + l->tail = NULL; + l->count = 0; + return(l); +} + +struct sync_modseq_item *sync_modseq_list_add(struct sync_modseq_list *l, + unsigned long uid, + modseq_t modseq) +{ + struct sync_modseq_item *result + = xzmalloc(sizeof(struct sync_modseq_item)); + + result->uid = uid; + result->modseq = modseq; + + if (l->tail) + l->tail = l->tail->next = result; + else + l->head = l->tail = result; + + l->count++; + return(result); +} + +void sync_modseq_list_free(struct sync_modseq_list **lp) +{ + struct sync_modseq_list *l = *lp; + struct sync_modseq_item *current, *next; + + current = l->head; + while (current) { + next = current->next; + free(current); + current = next; + } + free(l); + *lp = NULL; +} + +/* ====================================================================== */ + char *sync_sieve_get_path(char *userid, char *sieve_path, size_t psize) { char *domain; Index: imap/sync_support.h =================================================================== RCS file: /cvs/src/cyrus/imap/sync_support.h,v retrieving revision 1.9 diff -u -d -r1.9 sync_support.h --- imap/sync_support.h 5 Oct 2007 16:28:41 -0000 1.9 +++ imap/sync_support.h 11 Oct 2007 18:18:41 -0000 @@ -127,6 +127,7 @@ struct sync_msg *next; struct message_guid guid; unsigned long uid; + modseq_t modseq; struct sync_flags flags; }; @@ -134,11 +135,13 @@ struct sync_msg *head, *tail; unsigned long count; unsigned long last_uid; + modseq_t highestmodseq; struct sync_flags_meta meta; }; struct sync_msg_list *sync_msg_list_create(char **flagname, - unsigned long last_uid); + unsigned long last_uid, + modseq_t highestmodseq); struct sync_msg *sync_msg_list_add(struct sync_msg_list *l); @@ -418,6 +421,28 @@ /* ====================================================================== */ +struct sync_modseq_item { + struct sync_modseq_item *next; + unsigned long uid; + modseq_t modseq; +}; + +struct sync_modseq_list { + struct sync_modseq_item *head; + struct sync_modseq_item *tail; + unsigned long count; +}; + +struct sync_modseq_list *sync_modseq_list_create(); + +struct sync_modseq_item *sync_modseq_list_add(struct sync_modseq_list *l, + unsigned long uid, + modseq_t modseq); + +void sync_modseq_list_free(struct sync_modseq_list **lp); + +/* ====================================================================== */ + struct sync_sieve_item { struct sync_sieve_item *next; char *name;