commit 20ae5a798231c8e56a671ff5215bf0dd302907a7
parent 67aedbe11f07aeb23257f1406da4595b1d466746
Author: Damien Leone <damien.leone@fensalir.fr>
Date: Fri, 25 Jun 2010 18:17:08 +0200
Synchronize remote modifications from a Maildir source to sup
It is now possible to use multiple clients to handle Maildirs (and
IMAP server if you use offlineimap).
It works by using files' ctime instead of mtime when polling so we can
detect if the mail's flags have changed. Maildir#poll now appends
"subdir/ids" in the added and deleted arrays at each loop instead of
recreating them. Arrays are then compared to each other allowing us to
see if a mail has been updated or moved (from "cur/" to "new/" for
instance, this is what offlineimap does when you mark a read mail as
unread on an IMAP server).
The index is then updated and the display is refreshed. When a mail is
deleted it is now *really* deleted from xapian. Before, only its
location was removed but the mail could still be visible in search
results although we could'nt load its content ("An error occurred
while loading this message").
Beware that sup does NOT synchronize its modifications to the source.
Diffstat:
5 files changed, 140 insertions(+), 39 deletions(-)
diff --git a/lib/sup/index.rb b/lib/sup/index.rb
@@ -241,11 +241,11 @@ EOS
## Yield each message-id matching query
EACH_ID_PAGE = 100
- def each_id query={}
+ def each_id query={}, ignore_neg_terms = true
offset = 0
page = EACH_ID_PAGE
- xapian_query = build_xapian_query query
+ xapian_query = build_xapian_query query, ignore_neg_terms
while true
ids = run_query_ids xapian_query, offset, (offset+page)
ids.each { |id| yield id }
@@ -255,8 +255,8 @@ EOS
end
## Yield each message matching query
- def each_message query={}, &b
- each_id query do |id|
+ def each_message query={}, ignore_neg_terms = true, &b
+ each_id query, ignore_neg_terms do |id|
yield build_message(id)
end
end
@@ -301,9 +301,9 @@ EOS
## Yields (in lexicographical order) the source infos of all locations from
## the given source with the given source_info prefix
def each_source_info source_id, prefix='', &b
- prefix = mkterm :location, source_id, prefix
- each_prefixed_term prefix do |x|
- yield x[prefix.length..-1]
+ p = mkterm :location, source_id, prefix
+ each_prefixed_term p do |x|
+ yield prefix + x[p.length..-1]
end
end
@@ -593,7 +593,7 @@ EOS
end
Q = Xapian::Query
- def build_xapian_query opts
+ def build_xapian_query opts, ignore_neg_terms = true
labels = ([opts[:label]] + (opts[:labels] || [])).compact
neglabels = [:spam, :deleted, :killed].reject { |l| (labels.include? l) || opts.member?("load_#{l}".intern) }
pos_terms, neg_terms = [], []
@@ -609,7 +609,7 @@ EOS
pos_terms << Q.new(Q::OP_OR, participant_terms)
end
- neg_terms.concat(neglabels.map { |l| mkterm(:label,l) })
+ neg_terms.concat(neglabels.map { |l| mkterm(:label,l) }) if ignore_neg_terms
pos_query = Q.new(Q::OP_AND, pos_terms)
neg_query = Q.new(Q::OP_OR, neg_terms)
diff --git a/lib/sup/maildir.rb b/lib/sup/maildir.rb
@@ -20,7 +20,7 @@ class Maildir < Source
@dir = uri.path
@labels = Set.new(labels || [])
@mutex = Mutex.new
- @mtimes = { 'cur' => Time.at(0), 'new' => Time.at(0) }
+ @ctimes = { 'cur' => Time.at(0), 'new' => Time.at(0) }
end
def file_path; @dir end
@@ -87,33 +87,61 @@ class Maildir < Source
## XXX use less memory
def poll
- @mtimes.each do |d,prev_mtime|
+ added = []
+ deleted = []
+ updated = []
+ @ctimes.each do |d,prev_ctime|
subdir = File.join @dir, d
debug "polling maildir #{subdir}"
raise FatalSourceError, "#{subdir} not a directory" unless File.directory? subdir
- mtime = File.mtime subdir
- next if prev_mtime >= mtime
- @mtimes[d] = mtime
+ ctime = File.ctime subdir
+ next if prev_ctime >= ctime
+ @ctimes[d] = ctime
old_ids = benchmark(:maildir_read_index) { Enumerator.new(Index.instance, :each_source_info, self.id, "#{d}/").to_a }
- new_ids = benchmark(:maildir_read_dir) { Dir.glob("#{subdir}/*").map { |x| File.basename x }.sort }
- added = new_ids - old_ids
- deleted = old_ids - new_ids
+ new_ids = benchmark(:maildir_read_dir) { Dir.glob("#{subdir}/*").map { |x| File.join(d,File.basename(x)) }.sort }
+ added += new_ids - old_ids
+ deleted += old_ids - new_ids
debug "#{old_ids.size} in index, #{new_ids.size} in filesystem"
- debug "#{added.size} added, #{deleted.size} deleted"
+ end
- added.each_with_index do |id,i|
- yield :add,
- :info => File.join(d,id),
- :labels => @labels + maildir_labels(id) + [:inbox],
- :progress => i.to_f/(added.size+deleted.size)
+ ## find updated mails by checking if an id is in both added and
+ ## deleted arrays, meaning that its flags changed or that it has
+ ## been moved, these ids need to be removed from added and deleted
+ add_to_delete = del_to_delete = []
+ added.each do |id_add|
+ deleted.each do |id_del|
+ if maildir_data(id_add)[0] == maildir_data(id_del)[0]
+ updated.push [ id_del, id_add ]
+ add_to_delete.push id_add
+ del_to_delete.push id_del
+ end
end
+ end
+ added -= add_to_delete
+ deleted -= del_to_delete
+ debug "#{added.size} added, #{deleted.size} deleted, #{updated.size} updated"
+
+ added.each_with_index do |id,i|
+ yield :add,
+ :info => File.join(d,id),
+ :labels => @labels + maildir_labels(id) + [:inbox],
+ :progress => i.to_f/(added.size+deleted.size)
+ end
- deleted.each_with_index do |id,i|
- yield :delete,
- :info => File.join(d,id),
- :progress => (i.to_f+added.size)/(added.size+deleted.size)
- end
+ deleted.each_with_index do |id,i|
+ yield :delete,
+ :info => File.join(d,id),
+ :progress => (i.to_f+added.size)/(added.size+deleted.size)
+ end
+
+ # TODO: Fix this
+ updated.each do |id|
+ yield :update,
+ :old_info => id[0],
+ :new_info => id[1],
+ :labels => @labels + maildir_labels(id[1]),
+ :progress => 0.0
end
nil
end
@@ -159,6 +187,7 @@ private
end
def maildir_data id
+ id = File.basename id
id =~ %r{^([^:]+):([12]),([DFPRST]*)$}
[($1 || id), ($2 || "2"), ($3 || "")]
end
diff --git a/lib/sup/modes/thread-index-mode.rb b/lib/sup/modes/thread-index-mode.rb
@@ -200,6 +200,26 @@ EOS
BufferManager.draw_screen
end
+ def handle_updated_update sender, m
+ t = thread_containing(m) or return
+ l = @lines[t] or return
+ @ts_mutex.synchronize do
+ @ts.remove_message m
+ @ts.add_message m
+ end
+ Index.save_thread t
+ update_text_for_line l
+ end
+
+ def handle_location_deleted_update sender, m
+ t = thread_containing(m)
+ delete_thread t if t and t.first.id == m.id
+ @ts_mutex.synchronize do
+ @ts.delete_message m if t
+ end
+ update
+ end
+
def handle_single_message_deleted_update sender, m
@ts_mutex.synchronize do
return unless @ts.contains? m
@@ -756,6 +776,16 @@ protected
update
end
+ def delete_thread t
+ @mutex.synchronize do
+ i = @threads.index(t) or return
+ @threads.delete_at i
+ @size_widgets.delete_at i
+ @date_widgets.delete_at i
+ @tags.drop_tag_for t
+ end
+ end
+
def hide_thread t
@mutex.synchronize do
i = @threads.index(t) or return
diff --git a/lib/sup/poll.rb b/lib/sup/poll.rb
@@ -46,15 +46,23 @@ EOS
HookManager.run "before-poll"
BufferManager.flash "Polling for new messages..."
- num, numi, from_and_subj, from_and_subj_inbox, loaded_labels = @mode.poll
+ flash_msg = ""
+ num, numi, numu, numd, from_and_subj, from_and_subj_inbox, loaded_labels = @mode.poll
clear_running_totals if @should_clear_running_totals
@running_totals[:num] += num
@running_totals[:numi] += numi
+ @running_totals[:numu] += numu
+ @running_totals[:numd] += numd
@running_totals[:loaded_labels] += loaded_labels || []
- if @running_totals[:num] > 0
- BufferManager.flash "Loaded #{@running_totals[:num].pluralize 'new message'}, #{@running_totals[:numi]} to inbox. Labels: #{@running_totals[:loaded_labels].map{|l| l.to_s}.join(', ')}"
- else
+
+ flash_msg += "Loaded #{@running_totals[:num].pluralize 'new message'}, #{@running_totals[:numi]} to inbox, labels: #{@running_totals[:loaded_labels].map{|l| l.to_s}.join(', ')}. " if @running_totals[:num] > 0
+ flash_msg += "Updated #{@running_totals[:numu].pluralize 'message'}. " if @running_totals[:numu] > 0
+ flash_msg += "Deleted #{@running_totals[:numd].pluralize 'message'}." if @running_totals[:numd] > 0
+
+ if flash_msg == ""
BufferManager.flash "No new messages."
+ else
+ BufferManager.flash flash_msg
end
HookManager.run "after-poll", :num => num, :num_inbox => numi, :from_and_subj => from_and_subj, :from_and_subj_inbox => from_and_subj_inbox, :num_inbox_total_unread => lambda { Index.num_results_for :labels => [:inbox, :unread] }
@@ -94,7 +102,7 @@ EOS
end
def do_poll
- total_num = total_numi = 0
+ total_num = total_numi = total_numu = total_numd = 0
from_and_subj = []
from_and_subj_inbox = []
loaded_labels = Set.new
@@ -108,11 +116,14 @@ EOS
next
end
- num = 0
- numi = 0
+ msg = ""
+ num = numi = numu = numd = 0
poll_from source do |action,m,old_m,progress|
if action == :delete
yield "Deleting #{m.id}"
+ numd += 1
+ elsif action == :update
+ numu += 1
elsif action == :add
if old_m
new_locations = (m.locations - old_m.locations)
@@ -134,9 +145,14 @@ EOS
else fail
end
end
- yield "Found #{num} messages, #{numi} to inbox." unless num == 0
+ msg += "Found #{num} messages, #{numi} to inbox. " unless num == 0
+ msg += "Updated #{numu} messages. " unless numu == 0
+ msg += "Deleted #{numd} messages." unless numd == 0
+ yield msg unless msg == ""
total_num += num
total_numi += numi
+ total_numu += numu
+ total_numd += numd
end
loaded_labels = loaded_labels - LabelManager::HIDDEN_RESERVED_LABELS - [:inbox, :killed]
@@ -144,7 +160,7 @@ EOS
@last_poll = Time.now
@polling = false
end
- [total_num, total_numi, from_and_subj, from_and_subj_inbox, loaded_labels]
+ [total_num, total_numi, total_numu, total_numd, from_and_subj, from_and_subj_inbox, loaded_labels]
end
## like Source#poll, but yields successive Message objects, which have their
@@ -177,9 +193,29 @@ EOS
when :delete
Index.each_message :location => [source.id, args[:info]] do |m|
m.locations.delete Location.new(source, args[:info])
+<<<<<<< HEAD
yield :delete, m, [source,args[:info]], args[:progress] if block_given?
+=======
+>>>>>>> Synchronize remote modifications from a Maildir source to sup
Index.sync_message m, false
- #UpdateManager.relay self, :deleted, m
+ if m.locations.size == 0
+ yield :delete, m, [source,args[:info]] if block_given?
+ Index.delete m.id
+ UpdateManager.relay self, :location_deleted, m
+ end
+ end
+ when :update
+ Index.each_message({:location => [source.id, args[:old_info]]}, false) do |m|
+ m.locations.delete Location.new(source, args[:old_info])
+ m.locations.push Location.new(source, args[:new_info])
+ ## Update labels that might have been modified remotely
+ [:unread, :starred, :deleted].each do |l|
+ m.labels.delete l
+ end
+ m.labels += args[:labels]
+ yield :update, m
+ Index.sync_message m, true
+ UpdateManager.relay self, :updated, m
end
end
end
@@ -192,7 +228,7 @@ EOS
def handle_idle_update sender, idle_since; @should_clear_running_totals = false; end
def handle_unidle_update sender, idle_since; @should_clear_running_totals = true; clear_running_totals; end
- def clear_running_totals; @running_totals = {:num => 0, :numi => 0, :loaded_labels => Set.new}; end
+ def clear_running_totals; @running_totals = {:num => 0, :numi => 0, :numu => 0, :numd => 0, :loaded_labels => Set.new}; end
end
end
diff --git a/lib/sup/thread.rb b/lib/sup/thread.rb
@@ -387,6 +387,12 @@ class ThreadSet
m.refs.any? { |ref_id| @messages.member? ref_id }
end
+ def delete_message message
+ el = @messages[message.id]
+ return unless el.message
+ el.message = nil
+ end
+
## the heart of the threading code
def add_message message
el = @messages[message.id]