commit c358de5adfb83b94dceae912933663cd361da001
parent 735c8519f44ce452dec15cf226199a98a470d8a3
Author: Gaute Hope <eg@gaute.vetsj.com>
Date: Thu, 10 Oct 2013 09:18:30 +0200
Merge maildir-sync
Diffstat:
17 files changed, 556 insertions(+), 91 deletions(-)
diff --git a/bin/sup b/bin/sup
@@ -153,6 +153,7 @@ Index.lock_interactively or exit
begin
Redwood::start
Index.load
+ Redwood::check_syncback_settings
Index.start_sync_worker unless $opts[:no_threads]
$die = false
diff --git a/bin/sup-add b/bin/sup-add
@@ -30,6 +30,7 @@ Options are:
EOS
opt :archive, "Automatically archive all new messages from these sources."
opt :unusual, "Do not automatically poll these sources for new messages."
+ opt :sync_back, "Synchronize status flags back into messages, defaults to true (Maildir sources only).", :default => true
opt :labels, "A comma-separated set of labels to apply to all messages from this source", :type => String
opt :force_new, "Create a new account for this source, even if one already exists."
opt :force_account, "Reuse previously defined account user@hostname.", :type => String
@@ -99,7 +100,7 @@ begin
source =
case parsed_uri.scheme
when "maildir"
- Redwood::Maildir.new uri, !$opts[:unusual], $opts[:archive], nil, labels
+ Redwood::Maildir.new uri, !$opts[:unusual], $opts[:archive], $opts[:sync_back], nil, labels
when "mbox"
Redwood::MBox.new uri, !$opts[:unusual], $opts[:archive], nil, labels
when nil
diff --git a/bin/sup-config b/bin/sup-config
@@ -88,6 +88,8 @@ def add_source
usual = axe_yes "Does this source ever receive new messages?", "y"
archive = usual ? axe_yes("Should new messages be automatically archived? (I.e. not appear in your inbox, though still be accessible via search.)") : false
+ sync_back = (type == :maildir) ? axe_yes("Should the original Maildir messages be modified to reflect changes like read status, starred messages, etc.?", "y") : false
+
labels_str = axe("Enter any labels to be automatically added to all messages from this source, separated by spaces (or 'none')", default_labels.join(","))
labels = if labels_str =~ /^\s*none\s*$/i
@@ -99,6 +101,7 @@ def add_source
cmd = build_cmd "sup-add"
cmd += " --unusual" unless usual
cmd += " --archive" if archive
+ cmd += " --no-sync-back" unless sync_back
cmd += " --labels=#{labels.join(',')}" if labels && !labels.empty?
cmd += " #{uri}"
diff --git a/bin/sup-sync-back-maildir b/bin/sup-sync-back-maildir
@@ -0,0 +1,116 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'trollop'
+require "sup"
+
+opts = Trollop::options do
+ version "sup-sync-back-maildir (sup #{Redwood::VERSION})"
+ banner <<EOS
+Export Xapian entries to Maildir sources on disk.
+
+This script parses the Xapian entries for a given Maildir source and
+renames e-mail files on disk according to the labels stored in the
+index. It will hence export all the changes you made in Sup to your
+Maildirs so it can be propagated to your IMAP server with offlineimap
+for instance.
+
+It also merges some Maildir flags that were not supported by Sup such
+as R (replied) and P (passed, forwarded), for instance suppose you
+have an e-mail file like this: foo_bar:2,FRS (flags are favorite,
+replied, seen) and its Xapian entry has labels 'starred', the merge
+operation will add the 'replied' label to the Xapian entry.
+
+If you choose not to merge you will lose information because in the
+previous example the file will be renamed to foo_bar:2,FS.
+
+Running this script is *strongly* recommended when setting the
+"sync_back_to_maildir" option from false to true.
+
+Usage:
+ sup-sync-back-maildir [options] <source>*
+
+where <source>* is source URIs. If no source is given, the default
+behavior is to sync back all Maildir sources that have not disabled
+sync back using the configuration parameter sync_back = false.
+
+Options include:
+EOS
+ opt :no_confirm, "Don't ask for confirmation before synchronizing", :default => false, :short => "n"
+ opt :no_merge, "Don't merge new supported Maildir flags (R and P)", :default => false, :short => "m"
+ opt :list_sources, "List your Maildir sources and exit", :default => false, :short => "l"
+end
+
+def die msg
+ $stderr.puts "Error: #{msg}"
+ exit(-1)
+end
+
+Redwood::start true
+index = Redwood::Index.init
+index.lock_interactively or exit
+index.load
+
+## Force sync_back_to_maildir option otherwise nothing will happen
+$config[:sync_back_to_maildir] = true
+
+begin
+ sync_performed = File.readlines Redwood::SYNC_OK_FN
+ sources = []
+
+ ## Try to find out sources given in parameters
+ sources = ARGV.map do |uri|
+ s = Redwood::SourceManager.source_for(uri) or die "unknown source: #{uri}. Did you add it with sup-add first?"
+ s.is_a?(Redwood::Maildir) or die "#{uri} is not a Maildir source."
+ s.sync_back_enabled? or die "#{uri} has disabled sync back - check your configuration."
+ s
+ end unless opts[:list_sources]
+
+ ## Otherwise, check all sources in sources.yaml
+ if sources.empty? or opts[:list_sources] == true
+ sources = Redwood::SourceManager.usual_sources.select do |s|
+ s.is_a? Redwood::Maildir and s.sync_back_enabled?
+ end
+ end
+
+ if opts[:list_sources] == true
+ sources.each do |s|
+ puts "id: #{s.id}, uri: #{s.uri}"
+ end
+ else
+ sources.each do |s|
+ if opts[:no_confirm] == false
+ print "Are you sure you want to synchronize '#{s.uri}'? (Y/n) "
+ next if STDIN.gets.chomp.downcase == 'n'
+ end
+
+ infos = Enumerator.new(index, :each_source_info, s.id).to_a
+ counter = 0
+ infos.each do |info|
+ print "\rSynchronizing '#{s.uri}'... #{((counter += 1)/infos.size.to_f*100).to_i}%"
+ index.each_message({:location => [s.id, info]}, false) do |m|
+ if opts[:no_merge] == false
+ m.merge_labels_from_locations [:replied, :forwarded]
+ end
+
+ if Redwood::Index.message_joining_killed? m
+ m.labels += [:killed]
+ end
+
+ index.save_message m
+ end
+ end
+ print "\n"
+ sync_performed << s.uri
+ end
+ ## Write a flag file to tell sup that the synchronization has been performed
+ File.open(Redwood::SYNC_OK_FN, 'w') {|f| f.write(sync_performed.join("\n")) }
+ end
+rescue Exception => e
+ File.open("sup-exception-log.txt", "w") { |f| f.puts e.backtrace }
+ raise
+ensure
+ index.save_index
+ Redwood::finish
+ index.unlock
+end
diff --git a/lib/sup.rb b/lib/sup.rb
@@ -59,10 +59,12 @@ module Redwood
HOOK_DIR = File.join(BASE_DIR, "hooks")
SEARCH_FN = File.join(BASE_DIR, "searches.txt")
LOG_FN = File.join(BASE_DIR, "log")
+ SYNC_OK_FN = File.join(BASE_DIR, "sync-back-ok")
YAML_DOMAIN = "supmua.org"
LEGACY_YAML_DOMAIN = "masanjin.net"
YAML_DATE = "2006-10-01"
+ MAILDIR_SYNC_CHECK_SKIPPED = 'SKIPPED'
## record exceptions thrown in threads nicely
@exceptions = []
@@ -157,7 +159,7 @@ module Redwood
SourceManager SearchManager IdleManager).map { |x| Redwood.const_get x.to_sym }
end
- def start
+ def start bypass_sync_check = false
managers.each { |x| fail "#{x} already instantiated" if x.instantiated? }
FileUtils.mkdir_p Redwood::BASE_DIR
@@ -173,6 +175,61 @@ module Redwood
Redwood::SearchManager.init Redwood::SEARCH_FN
managers.each { |x| x.init unless x.instantiated? }
+
+ return if bypass_sync_check
+
+ if $config[:sync_back_to_maildir]
+ if not File.exists? Redwood::SYNC_OK_FN
+ Redwood.warn_syncback <<EOS
+It appears that the "sync_back_to_maildir" option has been changed
+from false to true since the last execution of sup.
+EOS
+ $stderr.puts <<EOS
+
+Should I complain about this again? (Y/n)
+EOS
+ File.open(Redwood::SYNC_OK_FN, 'w') {|f| f.write(Redwood::MAILDIR_SYNC_CHECK_SKIPPED) } if STDIN.gets.chomp.downcase == 'n'
+ elsif not $config[:sync_back_to_maildir] and File.exists? Redwood::SYNC_OK_FN
+ File.delete(Redwood::SYNC_OK_FN)
+ end
+ end
+ end
+
+ def check_syncback_settings
+ active_sync_sources = File.readlines(Redwood::SYNC_OK_FN).collect { |s| s.strip }
+ return if active_sync_sources.length == 1 and active_sync_sources[0] == Redwood::MAILDIR_SYNC_CHECK_SKIPPED
+ sources = SourceManager.sources
+ newly_synced = sources.select { |s| s.is_a? Maildir and s.sync_back_enabled? and not active_sync_sources.include? s.uri }
+ unless newly_synced.empty?
+ Redwood.warn_syncback <<EOS
+It appears that the option "sync_back" of the following source(s)
+has been changed from false to true since the last execution of
+sup:
+
+#{newly_synced.join("\n")}
+EOS
+ end
+ end
+
+ def self.warn_syncback details
+ $stderr.puts <<EOS
+WARNING
+-------
+
+#{details}
+
+It is *strongly* recommended that you run "sup-sync-back-maildir"
+before continuing, otherwise you might lose informations in your
+Xapian index.
+
+This script should be executed each time the "sync_back_to_maildir" is
+changed from false to true.
+
+Please run "sup-sync-back-maildir -h" to see why it is useful.
+
+Are you really sure you want to continue? (y/N)
+EOS
+ abort "Aborted" unless STDIN.gets.chomp.downcase == 'y'
end
def finish
@@ -262,7 +319,8 @@ EOM
:wrap_width => 0,
:slip_rows => 0,
:col_jump => 2,
- :stem_language => "english"
+ :stem_language => "english",
+ :sync_back_to_maildir => false
}
if File.exists? filename
config = Redwood::load_yaml_obj filename
@@ -303,7 +361,8 @@ EOM
end
module_function :save_yaml_obj, :load_yaml_obj, :start, :finish,
- :report_broken_sources, :load_config, :managers
+ :report_broken_sources, :load_config, :managers,
+ :check_syncback_settings
end
require 'sup/version'
diff --git a/lib/sup/index.rb b/lib/sup/index.rb
@@ -168,6 +168,32 @@ EOS
matchset.matches_estimated
end
+ ## check if a message is part of a killed thread
+ ## (warning: duplicates code below)
+ ## NOTE: We can be more efficient if we assume every
+ ## killed message that hasn't been initially added
+ ## to the indexi s this way
+ def message_joining_killed? m
+ return false unless doc = find_doc(m.id)
+ queue = doc.value(THREAD_VALUENO).split(',')
+ seen_threads = Set.new
+ seen_messages = Set.new [m.id]
+ while not queue.empty?
+ thread_id = queue.pop
+ next if seen_threads.member? thread_id
+ return true if thread_killed?(thread_id)
+ seen_threads << thread_id
+ docs = term_docids(mkterm(:thread, thread_id)).map { |x| @xapian.document x }
+ docs.each do |doc|
+ msgid = doc.value MSGID_VALUENO
+ next if seen_messages.member? msgid
+ seen_messages << msgid
+ queue.concat doc.value(THREAD_VALUENO).split(',')
+ end
+ end
+ false
+ end
+
## yield all messages in the thread containing 'm' by repeatedly
## querying the index. yields pairs of message ids and
## message-building lambdas, so that building an unwanted message
@@ -248,11 +274,11 @@ EOS
## Yield each message-id matching query
EACH_ID_PAGE = 100
- def each_id query={}
+ def each_id query={}, ignore_neg_terms = true
offset = 0
page = EACH_ID_PAGE
- xapian_query = build_xapian_query query
+ xapian_query = build_xapian_query query, ignore_neg_terms
while true
ids = run_query_ids xapian_query, offset, (offset+page)
ids.each { |id| yield id }
@@ -262,8 +288,12 @@ EOS
end
## Yield each message matching query
- def each_message query={}, &b
- each_id query do |id|
+ ## The ignore_neg_terms parameter is used to display result even if
+ ## it contains "forbidden" labels such as :deleted, it is used in
+ ## Poll#poll_from when we need to get the location of a message that
+ ## may contain these labels
+ def each_message query={}, ignore_neg_terms = true, &b
+ each_id query, ignore_neg_terms do |id|
yield build_message(id)
end
end
@@ -313,9 +343,9 @@ EOS
## Yields (in lexicographical order) the source infos of all locations from
## the given source with the given source_info prefix
def each_source_info source_id, prefix='', &b
- prefix = mkterm :location, source_id, prefix
- each_prefixed_term prefix do |x|
- yield x[prefix.length..-1]
+ p = mkterm :location, source_id, prefix
+ each_prefixed_term p do |x|
+ yield prefix + x[p.length..-1]
end
end
@@ -500,14 +530,18 @@ EOS
query
end
+ def save_message m
+ if @sync_worker
+ @sync_queue << m
+ else
+ update_message_state m
+ end
+ m.clear_dirty
+ end
+
def save_thread t
t.each_dirty_message do |m|
- if @sync_worker
- @sync_queue << m
- else
- update_message_state m
- end
- m.clear_dirty
+ save_message m
end
end
@@ -614,7 +648,7 @@ EOS
end
Q = Xapian::Query
- def build_xapian_query opts
+ def build_xapian_query opts, ignore_neg_terms = true
labels = ([opts[:label]] + (opts[:labels] || [])).compact
neglabels = [:spam, :deleted, :killed].reject { |l| (labels.include? l) || opts.member?("load_#{l}".intern) }
pos_terms, neg_terms = [], []
@@ -630,7 +664,7 @@ EOS
pos_terms << Q.new(Q::OP_OR, participant_terms)
end
- neg_terms.concat(neglabels.map { |l| mkterm(:label,l) })
+ neg_terms.concat(neglabels.map { |l| mkterm(:label,l) }) if ignore_neg_terms
pos_query = Q.new(Q::OP_AND, pos_terms)
neg_query = Q.new(Q::OP_OR, neg_terms)
@@ -643,6 +677,10 @@ EOS
end
def sync_message m, overwrite
+ ## TODO: we should not save the message if the sync_back failed
+ ## since it would overwrite the location field
+ m.sync_back
+
doc = synchronize { find_doc(m.id) }
existed = doc != nil
doc ||= Xapian::Document.new
diff --git a/lib/sup/label.rb b/lib/sup/label.rb
@@ -7,10 +7,10 @@ class LabelManager
## labels that have special semantics. user will be unable to
## add/remove these via normal label mechanisms.
- RESERVED_LABELS = [ :starred, :spam, :draft, :unread, :killed, :sent, :deleted, :inbox, :attachment ]
+ RESERVED_LABELS = [ :starred, :spam, :draft, :unread, :killed, :sent, :deleted, :inbox, :attachment, :forwarded, :replied ]
## labels that will typically be hidden from the user
- HIDDEN_RESERVED_LABELS = [ :starred, :unread, :attachment ]
+ HIDDEN_RESERVED_LABELS = [ :starred, :unread, :attachment, :forwarded, :replied ]
def initialize fn
@fn = fn
diff --git a/lib/sup/maildir.rb b/lib/sup/maildir.rb
@@ -1,4 +1,5 @@
require 'uri'
+require 'set'
module Redwood
@@ -7,8 +8,8 @@ class Maildir < Source
MYHOSTNAME = Socket.gethostname
## remind me never to use inheritance again.
- yaml_properties :uri, :usual, :archived, :id, :labels
- def initialize uri, usual=true, archived=false, id=nil, labels=[]
+ yaml_properties :uri, :usual, :archived, :sync_back, :id, :labels
+ def initialize uri, usual=true, archived=false, sync_back=true, id=nil, labels=[]
super uri, usual, archived, id
@expanded_uri = Source.expand_filesystem_uri(uri)
uri = URI(@expanded_uri)
@@ -17,16 +18,28 @@ class Maildir < Source
raise ArgumentError, "maildir URI cannot have a host: #{uri.host}" if uri.host
raise ArgumentError, "maildir URI must have a path component" unless uri.path
+ @sync_back = sync_back
+ # sync by default if not specified
+ @sync_back = true if @sync_back.nil?
+
@dir = uri.path
@labels = Set.new(labels || [])
@mutex = Mutex.new
- @mtimes = { 'cur' => Time.at(0), 'new' => Time.at(0) }
+ @ctimes = { 'cur' => Time.at(0), 'new' => Time.at(0) }
end
def file_path; @dir end
def self.suggest_labels_for path; [] end
def is_source_for? uri; super || (uri == @expanded_uri); end
+ def supported_labels?
+ [:draft, :starred, :forwarded, :replied, :unread, :deleted]
+ end
+
+ def sync_back_enabled?
+ @sync_back
+ end
+
def store_message date, from_email, &block
stored = false
new_fn = new_maildir_basefn + ':2,S'
@@ -71,6 +84,14 @@ class Maildir < Source
with_file_for(id) { |f| RMail::Parser.read f }
end
+ def sync_back id, labels
+ synchronize do
+ debug "syncing back maildir message #{id} with flags #{labels.to_a}"
+ flags = maildir_reconcile_flags id, labels
+ maildir_mark_file id, flags
+ end
+ end
+
def raw_header id
ret = ""
with_file_for(id) do |f|
@@ -87,41 +108,78 @@ class Maildir < Source
## XXX use less memory
def poll
- @mtimes.each do |d,prev_mtime|
+ added = []
+ deleted = []
+ updated = []
+ @ctimes.each do |d,prev_ctime|
subdir = File.join @dir, d
debug "polling maildir #{subdir}"
raise FatalSourceError, "#{subdir} not a directory" unless File.directory? subdir
- mtime = File.mtime subdir
- next if prev_mtime >= mtime
- @mtimes[d] = mtime
+ ctime = File.ctime subdir
+ next if prev_ctime >= ctime
+ @ctimes[d] = ctime
old_ids = benchmark(:maildir_read_index) { Enumerator.new(Index.instance, :each_source_info, self.id, "#{d}/").to_a }
- new_ids = benchmark(:maildir_read_dir) { Dir.glob("#{subdir}/*").map { |x| File.basename x }.sort }
- added = new_ids - old_ids
- deleted = old_ids - new_ids
+ new_ids = benchmark(:maildir_read_dir) { Dir.glob("#{subdir}/*").map { |x| File.join(d,File.basename(x)) }.sort }
+ added += new_ids - old_ids
+ deleted += old_ids - new_ids
debug "#{old_ids.size} in index, #{new_ids.size} in filesystem"
- debug "#{added.size} added, #{deleted.size} deleted"
+ end
- added.each_with_index do |id,i|
- yield :add,
- :info => File.join(d,id),
- :labels => @labels + maildir_labels(id) + [:inbox],
- :progress => i.to_f/(added.size+deleted.size)
- end
+ ## find updated mails by checking if an id is in both added and
+ ## deleted arrays, meaning that its flags changed or that it has
+ ## been moved, these ids need to be removed from added and deleted
+ add_to_delete = del_to_delete = []
+ map = Hash.new { |hash, key| hash[key] = [] }
+ deleted.each do |id_del|
+ map[maildir_data(id_del)[0]].push id_del
+ end
+ added.each do |id_add|
+ map[maildir_data(id_add)[0]].each do |id_del|
+ updated.push [ id_del, id_add ]
+ add_to_delete.push id_add
+ del_to_delete.push id_del
+ end
+ end
+ added -= add_to_delete
+ deleted -= del_to_delete
+ debug "#{added.size} added, #{deleted.size} deleted, #{updated.size} updated"
+ total_size = added.size+deleted.size+updated.size
- deleted.each_with_index do |id,i|
- yield :delete,
- :info => File.join(d,id),
- :progress => (i.to_f+added.size)/(added.size+deleted.size)
- end
+ added.each_with_index do |id,i|
+ yield :add,
+ :info => id,
+ :labels => @labels + maildir_labels(id) + [:inbox],
+ :progress => i.to_f/total_size
+ end
+
+ deleted.each_with_index do |id,i|
+ yield :delete,
+ :info => id,
+ :progress => (i.to_f+added.size)/total_size
+ end
+
+ updated.each_with_index do |id,i|
+ yield :update,
+ :old_info => id[0],
+ :new_info => id[1],
+ :labels => @labels + maildir_labels(id[1]),
+ :progress => (i.to_f+added.size+deleted.size)/total_size
end
nil
end
+ def labels? id
+ maildir_labels id
+ end
+
def maildir_labels id
(seen?(id) ? [] : [:unread]) +
(trashed?(id) ? [:deleted] : []) +
- (flagged?(id) ? [:starred] : [])
+ (flagged?(id) ? [:starred] : []) +
+ (passed?(id) ? [:forwarded] : []) +
+ (replied?(id) ? [:replied] : []) +
+ (draft?(id) ? [:draft] : [])
end
def draft? id; maildir_data(id)[2].include? "D"; end
@@ -131,13 +189,6 @@ class Maildir < Source
def seen? id; maildir_data(id)[2].include? "S"; end
def trashed? id; maildir_data(id)[2].include? "T"; end
- def mark_draft id; maildir_mark_file id, "D" unless draft? id; end
- def mark_flagged id; maildir_mark_file id, "F" unless flagged? id; end
- def mark_passed id; maildir_mark_file id, "P" unless passed? id; end
- def mark_replied id; maildir_mark_file id, "R" unless replied? id; end
- def mark_seen id; maildir_mark_file id, "S" unless seen? id; end
- def mark_trashed id; maildir_mark_file id, "T" unless trashed? id; end
-
def valid? id
File.exists? File.join(@dir, id)
end
@@ -159,25 +210,47 @@ private
end
def maildir_data id
- id =~ %r{^([^:]+):([12]),([DFPRST]*)$}
+ id = File.basename id
+ # Flags we recognize are DFPRST
+ id =~ %r{^([^:]+):([12]),([A-Za-z]*)$}
[($1 || id), ($2 || "2"), ($3 || "")]
end
- ## not thread-safe on msg
- def maildir_mark_file msg, flag
- orig_path = @ids_to_fns[msg]
- orig_base, orig_fn = File.split(orig_path)
- new_base = orig_base.slice(0..-4) + 'cur'
- tmp_base = orig_base.slice(0..-4) + 'tmp'
- md_base, md_ver, md_flags = maildir_data msg
- md_flags += flag; md_flags = md_flags.split(//).sort.join.squeeze
- new_path = File.join new_base, "#{md_base}:#{md_ver},#{md_flags}"
- tmp_path = File.join tmp_base, "#{md_base}:#{md_ver},#{md_flags}"
- File.link orig_path, tmp_path
- File.unlink orig_path
- File.link tmp_path, new_path
- File.unlink tmp_path
- @ids_to_fns[msg] = new_path
+ def maildir_reconcile_flags id, labels
+ new_flags = Set.new( maildir_data(id)[2].each_char )
+
+ # Set flags based on labels for the six flags we recognize
+ if labels.member? :draft then new_flags.add?( "D" ) else new_flags.delete?( "D" ) end
+ if labels.member? :starred then new_flags.add?( "F" ) else new_flags.delete?( "F" ) end
+ if labels.member? :forwarded then new_flags.add?( "P" ) else new_flags.delete?( "P" ) end
+ if labels.member? :replied then new_flags.add?( "R" ) else new_flags.delete?( "R" ) end
+ if not labels.member? :unread then new_flags.add?( "S" ) else new_flags.delete?( "S" ) end
+ if labels.member? :deleted or labels.member? :killed then new_flags.add?( "T" ) else new_flags.delete?( "T" ) end
+
+ ## Flags must be stored in ASCII order according to Maildir
+ ## documentation
+ new_flags.to_a.sort.join
+ end
+
+ def maildir_mark_file orig_path, flags
+ @mutex.synchronize do
+ new_base = (flags.include?("S")) ? "cur" : "new"
+ md_base, md_ver, md_flags = maildir_data orig_path
+
+ return if md_flags == flags
+
+ new_loc = File.join new_base, "#{md_base}:#{md_ver},#{flags}"
+ orig_path = File.join @dir, orig_path
+ new_path = File.join @dir, new_loc
+ tmp_path = File.join @dir, "tmp", "#{md_base}:#{md_ver},#{flags}"
+
+ File.link orig_path, tmp_path
+ File.unlink orig_path
+ File.link tmp_path, new_path
+ File.unlink tmp_path
+
+ new_loc
+ end
end
end
diff --git a/lib/sup/message.rb b/lib/sup/message.rb
@@ -291,6 +291,32 @@ EOS
location.each_raw_message_line &b
end
+ def sync_back
+ @locations.map { |l| l.sync_back @labels, self }.any? do
+ UpdateManager.relay self, :updated, self
+ end
+ end
+
+ def merge_labels_from_locations merge_labels
+ ## Get all labels from all locations
+ location_labels = Set.new([])
+
+ @locations.each do |l|
+ if l.valid?
+ location_labels = location_labels.union(l.labels?)
+ end
+ end
+
+ ## Add to the message labels the intersection between all location
+ ## labels and those we want to merge
+ location_labels = location_labels.intersection(merge_labels.to_set)
+
+ if not location_labels.empty?
+ @labels = @labels.union(location_labels)
+ @dirty = true
+ end
+ end
+
## returns all the content from a message that will be indexed
def indexable_content
load_from_source!
@@ -704,6 +730,24 @@ class Location
source.raw_message info
end
+ def sync_back labels, message
+ synced = false
+ return synced unless sync_back_enabled? and valid?
+ source.synchronize do
+ new_info = source.sync_back(@info, labels)
+ if new_info
+ @info = new_info
+ Index.sync_message message, true
+ synced = true
+ end
+ end
+ synced
+ end
+
+ def sync_back_enabled?
+ source.respond_to? :sync_back and $config[:sync_back_to_maildir] and source.sync_back_enabled?
+ end
+
## much faster than raw_message
def each_raw_message_line &b
source.each_raw_message_line info, &b
@@ -717,6 +761,10 @@ class Location
source.valid? info
end
+ def labels?
+ source.labels? info
+ end
+
def == o
o.source.id == source.id and o.info == info
end
diff --git a/lib/sup/modes/forward_mode.rb b/lib/sup/modes/forward_mode.rb
@@ -7,9 +7,10 @@ class ForwardMode < EditMessageMode
"From" => AccountManager.default_account.full_address,
}
+ @m = opts[:message]
header["Subject"] =
- if opts[:message]
- "Fwd: " + opts[:message].subj
+ if @m
+ "Fwd: " + @m.subj
elsif opts[:attachments]
"Fwd: " + opts[:attachments].keys.join(", ")
end
@@ -19,8 +20,8 @@ class ForwardMode < EditMessageMode
header["Bcc"] = opts[:bcc].map { |p| p.full_address }.join(", ") if opts[:bcc]
body =
- if opts[:message]
- forward_body_lines(opts[:message])
+ if @m
+ forward_body_lines @m
elsif opts[:attachments]
["Note: #{opts[:attachments].size.pluralize 'attachment'}."]
end
@@ -68,6 +69,14 @@ protected
m.quotable_header_lines + [""] + m.quotable_body_lines +
["--- End forwarded message ---"]
end
+
+ def send_message
+ return unless super # super returns true if the mail has been sent
+ if @m
+ @m.add_label :forwarded
+ Index.save_message @m
+ end
+ end
end
end
diff --git a/lib/sup/modes/reply_mode.rb b/lib/sup/modes/reply_mode.rb
@@ -217,6 +217,12 @@ protected
update
end
end
+
+ def send_message
+ return unless super # super returns true if the mail has been sent
+ @m.add_label :replied
+ Index.save_message @m
+ end
end
end
diff --git a/lib/sup/modes/thread_index_mode.rb b/lib/sup/modes/thread_index_mode.rb
@@ -200,6 +200,26 @@ EOS
BufferManager.draw_screen
end
+ def handle_updated_update sender, m
+ t = thread_containing(m) or return
+ l = @lines[t] or return
+ @ts_mutex.synchronize do
+ @ts.delete_message m
+ @ts.add_message m
+ end
+ Index.save_thread t
+ update_text_for_line l
+ end
+
+ def handle_location_deleted_update sender, m
+ t = thread_containing(m)
+ delete_thread t if t and t.first.id == m.id
+ @ts_mutex.synchronize do
+ @ts.delete_message m if t
+ end
+ update
+ end
+
def handle_single_message_deleted_update sender, m
@ts_mutex.synchronize do
return unless @ts.contains? m
@@ -755,6 +775,16 @@ protected
update
end
+ def delete_thread t
+ @mutex.synchronize do
+ i = @threads.index(t) or return
+ @threads.delete_at i
+ @size_widgets.delete_at i
+ @date_widgets.delete_at i
+ @tags.drop_tag_for t
+ end
+ end
+
def hide_thread t
@mutex.synchronize do
i = @threads.index(t) or return
diff --git a/lib/sup/modes/thread_view_mode.rb b/lib/sup/modes/thread_view_mode.rb
@@ -248,6 +248,8 @@ EOS
sm.puts m.raw_message
end
raise SendmailCommandFailed, "Couldn't execute #{cmd}" unless $? == 0
+ m.add_label :forwarded
+ Index.save_message m
rescue SystemCallError, SendmailCommandFailed => e
warn "problem sending mail: #{e.message}"
BufferManager.flash "Problem sending mail: #{e.message}"
diff --git a/lib/sup/poll.rb b/lib/sup/poll.rb
@@ -25,6 +25,9 @@ Variables:
num_total: the total number of messages
num_inbox_total: the total number of new messages in the inbox.
num_inbox_total_unread: the total number of unread messages in the inbox
+ num_updated: the total number of updated messages
+ num_deleted: the total number of deleted messages
+ labels: the labels that were applied
from_and_subj: an array of (from email address, subject) pairs
from_and_subj_inbox: an array of (from email address, subject) pairs for
only those messages appearing in the inbox
@@ -52,23 +55,32 @@ EOS
BufferManager.flash "Polling for new messages..."
end
- num, numi, from_and_subj, from_and_subj_inbox, loaded_labels = @mode.poll
+ num, numi, numu, numd, from_and_subj, from_and_subj_inbox, loaded_labels = @mode.poll
clear_running_totals if @should_clear_running_totals
@running_totals[:num] += num
@running_totals[:numi] += numi
+ @running_totals[:numu] += numu
+ @running_totals[:numd] += numd
@running_totals[:loaded_labels] += loaded_labels || []
if HookManager.enabled? "after-poll"
hook_args = { :num => num, :num_inbox => numi,
:num_total => @running_totals[:num], :num_inbox_total => @running_totals[:numi],
+ :num_updated => @running_totals[:numu],
+ :num_deleted => @running_totals[:numd],
+ :labels => @running_totals[:loaded_labels],
:from_and_subj => from_and_subj, :from_and_subj_inbox => from_and_subj_inbox,
:num_inbox_total_unread => lambda { Index.num_results_for :labels => [:inbox, :unread] } }
HookManager.run("after-poll", hook_args)
else
if @running_totals[:num] > 0
- BufferManager.flash "Loaded #{@running_totals[:num].pluralize 'new message'}, #{@running_totals[:numi]} to inbox. Labels: #{@running_totals[:loaded_labels].map{|l| l.to_s}.join(', ')}"
+ flash_msg = "Loaded #{@running_totals[:num].pluralize 'new message'}, #{@running_totals[:numi]} to inbox. " if @running_totals[:num] > 0
+ flash_msg += "Updated #{@running_totals[:numu].pluralize 'message'}. " if @running_totals[:numu] > 0
+ flash_msg += "Deleted #{@running_totals[:numd].pluralize 'message'}. " if @running_totals[:numd] > 0
+ flash_msg += "Labels: #{@running_totals[:loaded_labels].map{|l| l.to_s}.join(', ')}." if @running_totals[:loaded_labels].size > 0
+ BufferManager.flash flash_msg
else
BufferManager.flash "No new messages."
end
@@ -115,7 +127,7 @@ EOS
end
def do_poll
- total_num = total_numi = 0
+ total_num = total_numi = total_numu = total_numd = 0
from_and_subj = []
from_and_subj_inbox = []
loaded_labels = Set.new
@@ -129,16 +141,23 @@ EOS
next
end
- num = 0
- numi = 0
+ msg = ""
+ num = numi = numu = numd = 0
poll_from source do |action,m,old_m,progress|
if action == :delete
yield "Deleting #{m.id}"
+ loaded_labels.merge m.labels
+ numd += 1
+ elsif action == :update
+ yield "Message at #{m.source_info} is an update of an old message. Updating labels from #{old_m.labels.to_a * ','} => #{m.labels.to_a * ','}"
+ loaded_labels.merge m.labels
+ numu += 1
elsif action == :add
if old_m
new_locations = (m.locations - old_m.locations)
if not new_locations.empty?
- yield "Message at #{new_locations[0].info} is an update of an old message. Updating labels from #{old_m.labels.to_a * ','} => #{m.labels.to_a * ','}"
+ yield "Message at #{new_locations[0].info} has changed its source location. Updating labels from #{old_m.labels.to_a * ','} => #{m.labels.to_a * ','}"
+ numu += 1
else
yield "Skipping already-imported message at #{m.locations[-1].info}"
end
@@ -155,25 +174,29 @@ EOS
else fail
end
end
- yield "Found #{num} messages, #{numi} to inbox." unless num == 0
+ msg += "Found #{num} messages, #{numi} to inbox. " unless num == 0
+ msg += "Updated #{numu} messages. " unless numu == 0
+ msg += "Deleted #{numd} messages." unless numd == 0
+ yield msg unless msg == ""
total_num += num
total_numi += numi
+ total_numu += numu
+ total_numd += numd
end
loaded_labels = loaded_labels - LabelManager::HIDDEN_RESERVED_LABELS - [:inbox, :killed]
yield "Done polling; loaded #{total_num} new messages total"
@last_poll = Time.now
end
- [total_num, total_numi, from_and_subj, from_and_subj_inbox, loaded_labels]
+ [total_num, total_numi, total_numu, total_numd, from_and_subj, from_and_subj_inbox, loaded_labels]
end
## like Source#poll, but yields successive Message objects, which have their
## labels and locations set correctly. The Messages are saved to or removed
## from the index after being yielded.
def poll_from source, opts={}
- debug "trying to acquire poll lock for: #{source}.."
- if source.poll_lock.try_lock
- debug "lock acquired for: #{source}."
+ debug "trying to acquire poll lock for: #{source}..."
+ if source.try_lock
begin
source.poll do |sym, args|
case sym
@@ -191,18 +214,40 @@ EOS
yield :add, m, old_m, args[:progress] if block_given?
Index.sync_message m, true
+ if Index.message_joining_killed? m
+ m.labels += [:killed]
+ Index.sync_message m, true
+ end
+
## We need to add or unhide the message when it either did not exist
## before at all or when it was updated. We do *not* add/unhide when
## the same message was found at a different location
- if !old_m or not old_m.locations.member? m.location
+ if old_m
+ UpdateManager.relay self, :updated, m
+ elsif !old_m or not old_m.locations.member? m.location
UpdateManager.relay self, :added, m
end
when :delete
- Index.each_message :location => [source.id, args[:info]] do |m|
+ Index.each_message({:location => [source.id, args[:info]]}, false) do |m|
m.locations.delete Location.new(source, args[:info])
- yield :delete, m, [source,args[:info]], args[:progress] if block_given?
Index.sync_message m, false
- #UpdateManager.relay self, :deleted, m
+ if m.locations.size == 0
+ yield :delete, m, [source,args[:info]], args[:progress] if block_given?
+ Index.delete m.id
+ UpdateManager.relay self, :location_deleted, m
+ end
+ end
+ when :update
+ Index.each_message({:location => [source.id, args[:old_info]]}, false) do |m|
+ old_m = Index.build_message m.id
+ m.locations.delete Location.new(source, args[:old_info])
+ m.locations.push Location.new(source, args[:new_info])
+ ## Update labels that might have been modified remotely
+ m.labels -= source.supported_labels?
+ m.labels += args[:labels]
+ yield :update, m, old_m if block_given?
+ Index.sync_message m, true
+ UpdateManager.relay self, :updated, m
end
end
end
@@ -212,7 +257,7 @@ EOS
ensure
source.go_idle
- source.poll_lock.unlock
+ source.unlock
end
else
debug "source #{source} is already being polled."
@@ -221,7 +266,7 @@ EOS
def handle_idle_update sender, idle_since; @should_clear_running_totals = false; end
def handle_unidle_update sender, idle_since; @should_clear_running_totals = true; clear_running_totals; end
- def clear_running_totals; @running_totals = {:num => 0, :numi => 0, :loaded_labels => Set.new}; end
+ def clear_running_totals; @running_totals = {:num => 0, :numi => 0, :numu => 0, :numd => 0, :loaded_labels => Set.new}; end
end
end
diff --git a/lib/sup/sent.rb b/lib/sup/sent.rb
@@ -27,7 +27,7 @@ class SentManager
def write_sent_message date, from_email, &block
::Thread.new do
debug "store the sent message (locking sent source..)"
- @source.poll_lock.synchronize do
+ @source.synchronize do
@source.store_message date, from_email, &block
end
PollManager.poll_from @source
diff --git a/lib/sup/source.rb b/lib/sup/source.rb
@@ -1,4 +1,5 @@
require "sup/rfc2047"
+require "monitor"
module Redwood
@@ -54,7 +55,7 @@ class Source
bool_accessor :usual, :archived
attr_reader :uri
- attr_accessor :id, :poll_lock
+ attr_accessor :id
def initialize uri, usual=true, archived=false, id=nil
raise ArgumentError, "id must be an integer: #{id.inspect}" unless id.is_a? Fixnum if id
@@ -64,7 +65,7 @@ class Source
@archived = archived
@id = id
- @poll_lock = Mutex.new
+ @poll_lock = Monitor.new
end
## overwrite me if you have a disk incarnation (currently used only for sup-sync-back)
@@ -81,6 +82,14 @@ class Source
## leaks (esp. file descriptors).
def go_idle; end
+ ## Returns an array containing all the labels that are natively
+ ## supported by this source
+ def supported_labels?; [] end
+
+ ## Returns an array containing all the labels that are currently in
+ ## the location filename
+ def labels? info; [] end
+
## Yields values of the form [Symbol, Hash]
## add: info, labels, progress
## delete: info, progress
@@ -92,6 +101,25 @@ class Source
true
end
+ def synchronize &block
+ @poll_lock.synchronize &block
+ end
+
+ def try_lock
+ acquired = @poll_lock.try_enter
+ if acquired
+ debug "lock acquired for: #{self}"
+ else
+ debug "could not acquire lock for: #{self}"
+ end
+ acquired
+ end
+
+ def unlock
+ @poll_lock.exit
+ debug "lock released for: #{self}"
+ end
+
## utility method to read a raw email header from an IO stream and turn it
## into a hash of key-value pairs. minor special semantics for certain headers.
##
diff --git a/lib/sup/thread.rb b/lib/sup/thread.rb
@@ -387,6 +387,12 @@ class ThreadSet
m.refs.any? { |ref_id| @messages.member? ref_id }
end
+ def delete_message message
+ el = @messages[message.id]
+ return unless el.message
+ el.message = nil
+ end
+
## the heart of the threading code
def add_message message
el = @messages[message.id]