commit 50241e1db8ef21ba68c32dd5f3fe0ffbe321c65c
parent 84876cb6bdef609013e9be53069ae5268ba53663
Author: Rich Lane <rlane@club.cc.cmu.edu>
Date: Sat, 3 Apr 2010 10:38:09 -0700
Merge branch 'multiple-locations'
Diffstat:
4 files changed, 66 insertions(+), 32 deletions(-)
diff --git a/bin/sup-sync b/bin/sup-sync
@@ -153,7 +153,7 @@ begin
## skip this message if we're operating only on changed messages, the
## message is in the index, and it's unchanged from what the source is
## reporting.
- next if old_m && old_m.source.id == m.source.id && old_m.source_info == m.source_info
+ next if old_m && old_m.locations.member?([m.source, m.source_info])
when :restored
## skip if we're operating on restored messages, and this one
## ain't (or we wouldn't be making a change)
@@ -204,6 +204,8 @@ begin
end
end
+ m.locations = old_m.locations + m.locations if old_m
+
## now, actually do the operation
case dothis
when :add_message
diff --git a/lib/sup/index.rb b/lib/sup/index.rb
@@ -22,7 +22,7 @@ class Index
include InteractiveLock
STEM_LANGUAGE = "english"
- INDEX_VERSION = '2'
+ INDEX_VERSION = '3'
## dates are converted to integers for xapian, and are used for document ids,
## so we must ensure they're reasonably valid. this typically only affect
@@ -105,8 +105,8 @@ EOS
@xapian = Xapian::WritableDatabase.new(path, Xapian::DB_OPEN)
db_version = @xapian.get_metadata 'version'
db_version = '0' if db_version.empty?
- if db_version == '1'
- info "Upgrading index format 1 to 2"
+ if db_version == '1' || db_version == '2'
+ info "Upgrading index format #{db_version} to #{INDEX_VERSION}"
@xapian.set_metadata 'version', INDEX_VERSION
elsif db_version != INDEX_VERSION
fail "This Sup version expects a v#{INDEX_VERSION} index, but you have an existing v#{db_version} index. Please downgrade to your previous version and dump your labels before upgrading to this version (then run sup-sync --restore)."
@@ -194,11 +194,15 @@ EOS
entry = synchronize { get_entry id }
return unless entry
- source = SourceManager[entry[:source_id]]
- raise "invalid source #{entry[:source_id]}" unless source
+ locations = entry[:locations].map do |source_id,source_info|
+ source = SourceManager[source_id]
+ raise "invalid source #{source_id}" unless source
+ [source, source_info]
+ end
- m = Message.new :source => source, :source_info => entry[:source_info],
- :labels => entry[:labels], :snippet => entry[:snippet]
+ m = Message.new :locations => locations,
+ :labels => entry[:labels],
+ :snippet => entry[:snippet]
mk_person = lambda { |x| Person.new(*x.reverse!) }
entry[:from] = mk_person[entry[:from]]
@@ -454,6 +458,7 @@ EOS
'id' => 'Q',
'thread' => 'H',
'ref' => 'R',
+ 'location' => 'J',
}
PREFIX = NORMAL_PREFIX.merge BOOLEAN_PREFIX
@@ -515,7 +520,7 @@ EOS
def get_entry id
return unless doc = find_doc(id)
- Marshal.load doc.data
+ doc.entry
end
def thread_killed? thread_id
@@ -548,6 +553,7 @@ EOS
pos_terms.concat(labels.map { |l| mkterm(:label,l) })
pos_terms << opts[:qobj] if opts[:qobj]
pos_terms << mkterm(:source_id, opts[:source_id]) if opts[:source_id]
+ pos_terms << mkterm(:location, *opts[:location]) if opts[:location]
if opts[:participants]
participant_terms = opts[:participants].map { |p| [:from,:to].map { |d| mkterm(:email, d, (Redwood::Person === p) ? p.email : p) } }.flatten
@@ -576,8 +582,7 @@ EOS
entry = {
:message_id => m.id,
- :source_id => m.source.id,
- :source_info => m.source_info,
+ :locations => m.locations.map { |source,source_info| [source.id, source_info] },
:date => truncate_date(m.date),
:snippet => snippet,
:labels => m.labels.to_a,
@@ -596,6 +601,7 @@ EOS
index_message_static m, doc, entry
end
+ index_message_locations doc, entry, old_entry
index_message_threading doc, entry, old_entry
index_message_labels doc, entry[:labels], (do_index_static ? [] : old_entry[:labels])
doc.entry = entry
@@ -638,7 +644,6 @@ EOS
doc.add_term mkterm(:date, m.date) if m.date
doc.add_term mkterm(:type, 'mail')
doc.add_term mkterm(:msgid, m.id)
- doc.add_term mkterm(:source_id, m.source.id)
m.attachments.each do |a|
a =~ /\.(\w+)$/ or next
doc.add_term mkterm(:attachment_extension, $1)
@@ -655,6 +660,13 @@ EOS
doc.add_value DATE_VALUENO, date_value
end
+ def index_message_locations doc, entry, old_entry
+ old_entry[:locations].map { |x| x[0] }.uniq.each { |x| doc.remove_term mkterm(:source_id, x) } if old_entry
+ entry[:locations].map { |x| x[0] }.uniq.each { |x| doc.add_term mkterm(:source_id, x) }
+ old_entry[:locations].each { |x| (doc.remove_term mkterm(:location, *x) rescue nil) } if old_entry
+ entry[:locations].each { |x| doc.add_term mkterm(:location, *x) }
+ end
+
def index_message_labels doc, new_labels, old_labels
return if new_labels == old_labels
added = new_labels.to_a - old_labels.to_a
@@ -717,6 +729,8 @@ EOS
end + args[1].to_s.downcase
when :source_id
PREFIX['source_id'] + args[0].to_s.downcase
+ when :location
+ PREFIX['location'] + [args[0]].pack('n') + args[1].to_s
when :attachment_extension
PREFIX['attachment_extension'] + args[0].to_s.downcase
when :msgid, :ref, :thread
@@ -731,7 +745,13 @@ end
class Xapian::Document
def entry
- Marshal.load data
+ entry = Marshal.load data
+ if entry[:source_id]
+ entry[:locations] = [[entry[:source_id], entry[:source_info]]]
+ entry.delete :source_id
+ entry.delete :source_info
+ end
+ entry
end
def entry=(x)
diff --git a/lib/sup/message.rb b/lib/sup/message.rb
@@ -33,17 +33,18 @@ class Message
DEFAULT_SENDER = "(missing sender)"
MAX_HEADER_VALUE_SIZE = 4096
- attr_reader :id, :date, :from, :subj, :refs, :replytos, :to, :source,
+ attr_reader :id, :date, :from, :subj, :refs, :replytos, :to,
:cc, :bcc, :labels, :attachments, :list_address, :recipient_email, :replyto,
- :source_info, :list_subscribe, :list_unsubscribe
+ :list_subscribe, :list_unsubscribe
bool_reader :dirty, :source_marked_read, :snippet_contains_encrypted_content
+ attr_accessor :locations
+
## if you specify a :header, will use values from that. otherwise,
## will try and load the header from the source.
def initialize opts
- @source = opts[:source] or raise ArgumentError, "source can't be nil"
- @source_info = opts[:source_info] or raise ArgumentError, "source_info can't be nil"
+ @locations = opts[:locations] or raise ArgumentError, "locations can't be nil"
@snippet = opts[:snippet]
@snippet_contains_encrypted_content = false
@have_snippet = !(opts[:snippet].nil? || opts[:snippet].empty?)
@@ -170,10 +171,10 @@ class Message
attr_reader :snippet
def is_list_message?; !@list_address.nil?; end
- def is_draft?; @source.is_a? DraftLoader; end
+ def is_draft?; source.is_a? DraftLoader; end
def draft_filename
raise "not a draft" unless is_draft?
- @source.fn_for_offset @source_info
+ source.fn_for_offset source_info
end
## sanitize message ids by removing spaces and non-ascii characters.
@@ -224,11 +225,21 @@ class Message
@chunks
end
+ def source
+ fail if @locations.empty?
+ @locations.last[0]
+ end
+
+ def source_info
+ fail if @locations.empty?
+ @locations.last[1]
+ end
+
## this is called when the message body needs to actually be loaded.
def load_from_source!
@chunks ||=
- if @source.respond_to?(:has_errors?) && @source.has_errors?
- [Chunk::Text.new(error_message(@source.error.message).split("\n"))]
+ if source.respond_to?(:has_errors?) && source.has_errors?
+ [Chunk::Text.new(error_message(source.error.message).split("\n"))]
else
begin
## we need to re-read the header because it contains information
@@ -239,14 +250,14 @@ class Message
## bloat the index.
## actually, it's also the differentiation between to/cc/bcc,
## so i will keep this.
- rmsg = @source.load_message(@source_info)
+ rmsg = source.load_message(source_info)
parse_header rmsg.header
message_to_chunks rmsg
rescue SourceError, SocketError => e
- warn "problem getting messages from #{@source}: #{e.message}"
+ warn "problem getting messages from #{source}: #{e.message}"
## we need force_to_top here otherwise this window will cover
## up the error message one
- @source.error ||= e
+ source.error ||= e
Redwood::report_broken_sources :force_to_top => true
[Chunk::Text.new(error_message(e.message).split("\n"))]
end
@@ -264,7 +275,7 @@ class Message
should have popped up at some point.
The message location was:
- #@source##@source_info
+ #{source}##{source_info}
***********************************************************************
The error message was:
@@ -277,24 +288,24 @@ EOS
begin
yield
rescue SourceError => e
- warn "problem getting messages from #{@source}: #{e.message}"
- @source.error ||= e
+ warn "problem getting messages from #{source}: #{e.message}"
+ source.error ||= e
Redwood::report_broken_sources :force_to_top => true
error_message e.message
end
end
def raw_header
- with_source_errors_handled { @source.raw_header @source_info }
+ with_source_errors_handled { source.raw_header source_info }
end
def raw_message
- with_source_errors_handled { @source.raw_message @source_info }
+ with_source_errors_handled { source.raw_message source_info }
end
## much faster than raw_message
def each_raw_message_line &b
- with_source_errors_handled { @source.each_raw_message_line(@source_info, &b) }
+ with_source_errors_handled { source.each_raw_message_line(source_info, &b) }
end
## returns all the content from a message that will be indexed
@@ -336,7 +347,7 @@ EOS
end
def self.build_from_source source, source_info
- m = Message.new :source => source, :source_info => source_info
+ m = Message.new :locations => [[source, source_info]]
m.load_from_source!
m
end
diff --git a/lib/sup/poll.rb b/lib/sup/poll.rb
@@ -116,13 +116,14 @@ EOS
each_message_from source do |m|
old_m = Index.build_message m.id
if old_m
- if old_m.source.id != source.id || old_m.source_info != m.source_info
+ if not old_m.locations.member? [source, m.source_info]
## here we merge labels between new and old versions, but we don't let the new
## message add :unread or :inbox labels. (they can exist in the old version,
## just not be added.)
new_labels = old_m.labels + (m.labels - [:unread, :inbox])
yield "Message at #{m.source_info} is an updated of an old message. Updating labels from #{m.labels.to_a * ','} => #{new_labels.to_a * ','}"
m.labels = new_labels
+ m.locations = old_m.locations + m.locations
Index.update_message m
else
yield "Skipping already-imported message at #{m.source_info}"