commit 8401bef72bf71646259cb5a2c2d50c301ac2f2b4
parent 91cdc3494f3b20b621a7fcb26c934f8ee52aea4e
Author: wmorgan <wmorgan@5c8cc53c-5e98-4d25-b20a-d8db53a31250>
Date: Sun, 1 Apr 2007 02:28:01 +0000
moved sup-import to sup-sync and changed interface a lot. note that trollop 1.5 is now required
git-svn-id: svn://rubyforge.org/var/svn/sup/trunk@353 5c8cc53c-5e98-4d25-b20a-d8db53a31250
Diffstat:
15 files changed, 305 insertions(+), 244 deletions(-)
diff --git a/Manifest.txt b/Manifest.txt
@@ -6,7 +6,7 @@ README.txt
Rakefile
bin/sup
bin/sup-add
-bin/sup-import
+bin/sup-sync
bin/sup-recover-sources
doc/FAQ.txt
doc/Philosophy.txt
diff --git a/Rakefile b/Rakefile
@@ -16,7 +16,7 @@ Hoe.new('sup', Redwood::VERSION) do |p|
p.url = p.paragraphs_of('README.txt', 0).first.split(/\n/)[2].gsub(/^\s+/, "")
p.changes = p.paragraphs_of('History.txt', 0..0).join("\n\n")
p.email = "wmorgan-sup@masanjin.net"
- p.extra_deps = [['ferret', '>= 0.10.13'], ['ncurses', '>= 0.9.1'], ['rmail', '>= 0.17'], 'highline', 'net-ssh', 'trollop']
+ p.extra_deps = [['ferret', '>= 0.10.13'], ['ncurses', '>= 0.9.1'], ['rmail', '>= 0.17'], 'highline', 'net-ssh', ['trollop', '>= 1.5']]
end
rule 'ss?.png' => 'ss?-small.png' do |t|
diff --git a/bin/sup-import b/bin/sup-import
@@ -1,157 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'uri'
-require 'rubygems'
-require 'trollop'
-require "sup"
-
-class Float
- def to_s; sprintf '%.2f', self; end
-end
-
-class Numeric
- def to_time_s
- i = to_i
- sprintf "%d:%02d:%02d", i / 3600, (i / 60) % 60, i % 60
- end
-end
-
-def time
- startt = Time.now
- yield
- Time.now - startt
-end
-
-opts = Trollop::options do
- version "sup-import (sup #{Redwood::VERSION})"
- banner <<EOS
-Imports messages into the Sup index from one or more sources.
-
-Usage:
- sup-import [options] <source>*
-
-where <source>* is zero or more source URIs or mbox filenames. If no
-sources are given, imports messages from all sources marked as
-"usual".
-
-Options are:
-EOS
- opt :archive, "Automatically archive any imported messages."
- opt :read, "Automatically mark as read any imported messages."
- opt :verbose, "Print message ids as they're processed."
- opt :optimize, "As the last stage of the import, optimize the index."
- text <<EOS
-
-The following options allow sup-import to consider *all* messages in the
-source, not just new ones:
-EOS
- opt :rebuild, "Scan over the entire source and update the index to account for any messages that have been deleted, altered, or moved from another source."
- opt :full_rebuild, "Re-insert all messages in the source, not just ones that have changed or are new."
- opt :start_at, "For rescan and rebuild, start at the given offset.", :type => :int
- opt :overwrite_state, "For --full-rebuild, overwrite the message state to the default state for that source, obeying --archive and --read if given."
-end
-Trollop::die :start_at, "must be non-negative" if (opts[:start_at] || 0) < 0
-Trollop::die :start_at, "requires either --rebuild or --full-rebuild" if opts[:start_at] && !(opts[:rebuild] || opts[:full_rebuild])
-Trollop::die :overwrite_state, "requires --full-rebuild" if opts[:overwrite_state] && !opts[:full_rebuild]
-Trollop::die :force_rebuild, "cannot be specified with --rebuild" if opts[:full_rebuild] && opts[:rebuild]
-
-Redwood::start
-index = Redwood::Index.new
-index.load
-
-sources = ARGV.map do |uri|
- uri = "mbox://#{uri}" unless uri =~ %r!://!
- index.source_for uri or Trollop::die "Unknown source: #{uri}. Did you add it with sup-add first?"
-end
-
-sources = index.usual_sources if sources.empty?
-
-if opts[:rebuild] || opts[:full_rebuild]
- if opts[:start_at]
- sources.each { |s| s.seek_to! opts[:start_at] }
- else
- sources.each { |s| s.reset! }
- end
-end
-
-last_update = start = Time.now
-found = {}
-begin
- sources.each do |source|
- num_added = 0
- num_updated = 0
- puts "Scanning #{source}..."
- Redwood::PollManager.add_new_messages_from source do |m, offset, entry|
- ## if the entry exists on disk
- if entry && !opts[:overwrite_state]
- m.labels = entry[:label].split(/\s+/).map { |x| x.intern }
- else
- ## m.labels defaults to labels from the source
- m.labels -= [:inbox] if opts[:archive]
- m.labels -= [:unread] if opts[:read]
- end
-
- if Time.now - last_update > 60
- last_update = Time.now
- elapsed = last_update - start
- pctdone = source.respond_to?(:pct_done) ? source.pct_done : 100.0 * (source.cur_offset.to_f - source.start_offset).to_f / (source.end_offset - source.start_offset).to_f
- remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
- puts "## #{num_added + num_updated} (#{pctdone}% done) read; #{elapsed.to_time_s} elapsed; est. #{remaining.to_time_s} remaining"
- end
-
- ## update if...
- if entry.nil? # it's a new message; or
- puts "Adding message at #{offset}, labels: #{m.labels * ' '}" if opts[:verbose]
- num_added += 1
- found[m.id] = true
- m
- elsif opts[:full_rebuild] || # we're updating everyone; or
- (opts[:rebuild] && (entry[:source_id].to_i != source.id || entry[:source_info].to_i != offset)) # we're updating just the changed ones
- puts "Updating message at #{offset} (from #{m.from.longname}, subject '#{m.subj}'), source #{entry[:source_id]} => #{source.id}, offset #{entry[:source_info]} => #{offset}, labels: {#{m.labels * ', '}}" if opts[:verbose]
- num_updated += 1 unless found[m.id]
- found[m.id] = true
- m
- else
- found[m.id] = true
- nil
- end
- end
- puts "Added #{num_added}, updated #{num_updated} messages from #{source}."
- end
-ensure
- puts "Saving index and sources..."
- index.save
- Redwood::finish
-end
-
-## delete any messages in the index that claim they're from one of
-## these sources, but that we didn't see.
-##
-## kinda crappy code here, because we delve directly into the Ferret
-## API.
-##
-## TODO: move this to Index, i suppose.
-if opts[:rebuild] || opts[:full_rebuild]
- puts "Deleting missing messages from the index..."
- numdel = num = 0
- sources.each do |source|
- raise "no source id for #{source}" unless source.id
- q = "+source_id:#{source.id}"
- q += " +source_info: >= #{opts[:start_at]}" if opts[:start_at]
- num += index.index.search_each(q, :limit => :all) do |docid, score|
- mid = index.index[docid][:message_id]
-# puts "got #{mid}"
- next if found[mid]
- puts "Deleting #{mid}" if opts[:verbose]
- index.index.delete docid
- numdel += 1
- end
- end
- puts "Deleted #{numdel} / #{num} messages"
-end
-
-if opts[:optimize]
- puts "Optimizing index..."
- optt = time { index.index.optimize }
- puts "Optimized index of size #{index.size} in #{optt}s."
-end
diff --git a/bin/sup-sync b/bin/sup-sync
@@ -0,0 +1,235 @@
+#!/usr/bin/env ruby
+
+require 'uri'
+require 'rubygems'
+require 'trollop'
+require "sup"
+
+class Float
+ def to_s; sprintf '%.2f', self; end
+end
+
+class Numeric
+ def to_time_s
+ i = to_i
+ sprintf "%d:%02d:%02d", i / 3600, (i / 60) % 60, i % 60
+ end
+end
+
+def time
+ startt = Time.now
+ yield
+ Time.now - startt
+end
+
+opts = Trollop::options do
+ version "sup-sync (sup #{Redwood::VERSION})"
+ banner <<EOS
+Synchronizes the Sup index with one or more message sources by adding
+messages, deleting messages, or changing message state in the index as
+appropriate.
+
+"Message state" means read/unread, archived/inbox, starred/unstarred,
+and all user-defined labels on each message.
+
+"Default source state" refers to any state that a source itself has
+keeps about a message. Sup-sync uses this information when adding a
+new message to the index. The source state is typically limited to
+read/unread, archived/inbox status and a single label based on the
+source name. Messages using the default source state are placed in
+the inbox (i.e. not archived) and unstarred.
+
+Usage:
+ sup-sync [options] <source>*
+
+where <source>* is zero or more source URIs. If no sources are given,
+sync from all usual sources.
+
+Supported source URIs:
+ mbox://<path to mbox file>, e.g. mbox:///var/spool/mail/me
+ maildir://<path to maildir dir>, e.g. maildir:///home/me/Maildir
+ mbox+ssh://<machine>/<path to mbox file>
+ imap://<machine>/[<folder>]
+ imaps://<machine>/[<folder>]
+
+Options controlling WHICH messages sup-sync operates on:
+EOS
+ opt :new, "Operate on new messages only. Don't scan over the entire source. (Default.)", :short => :none
+ opt :changed, "Scan over the entire source for messages that have been deleted, altered, or moved from another source. (In the case of mbox sources, this includes all messages AFTER an altered message.)"
+ opt :restored, "Operate only on those messages included in a dump file as specified by --restore."
+ opt :all, "Operate on all messages in the source, regardless of newness or changedness."
+ opt :start_at, "For --changed and --all, start at a particular offset.", :type => :int
+
+text <<EOS
+
+Options controlling HOW message state is altered:
+EOS
+ opt :asis, "If the message is already in the index, preserve its state. Otherwise, use default source state. (Default.)", :short => :none
+ opt :restore, "Restore message state from a dump file created with sup-dump. If a message is not in this dumpfile, act as --asis.", :type => String, :short => :none
+ opt :discard, "Discard any message state in the index and use the default source state. Dangerous!", :short => :none
+ opt :archive, "When using the default source state, mark messages as archived.", :short => "-x"
+ opt :read, "When using the default source state, mark messages as read."
+ opt :extra_labels, "When using the default source state, also apply these user-defined labels. Should be a comma-separated list.", :type => String, :short => :none
+
+text <<EOS
+
+Other options:
+EOS
+ opt :verbose, "Print message ids as they're processed."
+ opt :optimize, "As the final operation, optimize the index."
+ opt :dry_run, "Don't actually modify the index. Probably only useful with --verbose.", :short => "-n"
+ opt :version, "Show version information", :short => :none
+
+ conflicts :changed, :all, :new, :restored
+ conflicts :asis, :restore, :discard
+end
+Trollop::die :restored, "requires --restore" if opts[:restore] unless opts[:restored]
+if opts[:start_at]
+ Trollop::die :start_at, "must be non-negative" if opts[:start_at] < 0
+ Trollop::die :start_at, "requires either --changed or --all" unless opts[:changed] || opts[:all]
+end
+
+target = [:new, :changed, :all, :restored].find { |x| opts[x] } || :new
+op = [:asis, :restore, :discard].find { |x| opts[x] } || :asis
+
+Redwood::start
+index = Redwood::Index.new
+index.load
+
+restored_state =
+ if opts[:restore]
+ dump = {}
+ $stderr.puts "Loading state dump from #{opts[:restore]}..."
+ IO.foreach opts[:restore] do |l|
+ l =~ /^(\S+) (\d+) (\d+) \((.*?)\)$/ or raise "Can't read dump line: #{l.inspect}"
+ mid, source_id, source_info, labels = $1, $2.to_i, $3.to_i, $4
+ dump[mid] = labels.split(" ").map { |x| x.intern }
+ end
+ $stderr.puts "Read #{dump.size} entries from dump file."
+ dump
+ else
+ {}
+ end
+
+sources = ARGV.map do |uri|
+ uri = "mbox://#{uri}" unless uri =~ %r!://!
+ index.source_for uri or Trollop::die "Unknown source: #{uri}. Did you add it with sup-add first?"
+end
+
+sources = index.usual_sources if sources.empty?
+
+unless target == :new
+ if opts[:start_at]
+ sources.each { |s| s.seek_to! opts[:start_at] }
+ else
+ sources.each { |s| s.reset! }
+ end
+end
+
+last_info_time = start_time = Time.now
+seen = {}
+begin
+ sources.each do |source|
+ num_added, num_updated, num_scanned = 0, 0, 0
+ $stderr.puts "Scanning #{source}..."
+
+ Redwood::PollManager.add_messages_from source do |m, offset, entry|
+ num_scanned += 1
+ seen[m.id] = true
+
+ ## skip if we're operating only on changed messages, the message
+ ## is in the index, and it's unchanged from what the source is
+ ## reporting.
+ next if target == :changed && entry && entry[:source_id].to_i == source.id && entry[:source_info].to_i == offset
+
+ ## skip if we're operating on restored messages, and this one
+ ## ain't.
+ next if target == :restored && !restored_state[m.id]
+
+ ## m.labels is the default source labels. tweak these according
+ ## to default source state modification flags.
+ m.labels -= [:inbox] if opts[:archive]
+ m.labels -= [:unread] if opts[:read]
+ m.labels += opts[:extra_labels].split(/\s*,\s*/).map { |x| x.intern } if opts[:extra_labels]
+
+ ## get the state currently in the index
+ index_state =
+ if entry
+ entry[:label].split(/\s+/).map { |x| x.intern }
+ else
+ nil
+ end
+
+ ## assign message labels based on the operation we're performing
+ case op
+ when :asis
+ m.labels = index_state if index_state
+ when :restore
+ ## if the entry exists on disk
+ if restored_state[m.id]
+ m.labels = restored_state[m.id]
+ elsif index_state
+ m.labels = index_state
+ end
+ when :discard
+ ## nothin! use default source labels
+ end
+
+ if Time.now - last_info_time > 60
+ last_info_time = Time.now
+ elapsed = last_info_time - start_time
+ pctdone = source.respond_to?(:pct_done) ? source.pct_done : 100.0 * (source.cur_offset.to_f - source.start_time_offset).to_f / (source.end_offset - source.start_time_offset).to_f
+ remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
+ puts "## #{num_added + num_updated} (#{pctdone}% done) read; #{elapsed.to_time_s} elapsed; est. #{remaining.to_time_s} remaining"
+ end
+
+ if index_state.nil?
+ puts "Adding message #{source}##{offset} with state {#{m.labels * ', '}}" if opts[:verbose]
+ num_added += 1
+ else
+ puts "Updating message #{source}##{offset}, source #{entry[:source_id]} => #{source.id}, offset #{entry[:source_info]} => #{offset}, state {#{index_state * ', '}} => {#{m.labels * ', '}}" if opts[:verbose]
+ num_updated += 1
+ end
+
+ opts[:dry_run] ? nil : m
+ end
+ $stderr.puts "Added #{num_added}, updated #{num_updated} messages from #{source}."
+ end
+ensure
+ $stderr.puts "Saving index and sources..."
+ index.save
+ Redwood::finish
+end
+
+## delete any messages in the index that claim they're from one of
+## these sources, but that we didn't see.
+##
+## kinda crappy code here, because we delve directly into the Ferret
+## API.
+##
+## TODO: move this to Index, i suppose.
+if target == :all || target == :changed
+ $stderr.puts "Deleting missing messages from the index..."
+ num_del, num_scanned = 0, 0
+ sources.each do |source|
+ raise "no source id for #{source}" unless source.id
+ q = "+source_id:#{source.id}"
+ q += " +source_info: >= #{opts[:start_at]}" if opts[:start_at]
+ index.index.search_each(q, :limit => :all) do |docid, score|
+ num_scanned += 1
+ mid = index.index[docid][:message_id]
+ unless seen[mid]
+ puts "Deleting #{mid}" if opts[:verbose]
+ index.index.delete docid unless opts[:dry_run]
+ num_del += 1
+ end
+ end
+ end
+ $stderr.puts "Deleted #{num_del} / #{num_scanned} messages"
+end
+
+if opts[:optimize]
+ $stderr.puts "Optimizing index..."
+ optt = time { index.index.optimize unless opts[:dry_run] }
+ $stderr.puts "Optimized index of size #{index.size} in #{optt}s."
+end
diff --git a/doc/TODO b/doc/TODO
@@ -3,24 +3,24 @@ for 0.0.8
_ split out threading & message chunk parsing to a separate library
_ ferret upgrade script (dump & restore)
_ nice little startup config program
-x maildir
-x bugfix: single-line messages come empty upon reply
_ bugfix: when one new message comes into an imap folder, we don't
catch it until a reload
_ bugfix: triggering a pageup when cursor scrolling up jumps to the
bottom of the page rather than the next line
-x compose in thread-view-mode auto-fills in person
_ bugfix: stars on messages with blue backgrounds still have green bgs
-x bugfix: mark messages as read immediately when t-v-m is opened
_ bugfix: m in thread-view-mode when a person is not selected should open up a
blank compose-mode rather than do nothing
_ Net::SMTP support (cuz I'm going to need it soon)
-x bugfix: 'N' in thread-view-mode (expand only new messages) crashes
-_ bugfix: detect source corruption at startup
_ bugfix: add new message counts until keypress
_ bugfix: attachment filenames sometimes not detected (filename=)
_ bugfix: final logging messages to stdout?
_ bugfix: mbox directory shouldn't generate an exception, just an error
+x bugfix: mark messages as read immediately when t-v-m is opened
+x compose in thread-view-mode auto-fills in person
+x bugfix: 'N' in thread-view-mode (expand only new messages) crashes
+x bugfix: detect source corruption at startup
+x maildir
+x bugfix: single-line messages come empty upon reply
for 0.0.9
---------
@@ -30,6 +30,7 @@ _ select all, starred, to me, etc
_ undo
_ gmail
_ warnings: top-posting, missing attachment, ruby-talk:XXXX detection
+_ mboxz (compressed mbox)
future
------
diff --git a/lib/sup.rb b/lib/sup.rb
@@ -97,6 +97,7 @@ module Redwood
## not really a good place for this, so I'll just dump it here.
def report_broken_sources
+ return unless BufferManager.instantiated?
broken_sources = Index.usual_sources.select { |s| s.broken? }
unless broken_sources.empty?
BufferManager.spawn "Broken source report", TextMode.new(<<EOM)
diff --git a/lib/sup/draft.rb b/lib/sup/draft.rb
@@ -22,7 +22,7 @@ class DraftManager
my_message = nil
@source.each do |thisoffset, theselabels|
m = Message.new :source => @source, :source_info => thisoffset, :labels => theselabels
- Index.add_message m
+ Index.sync_message m
UpdateManager.relay self, :add, m
my_message = m if thisoffset == offset
end
diff --git a/lib/sup/imap.rb b/lib/sup/imap.rb
@@ -208,7 +208,7 @@ private
e
end
- message += " It is likely that messages have been deleted from this IMAP mailbox. Please run sup-import --rebuild #{to_s} to correct this problem." if opts[:suggest_rebuild]
+ message += " It is likely that messages have been deleted from this IMAP mailbox. Please run sup-sync --changed #{to_s} to correct this problem." if opts[:suggest_rebuild]
self.broken_msg = message
Redwood::log message
diff --git a/lib/sup/index.rb b/lib/sup/index.rb
@@ -72,25 +72,46 @@ class Index
end
end
- ## Update the message state on disk, by deleting and re-adding it.
- ## The message must exist in the index. docid and entry are found
- ## unless given.
+ ## Syncs the message to the index: deleting if it's already there,
+ ## and adding either way. Index state will be determined by m.labels.
##
- ## Overwrites the labels on disk with the new labels in 'm', so that
- ## we can actually change message state.
- def update_message m, docid=nil, entry=nil
- unless docid && entry
- docid, entry = load_entry_for_id m.id
- raise ArgumentError, "cannot find #{m.id} in the index" unless entry
- end
+ ## docid and entry can be specified if they're already known.
+ def sync_message m, docid=nil, entry=nil
+ docid, entry = load_entry_for_id m.id unless docid && entry
- raise "no entry and no source info for message #{m.id}" unless m.source && m.source_info
+ raise "no source info for message #{m.id}" unless m.source && m.source_info
+ raise "trying deleting non-corresponding entry #{docid}" if docid && @index[docid][:message_id] != m.id
- raise "deleting non-corresponding entry #{docid}" unless @index[docid][:message_id] == m.id
+ source_id =
+ if m.source.is_a? Integer
+ raise "Debugging: integer source set"
+ m.source
+ else
+ m.source.id or raise "unregistered source #{m.source} (id #{m.source.id.inspect})"
+ end
- @index.delete docid
- add_message m
+ to = (m.to + m.cc + m.bcc).map { |x| x.email }.join(" ")
+ d = {
+ :message_id => m.id,
+ :source_id => source_id,
+ :source_info => m.source_info,
+ :date => m.date.to_indexable_s,
+ :body => m.content,
+ :snippet => m.snippet,
+ :label => m.labels.join(" "),
+ :from => m.from ? m.from.email : "",
+ :to => (m.to + m.cc + m.bcc).map { |x| x.email }.join(" "),
+ :subject => wrap_subj(Message.normalize_subj(m.subj)),
+ :refs => (m.refs + m.replytos).uniq.join(" "),
+ }
+
+ @index.delete docid if docid
+ @index.add_document d
+
docid, entry = load_entry_for_id m.id
+ ## this hasn't been triggered in a long time. TODO: decide whether it's still a problem.
+ raise "just added message #{m.id} but couldn't find it in a search" unless docid
+ true
end
def save_index fn=File.join(@dir, "ferret")
@@ -210,41 +231,6 @@ class Index
def wrap_subj subj; "__START_SUBJECT__ #{subj} __END_SUBJECT__"; end
def unwrap_subj subj; subj =~ /__START_SUBJECT__ (.*?) __END_SUBJECT__/ && $1; end
- ## Adds a message to the index. The message cannot already exist in
- ## the index.
- def add_message m
- raise ArgumentError, "index already contains #{m.id}" if contains? m
-
- source_id =
- if m.source.is_a? Integer
- m.source
- else
- m.source.id or raise "unregistered source #{m.source} (id #{m.source.id.inspect})"
- end
-
- to = (m.to + m.cc + m.bcc).map { |x| x.email }.join(" ")
- d = {
- :message_id => m.id,
- :source_id => source_id,
- :source_info => m.source_info,
- :date => m.date.to_indexable_s,
- :body => m.content,
- :snippet => m.snippet,
- :label => m.labels.join(" "),
- :from => m.from ? m.from.email : "",
- :to => (m.to + m.cc + m.bcc).map { |x| x.email }.join(" "),
- :subject => wrap_subj(Message.normalize_subj(m.subj)),
- :refs => (m.refs + m.replytos).uniq.join(" "),
- }
-
- @index.add_document d
-
- docid, entry = load_entry_for_id m.id
- ## this hasn't been triggered in a long time. TODO: decide whether it's still a problem.
- raise "just added message #{m.id} but couldn't find it in a search" unless docid
- true
- end
-
def drop_entry docno; @index.delete docno; end
def load_entry_for_id mid
diff --git a/lib/sup/maildir.rb b/lib/sup/maildir.rb
@@ -96,7 +96,7 @@ class Maildir < Source
private
def die message, opts={}
- message += " It is likely that messages have been deleted from this Maildir mailbox. Please run sup-import --rebuild #{to_s} to correct this problem." if opts[:suggest_rebuild]
+ message += " It is likely that messages have been deleted from this Maildir mailbox. Please run sup-sync --changed #{to_s} to correct this problem." if opts[:suggest_rebuild]
self.broken_msg = message
Redwood::log message
BufferManager.flash "Error communicating with Maildir. See log for details." if BufferManager.instantiated?
diff --git a/lib/sup/mbox/loader.rb b/lib/sup/mbox/loader.rb
@@ -24,7 +24,7 @@ class Loader < Source
end
if cur_offset > end_offset
- self.broken_msg = "mbox file is smaller than last recorded message offset. Messages have probably been deleted via another client. Run 'sup-import --rebuild #{to_s}' to correct this."
+ self.broken_msg = "mbox file is smaller than last recorded message offset. Messages have probably been deleted via another client. Run 'sup-sync --changed #{to_s}' to correct this."
end
end
@@ -38,7 +38,7 @@ class Loader < Source
l = @f.gets
unless l =~ BREAK_RE
Redwood::log "#{to_s}: offset mismatch in mbox file offset #{offset.inspect}: #{l.inspect}"
- self.broken_msg = "offset mismatch in mbox file offset #{offset.inspect}: #{l.inspect}. Run 'sup-import --rebuild #{to_s}' to correct this."
+ self.broken_msg = "offset mismatch in mbox file offset #{offset.inspect}: #{l.inspect}. Run 'sup-sync --changed #{to_s}' to correct this."
raise SourceError, self.broken_msg
end
header = MBox::read_header @f
diff --git a/lib/sup/message.rb b/lib/sup/message.rb
@@ -149,7 +149,7 @@ class Message
def save index
return if broken?
- index.update_message self if @dirty
+ index.sync_message self if @dirty
@dirty = false
end
diff --git a/lib/sup/poll.rb b/lib/sup/poll.rb
@@ -46,7 +46,7 @@ class PollManager
yield "Loading from #{source}... " unless source.done? || source.broken?
num = 0
numi = 0
- add_new_messages_from source do |m, offset, entry|
+ add_messages_from source do |m, offset, entry|
## always preserve the labels on disk.
m.labels = entry[:label].split(/\s+/).map { |x| x.intern } if entry
yield "Found message at #{offset} with labels {#{m.labels * ', '}}"
@@ -69,18 +69,19 @@ class PollManager
end
## this is the main mechanism for adding new messages to the
- ## index. it's called both by sup-import and by PollMode.
+ ## index. it's called both by sup-sync and by PollMode.
##
- ## for each new message in the source, this yields the message, the
- ## source offset, and the index entry on disk (if any). it expects
- ## the yield to return the message (possibly altered in some way),
- ## and then adds it (if new) or updates it (if previously seen).
+ ## for each message in the source, starting from the source's
+ ## starting offset, this methods yields the message, the source
+ ## offset, and the index entry on disk (if any). it expects the
+ ## yield to return the message (possibly altered in some way), and
+ ## then adds it (if new) or updates it (if previously seen).
##
- ## the labels of the yielded message are the source labels. it is
- ## likely that callers will want to replace these with the index
- ## labels, if they exist, so that state is not lost when e.g. a new
- ## version of a message from a mailing list comes in.
- def add_new_messages_from source
+ ## the labels of the yielded message are the default source
+ ## labels. it is likely that callers will want to replace these with
+ ## the index labels, if they exist, so that state is not lost when
+ ## e.g. a new version of a message from a mailing list comes in.
+ def add_messages_from source
return if source.done? || source.broken?
begin
@@ -101,22 +102,16 @@ class PollManager
end
docid, entry = Index.load_entry_for_id m.id
- m = yield m, offset, entry
- next unless m
- if entry
- Index.update_message m, docid, entry
- else
- Index.add_message m
- UpdateManager.relay self, :add, m
- end
- rescue MessageFormatError, SourceError => e
+ m = yield(m, offset, entry) or next
+ Index.sync_message m, docid, entry
+ UpdateManager.relay self, :add, m unless entry
+ rescue MessageFormatError => e
Redwood::log "ignoring erroneous message at #{source}##{offset}: #{e.message}"
- Redwood::report_broken_sources if BufferManager.instantiated?
end
end
rescue SourceError => e
Redwood::log "problem getting messages from #{source}: #{e.message}"
- Redwood::report_broken_sources if BufferManager.instantiated?
+ Redwood::report_broken_sources
end
end
end
diff --git a/lib/sup/sent.rb b/lib/sup/sent.rb
@@ -23,7 +23,7 @@ class SentManager
end
@source.each do |offset, labels|
m = Message.new :source => @source, :source_info => offset, :labels => @source.labels
- Index.add_message m
+ Index.sync_message m
UpdateManager.relay self, :add, m
end
end
diff --git a/lib/sup/thread.rb b/lib/sup/thread.rb
@@ -54,7 +54,7 @@ class Thread
## message can be a Message object, or :fake_root, or nil.
def each fake_root=false
adj = 0
- root = @containers.find_all { |c| !Message.subj_is_reply?(c) }.argmin { |c| c.date }
+ root = @containers.find_all { |c| !Message.subj_is_reply?(c) }.argmin { |c| c.date || 0 }
if root
adj = 1