From: rlane@club.cc.cmu.edu (Rich Lane)
Subject: [sup-talk] [PATCH 03/18] remove ferret entry from poll/sync interface
Date: Sat, 20 Jun 2009 13:50:02 -0700 [thread overview]
Message-ID: <1245531017-9907-4-git-send-email-rlane@club.cc.cmu.edu> (raw)
In-Reply-To: <1245531017-9907-3-git-send-email-rlane@club.cc.cmu.edu>
This leads to an extra index lookup in the sup-sync update path, but I think
it's worth it for the sake of API simplicity.
---
bin/sup-sync | 8 ++++----
bin/sup-sync-back | 6 +++---
lib/sup/index.rb | 18 ++++--------------
lib/sup/message.rb | 6 ++++++
lib/sup/poll.rb | 33 ++++++++++++++-------------------
lib/sup/sent.rb | 2 +-
6 files changed, 32 insertions(+), 41 deletions(-)
diff --git a/bin/sup-sync b/bin/sup-sync
index a759cbe..18a3cab 100755
--- a/bin/sup-sync
+++ b/bin/sup-sync
@@ -137,7 +137,7 @@ begin
num_added = num_updated = num_scanned = num_restored = 0
last_info_time = start_time = Time.now
- Redwood::PollManager.add_messages_from source, :force_overwrite => true do |m, offset, entry|
+ Redwood::PollManager.add_messages_from source, :force_overwrite => true do |m_old, m, offset|
num_scanned += 1
seen[m.id] = true
@@ -153,10 +153,10 @@ begin
## skip if we're operating only on changed messages, the message
## is in the index, and it's unchanged from what the source is
## reporting.
- next if target == :changed && entry && entry[:source_id].to_i == source.id && entry[:source_info].to_i == offset
+ next if target == :changed && m_old && m_old.source.id == source.id && m_old.source_info == offset
## get the state currently in the index
- index_state = entry[:label].symbolistize if entry
+ index_state = m_old.labels.dup if m_old
## skip if we're operating on restored messages, and this one
## ain't.
@@ -196,7 +196,7 @@ begin
puts "Adding message #{source}##{offset} from #{m.from} with state {#{m.labels * ', '}}" if opts[:verbose]
num_added += 1
else
- puts "Updating message #{source}##{offset}, source #{entry[:source_id]} => #{source.id}, offset #{entry[:source_info]} => #{offset}, state {#{index_state * ', '}} => {#{m.labels * ', '}}" if opts[:verbose]
+ puts "Updating message #{source}##{offset}, source #{m_old.source.id} => #{source.id}, offset #{m_old.source_info} => #{offset}, state {#{index_state * ', '}} => {#{m.labels * ', '}}" if opts[:verbose]
num_updated += 1
end
diff --git a/bin/sup-sync-back b/bin/sup-sync-back
index 4f1387e..1c746d2 100755
--- a/bin/sup-sync-back
+++ b/bin/sup-sync-back
@@ -105,11 +105,11 @@ EOS
num_dropped = num_moved = num_scanned = 0
out_fp = Tempfile.new "sup-sync-back-#{source.id}"
- Redwood::PollManager.add_messages_from source do |m, offset, entry|
+ Redwood::PollManager.add_messages_from source do |m_old, m, offset|
num_scanned += 1
- if entry
- labels = entry[:label].symbolistize.to_boolean_h
+ if m_old
+ labels = m_old.labels
if labels.member? :deleted
if opts[:drop_deleted]
diff --git a/lib/sup/index.rb b/lib/sup/index.rb
index b5d0e5d..89795da 100644
--- a/lib/sup/index.rb
+++ b/lib/sup/index.rb
@@ -174,16 +174,10 @@ EOS
## Syncs the message to the index, replacing any previous version. adding
## either way. Index state will be determined by the message's #labels
## accessor.
- ##
- ## if need_load is false, docid and entry are assumed to be set to the
- ## result of load_entry_for_id (which can be nil).
- def sync_message m, need_load=true, docid=nil, entry=nil, opts={}
- docid, entry = load_entry_for_id m.id if need_load
+ def sync_message m, opts={}
+ entry = @index[m.id]
raise "no source info for message #{m.id}" unless m.source && m.source_info
- @index_mutex.synchronize do
- raise "trying to delete non-corresponding entry #{docid} with index message-id #{@index[docid][:message_id].inspect} and parameter message id #{m.id.inspect}" if docid && @index[docid][:message_id] != m.id
- end
source_id = if m.source.is_a? Integer
m.source
@@ -256,13 +250,9 @@ EOS
}
@index_mutex.synchronize do
- @index.delete docid if docid
+ @index.delete m.id
@index.add_document d
end
-
- ## this hasn't been triggered in a long time.
- ## docid, entry = load_entry_for_id m.id
- ## raise "just added message #{m.id.inspect} but couldn't find it in a search" unless docid
end
def save_index fn=File.join(@dir, "ferret")
@@ -391,7 +381,7 @@ EOS
## builds a message object from a ferret result
def build_message docid
@index_mutex.synchronize do
- doc = @index[docid]
+ doc = @index[docid] or return
source = @source_mutex.synchronize { @sources[doc[:source_id].to_i] }
raise "invalid source #{doc[:source_id]}" unless source
diff --git a/lib/sup/message.rb b/lib/sup/message.rb
index 8525fdf..b667cb3 100644
--- a/lib/sup/message.rb
+++ b/lib/sup/message.rb
@@ -288,6 +288,12 @@ EOS
"Subject: #{@subj}"]
end
+ def self.build_from_source source, source_info
+ m = Message.new :source => source, :source_info => source_info
+ m.load_from_source!
+ m
+ end
+
private
## here's where we handle decoding mime attachments. unfortunately
diff --git a/lib/sup/poll.rb b/lib/sup/poll.rb
index 74f7d1c..bbad5f2 100644
--- a/lib/sup/poll.rb
+++ b/lib/sup/poll.rb
@@ -95,11 +95,11 @@ EOS
num = 0
numi = 0
- add_messages_from source do |m, offset, entry|
+ add_messages_from source do |m_old, m, offset|
## always preserve the labels on disk.
- m.labels = ((m.labels - [:unread, :inbox]) + entry[:label].symbolistize).uniq if entry
+ m.labels = ((m.labels - [:unread, :inbox]) + m_old.labels).uniq if m_old
yield "Found message at #{offset} with labels {#{m.labels * ', '}}"
- unless entry
+ unless m_old
num += 1
from_and_subj << [m.from && m.from.longname, m.subj]
if m.has_label?(:inbox) && ([:spam, :deleted, :killed] & m.labels).empty?
@@ -138,29 +138,24 @@ EOS
begin
return if source.done? || source.has_errors?
- source.each do |offset, labels|
+ source.each do |offset, default_labels|
if source.has_errors?
Redwood::log "error loading messages from #{source}: #{source.error.message}"
return
end
- labels << :sent if source.uri.eql?(SentManager.source_uri)
- labels.each { |l| LabelManager << l }
- labels = labels + (source.archived? ? [] : [:inbox])
+ m_new = Message.build_from_source source, offset
+ m_old = Index.build_message m_new.id
- m = Message.new :source => source, :source_info => offset, :labels => labels
- m.load_from_source!
+ m_new.labels = default_labels + (source.archived? ? [] : [:inbox])
+ m_new.labels << :sent if source.uri.eql?(SentManager.source_uri)
+ m_new.labels.delete :unread if m_new.source_marked_read?
+ m_new.labels.each { |l| LabelManager << l }
- if m.source_marked_read?
- m.remove_label :unread
- labels.delete :unread
- end
-
- docid, entry = Index.load_entry_for_id m.id
- HookManager.run "before-add-message", :message => m
- m = yield(m, offset, entry) or next if block_given?
- times = Index.sync_message m, false, docid, entry, opts
- UpdateManager.relay self, :added, m unless entry
+ HookManager.run "before-add-message", :message => m_new
+ m_ret = yield(m_old, m_new, offset) or next if block_given?
+ Index.sync_message m_ret, opts
+ UpdateManager.relay self, :added, m_ret unless m_old
end
rescue SourceError => e
Redwood::log "problem getting messages from #{source}: #{e.message}"
diff --git a/lib/sup/sent.rb b/lib/sup/sent.rb
index e6ae856..b750d71 100644
--- a/lib/sup/sent.rb
+++ b/lib/sup/sent.rb
@@ -30,7 +30,7 @@ class SentManager
def write_sent_message date, from_email, &block
@source.store_message date, from_email, &block
- PollManager.add_messages_from(@source) do |m, o, e|
+ PollManager.add_messages_from(@source) do |m_old, m, offset|
m.remove_label :unread
m
end
--
1.6.0.4
next prev parent reply other threads:[~2009-06-20 20:50 UTC|newest]
Thread overview: 44+ messages / expand[flat|nested] mbox.gz Atom feed top
2009-06-20 20:49 [sup-talk] [PATCH 0/18] Xapian-based index Rich Lane
2009-06-20 20:50 ` [sup-talk] [PATCH 01/18] remove load_entry_for_id call in sup-recover-sources Rich Lane
2009-06-20 20:50 ` [sup-talk] [PATCH 02/18] remove load_entry_for_id call in DraftManager.discard Rich Lane
2009-06-20 20:50 ` Rich Lane [this message]
2009-06-20 20:50 ` [sup-talk] [PATCH 04/18] index: remove unused method load_entry_for_id Rich Lane
2009-06-20 20:50 ` [sup-talk] [PATCH 05/18] switch DraftManager to use Message.build_from_source Rich Lane
2009-06-20 20:50 ` [sup-talk] [PATCH 06/18] index: move has_any_from_source_with_label? to sup-sync-back Rich Lane
2009-06-20 20:50 ` [sup-talk] [PATCH 07/18] move source-related methods to SourceManager Rich Lane
2009-06-20 20:50 ` [sup-talk] [PATCH 08/18] index: remove unused method fresh_thread_id Rich Lane
2009-06-20 20:50 ` [sup-talk] [PATCH 09/18] index: revert overeager opts->query rename in each_message_in_thread_for Rich Lane
2009-06-20 20:50 ` [sup-talk] [PATCH 10/18] index: make wrap_subj methods private Rich Lane
2009-06-20 20:50 ` [sup-talk] [PATCH 11/18] index: move Ferret-specific code to ferret_index.rb Rich Lane
2009-06-20 20:50 ` [sup-talk] [PATCH 12/18] remove last external uses of ferret docid Rich Lane
2009-06-20 20:50 ` [sup-talk] [PATCH 13/18] add Message.indexable_{body, chunks, subject} Rich Lane
2009-06-20 20:50 ` [sup-talk] [PATCH 14/18] index: choose index implementation with config entry or environment variable Rich Lane
2009-06-20 20:50 ` [sup-talk] [PATCH 15/18] index: add xapian implementation Rich Lane
2009-06-20 20:50 ` [sup-talk] [PATCH 16/18] fix String#ord monkeypatch Rich Lane
2009-06-20 20:50 ` [sup-talk] [PATCH 17/18] add limit argument to author_names_and_newness_for_thread Rich Lane
2009-06-20 20:50 ` [sup-talk] [PATCH 18/18] dont using SavingHash#[] for membership test Rich Lane
2009-06-22 14:46 ` Andrei Thorp
2009-06-24 16:30 ` [sup-talk] [PATCH 0/18] Xapian-based index William Morgan
2009-06-24 17:33 ` William Morgan
2009-06-26 2:00 ` Olly Betts
2009-06-26 13:49 ` William Morgan
2009-07-17 23:42 ` Richard Heycock
2009-07-23 10:23 ` Adeodato Simó
2009-07-25 4:53 ` Rich Lane
2009-07-25 9:21 ` Adeodato Simó
2009-07-25 19:59 ` Rich Lane
2009-07-25 23:28 ` Ingmar Vanhassel
2009-07-27 15:48 ` William Morgan
2009-07-27 16:56 ` Ingmar Vanhassel
2009-09-01 8:07 ` Ingmar Vanhassel
2009-09-03 16:52 ` Rich Lane
2009-07-27 17:06 ` Rich Lane
2009-07-31 16:20 ` Rich Lane
2009-08-12 13:05 ` Ingmar Vanhassel
2009-08-12 14:32 ` Nicolas Pouillard
2009-08-14 5:23 ` Rich Lane
2009-07-27 15:46 ` William Morgan
2009-07-28 16:53 ` Olly Betts
2009-07-28 17:01 ` William Morgan
2009-07-28 13:47 ` Olly Betts
2009-07-28 15:07 ` William Morgan
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=1245531017-9907-4-git-send-email-rlane@club.cc.cmu.edu \
--to=rlane@club.cc.cmu.edu \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox