sup

commit 5ea8a16105b90a2d98adacd30939c40b26b9f180
parent 23614190f51b7f98bffb64a879f2299d168d144c
Author: Rich Lane <rlane@club.cc.cmu.edu>
Date:   Wed, 24 Mar 2010 21:52:16 -0700

use Index to efficiently scan maildir

Diffstat:

1 file changed, 24 insertions(+), 87 deletions(-)
diff --git a/lib/sup/maildir.rb b/lib/sup/maildir.rb
@@ -3,14 +3,8 @@ require 'uri'
 
 module Redwood
 
-## Maildir doesn't provide an ordered unique id, which is what Sup
-## requires to be really useful. So we must maintain, in memory, a
-## mapping between Sup "ids" (timestamps, essentially) and the
-## pathnames on disk.
-
 class Maildir < Source
   include SerializeLabelsNicely
-  SCAN_INTERVAL = 30 # seconds
   MYHOSTNAME = Socket.gethostname
 
   ## remind me never to use inheritance again.
@@ -25,15 +19,7 @@ class Maildir < Source
 
     @dir = uri.path
     @labels = Set.new(labels || [])
-    @ids = []
-    @ids_to_fns = {}
-    @last_scan = nil
     @mutex = Mutex.new
-    #the mtime from the subdirs in the maildir with the unix epoch as default.
-    #these are used to determine whether scanning the directory for new mail
-    #is a worthwhile effort
-    @mtimes = { 'cur' => Time.at(0), 'new' => Time.at(0) }
-    @dir_ids = { 'cur' => [], 'new' => [] }
   end
 
   def file_path; @dir end
@@ -69,7 +55,6 @@ class Maildir < Source
   end
 
   def each_raw_message_line id
-    scan_mailbox
     with_file_for(id) do |f|
       until f.eof?
         yield f.gets
@@ -78,17 +63,14 @@ class Maildir < Source
   end
 
   def load_header id
-    scan_mailbox
     with_file_for(id) { |f| parse_raw_email_header f }
   end
 
   def load_message id
-    scan_mailbox
     with_file_for(id) { |f| RMail::Parser.read f }
   end
 
   def raw_header id
-    scan_mailbox
     ret = ""
     with_file_for(id) do |f|
       until f.eof? || (l = f.gets) =~ /^$/
@@ -99,101 +81,56 @@ class Maildir < Source
   end
 
   def raw_message id
-    scan_mailbox
     with_file_for(id) { |f| f.read }
   end
 
-  def scan_mailbox opts={}
-    return unless @ids.empty? || opts[:rescan]
-    return if @last_scan && (Time.now - @last_scan) < SCAN_INTERVAL
-
-    initial_poll = @ids.empty?
-
-    debug "scanning maildir #@dir..."
-    begin
-      @mtimes.each_key do |d|
-        subdir = File.join(@dir, d)
-        raise FatalSourceError, "#{subdir} not a directory" unless File.directory? subdir
-
-        mtime = File.mtime subdir
-
-        #only scan the dir if the mtime is more recent (or we haven't polled
-        #since startup)
-        if @mtimes[d] < mtime || initial_poll
-          @mtimes[d] = mtime
-          @dir_ids[d] = []
-          Dir[File.join(subdir, '*')].map do |fn|
-            id = make_id fn
-            @dir_ids[d] << id
-            @ids_to_fns[id] = fn
-          end
-        else
-          debug "no poll on #{d}.  mtime on indicates no new messages."
-        end
-      end
-      @ids = @dir_ids.values.flatten.uniq.sort!
-    rescue SystemCallError, IOError => e
-      raise FatalSourceError, "Problem scanning Maildir directories: #{e.message}."
-    end
-
-    debug "done scanning maildir"
-    @last_scan = Time.now
-  end
-  synchronized :scan_mailbox
-
+  ## XXX use less memory
   def each
-    scan_mailbox
-    @ids.each do |id|
+    old_ids = Enumerator.new(Index, :each_source_info, self.id).to_a
+    new_ids = Dir.glob("#{@dir}/*/*").map { |x| x[@dir.length..-1] }.sort
+    added = new_ids - old_ids
+    deleted = old_ids - new_ids
+    info "#{added.size} added, #{deleted.size} deleted"
+    added.each do |id|
       yield id, @labels + (seen?(id) ? [] : [:unread]) + (trashed?(id) ? [:deleted] : []) + (flagged?(id) ? [:starred] : [])
     end
+    nil
   end
 
   def pct_done; 100.0 * (0).to_f / (@ids.length - 1).to_f; end
 
-  def draft? msg; maildir_data(msg)[2].include? "D"; end
-  def flagged? msg; maildir_data(msg)[2].include? "F"; end
-  def passed? msg; maildir_data(msg)[2].include? "P"; end
-  def replied? msg; maildir_data(msg)[2].include? "R"; end
-  def seen? msg; maildir_data(msg)[2].include? "S"; end
-  def trashed? msg; maildir_data(msg)[2].include? "T"; end
+  def draft? id; maildir_data(id)[2].include? "D"; end
+  def flagged? id; maildir_data(id)[2].include? "F"; end
+  def passed? id; maildir_data(id)[2].include? "P"; end
+  def replied? id; maildir_data(id)[2].include? "R"; end
+  def seen? id; maildir_data(id)[2].include? "S"; end
+  def trashed? id; maildir_data(id)[2].include? "T"; end
 
-  def mark_draft msg; maildir_mark_file msg, "D" unless draft? msg; end
-  def mark_flagged msg; maildir_mark_file msg, "F" unless flagged? msg; end
-  def mark_passed msg; maildir_mark_file msg, "P" unless passed? msg; end
-  def mark_replied msg; maildir_mark_file msg, "R" unless replied? msg; end
-  def mark_seen msg; maildir_mark_file msg, "S" unless seen? msg; end
-  def mark_trashed msg; maildir_mark_file msg, "T" unless trashed? msg; end
-
-  def filename_for_id id; @ids_to_fns[id] end
+  def mark_draft id; maildir_mark_file id, "D" unless draft? id; end
+  def mark_flagged id; maildir_mark_file id, "F" unless flagged? id; end
+  def mark_passed id; maildir_mark_file id, "P" unless passed? id; end
+  def mark_replied id; maildir_mark_file id, "R" unless replied? id; end
+  def mark_seen id; maildir_mark_file id, "S" unless seen? id; end
+  def mark_trashed id; maildir_mark_file id, "T" unless trashed? id; end
 
 private
 
-  def make_id fn
-    #doing this means 1 syscall instead of 2 (File.mtime, File.size).
-    #makes a noticeable difference on nfs.
-    stat = File.stat(fn)
-    # use 7 digits for the size. why 7? seems nice.
-    sprintf("%d%07d", stat.mtime, stat.size % 10000000).to_i
-  end
-
   def new_maildir_basefn
     Kernel::srand()
     "#{Time.now.to_i.to_s}.#{$$}#{Kernel.rand(1000000)}.#{MYHOSTNAME}"
   end
 
   def with_file_for id
-    fn = @ids_to_fns[id] or raise OutOfSyncSourceError, "No such id: #{id.inspect}."
     begin
-      File.open(fn, 'rb') { |f| yield f }
+      File.open(File.join(@dir, id), 'rb') { |f| yield f }
     rescue SystemCallError, IOError => e
       raise FatalSourceError, "Problem reading file for id #{id.inspect}: #{fn.inspect}: #{e.message}."
     end
   end
 
-  def maildir_data msg
-    fn = File.basename @ids_to_fns[msg]
-    fn =~ %r{^([^:]+):([12]),([DFPRST]*)$}
-    [($1 || fn), ($2 || "2"), ($3 || "")]
+  def maildir_data id
+    id =~ %r{^([^:]+):([12]),([DFPRST]*)$}
+    [($1 || id), ($2 || "2"), ($3 || "")]
   end
 
   ## not thread-safe on msg