Subject: [sup-devel] [PATCH 1/3] remove mbox+ssh source

From: Rich Lane <[email-masked]>
Date: Sun, 17 Jan 2010 10:50:45 -0800

---
 README.txt                 |    1 -
 Rakefile                   |    1 -
 bin/sup-add                |    8 --
 bin/sup-config             |   13 ---
 doc/NewUserGuide.txt       |    1 -
 lib/sup/mbox.rb            |    2 -
 lib/sup/mbox/ssh-file.rb   |  254 --------------------------------------------
 lib/sup/mbox/ssh-loader.rb |   74 -------------
 8 files changed, 0 insertions(+), 354 deletions(-)
 delete mode 100644 lib/sup/mbox/ssh-file.rb
 delete mode 100644 lib/sup/mbox/ssh-loader.rb

diff --git a/README.txt b/README.txt
index 4204270..7819f0b 100644
--- a/README.txt
+++ b/README.txt
@@ -100,7 +100,6 @@ Current limitations which will be fixed:
  - ncurses >= 0.9.1
  - rmail >= 0.17
  - highline
- - net-ssh
  - trollop >= 1.12
  - lockfile
  - mime-types
diff --git a/Rakefile b/Rakefile
index 67cd0d2..9cf911b 100644
--- a/Rakefile
+++ b/Rakefile
@@ -51,7 +51,6 @@ spec = Gem::Specification.new do |s|
   s.add_dependency "ncurses", ">= 0.9.1"
   s.add_dependency "rmail", ">= 0.17"
   s.add_dependency "highline"
-  s.add_dependency "net-ssh"
   s.add_dependency "trollop", ">= 1.12"
   s.add_dependency "lockfile"
   s.add_dependency "mime-types", "~> 1"
diff --git a/bin/sup-add b/bin/sup-add
index a7a3752..b92b5ad 100755
--- a/bin/sup-add
+++ b/bin/sup-add
@@ -20,9 +20,6 @@ For mbox files on local disk, use the form:
     mbox:<path to mbox file>, or
     mbox://<path to mbox file>
 
-For mbox files on remote machines, use the form:
-    mbox+ssh://<machine name>/<path to mbox file>
-
 For IMAP folders, use the form (note no username or password!):
     imap://<machine name>/          # unsecure, "INBOX" folder  
     imap://<machine name>/<folder>  # unsecure, arbitrary
@@ -105,11 +102,6 @@ begin
 
     source = 
       case parsed_uri.scheme
-      when "mbox+ssh"
-        say "For SSH connections, if you will use public key authentication, you may leave the username and password blank."
-        say ""
-        username, password = get_login_info uri, Redwood::SourceManager.sources
-        Redwood::MBox::SSHLoader.new uri, username, password, nil, !$opts[:unusual], $opts[:archive], nil, labels
       when "imap", "imaps"
         username, password = get_login_info uri, Redwood::SourceManager.sources
         Redwood::IMAP.new uri, username, password, nil, !$opts[:unusual], $opts[:archive], nil, labels
diff --git a/bin/sup-config b/bin/sup-config
index b37e0b2..5541965 100755
--- a/bin/sup-config
+++ b/bin/sup-config
@@ -44,7 +44,6 @@ def add_source
     menu.prompt = "What type of mail source is it? "
     menu.choice("mbox file") { type = :mbox }
     menu.choice("maildir directory") { type = :maildir }
-    menu.choice("remote mbox file (accessible via ssh)") { type = :mboxssh }
     menu.choice("IMAP server (secure)") { type = :imaps }
     menu.choice("IMAP server (unsecure)") { type = :imap }
     menu.choice("Get me out of here!") { return }
@@ -70,18 +69,6 @@ def add_source
       $last_fn = fn
       [Redwood::Maildir.suggest_labels_for(fn),
        { :scheme => "maildir", :path => fn }]
-    when :mboxssh
-      $last_server ||= "localhost"
-      srv = axe "What machine is the mbox file located on?", $last_server
-      return if srv.nil? || srv.empty?
-      $last_server = srv
-
-      fn = axe "What's the path to the mbox file?", $last_fn
-      return if fn.nil? || fn.empty?
-      $last_fn = fn
-      fn = "/#{fn}" # lame
-      [Redwood::MBox::SSHLoader.suggest_labels_for(fn),
-       { :scheme => "mbox+ssh", :host => srv, :path => fn }]
     when :imap, :imaps
       $last_server ||= "localhost"
       srv = axe "What is the IMAP server (host, or host:port notation)?", $last_server
diff --git a/doc/NewUserGuide.txt b/doc/NewUserGuide.txt
index f22d2f1..4b46ae2 100644
--- a/doc/NewUserGuide.txt
+++ b/doc/NewUserGuide.txt
@@ -208,7 +208,6 @@ Instead of using sup-config to add a new source, you can manually run
 - maildir://path/to/a/filename, for a maildir directory on disk.
 - imap://imap.server/folder for an unsecure IMAP folder.
 - imaps://secure.imap.server/folder for a secure IMAP folder.
-- mbox+ssh://remote.machine/path/to/a/filename for a remote mbox file.
 
 Before you add the source, you need make three decisions. The first is
 whether you want Sup to regularly poll this source for new messages.
diff --git a/lib/sup/mbox.rb b/lib/sup/mbox.rb
index 7af40dd..57c371c 100644
--- a/lib/sup/mbox.rb
+++ b/lib/sup/mbox.rb
@@ -1,6 +1,4 @@
 require "sup/mbox/loader"
-require "sup/mbox/ssh-file"
-require "sup/mbox/ssh-loader"
 
 module Redwood
 
diff --git a/lib/sup/mbox/ssh-file.rb b/lib/sup/mbox/ssh-file.rb
deleted file mode 100644
index 4ae4bba..0000000
--- a/lib/sup/mbox/ssh-file.rb
+++ /dev/null
@@ -1,254 +0,0 @@
-require 'net/ssh'
-
-module Redwood
-module MBox
-
-class SSHFileError < StandardError; end
-
-## this is a file-like interface to a file that actually lives on the
-## other end of an ssh connection. it works by using wc, head and tail
-## to simulate (buffered) random access. on a fast connection, this
-## can have a good bandwidth, but the latency is pretty terrible:
-## about 1 second (!) per request.  luckily, we're either just reading
-## straight through the mbox (an import) or we're reading a few
-## messages at a time (viewing messages) so the latency is not a problem.
-
-## all of the methods here can throw SSHFileErrors, SocketErrors,
-## Net::SSH::Exceptions and Errno::ENOENTs.
-
-## a simple buffer of contiguous data
-class Buffer
-  def initialize
-    clear!
-  end
-
-  def clear!
-    @start = nil
-    @buf = ""
-  end
-
-  def empty?; @start.nil?; end
-  def start; @start; end
-  def endd; @start + @buf.length; end
-
-  def add data, offset=endd
-    #MBox::debug "+ adding #{data.length} bytes; size will be #{size + data.length}; limit #{SSHFile::MAX_BUF_SIZE}"
-
-    if start.nil?
-      @buf = data
-      @start = offset
-      return
-    end
-
-    raise "non-continguous data added to buffer (data #{offset}:#{offset + data.length}, buf range #{start}:#{endd})" if offset + data.length < start || offset > endd
-
-    if offset < start
-      @buf = data[0 ... (start - offset)] + @buf
-      @start = offset
-    else
-      return if offset + data.length < endd
-      @buf += data[(endd - offset) .. -1]
-    end
-  end
-
-  def [](o)
-    raise "only ranges supported due to programmer's laziness" unless o.is_a? Range
-    @buf[Range.new(o.first - @start, o.last - @start, o.exclude_end?)]
-  end
-
-  def index what, start=0
-    x = @buf.index(what, start - @start)
-    x.nil? ? nil : x + @start
-  end
-
-  def rindex what, start=0
-    x = @buf.rindex(what, start - @start)
-    x.nil? ? nil : x + @start
-  end
-
-  def size; empty? ? 0 : @buf.size; end
-  def to_s; empty? ? "<empty>" : "[#{start}, #{endd})"; end # for debugging
-end
-
-## sharing a ssh connection to one machines between sources seems to
-## create lots of broken situations: commands returning bizarre (large
-## positive integer) return codes despite working; commands
-## occasionally not working, etc. i suspect this is because of the
-## fragile nature of the ssh syncshell. 
-##
-## at any rate, we now open up one ssh connection per file, which is
-## probably silly in the extreme case.
-
-## the file-like interface to a remote file
-class SSHFile
-  MAX_BUF_SIZE = 1024 * 1024 # bytes
-  MAX_TRANSFER_SIZE = 1024 * 128
-  REASONABLE_TRANSFER_SIZE = 1024 * 32
-  SIZE_CHECK_INTERVAL = 60 * 1 # seconds
-
-  ## upon these errors we'll try to rereconnect a few times
-  RECOVERABLE_ERRORS = [ Errno::EPIPE, Errno::ETIMEDOUT ]
-
-  @@shells = {}
-  @@shells_mutex = Mutex.new
-
-  def initialize host, fn, ssh_opts={}
-    @buf = Buffer.new
-    @host = host
-    @fn = fn
-    @ssh_opts = ssh_opts
-    @file_size = nil
-    @offset = 0
-    @say_id = nil
-    @shell = nil
-    @shell_mutex = nil
-    @buf_mutex = Mutex.new
-  end
-
-  def to_s; "mbox+ssh://#@host/#@fn"; end ## TODO: remove this EVILness
-
-  def connect
-    do_remote nil
-  end
-
-  def eof?; @offset >= size; end
-  def eof; eof?; end # lame but IO's method is named this and rmail calls that
-  def seek loc; @offset = loc; end
-  def tell; @offset; end
-  def total; size; end
-  def path; @fn end
-
-  def size
-    if @file_size.nil? || (Time.now - @last_size_check) > SIZE_CHECK_INTERVAL
-      @last_size_check = Time.now
-      @file_size = do_remote("wc -c #@fn").split.first.to_i
-    end
-    @file_size
-  end
-
-  def gets
-    return nil if eof?
-    @buf_mutex.synchronize do
-      make_buf_include @offset
-      expand_buf_forward while @buf.index("\n", @offset).nil? && @buf.endd < size
-      returning(@buf[@offset .. (@buf.index("\n", @offset) || -1)]) { |line| @offset += line.length }
-    end
-  end
-
-  def read n
-    return nil if eof?
-    @buf_mutex.synchronize do
-      make_buf_include @offset, n
-      @buf[@offset ... (@offset += n)]
-    end
-  end
-
-private
-
-  ## TODO: share this code with imap
-  def say s
-    @say_id = BufferManager.say s, @say_id if BufferManager.instantiated?
-    info s
-  end
-
-  def shutup
-    BufferManager.clear @say_id if BufferManager.instantiated? && @say_id
-    @say_id = nil
-  end
-
-  def unsafe_connect
-    return if @shell
-
-    @key = [@host, @ssh_opts[:username]]
-    begin
-      @shell, @shell_mutex = @@shells_mutex.synchronize do
-        unless @@shells.member? @key
-          say "Opening SSH connection to #{@host} for #@fn..."
-          session = Net::SSH.start @host, @ssh_opts
-          say "Starting SSH shell..."
-          @@shells[@key] = [session.shell.sync, Mutex.new]
-        end
-        @@shells[@key]
-      end
-      
-      say "Checking for #@fn..."
-      @shell_mutex.synchronize { raise Errno::ENOENT, @fn unless @shell.test("-e #@fn").status == 0 }
-    ensure
-      shutup
-    end
-  end
-
-  def do_remote cmd, expected_size=0
-    retries = 0
-    result = nil
-
-    begin
-      unsafe_connect
-      if cmd
-        # MBox::debug "sending command: #{cmd.inspect}"
-        result = @shell_mutex.synchronize { x = @shell.send_command cmd; sleep 0.25; x }
-        raise SSHFileError, "Failure during remote command #{cmd.inspect}: #{(result.stderr || result.stdout || "")[0 .. 100]}" unless result.status == 0
-      end
-      ## Net::SSH::Exceptions seem to happen every once in a while for
-      ## no good reason.
-    rescue Net::SSH::Exception, *RECOVERABLE_ERRORS
-      if (retries += 1) <= 3
-        @@shells_mutex.synchronize do
-          @shell = nil
-          @@shells[@key] = nil
-        end
-        retry
-      end
-      raise
-    end
-
-    result.stdout if cmd
-  end
-
-  def get_bytes offset, size
-    do_remote "tail -c +#{offset + 1} #@fn | head -c #{size}", size
-  end
-
-  def expand_buf_forward n=REASONABLE_TRANSFER_SIZE
-    @buf.add get_bytes(@buf.endd, n)
-  end
-
-  ## try our best to transfer somewhere between
-  ## REASONABLE_TRANSFER_SIZE and MAX_TRANSFER_SIZE bytes
-  def make_buf_include offset, size=0
-    good_size = [size, REASONABLE_TRANSFER_SIZE].max
-
-    trans_start, trans_size = 
-      if @buf.empty?
-        [offset, good_size]
-      elsif offset < @buf.start
-        if @buf.start - offset <= good_size
-          start = [@buf.start - good_size, 0].max
-          [start, @buf.start - start]
-        elsif @buf.start - offset < MAX_TRANSFER_SIZE
-          [offset, @buf.start - offset]
-        else
-          MBox::debug "clearing SSH buffer because buf.start [email-masked]} - offset #{offset} >= #{MAX_TRANSFER_SIZE}"
-          @buf.clear!
-          [offset, good_size]
-        end
-      else
-        return if [offset + size, self.size].min <= @buf.endd # whoohoo!
-        if offset - @buf.endd <= good_size
-          [@buf.endd, good_size]
-        elsif offset - @buf.endd < MAX_TRANSFER_SIZE
-          [@buf.endd, offset - @buf.endd]
-        else
-          MBox::debug "clearing SSH buffer because offset #{offset} - buf.end [email-masked]} >= #{MAX_TRANSFER_SIZE}"
-          @buf.clear!
-          [offset, good_size]
-        end
-      end          
-
-    @buf.clear! if @buf.size > MAX_BUF_SIZE
-    @buf.add get_bytes(trans_start, trans_size), trans_start
-  end
-end
-
-end
-end
diff --git a/lib/sup/mbox/ssh-loader.rb b/lib/sup/mbox/ssh-loader.rb
deleted file mode 100644
index e422a48..0000000
--- a/lib/sup/mbox/ssh-loader.rb
+++ /dev/null
@@ -1,74 +0,0 @@
-require 'net/ssh'
-
-module Redwood
-module MBox
-
-class SSHLoader < Source
-  attr_accessor :username, :password
-
-  yaml_properties :uri, :username, :password, :cur_offset, :usual, 
-                  :archived, :id, :labels
-
-  def initialize uri, username=nil, password=nil, start_offset=nil, usual=true, archived=false, id=nil, labels=[]
-    raise ArgumentError, "not an mbox+ssh uri: #{uri.inspect}" unless uri =~ %r!^mbox\+ssh://!
-
-    super uri, start_offset, usual, archived, id
-
-    @parsed_uri = URI(uri)
-    @username = username
-    @password = password
-    @uri = uri
-    @cur_offset = start_offset
-    @labels = (labels || []).freeze
-
-    opts = {}
-    opts[:username] = @username if @username
-    opts[:password] = @password if @password
-    
-    @f = SSHFile.new host, filename, opts
-    @loader = Loader.new @f, start_offset, usual, archived, id
-    
-    ## heuristic: use the filename as a label, unless the file
-    ## has a path that probably represents an inbox.
-  end
-
-  def self.suggest_labels_for path; Loader.suggest_labels_for(path) end
-
-  def connect; safely { @f.connect }; end
-  def host; @parsed_uri.host; end
-  def filename; @parsed_uri.path[1..-1] end
-
-  def next
-    safely do
-      offset, labels = @loader.next
-      self.cur_offset = @loader.cur_offset # superclass keeps @cur_offset which is used by yaml
-      [offset, (labels + @labels).uniq] # add our labels
-    end
-  end
-
-  def end_offset
-    safely { @f.size }
-  end
-
-  def cur_offset= o; @cur_offset = @loader.cur_offset = o; @dirty = true; end
-  def id; @loader.id; end
-  def id= o; @id = @loader.id = o; end
-  # def cur_offset; @loader.cur_offset; end # think we'll be ok without this
-  def to_s; @parsed_uri.to_s; end
-
-  def safely
-    begin
-      yield
-    rescue Net::SSH::Exception, SocketError, SSHFileError, SystemCallError, IOError => e
-      m = "error communicating with SSH server #{host} (#{e.class.name}): #{e.message}"
-      raise FatalSourceError, m
-    end
-  end
-
-  [:start_offset, :load_header, :load_message, :raw_header, :raw_message].each do |meth|
-    define_method(meth) { |*a| safely { @loader.send meth, *a } }
-  end
-end
-
-end
-end
-- 
1.6.3.3

_______________________________________________
Sup-devel mailing list
[email-masked]
http://rubyforge.org/mailman/listinfo/sup-devel

---

From: Rich Lane <[email-masked]>
Date: Sun, 17 Jan 2010 10:50:47 -0800

---
 bin/sup-add |   40 ----------------------------------------
 1 files changed, 0 insertions(+), 40 deletions(-)

diff --git a/bin/sup-add b/bin/sup-add
index 059a4e2..e354ea8 100755
--- a/bin/sup-add
+++ b/bin/sup-add
@@ -30,50 +30,10 @@ EOS
   opt :unusual, "Do not automatically poll these sources for new messages."
   opt :labels, "A comma-separated set of labels to apply to all messages from this source", :type => String
   opt :force_new, "Create a new account for this source, even if one already exists."
-  opt :force_account, "Reuse previously defined account user@hostname.", :type => String
 end
 
 Trollop::die "require one or more sources" if ARGV.empty?
 
-## for sources that require login information, prompt the user for
-## that. also provide a list of previously-defined login info to
-## choose from, if any.
-def get_login_info uri, sources
-  uri = URI(uri)
-  accounts = sources.map do |s|
-    next unless s.respond_to?(:username)
-    suri = URI(s.uri)
-    [suri.host, s.username, s.password]
-  end.compact.uniq.sort_by { |h, u, p| h == uri.host ? 0 : 1 }
-
-  username, password = nil, nil
-  unless accounts.empty? || $opts[:force_new]
-    if $opts[:force_account]
-      host, username, password = accounts.find { |h, u, p| $opts[:force_account] == "#{u}@#{h}" }
-      unless username && password
-        say "No previous account #{$opts[:force_account].inspect} found."
-      end
-    else
-      say "Would you like to use the same account as for a previous source for #{uri}?"
-      choose do |menu|
-        accounts.each do |host, olduser, oldpw|
-          menu.choice("Use the account info for #{olduser}@#{host}") { username, password = olduser, oldpw }
-        end
-        menu.choice("Use a new account") { }
-        menu.prompt = "Account selection? "
-      end
-    end
-  end
-
-  unless username && password
-    username = ask("Username for #{uri.host}: ");
-    password = ask("Password for #{uri.host}: ") { |q| q.echo = false }
-    puts # why?
-  end
-
-  [username, password]
-end
-
 $terminal.wrap_at = :auto
 Redwood::start
 index = Redwood::Index.init
-- 
1.6.3.3

---

From: Rich Lane <[email-masked]>
Date: Sun, 17 Jan 2010 10:50:44 -0800

Every few days someone joins the IRC channel complaining of poor
performance or bugs with IMAP sources, and we have to tell them to use
offlineimap or fetchmail instead. This is embarrassing. There are likely
many more users who don't bother to ask on IRC and either unhappily
tolerate IMAP or give up on Sup.

Even with IMAP you still need to store a huge index on disk, so I can't
accept that you'd have enough space for that but not for the actual
messages. You still need to download every message to index it. Your
mails aren't any more secure from someone stealing your laptop. The same
arguments apply to mbox+ssh.

The benefit for me is that I won't have to deal with slow, blocking
sources when converting the UI to EventMachine.

If people agree that this change is a good idea, I assume William will
add a deprecation notice to master for 0.10 and merge this patchset to
next.
_______________________________________________
Sup-devel mailing list
[email-masked]
http://rubyforge.org/mailman/listinfo/sup-devel

---

From: William Morgan <[email-masked]>
Date: Mon, 18 Jan 2010 12:06:28 -0500

Reformatted excerpts from William Morgan's message of 2010-01-17:
> Yeah, another thing to add to 0.10... sigh.

On second thought, I'm going to wait to 0.12 to remove this, and add the
deprecation notice on 0.11. The poor guy with a ferret index and
multiple IMAP sources should have a fighting chance.

I'm also going to try and spend less time between releases...
-- 
William <[email-masked]>
_______________________________________________
Sup-devel mailing list
[email-masked]
http://rubyforge.org/mailman/listinfo/sup-devel

---

From: William Morgan <[email-masked]>
Date: Sun, 17 Jan 2010 15:39:58 -0500

Reformatted excerpts from Rich Lane's message of 2010-01-17:
> offlineimap or fetchmail instead.

I think this is a great idea. I'm sick of the same thing happening on
the mailing list. I actually currently use a plain IMAP source
successfully, but it's just a matter of time before it will start
bogging down.

---

From: Rich Lane <[email-masked]>
Date: Sun, 17 Jan 2010 10:50:46 -0800

---
 README.txt              |   10 +--
 bin/sup-add             |    9 --
 bin/sup-config          |   21 ---
 bin/sup-recover-sources |   13 +--
 doc/FAQ.txt             |    8 +-
 doc/NewUserGuide.txt    |   21 +--
 lib/sup.rb              |    1 -
 lib/sup/imap.rb         |  349 -----------------------------------------------
 lib/sup/source.rb       |    8 +-
 lib/sup/xapian_index.rb |    1 +
 10 files changed, 16 insertions(+), 425 deletions(-)
 delete mode 100644 lib/sup/imap.rb

diff --git a/README.txt b/README.txt
index 7819f0b..dba7298 100644
--- a/README.txt
+++ b/README.txt
@@ -12,8 +12,7 @@ email as an extension of your long-term memory, Sup is for you.
 Sup makes it easy to:
 - Handle massive amounts of email.
 
-- Mix email from different sources: mbox files (even across different
-  machines), Maildir directories, IMAP folders, and GMail accounts.
+- Mix email from different sources: mbox files and Maildir directories.
 
 - Instantaneously search over your entire email collection. Search over
   body text, or use a query language to combine search predicates in any
@@ -73,13 +72,6 @@ Current limitations which will be fixed:
   source (read, move, delete, etc) with another client Sup will punish
   you with a lengthy reindexing process.
 
-- Support for mbox, Maildir, and IMAP only at this point. No support for
-  POP or mh.
-
-- IMAP support is very slow due mostly to Ruby's IMAP library.  You may
-  consider something like offlineimap to mirror your IMAP folders with
-  local Maildir ones.
-
 - Unix-centrism in MIME attachment handling and in sendmail invocation.
 
 == SYNOPSYS:
diff --git a/bin/sup-add b/bin/sup-add
index b92b5ad..059a4e2 100755
--- a/bin/sup-add
+++ b/bin/sup-add
@@ -20,12 +20,6 @@ For mbox files on local disk, use the form:
     mbox:<path to mbox file>, or
     mbox://<path to mbox file>
 
-For IMAP folders, use the form (note no username or password!):
-    imap://<machine name>/          # unsecure, "INBOX" folder  
-    imap://<machine name>/<folder>  # unsecure, arbitrary
-    imaps://<machine name>/         # secure, "INBOX" folder
-    imaps://<machine name>/<folder> # secure, arbitrary folder 
-
 For Maildir folders, use the form:
     maildir:<path to Maildir directory>; or
     maildir://<path to Maildir directory>
@@ -102,9 +96,6 @@ begin
 
     source = 
       case parsed_uri.scheme
-      when "imap", "imaps"
-        username, password = get_login_info uri, Redwood::SourceManager.sources
-        Redwood::IMAP.new uri, username, password, nil, !$opts[:unusual], $opts[:archive], nil, labels
       when "maildir"
         Redwood::Maildir.new uri, nil, !$opts[:unusual], $opts[:archive], nil, labels
       when "mbox"
diff --git a/bin/sup-config b/bin/sup-config
index 5541965..059af6d 100755
--- a/bin/sup-config
+++ b/bin/sup-config
@@ -44,8 +44,6 @@ def add_source
     menu.prompt = "What type of mail source is it? "
     menu.choice("mbox file") { type = :mbox }
     menu.choice("maildir directory") { type = :maildir }
-    menu.choice("IMAP server (secure)") { type = :imaps }
-    menu.choice("IMAP server (unsecure)") { type = :imap }
     menu.choice("Get me out of here!") { return }
   end
 
@@ -69,25 +67,6 @@ def add_source
       $last_fn = fn
       [Redwood::Maildir.suggest_labels_for(fn),
        { :scheme => "maildir", :path => fn }]
-    when :imap, :imaps
-      $last_server ||= "localhost"
-      srv = axe "What is the IMAP server (host, or host:port notation)?", $last_server
-      return if srv.nil? || srv.empty?
-      $last_server = srv
-
-      $last_folder ||= "INBOX"
-      fn = axe "What's the folder path?", $last_folder
-      return if fn.nil? || fn.empty?
-      $last_folder = fn
-
-      fn = "/#{fn}"
-      if srv =~ /^(\S+):(\d+)$/
-        host, port = $1, $2.to_i
-      else
-        host, port = srv, nil
-      end
-      [Redwood::IMAP.suggest_labels_for(fn),
-       { :scheme => type.to_s, :host => host, :port => port, :path => fn }]
     end
 
     uri = begin
diff --git a/bin/sup-recover-sources b/bin/sup-recover-sources
index 43fa5f6..03ac97d 100755
--- a/bin/sup-recover-sources
+++ b/bin/sup-recover-sources
@@ -58,17 +58,8 @@ ARGV.each do |fn|
   next if Redwood::SourceManager.source_for fn
 
   ## TODO: merge this code with the same snippet in import
-  source = 
-    case fn
-    when %r!^imaps?://!
-      print "Username for #{fn}: "
-      username = $stdin.gets.chomp
-      print "Password for #{fn} (warning: cleartext): "
-      password = $stdin.gets.chomp
-      Redwood::IMAP.new(fn, username, password, nil, !$opts[:unusual], $opts[:archive])
-    else
-      Redwood::MBox::Loader.new(fn, nil, !$opts[:unusual], $opts[:archive])
-    end
+  ## XXX maildir?
+  source = Redwood::MBox::Loader.new(fn, nil, !$opts[:unusual], $opts[:archive])
 
   source_ids = Hash.new 0
   count = 0
diff --git a/doc/FAQ.txt b/doc/FAQ.txt
index 997a357..1a86211 100644
--- a/doc/FAQ.txt
+++ b/doc/FAQ.txt
@@ -105,13 +105,7 @@ S: The current solution is to directly modify RubyMail. Change line 159 of
 P: I see this error:
      /usr/local/lib/ruby/1.8/yaml.rb:133:in `transfer': allocator undefined for Bignum (TypeError)
 S: You need to upgrade to Ruby 1.8.5. YAML in earlier versions can't
-   parse BigNums, but Sup relies on that for Maildir and IMAP.
-
-P: I see this error:
-     /usr/lib/ruby/1.8/net/imap.rb:204: uninitialized constant Net::IMAP::SSL (NameError)
-S: You need to install a package called libssl-ruby or something similar.
-   Or, don't use imaps:// sources. Ruby's IMAP library otherwise fails in
-   this completely uninformative manner.
+   parse BigNums, but Sup relies on that for Maildir.
 
 P: When I run Sup remotely and view an HTML attachment, an existing
    Firefox on the *local* machine is redirected to the attachment
diff --git a/doc/NewUserGuide.txt b/doc/NewUserGuide.txt
index 4b46ae2..af18aeb 100644
--- a/doc/NewUserGuide.txt
+++ b/doc/NewUserGuide.txt
@@ -19,14 +19,14 @@ index is where Sup stores all message state (e.g. read or unread, any
 message labels), and all information necessary for searching and for
 threading messages. Sup only knows about messages in its index.
 
-We can add messages to the index by telling Sup about the "source"
-where the messages reside. Sources are things like IMAP folders, mbox
-folders, and maildir directories. Sup doesn't duplicate the actual
-message content in the index; it only stores whatever information is
-necessary for searching, threading and labelling. So when you search
-for messages or view your inbox, Sup talks only to the index (stored
-locally on disk). When you view a thread, Sup requests the full
-content of all the messages from the source.
+We can add messages to the index by telling Sup about the "source" where
+the messages reside. Sources are things like mbox folders and maildir
+directories. Sup doesn't duplicate the actual message content in the
+index; it only stores whatever information is necessary for searching,
+threading and labelling. So when you search for messages or view your
+inbox, Sup talks only to the index (stored locally on disk). When you
+view a thread, Sup requests the full content of all the messages from
+the source.
 
 The easiest way to set up all your sources is to run `sup-config`.
 This will interactively walk you through some basic configuration,
@@ -206,8 +206,6 @@ Instead of using sup-config to add a new source, you can manually run
 
 - mbox://path/to/a/filename, for an mbox file on disk.
 - maildir://path/to/a/filename, for a maildir directory on disk.
-- imap://imap.server/folder for an unsecure IMAP folder.
-- imaps://secure.imap.server/folder for a secure IMAP folder.
 
 Before you add the source, you need make three decisions. The first is
 whether you want Sup to regularly poll this source for new messages.
@@ -226,9 +224,6 @@ lists that you don't want polluting your inbox.
 The final decision is whether you want any labels automatically
 applied to messages from this source. You can use `--labels` to do this.
 
-If Sup requires account information, e.g. for IMAP servers and remote
-mbox files, `sup-add` will ask for it.
-
 Now that you've added the source, let's import all the current
 messages from it, by running sup-sync with the source URI. You can
 specify `--archive` to automatically archive all messages in this
diff --git a/lib/sup.rb b/lib/sup.rb
index ddc602e..ef1b49a 100644
--- a/lib/sup.rb
+++ b/lib/sup.rb
@@ -274,7 +274,6 @@ require "sup/message"
 require "sup/source"
 require "sup/mbox"
 require "sup/maildir"
-require "sup/imap"
 require "sup/person"
 require "sup/account"
 require "sup/thread"
diff --git a/lib/sup/imap.rb b/lib/sup/imap.rb
deleted file mode 100644
index bdb9e15..0000000
--- a/lib/sup/imap.rb
+++ /dev/null
@@ -1,349 +0,0 @@
-require 'uri'
-require 'net/imap'
-require 'stringio'
-require 'time'
-require 'rmail'
-require 'cgi'
-require 'set'
-
-## TODO: remove synchronized method protector calls; use a Monitor instead
-## (ruby's reentrant mutex)
-
-## fucking imap fucking sucks. what the FUCK kind of committee of dunces
-## designed this shit.
-##
-## imap talks about 'unique ids' for messages, to be used for
-## cross-session identification. great---just what sup needs! except it
-## turns out the uids can be invalidated every time the 'uidvalidity'
-## value changes on the server, and 'uidvalidity' can change without
-## restriction. it can change any time you log in. it can change EVERY
-## time you log in. of course the imap spec "strongly recommends" that it
-## never change, but there's nothing to stop people from just setting it
-## to the current timestamp, and in fact that's EXACTLY what the one imap
-## server i have at my disposal does. thus the so-called uids are
-## absolutely useless and imap provides no cross-session way of uniquely
-## identifying a message. but thanks for the "strong recommendation",
-## guys!
-##
-## so right now i'm using the 'internal date' and the size of each
-## message to uniquely identify it, and i scan over the entire mailbox
-## each time i open it to map those things to message ids. that can be
-## slow for large mailboxes, and we'll just have to hope that there are
-## no collisions. ho ho! a perfectly reasonable solution!
-##
-## and here's another thing. check out RFC2060 2.2.2 paragraph 5:
-##
-##   A client MUST be prepared to accept any server response at all
-##   times.  This includes server data that was not requested.
-##
-## yeah. that totally makes a lot of sense. and once again, the idiocy of
-## the spec actually happens in practice. you'll request flags for one
-## message, and get it interspersed with a random bunch of flags for some
-## other messages, including a different set of flags for the same
-## message! totally ok by the imap spec. totally retarded by any other
-## metric.
-##
-## fuck you, imap committee. you managed to design something nearly as
-## shitty as mbox but goddamn THIRTY YEARS LATER.
-module Redwood
-
-class IMAP < Source
-  include SerializeLabelsNicely
-  SCAN_INTERVAL = 60 # seconds
-
-  ## upon these errors we'll try to rereconnect a few times
-  RECOVERABLE_ERRORS = [ Errno::EPIPE, Errno::ETIMEDOUT, OpenSSL::SSL::SSLError ]
-
-  attr_accessor :username, :password
-  yaml_properties :uri, :username, :password, :cur_offset, :usual,
-                  :archived, :id, :labels
-
-  def initialize uri, username, password, last_idate=nil, usual=true, archived=false, id=nil, labels=[]
-    raise ArgumentError, "username and password must be specified" unless username && password
-    raise ArgumentError, "not an imap uri" unless uri =~ %r!imaps?://!
-
-    super uri, last_idate, usual, archived, id
-
-    @parsed_uri = URI(uri)
-    @username = username
-    @password = password
-    @imap = nil
-    @imap_state = {}
-    @ids = []
-    @last_scan = nil
-    @labels = Set.new((labels || []) - LabelManager::RESERVED_LABELS)
-    @say_id = nil
-    @mutex = Mutex.new
-  end
-
-  def self.suggest_labels_for path
-    path =~ /([^\/]*inbox[^\/]*)/i ? [$1.downcase.intern] : []
-  end
-
-  def host; @parsed_uri.host; end
-  def port; @parsed_uri.port || (ssl? ? 993 : 143); end
-  def mailbox
-    x = @parsed_uri.path[1..-1]
-    (x.nil? || x.empty?) ? 'INBOX' : CGI.unescape(x)
-  end
-  def ssl?; @parsed_uri.scheme == 'imaps' end
-
-  def check; end # do nothing because anything we do will be too slow,
-                 # and we'll catch the errors later.
-
-  ## is this necessary? TODO: remove maybe
-  def == o; o.is_a?(IMAP) && o.uri == self.uri && o.username == self.username; end
-
-  def load_header id
-    parse_raw_email_header StringIO.new(raw_header(id))
-  end
-
-  def load_message id
-    RMail::Parser.read raw_message(id)
-  end
-  
-  def each_raw_message_line id
-    StringIO.new(raw_message(id)).each { |l| yield l }
-  end
-
-  def raw_header id
-    unsynchronized_scan_mailbox
-    header, flags = get_imap_fields id, 'RFC822.HEADER'
-    header.gsub(/\r\n/, "\n")
-  end
-  synchronized :raw_header
-
-  def store_message date, from_email, &block
-    message = StringIO.new
-    yield message
-    message.string.gsub! /\n/, "\r\n"
-
-    safely { @imap.append mailbox, message.string, [:Seen], Time.now }
-  end
-
-  def raw_message id
-    unsynchronized_scan_mailbox
-    get_imap_fields(id, 'RFC822').first.gsub(/\r\n/, "\n")
-  end
-  synchronized :raw_message
-
-  def mark_as_deleted ids
-    ids = [ids].flatten # accept single arguments
-    unsynchronized_scan_mailbox
-    imap_ids = ids.map { |i| @imap_state[i] && @imap_state[i][:id] }.compact
-    return if imap_ids.empty?
-    @imap.store imap_ids, "+FLAGS", [:Deleted]
-  end
-  synchronized :mark_as_deleted
-
-  def expunge
-    @imap.expunge
-    unsynchronized_scan_mailbox true
-    true
-  end
-  synchronized :expunge
-
-  def connect
-    return if @imap
-    safely { } # do nothing!
-  end
-  synchronized :connect
-
-  def scan_mailbox force=false
-    return if !force && @last_scan && (Time.now - @last_scan) < SCAN_INTERVAL
-    last_id = safely do
-      @imap.examine mailbox
-      @imap.responses["EXISTS"].last
-    end
-    @last_scan = Time.now
-
-    @ids = [] if force
-    return if last_id == @ids.length
-
-    range = (@ids.length + 1) .. last_id
-    debug "fetching IMAP headers #{range}"
-    fetch(range, ['RFC822.SIZE', 'INTERNALDATE', 'FLAGS']).each do |v|
-      id = make_id v
-      @ids << id
-      @imap_state[id] = { :id => v.seqno, :flags => v.attr["FLAGS"] }
-    end
-    debug "done fetching IMAP headers"
-  end
-  synchronized :scan_mailbox
-
-  def each
-    return unless start_offset
-
-    ids = 
-      @mutex.synchronize do
-        unsynchronized_scan_mailbox
-        @ids
-      end
-
-    start = ids.index(cur_offset || start_offset) or raise OutOfSyncSourceError, "Unknown message id #{cur_offset || start_offset}."
-
-    start.upto(ids.length - 1) do |i|
-      id = ids[i]
-      state = @mutex.synchronize { @imap_state[id] } or next
-      self.cur_offset = id 
-      labels = { :Flagged => :starred,
-                 :Deleted => :deleted
-               }.inject(@labels) do |cur, (imap, sup)|
-        cur + (state[:flags].include?(imap) ? [sup] : [])
-      end
-
-      labels += [:unread] unless state[:flags].include?(:Seen)
-
-      yield id, labels
-    end
-  end
-
-  def start_offset
-    unsynchronized_scan_mailbox
-    @ids.first
-  end
-  synchronized :start_offset
-
-  def end_offset
-    unsynchronized_scan_mailbox
-    @ids.last + 1
-  end
-  synchronized :end_offset
-
-  def pct_done; 100.0 * (@ids.index(cur_offset) || 0).to_f / (@ids.length - 1).to_f; end
-
-private
-
-  def fetch ids, fields
-    results = safely { @imap.fetch ids, fields }
-    good_results = 
-      if ids.respond_to? :member?
-        results.find_all { |r| ids.member?(r.seqno) && fields.all? { |f| r.attr.member?(f) } }
-      else
-        results.find_all { |r| ids == r.seqno && fields.all? { |f| r.attr.member?(f) } }
-      end
-
-    if good_results.empty?
-      raise FatalSourceError, "no IMAP response for #{ids} containing all fields #{fields.join(', ')} (got #{results.size} results)"
-    elsif good_results.size < results.size
-      warn "Your IMAP server sucks. It sent #{results.size} results for a request for #{good_results.size} messages. What are you using, Binc?"
-    end
-
-    good_results
-  end
-
-  def unsafe_connect
-    say "Connecting to IMAP server #{host}:#{port}..."
-
-    ## apparently imap.rb does a lot of threaded stuff internally and if
-    ## an exception occurs, it will catch it and re-raise it on the
-    ## calling thread. but i can't seem to catch that exception, so i've
-    ## resorted to initializing it in its own thread. surely there's a
-    ## better way.
-    exception = nil
-    ::Thread.new do
-      begin
-        #raise Net::IMAP::ByeResponseError, "simulated imap failure"
-        @imap = Net::IMAP.new host, port, ssl?
-        say "Logging in..."
-
-        ## although RFC1730 claims that "If an AUTHENTICATE command fails
-        ## with a NO response, the client may try another", in practice
-        ## it seems like they can also send a BAD response.
-        begin
-          raise Net::IMAP::NoResponseError unless @imap.capability().member? "AUTH=CRAM-MD5"
-          @imap.authenticate 'CRAM-MD5', @username, @password
-        rescue Net::IMAP::BadResponseError, Net::IMAP::NoResponseError => e
-          debug "CRAM-MD5 authentication failed: #{e.class}. Trying LOGIN auth..."
-          begin
-            raise Net::IMAP::NoResponseError unless @imap.capability().member? "AUTH=LOGIN"
-            @imap.authenticate 'LOGIN', @username, @password
-          rescue Net::IMAP::BadResponseError, Net::IMAP::NoResponseError => e
-            debug "LOGIN authentication failed: #{e.class}. Trying plain-text LOGIN..."
-            @imap.login @username, @password
-          end
-        end
-        say "Successfully connected to #{@parsed_uri}."
-      rescue Exception => e
-        exception = e
-      ensure
-        shutup
-      end
-    end.join
-
-    raise exception if exception
-  end
-
-  def say s
-    @say_id = BufferManager.say s, @say_id if BufferManager.instantiated?
-    info s
-  end
-
-  def shutup
-    BufferManager.clear @say_id if BufferManager.instantiated?
-    @say_id = nil
-  end
-
-  def make_id imap_stuff
-    # use 7 digits for the size. why 7? seems nice.
-    %w(RFC822.SIZE INTERNALDATE).each do |w|
-      raise FatalSourceError, "requested data not in IMAP response: #{w}" unless imap_stuff.attr[w]
-    end
-
-    msize, mdate = imap_stuff.attr['RFC822.SIZE'] % 10000000, Time.parse(imap_stuff.attr["INTERNALDATE"])
-    sprintf("%d%07d", mdate.to_i, msize).to_i
-  end
-
-  def get_imap_fields id, *fields
-    raise OutOfSyncSourceError, "Unknown message id #{id}" unless @imap_state[id]
-
-    imap_id = @imap_state[id][:id]
-    result = fetch(imap_id, (fields + ['RFC822.SIZE', 'INTERNALDATE']).uniq).first
-    got_id = make_id result
-
-    ## I've turned off the following sanity check because Microsoft
-    ## Exchange fails it.  Exchange actually reports two different
-    ## INTERNALDATEs for the exact same message when queried at different
-    ## points in time.
-    ##
-    ## RFC2060 defines the semantics of INTERNALDATE for messages that
-    ## arrive via SMTP for via various IMAP commands, but states that
-    ## "All other cases are implementation defined.". Great, thanks guys,
-    ## yet another useless field.
-    ## 
-    ## Of course no OTHER imap server I've encountered returns DIFFERENT
-    ## values for the SAME message. But it's Microsoft; what do you
-    ## expect? If their programmers were any good they'd be working at
-    ## Google.
-
-    # raise OutOfSyncSourceError, "IMAP message mismatch: requested #{id}, got #{got_id}." unless got_id == id
-
-    fields.map { |f| result.attr[f] or raise FatalSourceError, "empty response from IMAP server: #{f}" }
-  end
-
-  ## execute a block, connected if unconnected, re-connected up to 3
-  ## times if a recoverable error occurs, and properly dying if an
-  ## unrecoverable error occurs.
-  def safely
-    retries = 0
-    begin
-      begin
-        unsafe_connect unless @imap
-        yield
-      rescue *RECOVERABLE_ERRORS => e
-        if (retries += 1) <= 3
-          @imap = nil
-          warn "got #{e.class.name}: #{e.message.inspect}"
-          sleep 2
-          retry
-        end
-        raise
-      end
-    rescue SocketError, Net::IMAP::Error, SystemCallError, IOError, OpenSSL::SSL::SSLError => e
-      raise FatalSourceError, "While communicating with IMAP server (type #{e.class.name}): #{e.message.inspect}"
-    end
-  end
-
-end
-
-end
diff --git a/lib/sup/source.rb b/lib/sup/source.rb
index 6fe7bfb..02d99c7 100644
--- a/lib/sup/source.rb
+++ b/lib/sup/source.rb
@@ -28,8 +28,7 @@ class Source
   ## (in the absence of some other client going in and fucking
   ## everything up). For example, for mboxes I use the file offset of
   ## the start of the message. If a source does NOT have that
-  ## capability, e.g. IMAP, then you have to do a little more work to
-  ## simulate it.
+  ## capability then you have to do a little more work to simulate it.
   ##
   ## To write a new source, subclass this class, and implement:
   ##
@@ -51,13 +50,12 @@ class Source
   ## OutOfSyncSourceErrors should be used for problems that a call to
   ## sup-sync will fix (namely someone's been playing with the source
   ## from another client); FatalSourceErrors can be used for anything
-  ## else (e.g. the imap server is down or the maildir is missing.)
+  ## else (e.g. the maildir is missing.)
   ##
   ## Finally, be sure the source is thread-safe, since it WILL be
   ## pummelled from multiple threads at once.
   ##
-  ## Examples for you to look at: mbox/loader.rb, imap.rb, and
-  ## maildir.rb.
+  ## Examples for you to look at: mbox/loader.rb and maildir.rb.
 
   ## let's begin!
   ##
diff --git a/lib/sup/xapian_index.rb b/lib/sup/xapian_index.rb
index c81dca4..e0c6ecf 100644
--- a/lib/sup/xapian_index.rb
+++ b/lib/sup/xapian_index.rb
@@ -1,5 +1,6 @@
 require 'xapian'
 require 'set'
+require 'monitor'
 
 module Redwood
 
-- 
1.6.3.3