commit 4ab9bb1d3173b7465ac22b92c77e6b99da845087
parent 628df59cde640295fc7fec1e87e0e3e10bba9552
Author: Gaute Hope <eg@gaute.vetsj.com>
Date: Thu, 7 Nov 2013 15:54:08 +0100
Merge branch 'develop'
Conflicts:
History.txt
ReleaseNotes
Diffstat:
33 files changed, 951 insertions(+), 323 deletions(-)
diff --git a/.travis.yml b/.travis.yml
@@ -10,6 +10,3 @@ before_install:
script: bundle exec rake travis
-matrix:
- allow_failures:
- - rvm: 2.0.0
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
@@ -3,7 +3,9 @@ Rich Lane <rlane at the club.cc.cmu dot edus>
Gaute Hope <eg at the gaute.vetsj dot coms>
Whyme Lyu <callme5long at the gmail dot coms>
Hamish Downer <dmishd at the gmail dot coms>
+Damien Leone <damien.leone at the fensalir dot frs>
Sascha Silbe <sascha-pgp at the silbe dot orgs>
+Eric Weikl <eric.weikl at the tngtech dot coms>
Ismo Puustinen <ismo at the iki dot fis>
Nicolas Pouillard <nicolas.pouillard at the gmail dot coms>
Michael Stapelberg <michael at the stapelberg dot des>
@@ -15,54 +17,54 @@ Clint Byrum <clint at the ubuntu dot coms>
Marcus Williams <marcus-sup at the bar-coded dot nets>
Lionel Ott <white.magic at the gmx dot des>
Gaudenz Steinlin <gaudenz at the soziologie dot chs>
-Damien Leone <damien.leone at the fensalir dot frs>
-Ingmar Vanhassel <ingmar at the exherbo dot orgs>
Mark Alexander <marka at the pobox dot coms>
-Eric Weikl <eric.weikl at the gmx dot nets>
+Ingmar Vanhassel <ingmar at the exherbo dot orgs>
+Edward Z. Yang <ezyang at the mit dot edus>
Christopher Warrington <chrisw at the rice dot edus>
W. Trevor King <wking at the drexel dot edus>
Richard Brown <rbrown at the exherbo dot orgs>
Anthony Martinez <pi+sup at the pihost dot uss>
Marc Hartstein <marc.hartstein at the alum.vassar dot edus>
-Israel Herraiz <israel.herraiz at the gmail dot coms>
Matthieu Rakotojaona <matthieu.rakotojaona at the gmail dot coms>
+Israel Herraiz <israel.herraiz at the gmail dot coms>
Bo Borgerson <gigabo at the gmail dot coms>
Michael Hamann <michael at the content-space dot des>
Jonathan Lassoff <jof at the thejof dot coms>
William Erik Baxter <web at the superscript dot coms>
Grant Hollingworth <grant at the antiflux dot orgs>
+Adeodato Simó <dato at the net.com.org dot ess>
Ico Doornekamp <ico at the pruts dot nls>
Markus Klinik <markus.klinik at the gmx dot des>
-Adeodato Simó <dato at the net.com.org dot ess>
Daniel Schoepe <daniel.schoepe at the googlemail dot coms>
Jason Petsod <jason at the petsod dot orgs>
-Edward Z. Yang <edwardzyang at the thewritingpot dot coms>
-Steve Goldman <sgoldman at the tower-research dot coms>
+James Taylor <james at the jamestaylor dot orgs>
Robin Burchell <viroteck at the viroteck dot nets>
+Steve Goldman <sgoldman at the tower-research dot coms>
Peter Harkins <ph at the malaprop dot orgs>
Decklin Foster <decklin at the red-bean dot coms>
Cameron Matheson <cam+sup at the cammunism dot orgs>
-Carl Worth <cworth at the cworth dot orgs>
Alex Vandiver <alex at the chmrr dot nets>
-Jeff Balogh <its.jeff.balogh at the gmail dot coms>
+Carl Worth <cworth at the cworth dot orgs>
Andrew Pimlott <andrew at the pimlott dot nets>
+Jeff Balogh <its.jeff.balogh at the gmail dot coms>
Matías Aguirre <matiasaguirre at the gmail dot coms>
Kornilios Kourtis <kkourt at the cslab.ece.ntua dot grs>
-Kevin Riggle <kevinr at the free-dissociation dot coms>
Giorgio Lando <patroclo7 at the gmail dot coms>
+Kevin Riggle <kevinr at the free-dissociation dot coms>
Benoît PIERRE <benoit.pierre at the gmail dot coms>
-Alvaro Herrera <alvherre at the alvh.no-ip dot orgs>
Steven Lawrance <stl at the koffein dot nets>
+Alvaro Herrera <alvherre at the alvh.no-ip dot orgs>
Jonah <Jonah at the GoodCoffee dot cas>
ian <itaylor at the uark dot edus>
+Per Andersson <avtobiff at the gmail dot coms>
Adam Lloyd <adam at the alloy-d dot nets>
Todd Eisenberger <teisenbe at the andrew.cmu dot edus>
Gregor Hoffleit <gregor at the sam.mediasupervision dot des>
MichaelRevell <mikearevell at the gmail dot coms>
-Per Andersson <avtobiff at the gmail dot coms>
Steven Walter <swalter at the monarch.(none)>
-Matthias Vallentin <vallentin at the icir dot orgs>
-Jon M. Dugan <jdugan at the es dot nets>
Stefan Lundström <lundst at the snabb.(none)>
Horacio Sanson <horacio at the skillupjapan.co dot jps>
+Jon M. Dugan <jdugan at the es dot nets>
+akojo <atte.kojo at the gmail dot coms>
+Matthias Vallentin <vallentin at the icir dot orgs>
Kirill Smelkov <kirr at the landau.phys.spbu dot rus>
diff --git a/History.txt b/History.txt
@@ -1,3 +1,47 @@
+== 0.15.0 / 2013-11-07
+
+* Maildir Syncback has now been merged into main sup! This is a
+ long-time waiting feature initially developed by Damien Leone,
+ then picked up by Edward Z. Yang who continued development. Additionally
+ several others have been contributing.
+
+ Eventually, recently, Eric Weikl has picked up this branch, modernized
+ it to current sup, maintained it and gotten it ready for release.
+
+ Main authors:
+
+ Damien Leone
+ Edward Z. Yang
+ Eric Weikl
+
+ Not all of the features initially proposed have been included. This is
+ to maintain compatibility with more operating systems and wait with
+ the more daring features to make sure sup is stable-ish.
+
+ This is a big change since sup now can modify your mail (!), please
+ back up your mail and your configuration before using the maildir
+ syncback feature. For instructions on how to migrate an existing
+ maildir source or how to set up a new one, refer to the wiki:
+
+ https://github.com/sup-heliotrope/sup/wiki/Using-sup-with-other-clients
+
+ It is possible to both disable maildir syncback globally (default:
+ disabled) and per-source (default: enabled).
+
+* Sup on Ruby 2.0.0 now works - but beware, this has not been very throughly
+ tested. Patches are welcome.
+
+* We are now using our own rmail-sup gem with fixes for Ruby 2.0.0 and
+ various warnings fixed.
+
+* sup-sync-back has been renamed to sup-sync-back-mbox to conform with
+ the other sync-back scripts.
+
+* You can now save attachments to directories without specifying the full
+ filename (default filename is used).
+
+* Various encoding fixes and minor bug fixes
+
== 0.14.1.1 / 2013-10-29
* SBU1: security release
diff --git a/README.md b/README.md
@@ -18,12 +18,16 @@ Features:
* [Ruby-programmable hooks][hooks]
* Automatically tracking recent contacts
-Current limitations which will be fixed:
+Current limitations:
-* [Doesn't run on Ruby 2.0][ruby20]
+* [Ruby 2.0 support][ruby20] is very fresh, consider it experimental. Patches
+ are welcome
-* Sup doesn't play nicely with other mail clients. Changes in Sup won't be
- synced back to mail source.
+* Sup does in general not play nicely with other mail clients, not all
+ changes can be synced back to the mail source. Refer to [Maildir Syncback][maildir-syncback]
+ in the wiki for this recently included feature. Maildir Syncback
+ allows you to sync back flag changes in messages and to write messages
+ to maildir sources.
* Unix-centrism in MIME attachment handling and in sendmail invocation.
@@ -68,3 +72,4 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
[ruby20]: https://github.com/sup-heliotrope/sup/wiki/Development#sup-014
[sup-talk]: http://rubyforge.org/mailman/listinfo/sup-talk
[sup-devel]: http://rubyforge.org/mailman/listinfo/sup-devel
+[maildir-syncback]: https://github.com/sup-heliotrope/sup/wiki/Using-sup-with-other-clients
diff --git a/ReleaseNotes b/ReleaseNotes
@@ -1,3 +1,11 @@
+Release 0.15.0:
+
+Maildir Syncback has been included. Refer to the wiki for more information on
+how to set it up.
+
+sup-sync-back has been moved to sup-sync-back-mbox, please make sure
+you make any needed changes.
+
Release 0.14.1.1:
See 0.13.2.1.
diff --git a/bin/sup b/bin/sup
@@ -153,6 +153,7 @@ Index.lock_interactively or exit
begin
Redwood::start
Index.load
+ Redwood::check_syncback_settings
Index.start_sync_worker unless $opts[:no_threads]
$die = false
diff --git a/bin/sup-add b/bin/sup-add
@@ -30,6 +30,7 @@ Options are:
EOS
opt :archive, "Automatically archive all new messages from these sources."
opt :unusual, "Do not automatically poll these sources for new messages."
+ opt :sync_back, "Synchronize status flags back into messages, defaults to true (Maildir sources only).", :default => true
opt :labels, "A comma-separated set of labels to apply to all messages from this source", :type => String
opt :force_new, "Create a new account for this source, even if one already exists."
opt :force_account, "Reuse previously defined account user@hostname.", :type => String
@@ -99,7 +100,7 @@ begin
source =
case parsed_uri.scheme
when "maildir"
- Redwood::Maildir.new uri, !$opts[:unusual], $opts[:archive], nil, labels
+ Redwood::Maildir.new uri, !$opts[:unusual], $opts[:archive], $opts[:sync_back], nil, labels
when "mbox"
Redwood::MBox.new uri, !$opts[:unusual], $opts[:archive], nil, labels
when nil
diff --git a/bin/sup-config b/bin/sup-config
@@ -88,6 +88,8 @@ def add_source
usual = axe_yes "Does this source ever receive new messages?", "y"
archive = usual ? axe_yes("Should new messages be automatically archived? (I.e. not appear in your inbox, though still be accessible via search.)") : false
+ sync_back = (type == :maildir) ? axe_yes("Should the original Maildir messages be modified to reflect changes like read status, starred messages, etc.?", "y") : false
+
labels_str = axe("Enter any labels to be automatically added to all messages from this source, separated by spaces (or 'none')", default_labels.join(","))
labels = if labels_str =~ /^\s*none\s*$/i
@@ -99,6 +101,7 @@ def add_source
cmd = build_cmd "sup-add"
cmd += " --unusual" unless usual
cmd += " --archive" if archive
+ cmd += " --no-sync-back" unless sync_back
cmd += " --labels=#{labels.join(',')}" if labels && !labels.empty?
cmd += " #{uri}"
diff --git a/bin/sup-sync-back b/bin/sup-sync-back
@@ -1,181 +0,0 @@
-#!/usr/bin/env ruby
-
-$:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
-
-require 'rubygems'
-require 'uri'
-require 'tempfile'
-require 'trollop'
-require "sup"
-
-fail "not working yet"
-
-## save a message 'm' to an open file pointer 'fp'
-def save m, fp
- m.source.each_raw_message_line(m.source_info) { |l| fp.print l }
-end
-def die msg
- $stderr.puts "Error: #{msg}"
- exit(-1)
-end
-def has_any_from_source_with_label? index, source, label
- query = { :source_id => source.id, :label => label, :limit => 1, :load_spam => true, :load_deleted => true, :load_killed => true }
- index.num_results_for(query) != 0
-end
-
-opts = Trollop::options do
- version "sup-sync-back (sup #{Redwood::VERSION})"
- banner <<EOS
-Drop or move messages from Sup sources that are marked as deleted or
-spam in the Sup index.
-
-Currently only works with mbox sources.
-
-Usage:
- sup-sync-back [options] <source>*
-
-where <source>* is zero or more source URIs. If no sources are given,
-sync back all usual sources.
-
-You almost certainly want to run sup-sync --changed after this command.
-Running this does not change the index.
-
-Options include:
-EOS
- opt :drop_deleted, "Drop deleted messages.", :default => false, :short => "d"
- opt :move_deleted, "Move deleted messages to a local mbox file.", :type => String, :short => :none
- opt :drop_spam, "Drop spam messages.", :default => false, :short => "s"
- opt :move_spam, "Move spam messages to a local mbox file.", :type => String, :short => :none
-
- opt :with_dotlockfile, "Specific dotlockfile location (mbox files only).", :default => "/usr/bin/dotlockfile", :short => :none
- opt :dont_use_dotlockfile, "Don't use dotlockfile to lock mbox files. Dangerous if other processes modify them concurrently.", :default => false, :short => :none
-
- opt :verbose, "Print message ids as they're processed."
- opt :dry_run, "Don't actually modify the index. Probably only useful with --verbose.", :short => "-n"
- opt :version, "Show version information", :short => :none
-
- conflicts :drop_deleted, :move_deleted
- conflicts :drop_spam, :move_spam
-end
-
-unless opts[:drop_deleted] || opts[:move_deleted] || opts[:drop_spam] || opts[:move_spam]
- puts <<EOS
-Nothing to do. Please specify at least one of --drop-deleted, --move-deleted,
---drop-spam, or --move-spam.
-EOS
-
- exit
-end
-
-Redwood::start
-index = Redwood::Index.init
-index.lock_interactively or exit
-
-deleted_fp, spam_fp = nil
-unless opts[:dry_run]
- deleted_fp = File.open(opts[:move_deleted], "a") if opts[:move_deleted]
- spam_fp = File.open(opts[:move_spam], "a") if opts[:move_spam]
-end
-
-dotlockfile = opts[:with_dotlockfile] || "/usr/bin/dotlockfile"
-
-begin
- index.load
-
- sources = ARGV.map do |uri|
- s = Redwood::SourceManager.source_for(uri) or die "unknown source: #{uri}. Did you add it with sup-add first?"
- s.is_a?(Redwood::MBox) or die "#{uri} is not an mbox source."
- s
- end
-
- if sources.empty?
- sources = Redwood::SourceManager.usual_sources.select { |s| s.is_a? Redwood::MBox }
- end
-
- unless sources.all? { |s| s.file_path.nil? } || File.executable?(dotlockfile) || opts[:dont_use_dotlockfile]
- die <<EOS
-can't execute dotlockfile binary: #{dotlockfile}. Specify --with-dotlockfile
-if it's in a nonstandard location, or, if you want to live dangerously, try
---dont-use-dotlockfile
-EOS
- end
-
- modified_sources = []
- sources.each do |source|
- $stderr.puts "Scanning #{source}..."
-
- unless ((opts[:drop_deleted] || opts[:move_deleted]) && has_any_from_source_with_label?(index, source, :deleted)) || ((opts[:drop_spam] || opts[:move_spam]) && has_any_from_source_with_label?(index, source, :spam))
- $stderr.puts "Nothing to do from this source; skipping"
- next
- end
-
- source.reset!
- num_dropped = num_moved = num_scanned = 0
-
- out_fp = Tempfile.new "sup-sync-back-#{source.id}"
- Redwood::PollManager.each_message_from source do |m|
- num_scanned += 1
-
- if(m_old = index.build_message(m.id))
- labels = m_old.labels
-
- if labels.member? :deleted
- if opts[:drop_deleted]
- puts "Dropping deleted message #{source}##{m.source_info}" if opts[:verbose]
- num_dropped += 1
- elsif opts[:move_deleted] && labels.member?(:deleted)
- puts "Moving deleted message #{source}##{m.source_info}" if opts[:verbose]
- save m, deleted_fp unless opts[:dry_run]
- num_moved += 1
- end
-
- elsif labels.member? :spam
- if opts[:drop_spam]
- puts "Dropping spam message #{source}##{m.source_info}" if opts[:verbose]
- num_dropped += 1
- elsif opts[:move_spam] && labels.member?(:spam)
- puts "Moving spam message #{source}##{m.source_info}" if opts[:verbose]
- save m, spam_fp unless opts[:dry_run]
- num_moved += 1
- end
- else
- save m, out_fp unless opts[:dry_run]
- end
- else
- save m, out_fp unless opts[:dry_run]
- end
- end
- $stderr.puts "Scanned #{num_scanned}, dropped #{num_dropped}, moved #{num_moved} messages from #{source}."
- modified_sources << source if num_dropped > 0 || num_moved > 0
- out_fp.close unless opts[:dry_run]
-
- unless opts[:dry_run] || (num_dropped == 0 && num_moved == 0)
- deleted_fp.flush if deleted_fp
- spam_fp.flush if spam_fp
- unless opts[:dont_use_dotlockfile]
- puts "Locking #{source.file_path}..."
- system "#{opts[:dotlockfile]} -l #{source.file_path}"
- puts "Writing #{source.file_path}..."
- FileUtils.cp out_fp.path, source.file_path
- puts "Unlocking #{source.file_path}..."
- system "#{opts[:dotlockfile]} -u #{source.file_path}"
- end
- end
- end
-
- unless opts[:dry_run]
- deleted_fp.close if deleted_fp
- spam_fp.close if spam_fp
- end
-
- $stderr.puts "Done."
- unless modified_sources.empty?
- $stderr.puts "You should now run: sup-sync --changed #{modified_sources.join(' ')}"
- end
-rescue Exception => e
- File.open("sup-exception-log.txt", "w") { |f| f.puts e.backtrace }
- raise
-ensure
- Redwood::finish
- index.unlock
-end
diff --git a/bin/sup-sync-back-maildir b/bin/sup-sync-back-maildir
@@ -0,0 +1,127 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+$:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
+
+require 'rubygems'
+require 'trollop'
+require "sup"
+
+opts = Trollop::options do
+ version "sup-sync-back-maildir (sup #{Redwood::VERSION})"
+ banner <<EOS
+Export Xapian entries to Maildir sources on disk.
+
+This script parses the Xapian entries for a given Maildir source and renames
+(changes maildir flags) e-mail files on disk according to the labels stored in
+the index. It will export all the changes you made in Sup to your
+Maildirs so that they can be propagated to your IMAP server with e.g. offlineimap.
+
+The script also merges some Maildir flags into Sup such
+as R (replied) and P (passed, forwarded), for instance suppose you
+have an e-mail file like this: foo_bar:2,FRS (flags are favorite,
+replied, seen) and its Xapian entry has labels 'starred', the merge
+operation will add the 'replied' label to the Xapian entry.
+
+If you choose not to merge (-m) you will lose information ('replied'), and in
+the previous example the file will be renamed to foo_bar:2,FS.
+
+Running this script is *strongly* recommended when setting the
+"sync_back_to_maildir" option from false to true in config.yaml or changing the
+"sync_back" flag to true for a source in sources.yaml.
+
+Usage:
+ sup-sync-back-maildir [options] <source>*
+
+where <source>* is source URIs. If no source is given, the default behavior is
+to sync back all Maildir sources marked as usual and that have not disabled
+sync back using the configuration parameter sync_back = false in sources.yaml.
+
+Options include:
+EOS
+ opt :no_confirm, "Don't ask for confirmation before synchronizing", :default => false, :short => "n"
+ opt :no_merge, "Don't merge new supported Maildir flags (R and P)", :default => false, :short => "m"
+ opt :list_sources, "List your Maildir sources and exit", :default => false, :short => "l"
+ opt :unusual_sources_too, "Sync unusual sources too if no specific source information is given", :default => false, :short => "u"
+end
+
+def die msg
+ $stderr.puts "Error: #{msg}"
+ exit(-1)
+end
+
+Redwood::start true
+index = Redwood::Index.init
+index.lock_interactively or exit
+index.load
+
+## Force sync_back_to_maildir option otherwise nothing will happen
+$config[:sync_back_to_maildir] = true
+
+begin
+ sync_performed = []
+ sync_performed = File.readlines(Redwood::SYNC_OK_FN).collect { |e| e.strip }.find_all { |e| not e.empty? } if File.exists? Redwood::SYNC_OK_FN
+ sources = []
+
+ ## Try to find out sources given in parameters
+ sources = ARGV.map do |uri|
+ s = Redwood::SourceManager.source_for(uri) or die "unknown source: #{uri}. Did you add it with sup-add first?"
+ s.is_a?(Redwood::Maildir) or die "#{uri} is not a Maildir source."
+ s.sync_back_enabled? or die "#{uri} has disabled sync back - check your configuration."
+ s
+ end unless opts[:list_sources]
+
+ ## Otherwise, check all sources in sources.yaml
+ if sources.empty? or opts[:list_sources] == true
+ if opts[:unusual_sources_too]
+ sources = Redwood::SourceManager.sources.select do |s|
+ s.is_a? Redwood::Maildir and s.sync_back_enabled?
+ end
+ else
+ sources = Redwood::SourceManager.usual_sources.select do |s|
+ s.is_a? Redwood::Maildir and s.sync_back_enabled?
+ end
+ end
+ end
+
+ if opts[:list_sources] == true
+ sources.each do |s|
+ puts "id: #{s.id}, uri: #{s.uri}"
+ end
+ else
+ sources.each do |s|
+ if opts[:no_confirm] == false
+ print "Are you sure you want to synchronize '#{s.uri}'? (Y/n) "
+ next if STDIN.gets.chomp.downcase == 'n'
+ end
+
+ infos = Enumerator.new(index, :each_source_info, s.id).to_a
+ counter = 0
+ infos.each do |info|
+ print "\rSynchronizing '#{s.uri}'... #{((counter += 1)/infos.size.to_f*100).to_i}%"
+ index.each_message({:location => [s.id, info]}, false) do |m|
+ if opts[:no_merge] == false
+ m.merge_labels_from_locations [:replied, :forwarded]
+ end
+
+ if Redwood::Index.message_joining_killed? m
+ m.labels += [:killed]
+ end
+
+ index.save_message m
+ end
+ end
+ print "\n"
+ sync_performed << s.uri
+ end
+ ## Write a flag file to tell sup that the synchronization has been performed
+ File.open(Redwood::SYNC_OK_FN, 'w') {|f| f.write(sync_performed.join("\n")) }
+ end
+rescue Exception => e
+ File.open("sup-exception-log.txt", "w") { |f| f.puts e.backtrace }
+ raise
+ensure
+ index.save_index
+ Redwood::finish
+ index.unlock
+end
diff --git a/bin/sup-sync-back-mbox b/bin/sup-sync-back-mbox
@@ -0,0 +1,181 @@
+#!/usr/bin/env ruby
+
+$:.unshift File.join(File.dirname(__FILE__), *%w[.. lib])
+
+require 'rubygems'
+require 'uri'
+require 'tempfile'
+require 'trollop'
+require "sup"
+
+fail "not working yet"
+
+## save a message 'm' to an open file pointer 'fp'
+def save m, fp
+ m.source.each_raw_message_line(m.source_info) { |l| fp.print l }
+end
+def die msg
+ $stderr.puts "Error: #{msg}"
+ exit(-1)
+end
+def has_any_from_source_with_label? index, source, label
+ query = { :source_id => source.id, :label => label, :limit => 1, :load_spam => true, :load_deleted => true, :load_killed => true }
+ index.num_results_for(query) != 0
+end
+
+opts = Trollop::options do
+ version "sup-sync-back-mbox (sup #{Redwood::VERSION})"
+ banner <<EOS
+Drop or move messages from Sup sources that are marked as deleted or
+spam in the Sup index.
+
+Currently only works with mbox sources.
+
+Usage:
+ sup-sync-back-mbox [options] <source>*
+
+where <source>* is zero or more source URIs. If no sources are given,
+sync back all usual sources.
+
+You almost certainly want to run sup-sync --changed after this command.
+Running this does not change the index.
+
+Options include:
+EOS
+ opt :drop_deleted, "Drop deleted messages.", :default => false, :short => "d"
+ opt :move_deleted, "Move deleted messages to a local mbox file.", :type => String, :short => :none
+ opt :drop_spam, "Drop spam messages.", :default => false, :short => "s"
+ opt :move_spam, "Move spam messages to a local mbox file.", :type => String, :short => :none
+
+ opt :with_dotlockfile, "Specific dotlockfile location (mbox files only).", :default => "/usr/bin/dotlockfile", :short => :none
+ opt :dont_use_dotlockfile, "Don't use dotlockfile to lock mbox files. Dangerous if other processes modify them concurrently.", :default => false, :short => :none
+
+ opt :verbose, "Print message ids as they're processed."
+ opt :dry_run, "Don't actually modify the index. Probably only useful with --verbose.", :short => "-n"
+ opt :version, "Show version information", :short => :none
+
+ conflicts :drop_deleted, :move_deleted
+ conflicts :drop_spam, :move_spam
+end
+
+unless opts[:drop_deleted] || opts[:move_deleted] || opts[:drop_spam] || opts[:move_spam]
+ puts <<EOS
+Nothing to do. Please specify at least one of --drop-deleted, --move-deleted,
+--drop-spam, or --move-spam.
+EOS
+
+ exit
+end
+
+Redwood::start
+index = Redwood::Index.init
+index.lock_interactively or exit
+
+deleted_fp, spam_fp = nil
+unless opts[:dry_run]
+ deleted_fp = File.open(opts[:move_deleted], "a") if opts[:move_deleted]
+ spam_fp = File.open(opts[:move_spam], "a") if opts[:move_spam]
+end
+
+dotlockfile = opts[:with_dotlockfile] || "/usr/bin/dotlockfile"
+
+begin
+ index.load
+
+ sources = ARGV.map do |uri|
+ s = Redwood::SourceManager.source_for(uri) or die "unknown source: #{uri}. Did you add it with sup-add first?"
+ s.is_a?(Redwood::MBox) or die "#{uri} is not an mbox source."
+ s
+ end
+
+ if sources.empty?
+ sources = Redwood::SourceManager.usual_sources.select { |s| s.is_a? Redwood::MBox }
+ end
+
+ unless sources.all? { |s| s.file_path.nil? } || File.executable?(dotlockfile) || opts[:dont_use_dotlockfile]
+ die <<EOS
+can't execute dotlockfile binary: #{dotlockfile}. Specify --with-dotlockfile
+if it's in a nonstandard location, or, if you want to live dangerously, try
+--dont-use-dotlockfile
+EOS
+ end
+
+ modified_sources = []
+ sources.each do |source|
+ $stderr.puts "Scanning #{source}..."
+
+ unless ((opts[:drop_deleted] || opts[:move_deleted]) && has_any_from_source_with_label?(index, source, :deleted)) || ((opts[:drop_spam] || opts[:move_spam]) && has_any_from_source_with_label?(index, source, :spam))
+ $stderr.puts "Nothing to do from this source; skipping"
+ next
+ end
+
+ source.reset!
+ num_dropped = num_moved = num_scanned = 0
+
+ out_fp = Tempfile.new "sup-sync-back-mbox-#{source.id}"
+ Redwood::PollManager.each_message_from source do |m|
+ num_scanned += 1
+
+ if(m_old = index.build_message(m.id))
+ labels = m_old.labels
+
+ if labels.member? :deleted
+ if opts[:drop_deleted]
+ puts "Dropping deleted message #{source}##{m.source_info}" if opts[:verbose]
+ num_dropped += 1
+ elsif opts[:move_deleted] && labels.member?(:deleted)
+ puts "Moving deleted message #{source}##{m.source_info}" if opts[:verbose]
+ save m, deleted_fp unless opts[:dry_run]
+ num_moved += 1
+ end
+
+ elsif labels.member? :spam
+ if opts[:drop_spam]
+ puts "Dropping spam message #{source}##{m.source_info}" if opts[:verbose]
+ num_dropped += 1
+ elsif opts[:move_spam] && labels.member?(:spam)
+ puts "Moving spam message #{source}##{m.source_info}" if opts[:verbose]
+ save m, spam_fp unless opts[:dry_run]
+ num_moved += 1
+ end
+ else
+ save m, out_fp unless opts[:dry_run]
+ end
+ else
+ save m, out_fp unless opts[:dry_run]
+ end
+ end
+ $stderr.puts "Scanned #{num_scanned}, dropped #{num_dropped}, moved #{num_moved} messages from #{source}."
+ modified_sources << source if num_dropped > 0 || num_moved > 0
+ out_fp.close unless opts[:dry_run]
+
+ unless opts[:dry_run] || (num_dropped == 0 && num_moved == 0)
+ deleted_fp.flush if deleted_fp
+ spam_fp.flush if spam_fp
+ unless opts[:dont_use_dotlockfile]
+ puts "Locking #{source.file_path}..."
+ system "#{opts[:dotlockfile]} -l #{source.file_path}"
+ puts "Writing #{source.file_path}..."
+ FileUtils.cp out_fp.path, source.file_path
+ puts "Unlocking #{source.file_path}..."
+ system "#{opts[:dotlockfile]} -u #{source.file_path}"
+ end
+ end
+ end
+
+ unless opts[:dry_run]
+ deleted_fp.close if deleted_fp
+ spam_fp.close if spam_fp
+ end
+
+ $stderr.puts "Done."
+ unless modified_sources.empty?
+ $stderr.puts "You should now run: sup-sync --changed #{modified_sources.join(' ')}"
+ end
+rescue Exception => e
+ File.open("sup-exception-log.txt", "w") { |f| f.puts e.backtrace }
+ raise
+ensure
+ Redwood::finish
+ index.unlock
+end
diff --git a/contrib/completion/_sup.zsh b/contrib/completion/_sup.zsh
@@ -1,8 +1,8 @@
-#compdef sup sup-add sup-config sup-dump sup-sync sup-sync-back sup-tweak-labels sup-recover-sources
+#compdef sup sup-add sup-config sup-dump sup-sync sup-sync-back-mbox sup-tweak-labels sup-recover-sources
# vim: set et sw=2 sts=2 ts=2 ft=zsh :
# TODO: sources completion: maildir://some/dir, mbox://some/file, ...
-# for sup-add, sup-sync, sup-sync-back, sup-tweak-labels
+# for sup-add, sup-sync, sup-sync-back-mbox, sup-tweak-labels
(( ${+functions[_sup_cmd]} )) ||
_sup_cmd()
diff --git a/lib/sup.rb b/lib/sup.rb
@@ -59,10 +59,12 @@ module Redwood
HOOK_DIR = File.join(BASE_DIR, "hooks")
SEARCH_FN = File.join(BASE_DIR, "searches.txt")
LOG_FN = File.join(BASE_DIR, "log")
+ SYNC_OK_FN = File.join(BASE_DIR, "sync-back-ok")
YAML_DOMAIN = "supmua.org"
LEGACY_YAML_DOMAIN = "masanjin.net"
YAML_DATE = "2006-10-01"
+ MAILDIR_SYNC_CHECK_SKIPPED = 'SKIPPED'
## record exceptions thrown in threads nicely
@exceptions = []
@@ -157,7 +159,7 @@ module Redwood
SourceManager SearchManager IdleManager).map { |x| Redwood.const_get x.to_sym }
end
- def start
+ def start bypass_sync_check = false
managers.each { |x| fail "#{x} already instantiated" if x.instantiated? }
FileUtils.mkdir_p Redwood::BASE_DIR
@@ -173,6 +175,74 @@ module Redwood
Redwood::SearchManager.init Redwood::SEARCH_FN
managers.each { |x| x.init unless x.instantiated? }
+
+ return if bypass_sync_check
+
+ if $config[:sync_back_to_maildir]
+ if not File.exists? Redwood::SYNC_OK_FN
+ Redwood.warn_syncback <<EOS
+It appears that the "sync_back_to_maildir" option has been changed
+from false to true since the last execution of sup.
+EOS
+ $stderr.puts <<EOS
+
+Should I complain about this again? (Y/n)
+EOS
+ File.open(Redwood::SYNC_OK_FN, 'w') {|f| f.write(Redwood::MAILDIR_SYNC_CHECK_SKIPPED) } if STDIN.gets.chomp.downcase == 'n'
+ end
+ elsif not $config[:sync_back_to_maildir] and File.exists? Redwood::SYNC_OK_FN
+ File.delete(Redwood::SYNC_OK_FN)
+ end
+ end
+
+ def check_syncback_settings
+ # don't check if syncback was never performed
+ return unless File.exists? Redwood::SYNC_OK_FN
+ active_sync_sources = File.readlines(Redwood::SYNC_OK_FN).collect { |e| e.strip }.find_all { |e| not e.empty? }
+ return if active_sync_sources.length == 1 and active_sync_sources[0] == Redwood::MAILDIR_SYNC_CHECK_SKIPPED
+ sources = SourceManager.sources
+ newly_synced = sources.select { |s| s.is_a? Maildir and s.sync_back_enabled? and not active_sync_sources.include? s.uri }
+ unless newly_synced.empty?
+
+ details =<<EOS
+It appears that the option "sync_back" of the following source(s)
+has been changed from false to true since the last execution of
+sup:
+
+EOS
+ newly_synced.each do |s|
+ details += "#{s} (usual: #{s.usual})\n"
+ end
+
+ Redwood.warn_syncback details
+ end
+ end
+
+ def self.warn_syncback details
+ $stderr.puts <<EOS
+WARNING
+-------
+
+#{details}
+
+It is *strongly* recommended that you run "sup-sync-back-maildir"
+before continuing, otherwise you might lose changes you have made in sup
+to your Xapian index.
+
+This script should be run each time you change the
+"sync_back_to_maildir" flag in config.yaml from false to true or
+the "sync_back" flag is changed to true for a source in sources.yaml.
+
+Please run "sup-sync-back-maildir -h" for more information and why this
+is needed.
+
+Note that if you have any sources that are not marked as 'ususal' in
+sources.yaml you need to manually specify them when running the
+sup-sync-back-maildir script.
+
+Are you really sure you want to continue? (y/N)
+EOS
+ abort "Aborted" unless STDIN.gets.chomp.downcase == 'y'
end
def finish
@@ -262,7 +332,8 @@ EOM
:wrap_width => 0,
:slip_rows => 0,
:col_jump => 2,
- :stem_language => "english"
+ :stem_language => "english",
+ :sync_back_to_maildir => false
}
if File.exists? filename
config = Redwood::load_yaml_obj filename
@@ -303,7 +374,8 @@ EOM
end
module_function :save_yaml_obj, :load_yaml_obj, :start, :finish,
- :report_broken_sources, :load_config, :managers
+ :report_broken_sources, :load_config, :managers,
+ :check_syncback_settings
end
require 'sup/version'
diff --git a/lib/sup/index.rb b/lib/sup/index.rb
@@ -168,6 +168,32 @@ EOS
matchset.matches_estimated
end
+ ## check if a message is part of a killed thread
+ ## (warning: duplicates code below)
+ ## NOTE: We can be more efficient if we assume every
+ ## killed message that hasn't been initially added
+ ## to the indexi s this way
+ def message_joining_killed? m
+ return false unless doc = find_doc(m.id)
+ queue = doc.value(THREAD_VALUENO).split(',')
+ seen_threads = Set.new
+ seen_messages = Set.new [m.id]
+ while not queue.empty?
+ thread_id = queue.pop
+ next if seen_threads.member? thread_id
+ return true if thread_killed?(thread_id)
+ seen_threads << thread_id
+ docs = term_docids(mkterm(:thread, thread_id)).map { |x| @xapian.document x }
+ docs.each do |doc|
+ msgid = doc.value MSGID_VALUENO
+ next if seen_messages.member? msgid
+ seen_messages << msgid
+ queue.concat doc.value(THREAD_VALUENO).split(',')
+ end
+ end
+ false
+ end
+
## yield all messages in the thread containing 'm' by repeatedly
## querying the index. yields pairs of message ids and
## message-building lambdas, so that building an unwanted message
@@ -248,11 +274,11 @@ EOS
## Yield each message-id matching query
EACH_ID_PAGE = 100
- def each_id query={}
+ def each_id query={}, ignore_neg_terms = true
offset = 0
page = EACH_ID_PAGE
- xapian_query = build_xapian_query query
+ xapian_query = build_xapian_query query, ignore_neg_terms
while true
ids = run_query_ids xapian_query, offset, (offset+page)
ids.each { |id| yield id }
@@ -262,8 +288,12 @@ EOS
end
## Yield each message matching query
- def each_message query={}, &b
- each_id query do |id|
+ ## The ignore_neg_terms parameter is used to display result even if
+ ## it contains "forbidden" labels such as :deleted, it is used in
+ ## Poll#poll_from when we need to get the location of a message that
+ ## may contain these labels
+ def each_message query={}, ignore_neg_terms = true, &b
+ each_id query, ignore_neg_terms do |id|
yield build_message(id)
end
end
@@ -313,9 +343,9 @@ EOS
## Yields (in lexicographical order) the source infos of all locations from
## the given source with the given source_info prefix
def each_source_info source_id, prefix='', &b
- prefix = mkterm :location, source_id, prefix
- each_prefixed_term prefix do |x|
- yield x[prefix.length..-1]
+ p = mkterm :location, source_id, prefix
+ each_prefixed_term p do |x|
+ yield prefix + x[p.length..-1]
end
end
@@ -492,7 +522,7 @@ EOS
raise ParseError, "xapian query parser error: #{e}"
end
- debug "parsed xapian query: #{Util::Query.describe(xapian_query)}"
+ debug "parsed xapian query: #{Util::Query.describe(xapian_query, subs)}"
raise ParseError if xapian_query.nil? or xapian_query.empty?
query[:qobj] = xapian_query
@@ -500,14 +530,18 @@ EOS
query
end
+ def save_message m
+ if @sync_worker
+ @sync_queue << m
+ else
+ update_message_state m
+ end
+ m.clear_dirty
+ end
+
def save_thread t
t.each_dirty_message do |m|
- if @sync_worker
- @sync_queue << m
- else
- update_message_state m
- end
- m.clear_dirty
+ save_message m
end
end
@@ -614,7 +648,7 @@ EOS
end
Q = Xapian::Query
- def build_xapian_query opts
+ def build_xapian_query opts, ignore_neg_terms = true
labels = ([opts[:label]] + (opts[:labels] || [])).compact
neglabels = [:spam, :deleted, :killed].reject { |l| (labels.include? l) || opts.member?("load_#{l}".intern) }
pos_terms, neg_terms = [], []
@@ -630,7 +664,7 @@ EOS
pos_terms << Q.new(Q::OP_OR, participant_terms)
end
- neg_terms.concat(neglabels.map { |l| mkterm(:label,l) })
+ neg_terms.concat(neglabels.map { |l| mkterm(:label,l) }) if ignore_neg_terms
pos_query = Q.new(Q::OP_AND, pos_terms)
neg_query = Q.new(Q::OP_OR, neg_terms)
@@ -643,6 +677,10 @@ EOS
end
def sync_message m, overwrite
+ ## TODO: we should not save the message if the sync_back failed
+ ## since it would overwrite the location field
+ m.sync_back
+
doc = synchronize { find_doc(m.id) }
existed = doc != nil
doc ||= Xapian::Document.new
diff --git a/lib/sup/label.rb b/lib/sup/label.rb
@@ -7,10 +7,10 @@ class LabelManager
## labels that have special semantics. user will be unable to
## add/remove these via normal label mechanisms.
- RESERVED_LABELS = [ :starred, :spam, :draft, :unread, :killed, :sent, :deleted, :inbox, :attachment ]
+ RESERVED_LABELS = [ :starred, :spam, :draft, :unread, :killed, :sent, :deleted, :inbox, :attachment, :forwarded, :replied ]
## labels that will typically be hidden from the user
- HIDDEN_RESERVED_LABELS = [ :starred, :unread, :attachment ]
+ HIDDEN_RESERVED_LABELS = [ :starred, :unread, :attachment, :forwarded, :replied ]
def initialize fn
@fn = fn
diff --git a/lib/sup/maildir.rb b/lib/sup/maildir.rb
@@ -1,4 +1,5 @@
require 'uri'
+require 'set'
module Redwood
@@ -7,8 +8,8 @@ class Maildir < Source
MYHOSTNAME = Socket.gethostname
## remind me never to use inheritance again.
- yaml_properties :uri, :usual, :archived, :id, :labels
- def initialize uri, usual=true, archived=false, id=nil, labels=[]
+ yaml_properties :uri, :usual, :archived, :sync_back, :id, :labels
+ def initialize uri, usual=true, archived=false, sync_back=true, id=nil, labels=[]
super uri, usual, archived, id
@expanded_uri = Source.expand_filesystem_uri(uri)
uri = URI(@expanded_uri)
@@ -17,16 +18,28 @@ class Maildir < Source
raise ArgumentError, "maildir URI cannot have a host: #{uri.host}" if uri.host
raise ArgumentError, "maildir URI must have a path component" unless uri.path
+ @sync_back = sync_back
+ # sync by default if not specified
+ @sync_back = true if @sync_back.nil?
+
@dir = uri.path
@labels = Set.new(labels || [])
@mutex = Mutex.new
- @mtimes = { 'cur' => Time.at(0), 'new' => Time.at(0) }
+ @ctimes = { 'cur' => Time.at(0), 'new' => Time.at(0) }
end
def file_path; @dir end
def self.suggest_labels_for path; [] end
def is_source_for? uri; super || (uri == @expanded_uri); end
+ def supported_labels?
+ [:draft, :starred, :forwarded, :replied, :unread, :deleted]
+ end
+
+ def sync_back_enabled?
+ @sync_back
+ end
+
def store_message date, from_email, &block
stored = false
new_fn = new_maildir_basefn + ':2,S'
@@ -44,7 +57,7 @@ class Maildir < Source
f.fsync
end
- File.link tmp_path, new_path
+ File.safe_link tmp_path, new_path
stored = true
ensure
File.unlink tmp_path if File.exists? tmp_path
@@ -71,6 +84,14 @@ class Maildir < Source
with_file_for(id) { |f| RMail::Parser.read f }
end
+ def sync_back id, labels
+ synchronize do
+ debug "syncing back maildir message #{id} with flags #{labels.to_a}"
+ flags = maildir_reconcile_flags id, labels
+ maildir_mark_file id, flags
+ end
+ end
+
def raw_header id
ret = ""
with_file_for(id) do |f|
@@ -87,41 +108,78 @@ class Maildir < Source
## XXX use less memory
def poll
- @mtimes.each do |d,prev_mtime|
+ added = []
+ deleted = []
+ updated = []
+ @ctimes.each do |d,prev_ctime|
subdir = File.join @dir, d
debug "polling maildir #{subdir}"
raise FatalSourceError, "#{subdir} not a directory" unless File.directory? subdir
- mtime = File.mtime subdir
- next if prev_mtime >= mtime
- @mtimes[d] = mtime
+ ctime = File.ctime subdir
+ next if prev_ctime >= ctime
+ @ctimes[d] = ctime
old_ids = benchmark(:maildir_read_index) { Enumerator.new(Index.instance, :each_source_info, self.id, "#{d}/").to_a }
- new_ids = benchmark(:maildir_read_dir) { Dir.glob("#{subdir}/*").map { |x| File.basename x }.sort }
- added = new_ids - old_ids
- deleted = old_ids - new_ids
+ new_ids = benchmark(:maildir_read_dir) { Dir.glob("#{subdir}/*").map { |x| File.join(d,File.basename(x)) }.sort }
+ added += new_ids - old_ids
+ deleted += old_ids - new_ids
debug "#{old_ids.size} in index, #{new_ids.size} in filesystem"
- debug "#{added.size} added, #{deleted.size} deleted"
+ end
- added.each_with_index do |id,i|
- yield :add,
- :info => File.join(d,id),
- :labels => @labels + maildir_labels(id) + [:inbox],
- :progress => i.to_f/(added.size+deleted.size)
- end
+ ## find updated mails by checking if an id is in both added and
+ ## deleted arrays, meaning that its flags changed or that it has
+ ## been moved, these ids need to be removed from added and deleted
+ add_to_delete = del_to_delete = []
+ map = Hash.new { |hash, key| hash[key] = [] }
+ deleted.each do |id_del|
+ map[maildir_data(id_del)[0]].push id_del
+ end
+ added.each do |id_add|
+ map[maildir_data(id_add)[0]].each do |id_del|
+ updated.push [ id_del, id_add ]
+ add_to_delete.push id_add
+ del_to_delete.push id_del
+ end
+ end
+ added -= add_to_delete
+ deleted -= del_to_delete
+ debug "#{added.size} added, #{deleted.size} deleted, #{updated.size} updated"
+ total_size = added.size+deleted.size+updated.size
- deleted.each_with_index do |id,i|
- yield :delete,
- :info => File.join(d,id),
- :progress => (i.to_f+added.size)/(added.size+deleted.size)
- end
+ added.each_with_index do |id,i|
+ yield :add,
+ :info => id,
+ :labels => @labels + maildir_labels(id) + [:inbox],
+ :progress => i.to_f/total_size
+ end
+
+ deleted.each_with_index do |id,i|
+ yield :delete,
+ :info => id,
+ :progress => (i.to_f+added.size)/total_size
+ end
+
+ updated.each_with_index do |id,i|
+ yield :update,
+ :old_info => id[0],
+ :new_info => id[1],
+ :labels => @labels + maildir_labels(id[1]),
+ :progress => (i.to_f+added.size+deleted.size)/total_size
end
nil
end
+ def labels? id
+ maildir_labels id
+ end
+
def maildir_labels id
(seen?(id) ? [] : [:unread]) +
(trashed?(id) ? [:deleted] : []) +
- (flagged?(id) ? [:starred] : [])
+ (flagged?(id) ? [:starred] : []) +
+ (passed?(id) ? [:forwarded] : []) +
+ (replied?(id) ? [:replied] : []) +
+ (draft?(id) ? [:draft] : [])
end
def draft? id; maildir_data(id)[2].include? "D"; end
@@ -131,13 +189,6 @@ class Maildir < Source
def seen? id; maildir_data(id)[2].include? "S"; end
def trashed? id; maildir_data(id)[2].include? "T"; end
- def mark_draft id; maildir_mark_file id, "D" unless draft? id; end
- def mark_flagged id; maildir_mark_file id, "F" unless flagged? id; end
- def mark_passed id; maildir_mark_file id, "P" unless passed? id; end
- def mark_replied id; maildir_mark_file id, "R" unless replied? id; end
- def mark_seen id; maildir_mark_file id, "S" unless seen? id; end
- def mark_trashed id; maildir_mark_file id, "T" unless trashed? id; end
-
def valid? id
File.exists? File.join(@dir, id)
end
@@ -159,25 +210,47 @@ private
end
def maildir_data id
- id =~ %r{^([^:]+):([12]),([DFPRST]*)$}
+ id = File.basename id
+ # Flags we recognize are DFPRST
+ id =~ %r{^([^:]+):([12]),([A-Za-z]*)$}
[($1 || id), ($2 || "2"), ($3 || "")]
end
- ## not thread-safe on msg
- def maildir_mark_file msg, flag
- orig_path = @ids_to_fns[msg]
- orig_base, orig_fn = File.split(orig_path)
- new_base = orig_base.slice(0..-4) + 'cur'
- tmp_base = orig_base.slice(0..-4) + 'tmp'
- md_base, md_ver, md_flags = maildir_data msg
- md_flags += flag; md_flags = md_flags.split(//).sort.join.squeeze
- new_path = File.join new_base, "#{md_base}:#{md_ver},#{md_flags}"
- tmp_path = File.join tmp_base, "#{md_base}:#{md_ver},#{md_flags}"
- File.link orig_path, tmp_path
- File.unlink orig_path
- File.link tmp_path, new_path
- File.unlink tmp_path
- @ids_to_fns[msg] = new_path
+ def maildir_reconcile_flags id, labels
+ new_flags = Set.new( maildir_data(id)[2].each_char )
+
+ # Set flags based on labels for the six flags we recognize
+ if labels.member? :draft then new_flags.add?( "D" ) else new_flags.delete?( "D" ) end
+ if labels.member? :starred then new_flags.add?( "F" ) else new_flags.delete?( "F" ) end
+ if labels.member? :forwarded then new_flags.add?( "P" ) else new_flags.delete?( "P" ) end
+ if labels.member? :replied then new_flags.add?( "R" ) else new_flags.delete?( "R" ) end
+ if not labels.member? :unread then new_flags.add?( "S" ) else new_flags.delete?( "S" ) end
+ if labels.member? :deleted or labels.member? :killed then new_flags.add?( "T" ) else new_flags.delete?( "T" ) end
+
+ ## Flags must be stored in ASCII order according to Maildir
+ ## documentation
+ new_flags.to_a.sort.join
+ end
+
+ def maildir_mark_file orig_path, flags
+ @mutex.synchronize do
+ new_base = (flags.include?("S")) ? "cur" : "new"
+ md_base, md_ver, md_flags = maildir_data orig_path
+
+ return if md_flags == flags
+
+ new_loc = File.join new_base, "#{md_base}:#{md_ver},#{flags}"
+ orig_path = File.join @dir, orig_path
+ new_path = File.join @dir, new_loc
+ tmp_path = File.join @dir, "tmp", "#{md_base}:#{md_ver},#{flags}"
+
+ File.safe_link orig_path, tmp_path
+ File.unlink orig_path
+ File.safe_link tmp_path, new_path
+ File.unlink tmp_path
+
+ new_loc
+ end
end
end
diff --git a/lib/sup/mbox.rb b/lib/sup/mbox.rb
@@ -120,7 +120,7 @@ class MBox < Source
## into memory with raw_message.
##
## i hoped never to have to move shit around on disk but
- ## sup-sync-back has to do it.
+ ## sup-sync-back-mbox has to do it.
def each_raw_message_line offset
@mutex.synchronize do
ensure_open
diff --git a/lib/sup/message.rb b/lib/sup/message.rb
@@ -291,6 +291,32 @@ EOS
location.each_raw_message_line &b
end
+ def sync_back
+ @locations.map { |l| l.sync_back @labels, self }.any? do
+ UpdateManager.relay self, :updated, self
+ end
+ end
+
+ def merge_labels_from_locations merge_labels
+ ## Get all labels from all locations
+ location_labels = Set.new([])
+
+ @locations.each do |l|
+ if l.valid?
+ location_labels = location_labels.union(l.labels?)
+ end
+ end
+
+ ## Add to the message labels the intersection between all location
+ ## labels and those we want to merge
+ location_labels = location_labels.intersection(merge_labels.to_set)
+
+ if not location_labels.empty?
+ @labels = @labels.union(location_labels)
+ @dirty = true
+ end
+ end
+
## returns all the content from a message that will be indexed
def indexable_content
load_from_source!
@@ -545,10 +571,18 @@ private
## (and possible signed) inline GPG messages
def inline_gpg_to_chunks body, encoding_to, encoding_from
lines = body.split("\n")
+
+ # First case: Message is enclosed between
+ #
+ # -----BEGIN PGP SIGNED MESSAGE-----
+ # and
+ # -----END PGP SIGNED MESSAGE-----
+ #
+ # In some cases, END PGP SIGNED MESSAGE doesn't appear
gpg = lines.between(GPG_SIGNED_START, GPG_SIGNED_END)
# between does not check if GPG_END actually exists
# Reference: http://permalink.gmane.org/gmane.mail.sup.devel/641
- if !gpg.empty? && !lines.index(GPG_END).nil?
+ if !gpg.empty?
msg = RMail::Message.new
msg.body = gpg.join("\n")
@@ -560,14 +594,21 @@ private
before = startidx != 0 ? lines[0 .. startidx-1] : []
after = endidx ? lines[endidx+1 .. lines.size] : []
+ # sig contains BEGIN PGP SIGNED MESSAGE and END PGP SIGNATURE, so
+ # we ditch them. sig may also contain the hash used by PGP (with a
+ # newline), so we also skip them
+ sig_start = sig[1].match(/^Hash:/) ? 3 : 1
+ sig_end = sig.size-2
payload = RMail::Message.new
- payload.body = sig[1, sig.size-2].join("\n")
+ payload.body = sig[sig_start, sig_end].join("\n")
return [text_to_chunks(before, false),
CryptoManager.verify(nil, msg, false),
message_to_chunks(payload),
text_to_chunks(after, false)].flatten.compact
end
+ # Second case: Message is encrypted
+
gpg = lines.between(GPG_START, GPG_END)
# between does not check if GPG_END actually exists
if !gpg.empty? && !lines.index(GPG_END).nil?
@@ -704,6 +745,24 @@ class Location
source.raw_message info
end
+ def sync_back labels, message
+ synced = false
+ return synced unless sync_back_enabled? and valid?
+ source.synchronize do
+ new_info = source.sync_back(@info, labels)
+ if new_info
+ @info = new_info
+ Index.sync_message message, true
+ synced = true
+ end
+ end
+ synced
+ end
+
+ def sync_back_enabled?
+ source.respond_to? :sync_back and $config[:sync_back_to_maildir] and source.sync_back_enabled?
+ end
+
## much faster than raw_message
def each_raw_message_line &b
source.each_raw_message_line info, &b
@@ -717,6 +776,10 @@ class Location
source.valid? info
end
+ def labels?
+ source.labels? info
+ end
+
def == o
o.source.id == source.id and o.info == info
end
diff --git a/lib/sup/modes/edit_message_mode.rb b/lib/sup/modes/edit_message_mode.rb
@@ -197,7 +197,15 @@ EOS
@file = Tempfile.new ["sup.#{self.class.name.gsub(/.*::/, '').camel_to_hyphy}", ".eml"]
@file.puts format_headers(@header - NON_EDITABLE_HEADERS).first
@file.puts
- @file.puts @body.join("\n")
+
+ begin
+ text = @body.join("\n")
+ rescue Encoding::CompatibilityError
+ text = @body.map { |x| x.fix_encoding! }.join("\n")
+ debug "encoding problem while writing message, trying to rescue, but expect errors: #{text}"
+ end
+
+ @file.puts text
@file.puts sig if ($config[:edit_signature] and !@sig_edited)
@file.close
end
@@ -205,13 +213,13 @@ EOS
def set_sig_edit_flag
sig = sig_lines.join("\n")
if $config[:edit_signature]
- pbody = @body.join("\n")
+ pbody = @body.map { |x| x.fix_encoding! }.join("\n").fix_encoding!
blen = pbody.length
slen = sig.length
if blen > slen and pbody[blen-slen..blen] == sig
@sig_edited = false
- @body = pbody[0..blen-slen].split("\n")
+ @body = pbody[0..blen-slen].fix_encoding!.split("\n")
else
@sig_edited = true
end
@@ -319,7 +327,7 @@ EOS
end
def delete_attachment
- i = curpos - @attachment_lines_offset - DECORATION_LINES - 2
+ i = curpos - @attachment_lines_offset - (@selectors.empty? ? 0 : DECORATION_LINES) - @selectors.size
if i >= 0 && i < @attachments.size && BufferManager.ask_yes_or_no("Delete attachment #{@attachment_names[i]}?")
@attachments.delete_at i
@attachment_names.delete_at i
@@ -551,9 +559,9 @@ protected
m.header[k] =
case v
when String
- (k.match(/subject/i) ? mime_encode_subject(v) : mime_encode_address(v)).fix_encoding!
+ (k.match(/subject/i) ? mime_encode_subject(v).dup.fix_encoding! : mime_encode_address(v)).dup.fix_encoding!
when Array
- (v.map { |v| mime_encode_address v }.join ", ").fix_encoding!
+ (v.map { |v| mime_encode_address v }.join ", ").dup.fix_encoding!
end
end
@@ -635,12 +643,12 @@ private
if HookManager.enabled? "mentions-attachments"
HookManager.run "mentions-attachments", :header => @header, :body => @body
else
- @body.any? { |l| l =~ /^[^>]/ && l =~ /\battach(ment|ed|ing|)\b/i }
+ @body.any? { |l| l.fix_encoding! =~ /^[^>]/ && l.fix_encoding! =~ /\battach(ment|ed|ing|)\b/i }
end
end
def top_posting?
- @body.join("\n") =~ /(\S+)\s*Excerpts from.*\n(>.*\n)+\s*\Z/
+ @body.map { |x| x.fix_encoding! }.join("\n").fix_encoding! =~ /(\S+)\s*Excerpts from.*\n(>.*\n)+\s*\Z/
end
def sig_lines
diff --git a/lib/sup/modes/forward_mode.rb b/lib/sup/modes/forward_mode.rb
@@ -7,9 +7,10 @@ class ForwardMode < EditMessageMode
"From" => AccountManager.default_account.full_address,
}
+ @m = opts[:message]
header["Subject"] =
- if opts[:message]
- "Fwd: " + opts[:message].subj
+ if @m
+ "Fwd: " + @m.subj
elsif opts[:attachments]
"Fwd: " + opts[:attachments].keys.join(", ")
end
@@ -19,8 +20,8 @@ class ForwardMode < EditMessageMode
header["Bcc"] = opts[:bcc].map { |p| p.full_address }.join(", ") if opts[:bcc]
body =
- if opts[:message]
- forward_body_lines(opts[:message])
+ if @m
+ forward_body_lines @m
elsif opts[:attachments]
["Note: #{opts[:attachments].size.pluralize 'attachment'}."]
end
@@ -68,6 +69,14 @@ protected
m.quotable_header_lines + [""] + m.quotable_body_lines +
["--- End forwarded message ---"]
end
+
+ def send_message
+ return unless super # super returns true if the mail has been sent
+ if @m
+ @m.add_label :forwarded
+ Index.save_message @m
+ end
+ end
end
end
diff --git a/lib/sup/modes/reply_mode.rb b/lib/sup/modes/reply_mode.rb
@@ -217,6 +217,12 @@ protected
update
end
end
+
+ def send_message
+ return unless super # super returns true if the mail has been sent
+ @m.add_label :replied
+ Index.save_message @m
+ end
end
end
diff --git a/lib/sup/modes/search_list_mode.rb b/lib/sup/modes/search_list_mode.rb
@@ -145,6 +145,12 @@ protected
def rename_selected_search
old_name, num_unread = @searches[curpos]
return unless old_name
+
+ if SearchManager.predefined_searches.has_key? old_name
+ BufferManager.flash "Cannot be edited: predefined search."
+ return
+ end
+
new_name = BufferManager.ask :save_search, "Rename this saved search: ", old_name
return unless new_name && new_name !~ /^\s*$/ && new_name != old_name
new_name.strip!
@@ -163,6 +169,12 @@ protected
def edit_selected_search
name, num_unread = @searches[curpos]
return unless name
+
+ if SearchManager.predefined_searches.has_key? name
+ BufferManager.flash "Cannot be edited: predefined search."
+ return
+ end
+
old_search_string = SearchManager.search_string_for name
new_search_string = BufferManager.ask :search, "Edit this saved search: ", (old_search_string + " ")
return unless new_search_string && new_search_string !~ /^\s*$/ && new_search_string != old_search_string
diff --git a/lib/sup/modes/thread_index_mode.rb b/lib/sup/modes/thread_index_mode.rb
@@ -200,6 +200,26 @@ EOS
BufferManager.draw_screen
end
+ def handle_updated_update sender, m
+ t = thread_containing(m) or return
+ l = @lines[t] or return
+ @ts_mutex.synchronize do
+ @ts.delete_message m
+ @ts.add_message m
+ end
+ Index.save_thread t
+ update_text_for_line l
+ end
+
+ def handle_location_deleted_update sender, m
+ t = thread_containing(m)
+ delete_thread t if t and t.first.id == m.id
+ @ts_mutex.synchronize do
+ @ts.delete_message m if t
+ end
+ update
+ end
+
def handle_single_message_deleted_update sender, m
@ts_mutex.synchronize do
return unless @ts.contains? m
@@ -755,6 +775,16 @@ protected
update
end
+ def delete_thread t
+ @mutex.synchronize do
+ i = @threads.index(t) or return
+ @threads.delete_at i
+ @size_widgets.delete_at i
+ @date_widgets.delete_at i
+ @tags.drop_tag_for t
+ end
+ end
+
def hide_thread t
@mutex.synchronize do
i = @threads.index(t) or return
diff --git a/lib/sup/modes/thread_view_mode.rb b/lib/sup/modes/thread_view_mode.rb
@@ -1,3 +1,5 @@
+require 'shellwords'
+
module Redwood
class ThreadViewMode < LineCursorMode
@@ -246,6 +248,8 @@ EOS
sm.puts m.raw_message
end
raise SendmailCommandFailed, "Couldn't execute #{cmd}" unless $? == 0
+ m.add_label :forwarded
+ Index.save_message m
rescue SystemCallError, SendmailCommandFailed => e
warn "problem sending mail: #{e.message}"
BufferManager.flash "Problem sending mail: #{e.message}"
@@ -359,8 +363,14 @@ EOS
when Chunk::Attachment
default_dir = $config[:default_attachment_save_dir]
default_dir = ENV["HOME"] if default_dir.nil? || default_dir.empty?
- default_fn = File.expand_path File.join(default_dir, chunk.filename)
- fn = BufferManager.ask_for_filename :filename, "Save attachment to file: ", default_fn
+ default_fn = File.expand_path File.join(default_dir, Shellwords.escape(chunk.filename))
+ fn = BufferManager.ask_for_filename :filename, "Save attachment to file or directory: ", default_fn, true
+
+ # if user selects directory use file name from message
+ if fn and File.directory? fn
+ fn = File.join(fn, Shellwords.escape(chunk.filename))
+ end
+
save_to_file(fn) { |f| f.print chunk.raw_content } if fn
else
m = @message_lines[curpos]
@@ -382,7 +392,7 @@ EOS
num_errors = 0
m.chunks.each do |chunk|
next unless chunk.is_a?(Chunk::Attachment)
- fn = File.join(folder, chunk.filename)
+ fn = File.join(folder, Shellwords.escape(chunk.filename))
num_errors += 1 unless save_to_file(fn, false) { |f| f.print chunk.raw_content }
num += 1
end
@@ -780,13 +790,13 @@ private
@person_lines[start] = m.from
[[prefix_widget, open_widget, new_widget, attach_widget, starred_widget,
[color,
- "#{m.from ? m.from.mediumname : '?'} to #{m.recipients.map { |l| l.shortname }.join(', ')} #{m.date.to_nice_s} (#{m.date.to_nice_distance_s})"]]]
+ "#{m.from ? m.from.mediumname.fix_encoding! : '?'} to #{m.recipients.map { |l| l.shortname.fix_encoding! }.join(', ')} #{m.date.to_nice_s.fix_encoding!} (#{m.date.to_nice_distance_s.fix_encoding!})"]]]
when :closed
@person_lines[start] = m.from
[[prefix_widget, open_widget, new_widget, attach_widget, starred_widget,
[color,
- "#{m.from ? m.from.mediumname : '?'}, #{m.date.to_nice_s} (#{m.date.to_nice_distance_s}) #{m.snippet}"]]]
+ "#{m.from ? m.from.mediumname.fix_encoding! : '?'}, #{m.date.to_nice_s.fix_encoding!} (#{m.date.to_nice_distance_s.fix_encoding!}) #{m.snippet ? m.snippet.fix_encoding! : ''}"]]]
when :detailed
@person_lines[start] = m.from
diff --git a/lib/sup/poll.rb b/lib/sup/poll.rb
@@ -25,6 +25,9 @@ Variables:
num_total: the total number of messages
num_inbox_total: the total number of new messages in the inbox.
num_inbox_total_unread: the total number of unread messages in the inbox
+ num_updated: the total number of updated messages
+ num_deleted: the total number of deleted messages
+ labels: the labels that were applied
from_and_subj: an array of (from email address, subject) pairs
from_and_subj_inbox: an array of (from email address, subject) pairs for
only those messages appearing in the inbox
@@ -52,23 +55,32 @@ EOS
BufferManager.flash "Polling for new messages..."
end
- num, numi, from_and_subj, from_and_subj_inbox, loaded_labels = @mode.poll
+ num, numi, numu, numd, from_and_subj, from_and_subj_inbox, loaded_labels = @mode.poll
clear_running_totals if @should_clear_running_totals
@running_totals[:num] += num
@running_totals[:numi] += numi
+ @running_totals[:numu] += numu
+ @running_totals[:numd] += numd
@running_totals[:loaded_labels] += loaded_labels || []
if HookManager.enabled? "after-poll"
hook_args = { :num => num, :num_inbox => numi,
:num_total => @running_totals[:num], :num_inbox_total => @running_totals[:numi],
+ :num_updated => @running_totals[:numu],
+ :num_deleted => @running_totals[:numd],
+ :labels => @running_totals[:loaded_labels],
:from_and_subj => from_and_subj, :from_and_subj_inbox => from_and_subj_inbox,
:num_inbox_total_unread => lambda { Index.num_results_for :labels => [:inbox, :unread] } }
HookManager.run("after-poll", hook_args)
else
if @running_totals[:num] > 0
- BufferManager.flash "Loaded #{@running_totals[:num].pluralize 'new message'}, #{@running_totals[:numi]} to inbox. Labels: #{@running_totals[:loaded_labels].map{|l| l.to_s}.join(', ')}"
+ flash_msg = "Loaded #{@running_totals[:num].pluralize 'new message'}, #{@running_totals[:numi]} to inbox. " if @running_totals[:num] > 0
+ flash_msg += "Updated #{@running_totals[:numu].pluralize 'message'}. " if @running_totals[:numu] > 0
+ flash_msg += "Deleted #{@running_totals[:numd].pluralize 'message'}. " if @running_totals[:numd] > 0
+ flash_msg += "Labels: #{@running_totals[:loaded_labels].map{|l| l.to_s}.join(', ')}." if @running_totals[:loaded_labels].size > 0
+ BufferManager.flash flash_msg
else
BufferManager.flash "No new messages."
end
@@ -115,7 +127,7 @@ EOS
end
def do_poll
- total_num = total_numi = 0
+ total_num = total_numi = total_numu = total_numd = 0
from_and_subj = []
from_and_subj_inbox = []
loaded_labels = Set.new
@@ -129,16 +141,23 @@ EOS
next
end
- num = 0
- numi = 0
+ msg = ""
+ num = numi = numu = numd = 0
poll_from source do |action,m,old_m,progress|
if action == :delete
yield "Deleting #{m.id}"
+ loaded_labels.merge m.labels
+ numd += 1
+ elsif action == :update
+ yield "Message at #{m.source_info} is an update of an old message. Updating labels from #{old_m.labels.to_a * ','} => #{m.labels.to_a * ','}"
+ loaded_labels.merge m.labels
+ numu += 1
elsif action == :add
if old_m
new_locations = (m.locations - old_m.locations)
if not new_locations.empty?
- yield "Message at #{new_locations[0].info} is an update of an old message. Updating labels from #{old_m.labels.to_a * ','} => #{m.labels.to_a * ','}"
+ yield "Message at #{new_locations[0].info} has changed its source location. Updating labels from #{old_m.labels.to_a * ','} => #{m.labels.to_a * ','}"
+ numu += 1
else
yield "Skipping already-imported message at #{m.locations[-1].info}"
end
@@ -155,25 +174,29 @@ EOS
else fail
end
end
- yield "Found #{num} messages, #{numi} to inbox." unless num == 0
+ msg += "Found #{num} messages, #{numi} to inbox. " unless num == 0
+ msg += "Updated #{numu} messages. " unless numu == 0
+ msg += "Deleted #{numd} messages." unless numd == 0
+ yield msg unless msg == ""
total_num += num
total_numi += numi
+ total_numu += numu
+ total_numd += numd
end
loaded_labels = loaded_labels - LabelManager::HIDDEN_RESERVED_LABELS - [:inbox, :killed]
yield "Done polling; loaded #{total_num} new messages total"
@last_poll = Time.now
end
- [total_num, total_numi, from_and_subj, from_and_subj_inbox, loaded_labels]
+ [total_num, total_numi, total_numu, total_numd, from_and_subj, from_and_subj_inbox, loaded_labels]
end
## like Source#poll, but yields successive Message objects, which have their
## labels and locations set correctly. The Messages are saved to or removed
## from the index after being yielded.
def poll_from source, opts={}
- debug "trying to acquire poll lock for: #{source}.."
- if source.poll_lock.try_lock
- debug "lock acquired for: #{source}."
+ debug "trying to acquire poll lock for: #{source}..."
+ if source.try_lock
begin
source.poll do |sym, args|
case sym
@@ -191,18 +214,40 @@ EOS
yield :add, m, old_m, args[:progress] if block_given?
Index.sync_message m, true
+ if Index.message_joining_killed? m
+ m.labels += [:killed]
+ Index.sync_message m, true
+ end
+
## We need to add or unhide the message when it either did not exist
## before at all or when it was updated. We do *not* add/unhide when
## the same message was found at a different location
- if !old_m or not old_m.locations.member? m.location
+ if old_m
+ UpdateManager.relay self, :updated, m
+ elsif !old_m or not old_m.locations.member? m.location
UpdateManager.relay self, :added, m
end
when :delete
- Index.each_message :location => [source.id, args[:info]] do |m|
+ Index.each_message({:location => [source.id, args[:info]]}, false) do |m|
m.locations.delete Location.new(source, args[:info])
- yield :delete, m, [source,args[:info]], args[:progress] if block_given?
Index.sync_message m, false
- #UpdateManager.relay self, :deleted, m
+ if m.locations.size == 0
+ yield :delete, m, [source,args[:info]], args[:progress] if block_given?
+ Index.delete m.id
+ UpdateManager.relay self, :location_deleted, m
+ end
+ end
+ when :update
+ Index.each_message({:location => [source.id, args[:old_info]]}, false) do |m|
+ old_m = Index.build_message m.id
+ m.locations.delete Location.new(source, args[:old_info])
+ m.locations.push Location.new(source, args[:new_info])
+ ## Update labels that might have been modified remotely
+ m.labels -= source.supported_labels?
+ m.labels += args[:labels]
+ yield :update, m, old_m if block_given?
+ Index.sync_message m, true
+ UpdateManager.relay self, :updated, m
end
end
end
@@ -212,7 +257,7 @@ EOS
ensure
source.go_idle
- source.poll_lock.unlock
+ source.unlock
end
else
debug "source #{source} is already being polled."
@@ -221,7 +266,7 @@ EOS
def handle_idle_update sender, idle_since; @should_clear_running_totals = false; end
def handle_unidle_update sender, idle_since; @should_clear_running_totals = true; clear_running_totals; end
- def clear_running_totals; @running_totals = {:num => 0, :numi => 0, :loaded_labels => Set.new}; end
+ def clear_running_totals; @running_totals = {:num => 0, :numi => 0, :numu => 0, :numd => 0, :loaded_labels => Set.new}; end
end
end
diff --git a/lib/sup/search.rb b/lib/sup/search.rb
@@ -7,6 +7,8 @@ class SearchManager
class ExpansionError < StandardError; end
+ attr_reader :predefined_searches
+
def initialize fn
@fn = fn
@searches = {}
@@ -43,24 +45,40 @@ class SearchManager
def add name, search_string
return unless valid_name? name
+ if @predefined_searches.has_key? name
+ warn "cannot add search: #{name} is already taken by a predefined search"
+ return
+ end
@searches[name] = search_string
@modified = true
end
def rename old, new
return unless @searches.has_key? old
+ if [old, new].any? { |x| @predefined_searches.has_key? x }
+ warn "cannot rename search: #{old} or #{new} is already taken by a predefined search"
+ return
+ end
search_string = @searches[old]
delete old if add new, search_string
end
def edit name, search_string
return unless @searches.has_key? name
+ if @predefined_searches.has_key? name
+ warn "cannot edit predefined search: #{name}."
+ return
+ end
@searches[name] = search_string
@modified = true
end
def delete name
return unless @searches.has_key? name
+ if @predefined_searches.has_key? name
+ warn "cannot delete predefined search: #{name}."
+ return
+ end
@searches.delete name
@modified = true
end
diff --git a/lib/sup/sent.rb b/lib/sup/sent.rb
@@ -27,7 +27,7 @@ class SentManager
def write_sent_message date, from_email, &block
::Thread.new do
debug "store the sent message (locking sent source..)"
- @source.poll_lock.synchronize do
+ @source.synchronize do
@source.store_message date, from_email, &block
end
PollManager.poll_from @source
diff --git a/lib/sup/source.rb b/lib/sup/source.rb
@@ -1,4 +1,5 @@
require "sup/rfc2047"
+require "monitor"
module Redwood
@@ -53,8 +54,8 @@ class Source
## Examples for you to look at: mbox.rb and maildir.rb.
bool_accessor :usual, :archived
- attr_reader :uri
- attr_accessor :id, :poll_lock
+ attr_reader :uri, :usual
+ attr_accessor :id
def initialize uri, usual=true, archived=false, id=nil
raise ArgumentError, "id must be an integer: #{id.inspect}" unless id.is_a? Fixnum if id
@@ -64,10 +65,10 @@ class Source
@archived = archived
@id = id
- @poll_lock = Mutex.new
+ @poll_lock = Monitor.new
end
- ## overwrite me if you have a disk incarnation (currently used only for sup-sync-back)
+ ## overwrite me if you have a disk incarnation (currently used only for sup-sync-back-mbox)
def file_path; nil end
def to_s; @uri.to_s; end
@@ -81,6 +82,14 @@ class Source
## leaks (esp. file descriptors).
def go_idle; end
+ ## Returns an array containing all the labels that are natively
+ ## supported by this source
+ def supported_labels?; [] end
+
+ ## Returns an array containing all the labels that are currently in
+ ## the location filename
+ def labels? info; [] end
+
## Yields values of the form [Symbol, Hash]
## add: info, labels, progress
## delete: info, progress
@@ -92,6 +101,25 @@ class Source
true
end
+ def synchronize &block
+ @poll_lock.synchronize &block
+ end
+
+ def try_lock
+ acquired = @poll_lock.try_enter
+ if acquired
+ debug "lock acquired for: #{self}"
+ else
+ debug "could not acquire lock for: #{self}"
+ end
+ acquired
+ end
+
+ def unlock
+ @poll_lock.exit
+ debug "lock released for: #{self}"
+ end
+
## utility method to read a raw email header from an IO stream and turn it
## into a hash of key-value pairs. minor special semantics for certain headers.
##
diff --git a/lib/sup/thread.rb b/lib/sup/thread.rb
@@ -387,6 +387,12 @@ class ThreadSet
m.refs.any? { |ref_id| @messages.member? ref_id }
end
+ def delete_message message
+ el = @messages[message.id]
+ return unless el.message
+ el.message = nil
+ end
+
## the heart of the threading code
def add_message message
el = @messages[message.id]
diff --git a/lib/sup/util.rb b/lib/sup/util.rb
@@ -8,6 +8,7 @@ require 'set'
require 'enumerator'
require 'benchmark'
require 'unicode'
+require 'fileutils'
## time for some monkeypatching!
class Symbol
@@ -45,6 +46,17 @@ class Lockfile
def touch_yourself; touch path end
end
+class File
+ # platform safe file.link which attempts a copy if hard-linking fails
+ def self.safe_link src, dest
+ begin
+ File.link src, dest
+ rescue
+ FileUtils.copy src, dest
+ end
+ end
+end
+
class Pathname
def human_size
s =
@@ -120,8 +132,6 @@ module RMail
# Convert to ASCII before trying to match with regexp
class Field
- EXTRACT_FIELD_NAME_RE = /\A([^\x00-\x1f\x7f-\xff :]+):\s*/no
-
class << self
def parse(field)
field = field.dup.to_s
diff --git a/lib/sup/util/query.rb b/lib/sup/util/query.rb
@@ -3,10 +3,13 @@ module Redwood
module Query
class QueryDescriptionError < ArgumentError; end
- def self.describe query
+ def self.describe(query, fallback = nil)
d = query.description.force_encoding("UTF-8")
- raise QueryDescriptionError.new(d) unless d.valid_encoding?
+ unless d.valid_encoding?
+ raise QueryDescriptionError.new(d) unless fallback
+ d = fallback
+ end
return d
end
end
diff --git a/sup.gemspec b/sup.gemspec
@@ -5,8 +5,8 @@ require 'sup/version'
# Files
SUP_EXECUTABLES = %w(sup sup-add sup-config sup-dump sup-import-dump
- sup-recover-sources sup-sync sup-sync-back sup-tweak-labels
- sup-psych-ify-config-files)
+ sup-recover-sources sup-sync sup-sync-back-maildir sup-sync-back-mbox
+ sup-tweak-labels sup-psych-ify-config-files)
SUP_EXTRA_FILES = %w(CONTRIBUTORS README.md LICENSE History.txt ReleaseNotes)
SUP_FILES =
SUP_EXTRA_FILES +
@@ -37,7 +37,7 @@ DESC
# TODO: might want to add index migrating script here, too
s.post_install_message = <<-EOF
SUP: If you are upgrading Sup from before version 0.14.0: Please
- run `sup-psych-ify-config-files` to migrate from 0.13 to 0.14.
+ run `sup-psych-ify-config-files` to migrate from 0.13.
Check https://github.com/sup-heliotrope/sup/wiki/Migration-0.13-to-0.14
for more detailed and up-to-date instructions.
@@ -49,7 +49,7 @@ SUP: If you are upgrading Sup from before version 0.14.0: Please
s.add_runtime_dependency "xapian-ruby", "~> 1.2.15"
s.add_runtime_dependency "ncursesw-sup", "~> 1.3.1"
- s.add_runtime_dependency "rmail", ">= 0.17"
+ s.add_runtime_dependency "rmail-sup", "~> 1.0.1"
s.add_runtime_dependency "highline"
s.add_runtime_dependency "trollop", ">= 1.12"
s.add_runtime_dependency "lockfile"
diff --git a/test/unit/util/test_query.rb b/test/unit/util/test_query.rb
@@ -33,5 +33,14 @@ describe Redwood::Util::Query do
_ = life + query.description
end
end
+
+ it "returns a valid UTF-8 fallback description of bad input" do
+ msg = "asdfa \xc3\x28 åasdf"
+ query = Xapian::Query.new msg
+
+ desc = Redwood::Util::Query.describe(query, "invalid query")
+
+ assert_equal("invalid query", desc)
+ end
end
end