commit eed695e92bef3cb32ce935aa965ebd89b1939494
parent aa6f2a3fdd245997a86500eb036ce3d76074e36d
Author: Eric Weikl <eric.weikl@gmx.net>
Date: Sun, 7 Jul 2013 12:58:42 +0200
Merge branch 'sup-heliotrope/develop' into maildir-sync
Diffstat:
17 files changed, 248 insertions(+), 146 deletions(-)
diff --git a/.travis.yml b/.travis.yml
@@ -3,7 +3,6 @@ language: ruby
rvm:
- 2.0.0
- 1.9.3
- - 1.9.2
before_install:
- sudo apt-get update -qq
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
@@ -1,8 +1,8 @@
-William Morgan <wmorgan-sup at the masanjin dot nets>
+William Morgan <william at the twitter dot coms>
Rich Lane <rlane at the club.cc.cmu dot edus>
Gaute Hope <eg at the gaute.vetsj dot coms>
-Hamish Downer <dmishd at the gmail dot coms>
Whyme Lyu <callme5long at the gmail dot coms>
+Hamish Downer <dmishd at the gmail dot coms>
Sascha Silbe <sascha-pgp at the silbe dot orgs>
Ismo Puustinen <ismo at the iki dot fis>
Nicolas Pouillard <nicolas.pouillard at the gmail dot coms>
@@ -21,47 +21,46 @@ Eric Weikl <eric.weikl at the tngtech dot coms>
Christopher Warrington <chrisw at the rice dot edus>
W. Trevor King <wking at the drexel dot edus>
Richard Brown <rbrown at the exherbo dot orgs>
+Anthony Martinez <pi+sup at the pihost dot uss>
Marc Hartstein <marc.hartstein at the alum.vassar dot edus>
Israel Herraiz <israel.herraiz at the gmail dot coms>
-Anthony Martinez <pi+sup at the pihost dot uss>
Bo Borgerson <gigabo at the gmail dot coms>
-William Erik Baxter <web at the superscript dot coms>
Michael Hamann <michael at the content-space dot des>
+William Erik Baxter <web at the superscript dot coms>
+Jonathan Lassoff <jof at the thejof dot coms>
Grant Hollingworth <grant at the antiflux dot orgs>
+Markus Klinik <markus.klinik at the gmx dot des>
+Ico Doornekamp <ico at the pruts dot nls>
Adeodato Simó <dato at the net.com.org dot ess>
Daniel Schoepe <daniel.schoepe at the googlemail dot coms>
Jason Petsod <jason at the petsod dot orgs>
-Steve Goldman <sgoldman at the tower-research dot coms>
+Edward Z. Yang <edwardzyang at the thewritingpot dot coms>
Robin Burchell <viroteck at the viroteck dot nets>
+Steve Goldman <sgoldman at the tower-research dot coms>
Peter Harkins <ph at the malaprop dot orgs>
-Edward Z. Yang <ezyang at the MIT dot EDUs>
Decklin Foster <decklin at the red-bean dot coms>
Cameron Matheson <cam+sup at the cammunism dot orgs>
Carl Worth <cworth at the cworth dot orgs>
+Alex Vandiver <alex at the chmrr dot nets>
Jeff Balogh <its.jeff.balogh at the gmail dot coms>
-Alex Vandiver <alexmv at the mit dot edus>
Andrew Pimlott <andrew at the pimlott dot nets>
Matías Aguirre <matiasaguirre at the gmail dot coms>
-Anthony Martinez <pi at the pihost dot uss>
Kornilios Kourtis <kkourt at the cslab.ece.ntua dot grs>
-Kevin Riggle <kevinr at the free-dissociation dot coms>
Giorgio Lando <patroclo7 at the gmail dot coms>
+Kevin Riggle <kevinr at the free-dissociation dot coms>
Benoît PIERRE <benoit.pierre at the gmail dot coms>
Alvaro Herrera <alvherre at the alvh.no-ip dot orgs>
-Eric Weikl <eric.weikl at the gmx dot nets>
+Steven Lawrance <stl at the koffein dot nets>
Jonah <Jonah at the GoodCoffee dot cas>
-ian <ian at the lorf dot orgs>
-Adam Lloyd <adam at the alloy-d dot nets>
+ian <itaylor at the uark dot edus>
+MichaelRevell <mikearevell at the gmail dot coms>
+Gregor Hoffleit <gregor at the sam.mediasupervision dot des>
Todd Eisenberger <teisenbe at the andrew.cmu dot edus>
+Adam Lloyd <adam at the alloy-d dot nets>
Steven Walter <swalter at the monarch.(none)>
-Alex Vandiver <alex at the chmrr dot nets>
-Gregor Hoffleit <gregor at the sam.mediasupervision dot des>
Jon M. Dugan <jdugan at the es dot nets>
Matthieu Rakotojaona <matthieu.rakotojaona at the gmail dot coms>
-Stefan Lundström <lundst at the snabb.(none)>
Matthias Vallentin <vallentin at the icir dot orgs>
-Steven Lawrance <stl at the redhat dot coms>
-Jonathan Lassoff <jof at the thejof dot coms>
-ian <itaylor at the uark dot edus>
-Gregor Hoffleit <gregor at the hoffleit dot des>
+Stefan Lundström <lundst at the snabb.(none)>
+Whyme.Lyu <callme5long at the gmail dot coms>
Kirill Smelkov <kirr at the landau.phys.spbu dot rus>
diff --git a/History.txt b/History.txt
@@ -1,3 +1,12 @@
+== 0.13.2 / 2013-06-26
+
+* FreeBSD 10 comptability
+* More threadsafe polling
+
+== 0.13.1 / 2013-06-21
+
+* Bugfixes
+
== 0.13.0 / 2013-05-15
* Bugfixes
diff --git a/ReleaseNotes b/ReleaseNotes
@@ -1,3 +1,11 @@
+Release 0.13.2:
+
+FreeBSD compatability and more thread safe polling.
+
+Release 0.13.1:
+
+Another ruby 1.8 compatible release, various fixes.
+
Release 0.13.0:
Collection of bugfixes and stability fixes since 0.12.1. We now depend on our
diff --git a/bin/sup b/bin/sup
@@ -106,8 +106,6 @@ end
## ncurses.so that's been compiled against libncursesw. (note the w.) why
## this works, i have no idea. much like pretty much every aspect of
## dealing with curses. cargo cult programming at its best.
-##
-## BSD users: if libc.so.6 is not found, try installing compat6x.
require 'dl/import'
require 'rbconfig'
module LibC
@@ -115,6 +113,7 @@ module LibC
setlocale_lib = case RbConfig::CONFIG['arch']
when /darwin/; "libc.dylib"
when /cygwin/; "cygwin1.dll"
+ when /freebsd/; "libc.so.7"
else; "libc.so.6"
end
@@ -127,9 +126,6 @@ module LibC
rescue RuntimeError => e
warn "cannot dlload setlocale(); ncurses wide character support probably broken."
warn "dlload error was #{e.class}: #{e.message}"
- if RbConfig::CONFIG['arch'] =~ /bsd/
- warn "BSD variant detected. You may have to install a compat6x package to acquire libc."
- end
end
end
diff --git a/doc/FAQ.txt b/doc/FAQ.txt
@@ -21,8 +21,8 @@ A: I hate ads, I hate using a mouse, and I hate non-programmability and
Q: Why the console?
A: Because a keystroke is worth a hundred mouse clicks, as any Unix
- user knows. Because you don't need web browser. Because you get
- instantaneous response and a simple interface.
+ user knows. Because you don't need a web browser. Because you get
+ an instantaneous response and a simple interface.
Q: How does Sup deal with spam?
A: You can manually mark messages as spam, which prevents them from
diff --git a/lib/sup/account.rb b/lib/sup/account.rb
@@ -51,7 +51,7 @@ class AccountManager
end
hash[:alternates] ||= []
- [:name, :signature].each { |x| hash[x].force_encoding Encoding::UTF_8 if hash[x].respond_to? :encoding }
+ [:name, :signature].each { |x| hash[x] ? hash[x].fix_encoding : nil }
a = Account.new hash
@accounts[a] = true
diff --git a/lib/sup/crypto.rb b/lib/sup/crypto.rb
@@ -74,7 +74,7 @@ EOS
end
unless @gpgme_present
- @not_working_reason = ['gpgme gem not present',
+ @not_working_reason = ['gpgme gem not present',
'Install the gpgme gem in order to use signed and encrypted emails']
return
end
@@ -85,7 +85,7 @@ EOS
else
# check if the gpg-options hook uses the passphrase_callback
# if it doesn't then check if gpg agent is present
- gpg_opts = HookManager.run("gpg-options",
+ gpg_opts = HookManager.run("gpg-options",
{:operation => "sign", :options => {}}) || {}
if gpg_opts[:passphrase_callback].nil?
if ENV['GPG_AGENT_INFO'].nil?
@@ -116,7 +116,7 @@ EOS
gpg_opts = {:protocol => GPGME::PROTOCOL_OpenPGP, :armor => true, :textmode => true}
gpg_opts.merge!(gen_sign_user_opts(from))
- gpg_opts = HookManager.run("gpg-options",
+ gpg_opts = HookManager.run("gpg-options",
{:operation => "sign", :options => gpg_opts}) || gpg_opts
begin
@@ -125,7 +125,7 @@ EOS
raise Error, gpgme_exc_msg(exc.message)
end
- # if the key (or gpg-agent) is not available GPGME does not complain
+ # if the key (or gpg-agent) is not available GPGME does not complain
# but just returns a zero length string. Let's catch that
if sig.length == 0
raise Error, gpgme_exc_msg("GPG failed to generate signature: check that gpg-agent is running and your key is available.")
@@ -145,7 +145,7 @@ EOS
gpg_opts = {:protocol => GPGME::PROTOCOL_OpenPGP, :armor => true, :textmode => true}
if sign
- gpg_opts.merge!(gen_sign_user_opts(from))
+ gpg_opts.merge!(gen_sign_user_opts(from))
gpg_opts.merge!({:sign => true})
end
gpg_opts = HookManager.run("gpg-options",
@@ -158,7 +158,7 @@ EOS
raise Error, gpgme_exc_msg(exc.message)
end
- # if the key (or gpg-agent) is not available GPGME does not complain
+ # if the key (or gpg-agent) is not available GPGME does not complain
# but just returns a zero length string. Let's catch that
if cipher.length == 0
raise Error, gpgme_exc_msg("GPG failed to generate cipher text: check that gpg-agent is running and your key is available.")
@@ -290,7 +290,7 @@ EOS
# Look for Charset, they are put before the base64 crypted part
charsets = payload.body.split("\n").grep(/^Charset:/)
if !charsets.empty? and charsets[0] =~ /^Charset: (.+)$/
- output = Iconv.easy_decode($encoding, $1, output)
+ output.transcode($encoding, $1)
end
msg.body = output
else
@@ -362,7 +362,7 @@ private
else
first_sig = "Unknown error or empty signature"
end
- rescue EOFError
+ rescue EOFError
from_key = nil
first_sig = "No public key available for #{signature.fingerprint}"
end
diff --git a/lib/sup/index.rb b/lib/sup/index.rb
@@ -478,7 +478,7 @@ EOS
raise ParseError, "xapian query parser error: #{e}"
end
- debug "parsed xapian query: #{xapian_query.description}"
+ debug "parsed xapian query: #{Util::Query.describe(xapian_query)}"
raise ParseError if xapian_query.nil? or xapian_query.empty?
query[:qobj] = xapian_query
diff --git a/lib/sup/message.rb b/lib/sup/message.rb
@@ -69,7 +69,9 @@ class Message
return unless v
return v unless v.is_a? String
return unless v.size < MAX_HEADER_VALUE_SIZE # avoid regex blowup on spam
- Rfc2047.decode_to $encoding, Iconv.easy_decode($encoding, 'ASCII', v)
+ d = v.dup
+ d = d.transcode($encoding, 'ASCII')
+ Rfc2047.decode_to $encoding, d
end
def parse_header encoded_header
@@ -109,7 +111,9 @@ class Message
Time.now
end
- @subj = header["subject"] ? header["subject"].gsub(/\s+/, " ").gsub(/\s+$/, "") : DEFAULT_SUBJECT
+ subj = header["subject"]
+ subj = subj ? subj.fix_encoding : nil
+ @subj = subj ? subj.gsub(/\s+/, " ").gsub(/\s+$/, "") : DEFAULT_SUBJECT
@to = Person.from_address_list header["to"]
@cc = Person.from_address_list header["cc"]
@bcc = Person.from_address_list header["bcc"]
@@ -553,7 +557,7 @@ private
## if there's no charset, use the current encoding as the charset.
## this ensures that the body is normalized to avoid non-displayable
## characters
- body = Iconv.easy_decode($encoding, m.charset || $encoding, m.decode)
+ body = m.decode.transcode($encoding, m.charset)
else
body = ""
end
@@ -569,11 +573,13 @@ private
def inline_gpg_to_chunks body, encoding_to, encoding_from
lines = body.split("\n")
gpg = lines.between(GPG_SIGNED_START, GPG_SIGNED_END)
- if !gpg.empty?
+ # between does not check if GPG_END actually exists
+ # Reference: http://permalink.gmane.org/gmane.mail.sup.devel/641
+ if !gpg.empty? && !lines.index(GPG_END).nil?
msg = RMail::Message.new
msg.body = gpg.join("\n")
- body = Iconv.easy_decode(encoding_to, encoding_from, body)
+ body = body.transcode(encoding_to, encoding_from)
lines = body.split("\n")
sig = lines.between(GPG_SIGNED_START, GPG_SIG_START)
startidx = lines.index(GPG_SIGNED_START)
diff --git a/lib/sup/message_chunks.rb b/lib/sup/message_chunks.rb
@@ -124,7 +124,7 @@ EOS
@lines = nil
if text
- text = text.transcode(encoded_content.charset || $encoding)
+ text = text.transcode(encoded_content.charset || $encoding, text.encoding)
@lines = text.gsub("\r\n", "\n").gsub(/\t/, " ").gsub(/\r/, "").split("\n")
@quotable = true
end
diff --git a/lib/sup/poll.rb b/lib/sup/poll.rb
@@ -22,7 +22,7 @@ Variables:
num: the total number of new messages added in this poll
num_inbox: the number of new messages added in this poll which
appear in the inbox (i.e. were not auto-archived).
- num_total: the total number of messages
+ num_total: the total number of messages
num_inbox_total: the total number of new messages in the inbox.
num_inbox_total_unread: the total number of unread messages in the inbox
num_updated: the total number of updated messages
@@ -38,7 +38,7 @@ EOS
@mutex = Mutex.new
@thread = nil
@last_poll = nil
- @polling = false
+ @polling = Mutex.new
@poll_sources = nil
@mode = nil
@should_clear_running_totals = false
@@ -89,21 +89,27 @@ EOS
end
def poll
- return if @polling
- @polling = true
- @poll_sources = SourceManager.usual_sources
- num, numi = poll_with_sources
- @polling = false
- [num, numi]
+ if @polling.try_lock
+ @poll_sources = SourceManager.usual_sources
+ num, numi = poll_with_sources
+ @polling.unlock
+ [num, numi]
+ else
+ debug "poll already in progress."
+ return
+ end
end
def poll_unusual
- return if @polling
- @polling = true
- @poll_sources = SourceManager.unusual_sources
- num, numi = poll_with_sources
- @polling = false
- [num, numi]
+ if @polling.try_lock
+ @poll_sources = SourceManager.unusual_sources
+ num, numi = poll_with_sources
+ @polling.unlock
+ [num, numi]
+ else
+ debug "poll_unusual already in progress."
+ return
+ end
end
def start
@@ -181,7 +187,6 @@ EOS
loaded_labels = loaded_labels - LabelManager::HIDDEN_RESERVED_LABELS - [:inbox, :killed]
yield "Done polling; loaded #{total_num} new messages total"
@last_poll = Time.now
- @polling = false
end
[total_num, total_numi, total_numu, total_numd, from_and_subj, from_and_subj_inbox, loaded_labels]
end
@@ -190,64 +195,73 @@ EOS
## labels and locations set correctly. The Messages are saved to or removed
## from the index after being yielded.
def poll_from source, opts={}
- begin
- source.poll do |sym, args|
- case sym
- when :add
- m = Message.build_from_source source, args[:info]
- old_m = Index.build_message m.id
- m.labels += args[:labels]
- m.labels.delete :inbox if source.archived?
- m.labels.delete :unread if source.read?
- m.labels.delete :unread if m.source_marked_read? # preserve read status if possible
- m.labels.each { |l| LabelManager << l }
- m.labels = old_m.labels + (m.labels - [:unread, :inbox]) if old_m
- m.locations = old_m.locations + m.locations if old_m
- HookManager.run "before-add-message", :message => m
- yield :add, m, old_m, args[:progress] if block_given?
- Index.sync_message m, true
-
- if Index.message_joining_killed? m
- m.labels += [:killed]
- Index.sync_message m, true
- end
-
- ## We need to add or unhide the message when it either did not exist
- ## before at all or when it was updated. We do *not* add/unhide when
- ## the same message was found at a different location
- if old_m
- UpdateManager.relay self, :updated, m
- elsif !old_m or not old_m.locations.member? m.location
- UpdateManager.relay self, :added, m
- end
- when :delete
- Index.each_message({:location => [source.id, args[:info]]}, false) do |m|
- m.locations.delete Location.new(source, args[:info])
- Index.sync_message m, false
- if m.locations.size == 0
- yield :delete, m, [source,args[:info]], args[:progress] if block_given?
- Index.delete m.id
- UpdateManager.relay self, :location_deleted, m
- end
- end
- when :update
- Index.each_message({:location => [source.id, args[:old_info]]}, false) do |m|
+ debug "trying to acquiring poll lock for: #{source}.."
+ if source.poll_lock.try_lock
+ debug "lock acquired for: #{source}."
+ begin
+ source.poll do |sym, args|
+ case sym
+ when :add
+ m = Message.build_from_source source, args[:info]
old_m = Index.build_message m.id
- m.locations.delete Location.new(source, args[:old_info])
- m.locations.push Location.new(source, args[:new_info])
- ## Update labels that might have been modified remotely
- m.labels -= source.supported_labels?
m.labels += args[:labels]
- yield :update, m, old_m if block_given?
+ m.labels.delete :inbox if source.archived?
+ m.labels.delete :unread if source.read?
+ m.labels.delete :unread if m.source_marked_read? # preserve read status if possible
+ m.labels.each { |l| LabelManager << l }
+ m.labels = old_m.labels + (m.labels - [:unread, :inbox]) if old_m
+ m.locations = old_m.locations + m.locations if old_m
+ HookManager.run "before-add-message", :message => m
+ yield :add, m, old_m, args[:progress] if block_given?
Index.sync_message m, true
- UpdateManager.relay self, :updated, m
+
+ if Index.message_joining_killed? m
+ m.labels += [:killed]
+ Index.sync_message m, true
+ end
+
+ ## We need to add or unhide the message when it either did not exist
+ ## before at all or when it was updated. We do *not* add/unhide when
+ ## the same message was found at a different location
+ if old_m
+ UpdateManager.relay self, :updated, m
+ elsif !old_m or not old_m.locations.member? m.location
+ UpdateManager.relay self, :added, m
+ end
+ when :delete
+ Index.each_message({:location => [source.id, args[:info]]}, false) do |m|
+ m.locations.delete Location.new(source, args[:info])
+ Index.sync_message m, false
+ if m.locations.size == 0
+ yield :delete, m, [source,args[:info]], args[:progress] if block_given?
+ Index.delete m.id
+ UpdateManager.relay self, :location_deleted, m
+ end
+ end
+ when :update
+ Index.each_message({:location => [source.id, args[:old_info]]}, false) do |m|
+ old_m = Index.build_message m.id
+ m.locations.delete Location.new(source, args[:old_info])
+ m.locations.push Location.new(source, args[:new_info])
+ ## Update labels that might have been modified remotely
+ m.labels -= source.supported_labels?
+ m.labels += args[:labels]
+ yield :update, m, old_m if block_given?
+ Index.sync_message m, true
+ UpdateManager.relay self, :updated, m
+ end
end
end
- end
- source.go_idle
- rescue SourceError => e
- warn "problem getting messages from #{source}: #{e.message}"
+ rescue SourceError => e
+ warn "problem getting messages from #{source}: #{e.message}"
+
+ ensure
+ source.go_idle
+ source.poll_lock.unlock
+ end
+ else
+ debug "source #{source} is already being polled."
end
end
diff --git a/lib/sup/rfc2047.rb b/lib/sup/rfc2047.rb
@@ -16,8 +16,6 @@
#
# This file is distributed under the same terms as Ruby.
-require 'iconv'
-
module Rfc2047
WORD = %r{=\?([!\#$%&'*+-/0-9A-Z\\^\`a-z{|}~]+)\?([BbQq])\?([!->@-~]+)\?=} # :nodoc: 'stupid ruby-mode
WORDSEQ = %r{(#{WORD.source})\s+(?=#{WORD.source})}
@@ -52,7 +50,7 @@ module Rfc2047
# WORD.
end
- Iconv.easy_decode(target, charset, text)
+ text.transcode(target, charset)
end
end
end
diff --git a/lib/sup/source.rb b/lib/sup/source.rb
@@ -62,7 +62,7 @@ class Source
bool_accessor :usual, :archived
attr_reader :uri
- attr_accessor :id
+ attr_accessor :id, :poll_lock
def initialize uri, usual=true, archived=false, id=nil
raise ArgumentError, "id must be an integer: #{id.inspect}" unless id.is_a? Fixnum if id
@@ -71,6 +71,8 @@ class Source
@usual = usual
@archived = archived
@id = id
+
+ @poll_lock = Mutex.new
end
## overwrite me if you have a disk incarnation (currently used only for sup-sync-back)
diff --git a/lib/sup/util.rb b/lib/sup/util.rb
@@ -1,3 +1,5 @@
+# encoding: utf-8
+
require 'thread'
require 'lockfile'
require 'mime/types'
@@ -5,7 +7,6 @@ require 'pathname'
require 'set'
require 'enumerator'
require 'benchmark'
-require 'iconv'
## time for some monkeypatching!
class Symbol
@@ -31,7 +32,7 @@ class Lockfile
def dump_lock_id lock_id = @lock_id
"host: %s\npid: %s\nppid: %s\ntime: %s\nuser: %s\npname: %s\n" %
lock_id.values_at('host','pid','ppid','time','user', 'pname')
- end
+ end
def lockinfo_on_disk
h = load_lock_id IO.read(path)
@@ -114,6 +115,25 @@ module RMail
end
class Header
+
+ # Convert to ASCII before trying to match with regexp
+ class Field
+
+ EXTRACT_FIELD_NAME_RE = /\A([^\x00-\x1f\x7f-\xff :]+):\s*/no
+
+ class << self
+ def parse(field)
+ field = field.dup.to_s
+ field = field.fix_encoding.ascii
+ if field =~ EXTRACT_FIELD_NAME_RE
+ [ $1, $'.chomp ]
+ else
+ [ "", Field.value_strip(field) ]
+ end
+ end
+ end
+ end
+
## Be more cautious about invalid content-type headers
## the original RMail code calls
## value.strip.split(/\s*;\s*/)[0].downcase
@@ -341,7 +361,55 @@ class String
ret << s
end
+ # Fix the damn string! make sure it is valid utf-8, then convert to
+ # user encoding.
+ #
+ # Not Ruby 1.8 compatible
+ def fix_encoding
+ encode!('UTF-8', :invalid => :replace, :undef => :replace)
+
+ # do this anyway in case string is set to be UTF-8, encoding to
+ # something else (UTF-16 which can fully represent UTF-8) and back
+ # ensures invalid chars are replaced.
+ encode!('UTF-16', 'UTF-8', :invalid => :replace, :undef => :replace)
+ encode!('UTF-8', 'UTF-16', :invalid => :replace, :undef => :replace)
+
+ fail "Could not create valid UTF-8 string out of: '#{self.to_s}'." unless valid_encoding?
+
+ # now convert to $encoding
+ encode!($encoding, :invalid => :replace, :undef => :replace)
+
+ fail "Could not create valid #{$encoding.inspect?} string out of: '#{self.to_s}'." unless valid_encoding?
+
+ self
+ end
+
+ # transcode the string if original encoding is know
+ # fix if broken.
+ #
+ # Not Ruby 1.8 compatible
+ def transcode to_encoding, from_encoding
+ begin
+ encode!(to_encoding, from_encoding, :invalid => :replace, :undef => :replace)
+
+ unless valid_encoding?
+ # fix encoding (through UTF-8)
+ encode!('UTF-16', from_encoding, :invalid => :replace, :undef => :replace)
+ encode!(to_encoding, 'UTF-16', :invalid => :replace, :undef => :replace)
+ end
+
+ rescue Encoding::ConverterNotFoundError
+ debug "Encoding converter not found for #{from_encoding.inspect} or #{to_encoding.inspect}, fixing string: '#{self.to_s}', but expect weird characters."
+ fix_encoding
+ end
+
+ fail "Could not create valid #{to_encoding.inspect?} string out of: '#{self.to_s}'." unless valid_encoding?
+
+ self
+ end
+
def normalize_whitespace
+ fix_encoding
gsub(/\t/, " ").gsub(/\r/, "")
end
@@ -383,12 +451,8 @@ class String
out << b.chr
end
end
- out.force_encoding Encoding::UTF_8 if out.respond_to? :force_encoding
- out
- end
-
- def transcode src_encoding=$encoding
- Iconv.easy_decode $encoding, src_encoding, self
+ out = out.fix_encoding # this should now be an utf-8 string of ascii
+ # compat chars.
end
unless method_defined? :ascii_only?
@@ -659,27 +723,3 @@ class FinishLine
end
end
-class Iconv
- def self.easy_decode target, orig_charset, text
- if text.respond_to? :force_encoding
- text = text.dup
- text.force_encoding Encoding::BINARY
- end
- charset = case orig_charset
- when /UTF[-_ ]?8/i then "utf-8"
- when /(iso[-_ ])?latin[-_ ]?1$/i then "ISO-8859-1"
- when /iso[-_ ]?8859[-_ ]?15/i then 'ISO-8859-15'
- when /unicode[-_ ]1[-_ ]1[-_ ]utf[-_]7/i then "utf-7"
- when /^euc$/i then 'EUC-JP' # XXX try them all?
- when /^(x-unknown|unknown[-_ ]?8bit|ascii[-_ ]?7[-_ ]?bit)$/i then 'ASCII'
- else orig_charset
- end
-
- begin
- returning(Iconv.iconv(target + "//IGNORE", charset, text + " ").join[0 .. -2]) { |str| str.check }
- rescue Errno::EINVAL, Iconv::InvalidEncoding, Iconv::InvalidCharacter, Iconv::IllegalSequence, String::CheckError
- debug "couldn't transcode text from #{orig_charset} (#{charset}) to #{target} (#{text[0 ... 20].inspect}...): got #{$!.class} (#{$!.message})"
- text.ascii
- end
- end
-end
diff --git a/lib/sup/util/query.rb b/lib/sup/util/query.rb
@@ -0,0 +1,9 @@
+module Redwood
+ module Util
+ module Query
+ def self.describe query
+ query.description.force_encoding("UTF-8")
+ end
+ end
+ end
+end
diff --git a/test/unit/util/test_query.rb b/test/unit/util/test_query.rb
@@ -0,0 +1,22 @@
+# encoding: utf-8
+
+require "test_helper"
+
+require "sup/util/query"
+require "xapian"
+
+describe Redwood::Util::Query do
+ describe ".describe" do
+ it "returns a UTF-8 description of query" do
+ query = Xapian::Query.new "テスト"
+ life = "生活: "
+
+ assert_raises Encoding::CompatibilityError do
+ _ = life + query.description
+ end
+
+ desc = Redwood::Util::Query.describe(query)
+ _ = (life + desc) # No exception thrown
+ end
+ end
+end