commit d66683c3375b48810d8f6ca49805943f9997c84f
parent 19c702767bdb0e4ab74da3bb49604135fbc19cf4
Author: William Morgan <wmorgan-sup@masanjin.net>
Date: Tue, 19 May 2009 19:52:52 -0400
Merge commit 'origin/sup-sync-improvements'
Diffstat:
2 files changed, 28 insertions(+), 13 deletions(-)
diff --git a/bin/sup-sync b/bin/sup-sync
@@ -5,6 +5,8 @@ require 'rubygems'
require 'trollop'
require "sup"
+PROGRESS_UPDATE_INTERVAL = 15 # seconds
+
class Float
def to_s; sprintf '%.2f', self; end
def to_time_s
@@ -122,7 +124,9 @@ begin
unless target == :new
if opts[:start_at]
- sources.each { |s| s.seek_to! opts[:start_at] }
+ Trollop::die :start_at, "can only be used on one source" unless sources.size == 1
+ sources.first.seek_to! opts[:start_at]
+ sources.first.correct_offset! if sources.first.respond_to?(:correct_offset!)
else
sources.each { |s| s.reset! }
end
@@ -137,6 +141,15 @@ begin
num_scanned += 1
seen[m.id] = true
+ if Time.now - last_info_time > PROGRESS_UPDATE_INTERVAL
+ last_info_time = Time.now
+ elapsed = last_info_time - start_time
+ start = opts[:start_at] || source.start_offset
+ pctdone = 100.0 * (source.cur_offset - start).to_f / (source.end_offset - start).to_f
+ remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
+ $stderr.printf "## read %dm (about %.0f%%) @ %.1fm/s. %s elapsed, about %s remaining\n", num_scanned, pctdone, num_scanned / elapsed, elapsed.to_time_s, remaining.to_time_s
+ end
+
## skip if we're operating only on changed messages, the message
## is in the index, and it's unchanged from what the source is
## reporting.
@@ -171,16 +184,8 @@ begin
## nothin! use default source labels
end
- if Time.now - last_info_time > 60
- last_info_time = Time.now
- elapsed = last_info_time - start_time
- pctdone = source.respond_to?(:pct_done) ? source.pct_done : 100.0 * (source.cur_offset.to_f - source.start_offset).to_f / (source.end_offset - source.start_offset).to_f
- remaining = (100.0 - pctdone) * (elapsed.to_f / pctdone)
- $stderr.puts "## #{num_scanned} (#{pctdone}%) read; #{elapsed.to_time_s} elapsed; #{remaining.to_time_s} remaining"
- end
-
if index_state.nil?
- puts "Adding message #{source}##{offset} with state {#{m.labels * ', '}}" if opts[:verbose]
+ puts "Adding message #{source}##{offset} from #{m.from} with state {#{m.labels * ', '}}" if opts[:verbose]
num_added += 1
else
puts "Updating message #{source}##{offset}, source #{entry[:source_id]} => #{source.id}, offset #{entry[:source_info]} => #{offset}, state {#{index_state * ', '}} => {#{m.labels * ', '}}" if opts[:verbose]
@@ -200,9 +205,7 @@ begin
## API.
##
## TODO: move this to Index, i suppose.
-
-
- if target == :all || target == :changed
+ if (target == :all || target == :changed) && !opts[:start_at]
$stderr.puts "Deleting missing messages from the index..."
num_del, num_scanned = 0, 0
sources.each do |source|
diff --git a/lib/sup/mbox/loader.rb b/lib/sup/mbox/loader.rb
@@ -81,6 +81,18 @@ class Loader < Source
end
end
+ ## scan forward until we're at the valid start of a message
+ def correct_offset!
+ @mutex.synchronize do
+ @f.seek cur_offset
+ string = ""
+ until @f.eof? || (l = @f.gets) =~ BREAK_RE
+ string << l
+ end
+ self.cur_offset += string.length
+ end
+ end
+
def raw_header offset
ret = ""
@mutex.synchronize do