Re: [sup-devel] [PATCH] Inotify support for Maildirs. (FIRST DRAFT)

From: "Edward Z. Yang" <ezyang@MIT.EDU>
To: sup-devel <sup-devel@rubyforge.org>, alvherre <alvherre@alvh.no-ip.org>
Subject: Re: [sup-devel] [PATCH] Inotify support for Maildirs. (FIRST DRAFT)
Date: Mon, 03 Sep 2012 01:00:59 -0400	[thread overview]
Message-ID: <1346648395-sup-4546@javelin> (raw)
In-Reply-To: <1346648371-12305-1-git-send-email-ezyang@mit.edu>

The locking is a downright crime (where's the STM when you need it),
and it's still racy, but it should work OK.

Excerpts from Edward Z. Yang's message of Mon Sep 03 00:59:31 -0400 2012:
> From: "Edward Z. Yang" <ezyang@mit.edu>
> 
> Signed-off-by: Edward Z. Yang <ezyang@mit.edu>
> ---
>  lib/sup/maildir.rb | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++--
>  lib/sup/poll.rb    | 33 ++++++++++++++++++++++++++-------
>  lib/sup/source.rb  |  4 ++++
>  3 files changed, 80 insertions(+), 9 deletions(-)
> 
> diff --git a/lib/sup/maildir.rb b/lib/sup/maildir.rb
> index 2a91f05..743156d 100644
> --- a/lib/sup/maildir.rb
> +++ b/lib/sup/maildir.rb
> @@ -1,5 +1,6 @@
>  require 'uri'
>  require 'set'
> +require 'inotify'
>  
>  module Redwood
>  
> @@ -184,6 +185,45 @@ class Maildir < Source
>      nil
>    end
>  
> +  def continuous_poll poll_mutex
> +    i = Inotify.new
> +    watches = {}
> +    @ctimes.each do |d,prev_ctime|
> +      subdir = File.join @dir, d
> +      wd = i.add_watch(subdir, Inotify::CREATE | Inotify::DELETE | Inotify::MOVE)
> +      watches[wd] = d
> +    end
> +    i.each_event do |ev|
> +      poll_mutex.synchronize do
> +        @mutex.synchronize do
> +          begin
> +            ::Thread.current[@dir] = true
> +            id = File.join watches[ev.wd], ev.name
> +            # check if inotify is stale
> +            # since we have @mutex, there is no race (except for
> +            # an external program fucking us over)
> +            next unless File.exists? File.join(@dir, id)
> +            x = Enumerator.new(Index.instance, :each_source_info, self.id, "#{id}").to_a
> +            if ev.mask & Inotify::CREATE or ev.mask & Inotify::MOVE_TO
> +              next unless x.empty?
> +              yield :add,
> +                :info => id,
> +                :labels => @labels + maildir_labels(id) + [:inbox],
> +                :progress => 0
> +            elsif ev.mask & Inotify::DELETE or ev.mask & Inotify::MOVE_FROM
> +              next unless !x.empty?
> +              yield :delete,
> +                :info => id,
> +                :progress => 0
> +            end
> +          ensure
> +            ::Thread.current[@dir] = nil
> +          end
> +        end
> +      end
> +    end
> +  end
> +
>    def labels? id
>      maildir_labels id
>    end
> @@ -248,7 +288,16 @@ private
>    end
>  
>    def maildir_move_file orig_path, new_source_id, flags
> -    @mutex.synchronize do
> +    if ::Thread.current[@dir]
> +      _maildir_move_file orig_path, new_source_id, flags
> +    else
> +      @mutex.synchronize do
> +        _maildir_move_file orig_path, new_source_id, flags
> +      end
> +    end
> +  end
> +
> +  def _maildir_move_file orig_path, new_source_id, flags
>        new_base = (flags.include?("S")) ? "cur" : "new"
>        md_base, md_ver, md_flags = maildir_data orig_path
>  
> @@ -292,7 +341,6 @@ private
>        end
>  
>        [new_source, new_loc]
> -    end
>    end
>  end
>  
> diff --git a/lib/sup/poll.rb b/lib/sup/poll.rb
> index dbd351f..51e0afa 100644
> --- a/lib/sup/poll.rb
> +++ b/lib/sup/poll.rb
> @@ -94,11 +94,27 @@ EOS
>          poll if @last_poll.nil? || (Time.now - @last_poll) >= @delay
>        end
>      end
> +    # XXX dup dup
> +    SourceManager.usual_sources.each do |source|
> +      Redwood::reporting_thread("inotify poll for #{source}") do
> +        source.continuous_poll @mutex do |sym, args|
> +          poll_handler source, sym, args
> +        end
> +      end
> +    end
> +    SourceManager.unusual_sources.each do |source|
> +      Redwood::reporting_thread("inotify poll for #{source}") do
> +        source.continuous_poll @mutex do |sym, args|
> +          poll_handler source, sym, args
> +        end
> +      end
> +    end
>    end
>  
>    def stop
>      @thread.kill if @thread
>      @thread = nil
> +    # handle inotify polls
>    end
>  
>    def do_poll
> @@ -172,7 +188,16 @@ EOS
>    ## from the index after being yielded.
>    def poll_from source, opts={}
>      begin
> -      source.poll do |sym, args|
> +      source.poll do |sym,args|
> +        poll_handler source, sym, args
> +      end
> +      source.go_idle
> +    rescue SourceError => e
> +      warn "problem getting messages from #{source}: #{e.message}"
> +    end
> +  end
> +
> +  def poll_handler source, sym, args
>          case sym
>          when :add
>            m = Message.build_from_source source, args[:info]
> @@ -224,12 +249,6 @@ EOS
>              UpdateManager.relay self, :updated, m
>            end
>          end
> -      end
> -
> -      source.go_idle
> -    rescue SourceError => e
> -      warn "problem getting messages from #{source}: #{e.message}"
> -    end
>    end
>  
>    def handle_idle_update sender, idle_since; @should_clear_running_totals = false; end
> diff --git a/lib/sup/source.rb b/lib/sup/source.rb
> index 06b6e6b..073a10a 100644
> --- a/lib/sup/source.rb
> +++ b/lib/sup/source.rb
> @@ -102,6 +102,10 @@ class Source
>      unimplemented
>    end
>  
> +  ## Like poll, but never returns (it is continuous, and uses something
> +  ## like inotify. Will always be run in another thread.)
> +  def continuous_poll poll_mutex; [] end
> +
>    def valid? info
>      true
>    end
_______________________________________________
Sup-devel mailing list
Sup-devel@rubyforge.org
http://rubyforge.org/mailman/listinfo/sup-devel