Posts tagged with 'ubuntuone'

So yet again I have been confronted with broken tests in Ubuntu One. As I have already mentioned before I have spent a significant amount of time ensuring that the tests of Ubuntu One (which use twisted a lot) are deterministic and we do not leave a dirty reactor in the way. In order to do that a few week a go I wrote the following code that will help the rest of the team write such tests:

importosimportshutilimporttempfilefrom twisted.internetimport defer, endpoints, protocol
from twisted.spreadimport pb
from ubuntuone.devtools.testcasesimport BaseTestCase
# no init method + twisted common warnings# pylint: disable=W0232, C0103, E1101def server_protocol_factory(cls):
"""Factory to create tidy protocols."""if cls isNone:
cls = protocol.Protocolclass ServerTidyProtocol(cls):
"""A tidy protocol."""def connectionLost(self, *args):
"""Lost the connection."""
cls.connectionLost(self, *args)# lets tell everyone# pylint: disable=W0212if(self.factory._disconnecting
andself.factory.testserver_on_connection_lostisnotNoneandnotself.factory.testserver_on_connection_lost.called):
self.factory.testserver_on_connection_lost.callback(self)# pylint: enable=W0212return ServerTidyProtocol
def client_protocol_factory(cls):
"""Factory to create tidy protocols."""if cls isNone:
cls = protocol.Protocolclass ClientTidyProtocol(cls):
"""A tidy protocol."""def connectionLost(self, *a):
"""Connection list."""# pylint: disable=W0212if(self.factory._disconnecting
andself.factory.testserver_on_connection_lostisnotNoneandnotself.factory.testserver_on_connection_lost.called):
self.factory.testserver_on_connection_lost.callback(self)# pylint: enable=W0212
cls.connectionLost(self, *a)return ClientTidyProtocol
class TidySocketServer(object):
"""Ensure that twisted servers are correctly managed in tests.
Closing a twisted server is a complicated matter. In order to do so you
have to ensure that three different deferreds are fired:
1. The server must stop listening.
2. The client connection must disconnect.
3. The server connection must disconnect.
This class allows to create a server and a client that will ensure that
the reactor is left clean by following the pattern described at
http://mumak.net/stuff/twisted-disconnect.html
"""def__init__(self):
"""Create a new instance."""self.listener=Noneself.server_factory=Noneself.connector=Noneself.client_factory=Nonedef get_server_endpoint(self):
"""Return the server endpoint description."""raiseNotImplementedError('To be implemented by child classes.')def get_client_endpoint(self):
"""Return the client endpoint description."""raiseNotImplementedError('To be implemented by child classes.')@defer.inlineCallbacksdef listen_server(self, server_class, *args, **kwargs):
"""Start a server in a random port."""from twisted.internetimport reactor
self.server_factory= server_class(*args, **kwargs)self.server_factory._disconnecting =Falseself.server_factory.testserver_on_connection_lost= defer.Deferred()self.server_factory.protocol= server_protocol_factory(self.server_factory.protocol)
endpoint = endpoints.serverFromString(reactor,self.get_server_endpoint())self.listener=yield endpoint.listen(self.server_factory)
defer.returnValue(self.server_factory)@defer.inlineCallbacksdef connect_client(self, client_class, *args, **kwargs):
"""Conect a client to a given server."""from twisted.internetimport reactor
ifself.server_factoryisNone:
raiseValueError('Server Factory was not provided.')ifself.listenerisNone:
raiseValueError('%s has not started listening.',self.server_factory)self.client_factory= client_class(*args, **kwargs)self.client_factory._disconnecting =Falseself.client_factory.protocol= client_protocol_factory(self.client_factory.protocol)self.client_factory.testserver_on_connection_lost= defer.Deferred()
endpoint = endpoints.clientFromString(reactor,self.get_client_endpoint())self.connector=yield endpoint.connect(self.client_factory)
defer.returnValue(self.client_factory)def clean_up(self):
"""Action to be performed for clean up."""ifself.server_factoryisNoneorself.listenerisNone:
# nothing to cleanreturn defer.succeed(None)ifself.listenerandself.connector:
# clean client and serverself.server_factory._disconnecting =Trueself.client_factory._disconnecting =Trueself.connector.transport.loseConnection()
d = defer.maybeDeferred(self.listener.stopListening)return defer.gatherResults([d,self.client_factory.testserver_on_connection_lost,self.server_factory.testserver_on_connection_lost])ifself.listener:
# just clean the server since there is no clientself.server_factory._disconnecting =Truereturn defer.maybeDeferred(self.listener.stopListening)class TidyTCPServer(TidySocketServer):
"""A tidy tcp domain sockets server."""
client_endpoint_pattern ='tcp:host=127.0.0.1:port=%s'
server_endpoint_pattern ='tcp:0:interface=127.0.0.1'def get_server_endpoint(self):
"""Return the server endpoint description."""returnself.server_endpoint_patterndef get_client_endpoint(self):
"""Return the client endpoint description."""ifself.server_factoryisNone:
raiseValueError('Server Factory was not provided.')ifself.listenerisNone:
raiseValueError('%s has not started listening.',self.server_factory)returnself.client_endpoint_pattern % self.listener.getHost().portclass TidyUnixServer(TidySocketServer):
"""A tidy unix domain sockets server."""
client_endpoint_pattern ='unix:path=%s'
server_endpoint_pattern ='unix:%s'def__init__(self):
"""Create a new instance."""super(TidyUnixServer,self).__init__()self.temp_dir=tempfile.mkdtemp()self.path=os.path.join(self.temp_dir,'tidy_unix_server')def get_server_endpoint(self):
"""Return the server endpoint description."""returnself.server_endpoint_pattern % self.pathdef get_client_endpoint(self):
"""Return the client endpoint description."""returnself.client_endpoint_pattern % self.pathdef clean_up(self):
"""Action to be performed for clean up."""
result =super(TidyUnixServer,self).clean_up()# remove the dir once we are disconnected
result.addCallback(lambda _: shutil.rmtree(self.temp_dir))return result
class ServerTestCase(BaseTestCase):
"""Base test case for tidy servers."""@defer.inlineCallbacksdef setUp(self):
"""Set the diff tests."""yieldsuper(ServerTestCase,self).setUp()try:
self.server_runner=self.get_server()exceptNotImplementedError:
self.server_runner=Noneself.server_factory=Noneself.client_factory=Noneself.server_disconnected=Noneself.client_connected=Noneself.client_disconnected=Noneself.listener=Noneself.connector=Noneself.addCleanup(self.tear_down_server_client)def get_server(self):
"""Return the server to be used to run the tests."""raiseNotImplementedError('To be implemented by child classes.')@defer.inlineCallbacksdef listen_server(self, server_class, *args, **kwargs):
"""Listen a server.
The method takes the server class and the arguments that should be
passed to the server constructor.
"""self.server_factory=yieldself.server_runner.listen_server(
server_class, *args, **kwargs)self.server_disconnected=self.server_factory.testserver_on_connection_lostself.listener=self.server_runner.listener@defer.inlineCallbacksdef connect_client(self, client_class, *args, **kwargs):
"""Connect the client.
The method takes the client factory class and the arguments that
should be passed to the client constructor.
"""self.client_factory=yieldself.server_runner.connect_client(
client_class, *args, **kwargs)self.client_disconnected=self.client_factory.testserver_on_connection_lostself.connector=self.server_runner.connectordef tear_down_server_client(self):
"""Clean the server and client."""ifself.server_runner:
returnself.server_runner.clean_up()class TCPServerTestCase(ServerTestCase):
"""Test that uses a single twisted server."""def get_server(self):
"""Return the server to be used to run the tests."""return TidyTCPServer()class UnixServerTestCase(ServerTestCase):
"""Test that uses a single twisted server."""def get_server(self):
"""Return the server to be used to run the tests."""return TidyUnixServer()class PbServerTestCase(ServerTestCase):
"""Test a pb server."""def get_server(self):
"""Return the server to be used to run the tests."""raiseNotImplementedError('To be implemented by child classes.')@defer.inlineCallbacksdef listen_server(self, *args, **kwargs):
"""Listen a pb server."""yieldsuper(PbServerTestCase,self).listen_server(pb.PBServerFactory,
*args, **kwargs)@defer.inlineCallbacksdef connect_client(self, *args, **kwargs):
"""Connect a pb client."""yieldsuper(PbServerTestCase,self).connect_client(pb.PBClientFactory,
*args, **kwargs)class TCPPbServerTestCase(PbServerTestCase):
"""Test a pb server over TCP."""def get_server(self):
"""Return the server to be used to run the tests."""return TidyTCPServer()class UnixPbServerTestCase(PbServerTestCase):
"""Test a pb server over Unix domain sockets."""def get_server(self):
"""Return the server to be used to run the tests."""return TidyUnixServer()

The idea of the code is that developers do not need to worry about how to stop listening ports in their tests and just write tests like the following:

As you can see those tests do not give a rats ass about ensuring that the clients lose connection or we stop listening ports… Or so I though because the following code made such approach break in Mac OS X (although I suspect it was broken on Linux and Windows but we never experienced it):

While in all the other platforms the tests passed with no problems on Mac OS X the tests would block in the clean_up method from the server because the deferred that was called in the connectionLost from the ServerTidyProtocol was never fired… Interesting.. After digging in the code I realized that the main issue with the approach of the clean_up code was wrong. The problem relies on the way in which the NullProtocol works. As you can see in the code the protocol loses its connections as soon as it made. This results in to possible things:

The server does know that we have a client connected and calls buildProtocol.

The connection is lost so fast that the buildProtocol on the ServerFactory does not get call.

When running the tests on Windows and Linux we were always facing the first scenario, buildProtocol was called which meant that connectionLost in the server protocol would be called. On the other hand, on Mac OS X, 1 out of 10 runs of the tests would block in the clean up because we would be in the second scenario, that is, no protocol would be build in the ServerFactory which results in the connectionLost never being called because it was no needed. The work around this issue is quite simple once you understand what is going on. The ServerFactory has to be modified to set the deferred when buildProtocol is called and not before ensuring that when we cleanup we check if the deferred is None and if it is not we wait for it to be fired. The fixed version of the helper code is the following:

In the past days I have been working on implementing a python TestCase that can be used to perform integration tests to the future implementation of proxy support that will be landing in Ubuntu One. The idea of the TestCase is the following:

Start a proxy so that connections go throw it. The proxy has to be listening to two different ports, one in which auth is not required and a second one in which auth is required. At the moment the only supported proxy is Squid using base auth.

The test case should provide a way to access to the proxy details for subclasses to use.

The test case should integrate with the ubuntuone-dev-tools.

Initially, one of the major problems I had was to start squid in two different ports so that:

Port A accepts non-auth requests.

Port A rejects auth requests.

Port B accepts auth requests.

The idea is simple, if you use port A you should never auth while you must in port B, and example configuration of the ACLs and ports is the following:

Once the above was achieved the code of the test case was quite simple for Ubuntu O, unfortunatly, it was not that issues in Ubuntu P because there we have squid3 which supports http 1.1 and keeps the proxy keeps the connection alive. The fact that the connection is kept alive means that the reactor has a selectable running because the proxy keep it there. In order to solve the issue I wrote the code so that the server could say that the connection timedout. Here is the code that does it:

class SaveHTTPChannel(http.HTTPChannel):
"""A save protocol to be used in tests."""
protocolInstance = Nonedef connectionMade(self):
"""Keep track of the given protocol."""
SaveHTTPChannel.protocolInstance = self
http.HTTPChannel.connectionMade(self)class SaveSite(server.Site):
"""A site that let us know when it closed."""
protocol = SaveHTTPChannel
def__init__(self, *args, **kwargs):
"""Create a new instance."""
server.Site.__init__(self, *args, **kwargs)# we disable the timeout in the tests, we will deal with it manually.self.timeOut = None

The above defines a protocol that will know the instance that it was used so that we can trigger the time out in a clean up function.

At the moment we are working on providing support for proxy on Ubuntu One. In order to test this correctly I have been setting up a LAN in my office so that I can test as many scenarion as possible. On of those scenarios is the one in which the auth if the proxy uses Active Directory.

Because I use bind9 to set one of my boxed for the DNS I had to dig out how to configure it to work with AD. In order to do that I did the following:

Note:Is important to remember that the computer name of the server that has the AD role is dc1, if we used a diff name we have to change the configuration accordingly.

Restart the bind9 service:

sudo /etc/init.d/bind9 restart

Install the AD server and specify that you DO NOT want to set that server as a DNS server too.

Set the AD server to use your Ubuntu with your bind9 as the DNS server.

There are lots of things missing if you wanted to use this a set up for a corporate network, but it does the trick in my LAN since I do not have AD duplication or other fancy things. Maybe is useful for you home, who knows..

At Ubuntu One we required to be able to use named pipes on windows for IPC. This is a ver normal process in multi-process applications like the one we are going to provide, but in our case we had a twist, we are using twisted. As some of you may know there is not default reactor that would allow you to write a protocol in twisted and allows to use named pipes as the transport of the protocol. Well this was until very recently.

Txnamedpipes (lp:txnamedpipes) is a project that provides a ICOP based reactor that allows to use namedpipes for the transport of your protocol. At the moment we are confident that the implementation would allow you to use spred.pb or a custom protocol on twisted 10 and later on Windows 7 (we have been able to find a number of issues on Windows XP). The following is a small example of a spread.pb service and client that uses a named pipe for communication.

Pywin32 is a very cool project that allows you to access the win api without having to go through ctypes and deal with all the crazy parameters that COM is famous for. Unfortunately sometimes it has som issues which you face only a few times in your life.

This case I found a bug where GetFileSecurity does not use the GetFileSecurityW method but the w-less version. For those who don’t have to deal with this terrible details, the W usually means that the functions knows how to deal with utf-8 strings (backward compatibility can be a problem sometimes). I have reported the bug but for those that are in a hurry here is the patch:

Before I introduce the code, let me say that this is not a 100% exact implementation of the interfaces that can be found in pyinotify but the implementation of a subset that matches my needs. The main idea of creating this post is to give an example of the implementation of such a library for Windows trying to reuse the code that can be found in pyinotify.

Once I have excused my self, let get into the code. First of all, there are a number of classes from pyinotify that we can use in our code. That subset of classes is the below code which I grabbed from pyinotify git:

Unfortunatly we need to implement the code that talks with the Win32 API to be able to retrieve the events in the file system. In my design this is done by the Watch class that looks like this:

# Author: Manuel de la Pena <manuel@canonical.com>## Copyright 2011 Canonical Ltd.## This program is free software: you can redistribute it and/or modify it# under the terms of the GNU General Public License version 3, as published# by the Free Software Foundation.## This program is distributed in the hope that it will be useful, but# WITHOUT ANY WARRANTY; without even the implied warranties of# MERCHANTABILITY, SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR# PURPOSE. See the GNU General Public License for more details.## You should have received a copy of the GNU General Public License along# with this program. If not, see <http://www.gnu.org/licenses/>."""File notifications on windows."""importloggingimportosimportreimport winerror
fromQueueimportQueue, Empty
fromthreadingimport Thread
from uuid import uuid4
from twisted.internetimport task, reactor
from win32con import(
FILE_SHARE_READ,
FILE_SHARE_WRITE,
FILE_FLAG_BACKUP_SEMANTICS,
FILE_NOTIFY_CHANGE_FILE_NAME,
FILE_NOTIFY_CHANGE_DIR_NAME,
FILE_NOTIFY_CHANGE_ATTRIBUTES,
FILE_NOTIFY_CHANGE_SIZE,
FILE_NOTIFY_CHANGE_LAST_WRITE,
FILE_NOTIFY_CHANGE_SECURITY,
OPEN_EXISTING
)from win32file import CreateFile, ReadDirectoryChangesW
from ubuntuone.platform.windows.pyinotifyimport(
Event,
WatchManagerError,
ProcessEvent,
PrintAllEvents,
IN_OPEN,
IN_CLOSE_NOWRITE,
IN_CLOSE_WRITE,
IN_CREATE,
IN_ISDIR,
IN_DELETE,
IN_MOVED_FROM,
IN_MOVED_TO,
IN_MODIFY,
IN_IGNORED
)from ubuntuone.syncdaemon.filesystem_notificationsimport(
GeneralINotifyProcessor
)from ubuntuone.platform.windows.os_helperimport(
LONG_PATH_PREFIX,
abspath,
listdir
)# constant found in the msdn documentation:# http://msdn.microsoft.com/en-us/library/ff538834(v=vs.85).aspx
FILE_LIST_DIRECTORY = 0x0001
FILE_NOTIFY_CHANGE_LAST_ACCESS = 0x00000020
FILE_NOTIFY_CHANGE_CREATION = 0x00000040
# a map between the few events that we have on windows and those# found in pyinotify
WINDOWS_ACTIONS = {1: IN_CREATE,
2: IN_DELETE,
3: IN_MODIFY,
4: IN_MOVED_FROM,
5: IN_MOVED_TO
}# translates quickly the event and it's is_dir state to our standard events
NAME_TRANSLATIONS = {
IN_OPEN: 'FS_FILE_OPEN',
IN_CLOSE_NOWRITE: 'FS_FILE_CLOSE_NOWRITE',
IN_CLOSE_WRITE: 'FS_FILE_CLOSE_WRITE',
IN_CREATE: 'FS_FILE_CREATE',
IN_CREATE | IN_ISDIR: 'FS_DIR_CREATE',
IN_DELETE: 'FS_FILE_DELETE',
IN_DELETE | IN_ISDIR: 'FS_DIR_DELETE',
IN_MOVED_FROM: 'FS_FILE_DELETE',
IN_MOVED_FROM | IN_ISDIR: 'FS_DIR_DELETE',
IN_MOVED_TO: 'FS_FILE_CREATE',
IN_MOVED_TO | IN_ISDIR: 'FS_DIR_CREATE',
}# the default mask to be used in the watches added by the FilesystemMonitor# class
FILESYSTEM_MONITOR_MASK = FILE_NOTIFY_CHANGE_FILE_NAME | \
FILE_NOTIFY_CHANGE_DIR_NAME | \
FILE_NOTIFY_CHANGE_ATTRIBUTES | \
FILE_NOTIFY_CHANGE_SIZE | \
FILE_NOTIFY_CHANGE_LAST_WRITE | \
FILE_NOTIFY_CHANGE_SECURITY | \
FILE_NOTIFY_CHANGE_LAST_ACCESS
# The implementation of the code that is provided as the pyinotify# substituteclass Watch(object):
"""Implement the same functions as pyinotify.Watch."""def__init__(self, watch_descriptor, path, mask, auto_add,
events_queue=None, exclude_filter=None, proc_fun=None):
super(Watch, self).__init__()self.log = logging.getLogger('ubuntuone.platform.windows.' +
'filesystem_notifications.Watch')self._watching = Falseself._descriptor = watch_descriptor
self._auto_add = auto_add
self.exclude_filter = Noneself._proc_fun = proc_fun
self._cookie = Noneself._source_pathname = None# remember the subdirs we have so that when we have a delete we can# check if it was a removeself._subdirs = []# ensure that we work with an abspath and that we can deal with# long paths over 260 chars.self._path = os.path.abspath(path)ifnotself._path.startswith(LONG_PATH_PREFIX):
self._path = LONG_PATH_PREFIX + self._path
self._mask = mask
# lets make the q as big as possibleself._raw_events_queue = Queue()ifnot events_queue:
events_queue = Queue()self.events_queue = events_queue
def _path_is_dir(self, path):
""""Check if the path is a dir and update the local subdir list."""self.log.debug('Testing if path "%s" is a dir', path)
is_dir = Falseifos.path.exists(path):
is_dir = os.path.isdir(path)else:
self.log.debug('Path "%s" was deleted subdirs are %s.',
path, self._subdirs)# we removed the path, we look in the internal listif path inself._subdirs:
is_dir = Trueself._subdirs.remove(path)if is_dir:
self.log.debug('Adding %s to subdirs %s', path, self._subdirs)self._subdirs.append(path)return is_dir
def _process_events(self):
"""Process the events form the queue."""# we transform the events to be the same as the one in pyinotify# and then use the proc_funwhileself._watching ornotself._raw_events_queue.empty():
file_name, action = self._raw_events_queue.get()# map the windows events to the pyinotify ones, tis is dirty but# makes the multiplatform better, linux was first :P
is_dir = self._path_is_dir(file_name)ifos.path.exists(file_name):
is_dir = os.path.isdir(file_name)else:
# we removed the path, we look in the internal listif file_name inself._subdirs:
is_dir = Trueself._subdirs.remove(file_name)if is_dir:
self._subdirs.append(file_name)
mask = WINDOWS_ACTIONS[action]
head, tail = os.path.split(file_name)if is_dir:
mask |= IN_ISDIR
event_raw_data = {'wd': self._descriptor,
'dir': is_dir,
'mask': mask,
'name': tail,
'path': head.replace(self.path, '.')}# by the way in which the win api fires the events we know for# sure that no move events will be added in the wrong order, this# is kind of hacky, I dont like it too muchif WINDOWS_ACTIONS[action] == IN_MOVED_FROM:
self._cookie = str(uuid4())self._source_pathname = tail
event_raw_data['cookie'] = self._cookie
if WINDOWS_ACTIONS[action] == IN_MOVED_TO:
event_raw_data['src_pathname'] = self._source_pathname
event_raw_data['cookie'] = self._cookie
event = Event(event_raw_data)# FIXME: event deduces the pathname wrong and we need manually# set it
event.pathname = file_name
# add the event only if we do not have an exclude filter or# the exclude filter returns False, that is, the event will not# be excludedifnotself.exclude_filterornotself.exclude_filter(event):
self.log.debug('Addding event %s to queue.', event)self.events_queue.put(event)def _watch(self):
"""Watch a path that is a directory."""# we are going to be using the ReadDirectoryChangesW whihc requires# a direcotry handle and the mask to be used.
handle = CreateFile(self._path,
FILE_LIST_DIRECTORY,
FILE_SHARE_READ | FILE_SHARE_WRITE,
None,
OPEN_EXISTING,
FILE_FLAG_BACKUP_SEMANTICS,
None)self.log.debug('Watchng path %s.', self._path)whileself._watching:
# important information to know about the parameters:# param 1: the handle to the dir# param 2: the size to be used in the kernel to store events# that might be lost whilw the call is being performed. This# is complicates to fine tune since if you make lots of watcher# you migh used to much memory and make your OS to BSOD
results = ReadDirectoryChangesW(
handle,
1024,
self._auto_add,
self._mask,
None,
None)# add the diff events to the q so that the can be processed no# matter the speed.for action, filein results:
full_filename = os.path.join(self._path, file)self._raw_events_queue.put((full_filename, action))self.log.debug('Added %s to raw events queue.',
(full_filename, action))def start_watching(self):
"""Tell the watch to start processing events."""# get the diff dirs in the pathfor current_child in listdir(self._path):
full_child_path = os.path.join(self._path, current_child)ifos.path.isdir(full_child_path):
self._subdirs.append(full_child_path)# start to diff threads, one to watch the path, the other to# process the events.self.log.debug('Sart watching path.')self._watching = True
watch_thread = Thread(target=self._watch,
name='Watch(%s)'%self._path)
process_thread = Thread(target=self._process_events,
name='Process(%s)'%self._path)
process_thread.start()
watch_thread.start()def stop_watching(self):
"""Tell the watch to stop processing events."""self._watching = Falseself._subdirs = []def update(self, mask, proc_fun=None, auto_add=False):
"""Update the info used by the watcher."""self.log.debug('update(%s, %s, %s)', mask, proc_fun, auto_add)self._mask = mask
self._proc_fun = proc_fun
self._auto_add = auto_add
@propertydef path(self):
"""Return the patch watched."""returnself._path
@propertydef auto_add(self):
returnself._auto_add
@propertydef proc_fun(self):
returnself._proc_fun
class WatchManager(object):
"""Implement the same functions as pyinotify.WatchManager."""def__init__(self, exclude_filter=lambda path: False):
"""Init the manager to keep trak of the different watches."""super(WatchManager, self).__init__()self.log = logging.getLogger('ubuntuone.platform.windows.'
+ 'filesystem_notifications.WatchManager')self._wdm = {}self._wd_count = 0self._exclude_filter = exclude_filter
self._events_queue = Queue()self._ignored_paths = []def stop(self):
"""Close the manager and stop all watches."""self.log.debug('Stopping watches.')for current_wd inself._wdm:
self._wdm[current_wd].stop_watching()self.log.debug('Watch for %s stopped.', self._wdm[current_wd].path)def get_watch(self, wd):
"""Return the watch with the given descriptor."""returnself._wdm[wd]def del_watch(self, wd):
"""Delete the watch with the given descriptor."""try:
watch = self._wdm[wd]
watch.stop_watching()delself._wdm[wd]self.log.debug('Watch %s removed.', wd)exceptKeyError, e:
logging.error(str(e))def _add_single_watch(self, path, mask, proc_fun=None, auto_add=False,
quiet=True, exclude_filter=None):
self.log.debug('add_single_watch(%s, %s, %s, %s, %s, %s)', path, mask,
proc_fun, auto_add, quiet, exclude_filter)self._wdm[self._wd_count] = Watch(self._wd_count, path, mask,
auto_add, events_queue=self._events_queue,
exclude_filter=exclude_filter, proc_fun=proc_fun)self._wdm[self._wd_count].start_watching()self._wd_count += 1self.log.debug('Watch count increased to %s', self._wd_count)def add_watch(self, path, mask, proc_fun=None, auto_add=False,
quiet=True, exclude_filter=None):
ifhasattr(path, '__iter__'):
self.log.debug('Added collection of watches.')# we are dealing with a collection of pathsfor current_path in path:
ifnotself.get_wd(current_path):
self._add_single_watch(current_path, mask, proc_fun,
auto_add, quiet, exclude_filter)elifnotself.get_wd(path):
self.log.debug('Adding single watch.')self._add_single_watch(path, mask, proc_fun, auto_add,
quiet, exclude_filter)def update_watch(self, wd, mask=None, proc_fun=None, rec=False,
auto_add=False, quiet=True):
try:
watch = self._wdm[wd]
watch.stop_watching()self.log.debug('Stopped watch on %s for update.', watch.path)# update the data and restart watching
auto_add = auto_add or rec
watch.update(mask, proc_fun=proc_fun, auto_add=auto_add)# only start the watcher again if the mask was given, otherwhise# we are not watchng and therefore do not careif mask:
watch.start_watching()exceptKeyError, e:
self.log.error(str(e))ifnot quiet:
raise WatchManagerError('Watch %s was not found'% wd, {})def get_wd(self, path):
"""Return the watcher that is used to watch the given path."""for current_wd inself._wdm:
ifself._wdm[current_wd].pathin path and \
self._wdm[current_wd].auto_add:
return current_wd
def get_path(self, wd):
"""Return the path watched by the wath with the given wd."""
watch_ = self._wmd.get(wd)if watch:
return watch.pathdef rm_watch(self, wd, rec=False, quiet=True):
"""Remove the the watch with the given wd."""try:
watch = self._wdm[wd]
watch.stop_watching()delself._wdm[wd]except KeyrError, err:
self.log.error(str(err))ifnot quiet:
raise WatchManagerError('Watch %s was not found'% wd, {})def rm_path(self, path):
"""Remove a watch to the given path."""# it would be very tricky to remove a subpath from a watcher that is# looking at changes in ther kids. To make it simpler and less error# prone (and even better performant since we use less threads) we will# add a filter to the events in the watcher so that the events from# that child are not received :)def ignore_path(event):
"""Ignore an event if it has a given path."""
is_ignored = Falsefor ignored_path inself._ignored_paths:
if ignore_path in event.pathname:
returnTruereturnFalse
wd = self.get_wd(path)if wd:
ifself._wdm[wd].path == path:
self.log.debug('Removing watch for path "%s"', path)self.rm_watch(wd)else:
self.log.debug('Adding exclude filter for "%s"', path)# we have a watch that cotains the path as a child pathifnot path inself._ignored_paths:
self._ignored_paths.append(path)# FIXME: This assumes that we do not have other function# which in our usecase is correct, but what is we move this# to other projects evet?!? Maybe using the manager# exclude_filter is betterifnotself._wdm[wd].exclude_filter:
self._wdm[wd].exclude_filter = ignore_path
@propertydef watches(self):
"""Return a reference to the dictionary that contains the watches."""returnself._wdm
@propertydef events_queue(self):
"""Return the queue with the events that the manager contains."""returnself._events_queue
class Notifier(object):
"""
Read notifications, process events. Inspired by the pyinotify.Notifier
"""def__init__(self, watch_manager, default_proc_fun=None, read_freq=0,
threshold=10, timeout=-1):
"""Init to process event according to the given timeout & threshold."""super(Notifier, self).__init__()self.log = logging.getLogger('ubuntuone.platform.windows.'
+ 'filesystem_notifications.Notifier')# Watch Manager instanceself._watch_manager = watch_manager
# Default processing methodself._default_proc_fun = default_proc_fun
if default_proc_fun isNone:
self._default_proc_fun = PrintAllEvents()# Loop parametersself._read_freq = read_freq
self._threshold = threshold
self._timeout = timeout
def proc_fun(self):
returnself._default_proc_fun
def process_events(self):
"""
Process the event given the threshold and the timeout.
"""self.log.debug('Processing events with threashold: %s and timeout: %s',
self._threshold, self._timeout)# we will process an amount of events equal to the threshold of# the notifier and will block for the amount given by the timeout
processed_events = 0while processed_events <self._threshold:
try:
raw_event = Noneifnotself._timeout orself._timeout <0:
raw_event = self._watch_manager.events_queue.get(
block=False)else:
raw_event = self._watch_manager.events_queue.get(
timeout=self._timeout)
watch = self._watch_manager.get_watch(raw_event.wd)if watch isNone:
# Not really sure how we ended up here, nor how we should# handle these types of events and if it is appropriate to# completly skip them (like we are doing here).self.log.warning('Unable to retrieve Watch object '
+ 'associated to %s', raw_event)
processed_events += 1continueif watch and watch.proc_fun:
self.log.debug('Executing proc_fun from watch.')
watch.proc_fun(raw_event)# user processingselse:
self.log.debug('Executing default_proc_fun')self._default_proc_fun(raw_event)
processed_events += 1except Empty:
# increase the number of processed events, and continue
processed_events += 1continuedef stop(self):
"""Stop processing events and the watch manager."""self._watch_manager.stop()

While one of the threads is retrieving the events from the file system, the second one process them so that the will be exposed as pyinotify events. I have done so because I did not want to deal with OVERLAP structures for asyn operations in Win32 and because I wanted to use pyinotify events so that if someone with experience in pyinotify looks at the output, he can easily understand it. I really like this approach because it allowed me to reuse a fair amount of logic hat we had in the Ubuntu client and to approach the port in a very TDD way since the tests I’ve used are the same ones as the ones found on Ubuntu