[Twisted-Python] Twisted CallLater: Advanced Usage

[Clark C. Evans]

On Sun, Jul 13, 2003 at 12:24:34PM -0000, Moshe Zadka wrote:
| from __future__ import generators
| def os_path_walk(path, visit, arg):
|     d = defer.Deferred()
|     def walk(path):
|         files = os.listdir(path)
|         visit(arg, path, files)
|         for file in files:
|             yield None
|             if os.path.isdir(file):
|                 for _ in walk(os.path.join(path, file):
|                     pass
|     next = walk(path).next
|     def caller():
|         try:
|             next()
|         except StopIteration:
|             d.callback(None)
|         except:
|             d.errback(sys.exc_info()[1])
|         else:
|             reactor.callLater(0, caller)
|     caller()
|     return d

You could rewrite this using flow:

    from __future__ import generators
    import os
    from twisted.flow import flow
    
    def hierlistdir(path):
        names = os.listdir(path)
        yield (path, names)
        for name in names:
            subpath = os.path.join(path,name)
            if os.path.isdir(subpath):
                for x in hierlistdir(subpath):
                    yield x
    
    def process(path, visit, arg):
        lst = flow.wrap(hierlistdir(path))
        yield lst
        for (path, names) in lst:
            visit(arg, path, names)
            yield lst
    
    def os_path_walk(path, visit, arg):
        return flow.Deferred(process(path, visit, arg))

Although, really, if you were to use flow, you would not
bother to use a vistor pattern, you would just write
your processor as a generator instead.

    def print_if_contains(path, match):
        lst = flow.wrap(hierlistdir(path))
        nMatch = 0
        yield lst
        for (path, names) in lst:
            if match in names:
                nMatch += 1
                print nMatch, path
            yield lst
    
    d = flow.Deferred(print_if_contains("/some/path","someFile"))
    d.addCallback(lamda _: reactor.stop())

Where flow "shines" is when the input source blocks, for example,
when reading from a socket (see flow.Protocol), or via an async 
PostgreSQL database, or an LDAP connection.   Or when there are 
multiple stages of a given flow (a -> b -> c), or when you have
two stages (async PostgreSQL and LDAP queries) which need to
be merged to generate a HTML page...

Best,

Clark

P.S.  Here is the other example using flow

from __future__ import generators
from twisted.internet import reactor
from twisted.flow import flow
from twisted.python import util

def flowFactorial(n, steps = 10):
    coop = flow.Cooperate()
    acc = 1L
    while True:
        for i in range(steps):
            if 1 == n:
                yield acc
                return
            n, acc = n - 1, acc * n
        yield coop

d = flow.Deferred(flowFactorial(13))
d.addCallback(util.println)
d.addCallback(lambda _: reactor.stop())
reactor.run()