add mitogen - requires ansible 2.7

mitogen-0.2.7/mitogen/__init__.py

@@ -0,0 +1,120 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+"""
+On the Mitogen master, this is imported from ``mitogen/__init__.py`` as would
+be expected. On the slave, it is built dynamically during startup.
+"""
+
+
+#: Library version as a tuple.
+__version__ = (0, 2, 7)
+
+
+#: This is :data:`False` in slave contexts. Previously it was used to prevent
+#: re-execution of :mod:`__main__` in single file programs, however that now
+#: happens automatically.
+is_master = True
+
+
+#: This is `0` in a master, otherwise it is the master-assigned ID unique to
+#: the slave context used for message routing.
+context_id = 0
+
+
+#: This is :data:`None` in a master, otherwise it is the master-assigned ID
+#: unique to the slave's parent context.
+parent_id = None
+
+
+#: This is an empty list in a master, otherwise it is a list of parent context
+#: IDs ordered from most direct to least direct.
+parent_ids = []
+
+
+import os
+_default_profiling = os.environ.get('MITOGEN_PROFILING') is not None
+del os
+
+
+def main(log_level='INFO', profiling=_default_profiling):
+    """
+    Convenience decorator primarily useful for writing discardable test
+    scripts.
+
+    In the master process, when `func` is defined in the :mod:`__main__`
+    module, arranges for `func(router)` to be invoked immediately, with
+    :py:class:`mitogen.master.Router` construction and destruction handled just
+    as in :py:func:`mitogen.utils.run_with_router`. In slaves, this function
+    does nothing.
+
+    :param str log_level:
+        Logging package level to configure via
+        :py:func:`mitogen.utils.log_to_file`.
+
+    :param bool profiling:
+        If :py:data:`True`, equivalent to setting
+        :py:attr:`mitogen.master.Router.profiling` prior to router
+        construction. This causes ``/tmp`` files to be created everywhere at
+        the end of a successful run with :py:mod:`cProfile` output for every
+        thread.
+
+    Example:
+
+    ::
+
+        import mitogen
+        import requests
+
+        def get_url(url):
+            return requests.get(url).text
+
+        @mitogen.main()
+        def main(router):
+            z = router.ssh(hostname='k3')
+            print(z.call(get_url, 'https://example.org/')))))
+
+    """
+
+    def wrapper(func):
+        if func.__module__ != '__main__':
+            return func
+        import mitogen.parent
+        import mitogen.utils
+        if profiling:
+            mitogen.core.enable_profiling()
+            mitogen.master.Router.profiling = profiling
+        utils.log_to_file(level=log_level)
+        return mitogen.core._profile_hook(
+            'app.main',
+            utils.run_with_router,
+            func,
+        )
+    return wrapper

mitogen-0.2.7/mitogen/compat/pkgutil.py

@@ -0,0 +1,593 @@
+"""Utilities to support packages."""
+
+# !mitogen: minify_safe
+
+# NOTE: This module must remain compatible with Python 2.3, as it is shared
+# by setuptools for distribution with Python 2.3 and up.
+
+import os
+import sys
+import imp
+import os.path
+from types import ModuleType
+
+__all__ = [
+    'get_importer', 'iter_importers', 'get_loader', 'find_loader',
+    'walk_packages', 'iter_modules', 'get_data',
+    'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
+]
+
+def read_code(stream):
+    # This helper is needed in order for the PEP 302 emulation to
+    # correctly handle compiled files
+    import marshal
+
+    magic = stream.read(4)
+    if magic != imp.get_magic():
+        return None
+
+    stream.read(4) # Skip timestamp
+    return marshal.load(stream)
+
+
+def simplegeneric(func):
+    """Make a trivial single-dispatch generic function"""
+    registry = {}
+    def wrapper(*args, **kw):
+        ob = args[0]
+        try:
+            cls = ob.__class__
+        except AttributeError:
+            cls = type(ob)
+        try:
+            mro = cls.__mro__
+        except AttributeError:
+            try:
+                class cls(cls, object):
+                    pass
+                mro = cls.__mro__[1:]
+            except TypeError:
+                mro = object,   # must be an ExtensionClass or some such  :(
+        for t in mro:
+            if t in registry:
+                return registry[t](*args, **kw)
+        else:
+            return func(*args, **kw)
+    try:
+        wrapper.__name__ = func.__name__
+    except (TypeError, AttributeError):
+        pass    # Python 2.3 doesn't allow functions to be renamed
+
+    def register(typ, func=None):
+        if func is None:
+            return lambda f: register(typ, f)
+        registry[typ] = func
+        return func
+
+    wrapper.__dict__ = func.__dict__
+    wrapper.__doc__ = func.__doc__
+    wrapper.register = register
+    return wrapper
+
+
+def walk_packages(path=None, prefix='', onerror=None):
+    """Yields (module_loader, name, ispkg) for all modules recursively
+    on path, or, if path is None, all accessible modules.
+
+    'path' should be either None or a list of paths to look for
+    modules in.
+
+    'prefix' is a string to output on the front of every module name
+    on output.
+
+    Note that this function must import all *packages* (NOT all
+    modules!) on the given path, in order to access the __path__
+    attribute to find submodules.
+
+    'onerror' is a function which gets called with one argument (the
+    name of the package which was being imported) if any exception
+    occurs while trying to import a package.  If no onerror function is
+    supplied, ImportErrors are caught and ignored, while all other
+    exceptions are propagated, terminating the search.
+
+    Examples:
+
+    # list all modules python can access
+    walk_packages()
+
+    # list all submodules of ctypes
+    walk_packages(ctypes.__path__, ctypes.__name__+'.')
+    """
+
+    def seen(p, m={}):
+        if p in m:
+            return True
+        m[p] = True
+
+    for importer, name, ispkg in iter_modules(path, prefix):
+        yield importer, name, ispkg
+
+        if ispkg:
+            try:
+                __import__(name)
+            except ImportError:
+                if onerror is not None:
+                    onerror(name)
+            except Exception:
+                if onerror is not None:
+                    onerror(name)
+                else:
+                    raise
+            else:
+                path = getattr(sys.modules[name], '__path__', None) or []
+
+                # don't traverse path items we've seen before
+                path = [p for p in path if not seen(p)]
+
+                for item in walk_packages(path, name+'.', onerror):
+                    yield item
+
+
+def iter_modules(path=None, prefix=''):
+    """Yields (module_loader, name, ispkg) for all submodules on path,
+    or, if path is None, all top-level modules on sys.path.
+
+    'path' should be either None or a list of paths to look for
+    modules in.
+
+    'prefix' is a string to output on the front of every module name
+    on output.
+    """
+
+    if path is None:
+        importers = iter_importers()
+    else:
+        importers = map(get_importer, path)
+
+    yielded = {}
+    for i in importers:
+        for name, ispkg in iter_importer_modules(i, prefix):
+            if name not in yielded:
+                yielded[name] = 1
+                yield i, name, ispkg
+
+
+#@simplegeneric
+def iter_importer_modules(importer, prefix=''):
+    if not hasattr(importer, 'iter_modules'):
+        return []
+    return importer.iter_modules(prefix)
+
+iter_importer_modules = simplegeneric(iter_importer_modules)
+
+
+class ImpImporter:
+    """PEP 302 Importer that wraps Python's "classic" import algorithm
+
+    ImpImporter(dirname) produces a PEP 302 importer that searches that
+    directory.  ImpImporter(None) produces a PEP 302 importer that searches
+    the current sys.path, plus any modules that are frozen or built-in.
+
+    Note that ImpImporter does not currently support being used by placement
+    on sys.meta_path.
+    """
+
+    def __init__(self, path=None):
+        self.path = path
+
+    def find_module(self, fullname, path=None):
+        # Note: we ignore 'path' argument since it is only used via meta_path
+        subname = fullname.split(".")[-1]
+        if subname != fullname and self.path is None:
+            return None
+        if self.path is None:
+            path = None
+        else:
+            path = [os.path.realpath(self.path)]
+        try:
+            file, filename, etc = imp.find_module(subname, path)
+        except ImportError:
+            return None
+        return ImpLoader(fullname, file, filename, etc)
+
+    def iter_modules(self, prefix=''):
+        if self.path is None or not os.path.isdir(self.path):
+            return
+
+        yielded = {}
+        import inspect
+        try:
+            filenames = os.listdir(self.path)
+        except OSError:
+            # ignore unreadable directories like import does
+            filenames = []
+        filenames.sort()  # handle packages before same-named modules
+
+        for fn in filenames:
+            modname = inspect.getmodulename(fn)
+            if modname=='__init__' or modname in yielded:
+                continue
+
+            path = os.path.join(self.path, fn)
+            ispkg = False
+
+            if not modname and os.path.isdir(path) and '.' not in fn:
+                modname = fn
+                try:
+                    dircontents = os.listdir(path)
+                except OSError:
+                    # ignore unreadable directories like import does
+                    dircontents = []
+                for fn in dircontents:
+                    subname = inspect.getmodulename(fn)
+                    if subname=='__init__':
+                        ispkg = True
+                        break
+                else:
+                    continue    # not a package
+
+            if modname and '.' not in modname:
+                yielded[modname] = 1
+                yield prefix + modname, ispkg
+
+
+class ImpLoader:
+    """PEP 302 Loader that wraps Python's "classic" import algorithm
+    """
+    code = source = None
+
+    def __init__(self, fullname, file, filename, etc):
+        self.file = file
+        self.filename = filename
+        self.fullname = fullname
+        self.etc = etc
+
+    def load_module(self, fullname):
+        self._reopen()
+        try:
+            mod = imp.load_module(fullname, self.file, self.filename, self.etc)
+        finally:
+            if self.file:
+                self.file.close()
+        # Note: we don't set __loader__ because we want the module to look
+        # normal; i.e. this is just a wrapper for standard import machinery
+        return mod
+
+    def get_data(self, pathname):
+        return open(pathname, "rb").read()
+
+    def _reopen(self):
+        if self.file and self.file.closed:
+            mod_type = self.etc[2]
+            if mod_type==imp.PY_SOURCE:
+                self.file = open(self.filename, 'rU')
+            elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
+                self.file = open(self.filename, 'rb')
+
+    def _fix_name(self, fullname):
+        if fullname is None:
+            fullname = self.fullname
+        elif fullname != self.fullname:
+            raise ImportError("Loader for module %s cannot handle "
+                              "module %s" % (self.fullname, fullname))
+        return fullname
+
+    def is_package(self, fullname):
+        fullname = self._fix_name(fullname)
+        return self.etc[2]==imp.PKG_DIRECTORY
+
+    def get_code(self, fullname=None):
+        fullname = self._fix_name(fullname)
+        if self.code is None:
+            mod_type = self.etc[2]
+            if mod_type==imp.PY_SOURCE:
+                source = self.get_source(fullname)
+                self.code = compile(source, self.filename, 'exec')
+            elif mod_type==imp.PY_COMPILED:
+                self._reopen()
+                try:
+                    self.code = read_code(self.file)
+                finally:
+                    self.file.close()
+            elif mod_type==imp.PKG_DIRECTORY:
+                self.code = self._get_delegate().get_code()
+        return self.code
+
+    def get_source(self, fullname=None):
+        fullname = self._fix_name(fullname)
+        if self.source is None:
+            mod_type = self.etc[2]
+            if mod_type==imp.PY_SOURCE:
+                self._reopen()
+                try:
+                    self.source = self.file.read()
+                finally:
+                    self.file.close()
+            elif mod_type==imp.PY_COMPILED:
+                if os.path.exists(self.filename[:-1]):
+                    f = open(self.filename[:-1], 'rU')
+                    self.source = f.read()
+                    f.close()
+            elif mod_type==imp.PKG_DIRECTORY:
+                self.source = self._get_delegate().get_source()
+        return self.source
+
+
+    def _get_delegate(self):
+        return ImpImporter(self.filename).find_module('__init__')
+
+    def get_filename(self, fullname=None):
+        fullname = self._fix_name(fullname)
+        mod_type = self.etc[2]
+        if self.etc[2]==imp.PKG_DIRECTORY:
+            return self._get_delegate().get_filename()
+        elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
+            return self.filename
+        return None
+
+
+try:
+    import zipimport
+    from zipimport import zipimporter
+
+    def iter_zipimport_modules(importer, prefix=''):
+        dirlist = zipimport._zip_directory_cache[importer.archive].keys()
+        dirlist.sort()
+        _prefix = importer.prefix
+        plen = len(_prefix)
+        yielded = {}
+        import inspect
+        for fn in dirlist:
+            if not fn.startswith(_prefix):
+                continue
+
+            fn = fn[plen:].split(os.sep)
+
+            if len(fn)==2 and fn[1].startswith('__init__.py'):
+                if fn[0] not in yielded:
+                    yielded[fn[0]] = 1
+                    yield fn[0], True
+
+            if len(fn)!=1:
+                continue
+
+            modname = inspect.getmodulename(fn[0])
+            if modname=='__init__':
+                continue
+
+            if modname and '.' not in modname and modname not in yielded:
+                yielded[modname] = 1
+                yield prefix + modname, False
+
+    iter_importer_modules.register(zipimporter, iter_zipimport_modules)
+
+except ImportError:
+    pass
+
+
+def get_importer(path_item):
+    """Retrieve a PEP 302 importer for the given path item
+
+    The returned importer is cached in sys.path_importer_cache
+    if it was newly created by a path hook.
+
+    If there is no importer, a wrapper around the basic import
+    machinery is returned. This wrapper is never inserted into
+    the importer cache (None is inserted instead).
+
+    The cache (or part of it) can be cleared manually if a
+    rescan of sys.path_hooks is necessary.
+    """
+    try:
+        importer = sys.path_importer_cache[path_item]
+    except KeyError:
+        for path_hook in sys.path_hooks:
+            try:
+                importer = path_hook(path_item)
+                break
+            except ImportError:
+                pass
+        else:
+            importer = None
+        sys.path_importer_cache.setdefault(path_item, importer)
+
+    if importer is None:
+        try:
+            importer = ImpImporter(path_item)
+        except ImportError:
+            importer = None
+    return importer
+
+
+def iter_importers(fullname=""):
+    """Yield PEP 302 importers for the given module name
+
+    If fullname contains a '.', the importers will be for the package
+    containing fullname, otherwise they will be importers for sys.meta_path,
+    sys.path, and Python's "classic" import machinery, in that order.  If
+    the named module is in a package, that package is imported as a side
+    effect of invoking this function.
+
+    Non PEP 302 mechanisms (e.g. the Windows registry) used by the
+    standard import machinery to find files in alternative locations
+    are partially supported, but are searched AFTER sys.path. Normally,
+    these locations are searched BEFORE sys.path, preventing sys.path
+    entries from shadowing them.
+
+    For this to cause a visible difference in behaviour, there must
+    be a module or package name that is accessible via both sys.path
+    and one of the non PEP 302 file system mechanisms. In this case,
+    the emulation will find the former version, while the builtin
+    import mechanism will find the latter.
+
+    Items of the following types can be affected by this discrepancy:
+        imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY
+    """
+    if fullname.startswith('.'):
+        raise ImportError("Relative module names not supported")
+    if '.' in fullname:
+        # Get the containing package's __path__
+        pkg = '.'.join(fullname.split('.')[:-1])
+        if pkg not in sys.modules:
+            __import__(pkg)
+        path = getattr(sys.modules[pkg], '__path__', None) or []
+    else:
+        for importer in sys.meta_path:
+            yield importer
+        path = sys.path
+    for item in path:
+        yield get_importer(item)
+    if '.' not in fullname:
+        yield ImpImporter()
+
+def get_loader(module_or_name):
+    """Get a PEP 302 "loader" object for module_or_name
+
+    If the module or package is accessible via the normal import
+    mechanism, a wrapper around the relevant part of that machinery
+    is returned.  Returns None if the module cannot be found or imported.
+    If the named module is not already imported, its containing package
+    (if any) is imported, in order to establish the package __path__.
+
+    This function uses iter_importers(), and is thus subject to the same
+    limitations regarding platform-specific special import locations such
+    as the Windows registry.
+    """
+    if module_or_name in sys.modules:
+        module_or_name = sys.modules[module_or_name]
+    if isinstance(module_or_name, ModuleType):
+        module = module_or_name
+        loader = getattr(module, '__loader__', None)
+        if loader is not None:
+            return loader
+        fullname = module.__name__
+    else:
+        fullname = module_or_name
+    return find_loader(fullname)
+
+def find_loader(fullname):
+    """Find a PEP 302 "loader" object for fullname
+
+    If fullname contains dots, path must be the containing package's __path__.
+    Returns None if the module cannot be found or imported. This function uses
+    iter_importers(), and is thus subject to the same limitations regarding
+    platform-specific special import locations such as the Windows registry.
+    """
+    for importer in iter_importers(fullname):
+        loader = importer.find_module(fullname)
+        if loader is not None:
+            return loader
+
+    return None
+
+
+def extend_path(path, name):
+    """Extend a package's path.
+
+    Intended use is to place the following code in a package's __init__.py:
+
+        from pkgutil import extend_path
+        __path__ = extend_path(__path__, __name__)
+
+    This will add to the package's __path__ all subdirectories of
+    directories on sys.path named after the package.  This is useful
+    if one wants to distribute different parts of a single logical
+    package as multiple directories.
+
+    It also looks for *.pkg files beginning where * matches the name
+    argument.  This feature is similar to *.pth files (see site.py),
+    except that it doesn't special-case lines starting with 'import'.
+    A *.pkg file is trusted at face value: apart from checking for
+    duplicates, all entries found in a *.pkg file are added to the
+    path, regardless of whether they are exist the filesystem.  (This
+    is a feature.)
+
+    If the input path is not a list (as is the case for frozen
+    packages) it is returned unchanged.  The input path is not
+    modified; an extended copy is returned.  Items are only appended
+    to the copy at the end.
+
+    It is assumed that sys.path is a sequence.  Items of sys.path that
+    are not (unicode or 8-bit) strings referring to existing
+    directories are ignored.  Unicode items of sys.path that cause
+    errors when used as filenames may cause this function to raise an
+    exception (in line with os.path.isdir() behavior).
+    """
+
+    if not isinstance(path, list):
+        # This could happen e.g. when this is called from inside a
+        # frozen package.  Return the path unchanged in that case.
+        return path
+
+    pname = os.path.join(*name.split('.')) # Reconstitute as relative path
+    # Just in case os.extsep != '.'
+    sname = os.extsep.join(name.split('.'))
+    sname_pkg = sname + os.extsep + "pkg"
+    init_py = "__init__" + os.extsep + "py"
+
+    path = path[:] # Start with a copy of the existing path
+
+    for dir in sys.path:
+        if not isinstance(dir, basestring) or not os.path.isdir(dir):
+            continue
+        subdir = os.path.join(dir, pname)
+        # XXX This may still add duplicate entries to path on
+        # case-insensitive filesystems
+        initfile = os.path.join(subdir, init_py)
+        if subdir not in path and os.path.isfile(initfile):
+            path.append(subdir)
+        # XXX Is this the right thing for subpackages like zope.app?
+        # It looks for a file named "zope.app.pkg"
+        pkgfile = os.path.join(dir, sname_pkg)
+        if os.path.isfile(pkgfile):
+            try:
+                f = open(pkgfile)
+            except IOError, msg:
+                sys.stderr.write("Can't open %s: %s\n" %
+                                 (pkgfile, msg))
+            else:
+                for line in f:
+                    line = line.rstrip('\n')
+                    if not line or line.startswith('#'):
+                        continue
+                    path.append(line) # Don't check for existence!
+                f.close()
+
+    return path
+
+def get_data(package, resource):
+    """Get a resource from a package.
+
+    This is a wrapper round the PEP 302 loader get_data API. The package
+    argument should be the name of a package, in standard module format
+    (foo.bar). The resource argument should be in the form of a relative
+    filename, using '/' as the path separator. The parent directory name '..'
+    is not allowed, and nor is a rooted name (starting with a '/').
+
+    The function returns a binary string, which is the contents of the
+    specified resource.
+
+    For packages located in the filesystem, which have already been imported,
+    this is the rough equivalent of
+
+        d = os.path.dirname(sys.modules[package].__file__)
+        data = open(os.path.join(d, resource), 'rb').read()
+
+    If the package cannot be located or loaded, or it uses a PEP 302 loader
+    which does not support get_data(), then None is returned.
+    """
+
+    loader = get_loader(package)
+    if loader is None or not hasattr(loader, 'get_data'):
+        return None
+    mod = sys.modules.get(package) or loader.load_module(package)
+    if mod is None or not hasattr(mod, '__file__'):
+        return None
+
+    # Modify the resource name to be compatible with the loader.get_data
+    # signature - an os.path format "filename" starting with the dirname of
+    # the package's __file__
+    parts = resource.split('/')
+    parts.insert(0, os.path.dirname(mod.__file__))
+    resource_name = os.path.join(*parts)
+    return loader.get_data(resource_name)

mitogen-0.2.7/mitogen/compat/tokenize.py

@@ -0,0 +1,453 @@
+"""Tokenization help for Python programs.
+
+generate_tokens(readline) is a generator that breaks a stream of
+text into Python tokens.  It accepts a readline-like method which is called
+repeatedly to get the next line of input (or "" for EOF).  It generates
+5-tuples with these members:
+
+    the token type (see token.py)
+    the token (a string)
+    the starting (row, column) indices of the token (a 2-tuple of ints)
+    the ending (row, column) indices of the token (a 2-tuple of ints)
+    the original line (string)
+
+It is designed to match the working of the Python tokenizer exactly, except
+that it produces COMMENT tokens for comments and gives type OP for all
+operators
+
+Older entry points
+    tokenize_loop(readline, tokeneater)
+    tokenize(readline, tokeneater=printtoken)
+are the same, except instead of generating tokens, tokeneater is a callback
+function to which the 5 fields described above are passed as 5 arguments,
+each time a new token is found."""
+
+# !mitogen: minify_safe
+
+__author__ = 'Ka-Ping Yee <ping@lfw.org>'
+__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
+               'Skip Montanaro, Raymond Hettinger')
+
+from itertools import chain
+import string, re
+from token import *
+
+import token
+__all__ = [x for x in dir(token) if not x.startswith("_")]
+__all__ += ["COMMENT", "tokenize", "generate_tokens", "NL", "untokenize"]
+del token
+
+COMMENT = N_TOKENS
+tok_name[COMMENT] = 'COMMENT'
+NL = N_TOKENS + 1
+tok_name[NL] = 'NL'
+N_TOKENS += 2
+
+def group(*choices): return '(' + '|'.join(choices) + ')'
+def any(*choices): return group(*choices) + '*'
+def maybe(*choices): return group(*choices) + '?'
+
+Whitespace = r'[ \f\t]*'
+Comment = r'#[^\r\n]*'
+Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
+Name = r'[a-zA-Z_]\w*'
+
+Hexnumber = r'0[xX][\da-fA-F]+[lL]?'
+Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?'
+Binnumber = r'0[bB][01]+[lL]?'
+Decnumber = r'[1-9]\d*[lL]?'
+Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
+Exponent = r'[eE][-+]?\d+'
+Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
+Expfloat = r'\d+' + Exponent
+Floatnumber = group(Pointfloat, Expfloat)
+Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]')
+Number = group(Imagnumber, Floatnumber, Intnumber)
+
+# Tail end of ' string.
+Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
+# Tail end of " string.
+Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
+# Tail end of ''' string.
+Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
+# Tail end of """ string.
+Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
+Triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""')
+# Single-line ' or " string.
+String = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
+               r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
+
+# Because of leftmost-then-longest match semantics, be sure to put the
+# longest operators first (e.g., if = came before ==, == would get
+# recognized as two instances of =).
+Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
+                 r"//=?",
+                 r"[+\-*/%&|^=<>]=?",
+                 r"~")
+
+Bracket = '[][(){}]'
+Special = group(r'\r?\n', r'[:;.,`@]')
+Funny = group(Operator, Bracket, Special)
+
+PlainToken = group(Number, Funny, String, Name)
+Token = Ignore + PlainToken
+
+# First (or only) line of ' or " string.
+ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
+                group("'", r'\\\r?\n'),
+                r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
+                group('"', r'\\\r?\n'))
+PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
+PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
+
+tokenprog, pseudoprog, single3prog, double3prog = map(
+    re.compile, (Token, PseudoToken, Single3, Double3))
+endprogs = {"'": re.compile(Single), '"': re.compile(Double),
+            "'''": single3prog, '"""': double3prog,
+            "r'''": single3prog, 'r"""': double3prog,
+            "u'''": single3prog, 'u"""': double3prog,
+            "ur'''": single3prog, 'ur"""': double3prog,
+            "R'''": single3prog, 'R"""': double3prog,
+            "U'''": single3prog, 'U"""': double3prog,
+            "uR'''": single3prog, 'uR"""': double3prog,
+            "Ur'''": single3prog, 'Ur"""': double3prog,
+            "UR'''": single3prog, 'UR"""': double3prog,
+            "b'''": single3prog, 'b"""': double3prog,
+            "br'''": single3prog, 'br"""': double3prog,
+            "B'''": single3prog, 'B"""': double3prog,
+            "bR'''": single3prog, 'bR"""': double3prog,
+            "Br'''": single3prog, 'Br"""': double3prog,
+            "BR'''": single3prog, 'BR"""': double3prog,
+            'r': None, 'R': None, 'u': None, 'U': None,
+            'b': None, 'B': None}
+
+triple_quoted = {}
+for t in ("'''", '"""',
+          "r'''", 'r"""', "R'''", 'R"""',
+          "u'''", 'u"""', "U'''", 'U"""',
+          "ur'''", 'ur"""', "Ur'''", 'Ur"""',
+          "uR'''", 'uR"""', "UR'''", 'UR"""',
+          "b'''", 'b"""', "B'''", 'B"""',
+          "br'''", 'br"""', "Br'''", 'Br"""',
+          "bR'''", 'bR"""', "BR'''", 'BR"""'):
+    triple_quoted[t] = t
+single_quoted = {}
+for t in ("'", '"',
+          "r'", 'r"', "R'", 'R"',
+          "u'", 'u"', "U'", 'U"',
+          "ur'", 'ur"', "Ur'", 'Ur"',
+          "uR'", 'uR"', "UR'", 'UR"',
+          "b'", 'b"', "B'", 'B"',
+          "br'", 'br"', "Br'", 'Br"',
+          "bR'", 'bR"', "BR'", 'BR"' ):
+    single_quoted[t] = t
+
+tabsize = 8
+
+class TokenError(Exception): pass
+
+class StopTokenizing(Exception): pass
+
+def printtoken(type, token, srow_scol, erow_ecol, line): # for testing
+    srow, scol = srow_scol
+    erow, ecol = erow_ecol
+    print("%d,%d-%d,%d:\t%s\t%s" % \
+        (srow, scol, erow, ecol, tok_name[type], repr(token)))
+
+def tokenize(readline, tokeneater=printtoken):
+    """
+    The tokenize() function accepts two parameters: one representing the
+    input stream, and one providing an output mechanism for tokenize().
+
+    The first parameter, readline, must be a callable object which provides
+    the same interface as the readline() method of built-in file objects.
+    Each call to the function should return one line of input as a string.
+
+    The second parameter, tokeneater, must also be a callable object. It is
+    called once for each token, with five arguments, corresponding to the
+    tuples generated by generate_tokens().
+    """
+    try:
+        tokenize_loop(readline, tokeneater)
+    except StopTokenizing:
+        pass
+
+# backwards compatible interface
+def tokenize_loop(readline, tokeneater):
+    for token_info in generate_tokens(readline):
+        tokeneater(*token_info)
+
+class Untokenizer:
+
+    def __init__(self):
+        self.tokens = []
+        self.prev_row = 1
+        self.prev_col = 0
+
+    def add_whitespace(self, start):
+        row, col = start
+        if row < self.prev_row or row == self.prev_row and col < self.prev_col:
+            raise ValueError("start ({},{}) precedes previous end ({},{})"
+                             .format(row, col, self.prev_row, self.prev_col))
+        row_offset = row - self.prev_row
+        if row_offset:
+            self.tokens.append("\\\n" * row_offset)
+            self.prev_col = 0
+        col_offset = col - self.prev_col
+        if col_offset:
+            self.tokens.append(" " * col_offset)
+
+    def untokenize(self, iterable):
+        it = iter(iterable)
+        indents = []
+        startline = False
+        for t in it:
+            if len(t) == 2:
+                self.compat(t, it)
+                break
+            tok_type, token, start, end, line = t
+            if tok_type == ENDMARKER:
+                break
+            if tok_type == INDENT:
+                indents.append(token)
+                continue
+            elif tok_type == DEDENT:
+                indents.pop()
+                self.prev_row, self.prev_col = end
+                continue
+            elif tok_type in (NEWLINE, NL):
+                startline = True
+            elif startline and indents:
+                indent = indents[-1]
+                if start[1] >= len(indent):
+                    self.tokens.append(indent)
+                    self.prev_col = len(indent)
+                startline = False
+            self.add_whitespace(start)
+            self.tokens.append(token)
+            self.prev_row, self.prev_col = end
+            if tok_type in (NEWLINE, NL):
+                self.prev_row += 1
+                self.prev_col = 0
+        return "".join(self.tokens)
+
+    def compat(self, token, iterable):
+        indents = []
+        toks_append = self.tokens.append
+        startline = token[0] in (NEWLINE, NL)
+        prevstring = False
+
+        for tok in chain([token], iterable):
+            toknum, tokval = tok[:2]
+
+            if toknum in (NAME, NUMBER):
+                tokval += ' '
+
+            # Insert a space between two consecutive strings
+            if toknum == STRING:
+                if prevstring:
+                    tokval = ' ' + tokval
+                prevstring = True
+            else:
+                prevstring = False
+
+            if toknum == INDENT:
+                indents.append(tokval)
+                continue
+            elif toknum == DEDENT:
+                indents.pop()
+                continue
+            elif toknum in (NEWLINE, NL):
+                startline = True
+            elif startline and indents:
+                toks_append(indents[-1])
+                startline = False
+            toks_append(tokval)
+
+def untokenize(iterable):
+    """Transform tokens back into Python source code.
+
+    Each element returned by the iterable must be a token sequence
+    with at least two elements, a token number and token value.  If
+    only two tokens are passed, the resulting output is poor.
+
+    Round-trip invariant for full input:
+        Untokenized source will match input source exactly
+
+    Round-trip invariant for limited intput:
+        # Output text will tokenize the back to the input
+        t1 = [tok[:2] for tok in generate_tokens(f.readline)]
+        newcode = untokenize(t1)
+        readline = iter(newcode.splitlines(1)).next
+        t2 = [tok[:2] for tok in generate_tokens(readline)]
+        assert t1 == t2
+    """
+    ut = Untokenizer()
+    return ut.untokenize(iterable)
+
+def generate_tokens(readline):
+    """
+    The generate_tokens() generator requires one argument, readline, which
+    must be a callable object which provides the same interface as the
+    readline() method of built-in file objects. Each call to the function
+    should return one line of input as a string.  Alternately, readline
+    can be a callable function terminating with StopIteration:
+        readline = open(myfile).next    # Example of alternate readline
+
+    The generator produces 5-tuples with these members: the token type; the
+    token string; a 2-tuple (srow, scol) of ints specifying the row and
+    column where the token begins in the source; a 2-tuple (erow, ecol) of
+    ints specifying the row and column where the token ends in the source;
+    and the line on which the token was found. The line passed is the
+    logical line; continuation lines are included.
+    """
+    lnum = parenlev = continued = 0
+    namechars, numchars = string.ascii_letters + '_', '0123456789'
+    contstr, needcont = '', 0
+    contline = None
+    indents = [0]
+
+    while 1:                                   # loop over lines in stream
+        try:
+            line = readline()
+        except StopIteration:
+            line = ''
+        lnum += 1
+        pos, max = 0, len(line)
+
+        if contstr:                            # continued string
+            if not line:
+                raise TokenError("EOF in multi-line string", strstart)
+            endmatch = endprog.match(line)
+            if endmatch:
+                pos = end = endmatch.end(0)
+                yield (STRING, contstr + line[:end],
+                       strstart, (lnum, end), contline + line)
+                contstr, needcont = '', 0
+                contline = None
+            elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
+                yield (ERRORTOKEN, contstr + line,
+                           strstart, (lnum, len(line)), contline)
+                contstr = ''
+                contline = None
+                continue
+            else:
+                contstr = contstr + line
+                contline = contline + line
+                continue
+
+        elif parenlev == 0 and not continued:  # new statement
+            if not line: break
+            column = 0
+            while pos < max:                   # measure leading whitespace
+                if line[pos] == ' ':
+                    column += 1
+                elif line[pos] == '\t':
+                    column = (column//tabsize + 1)*tabsize
+                elif line[pos] == '\f':
+                    column = 0
+                else:
+                    break
+                pos += 1
+            if pos == max:
+                break
+
+            if line[pos] in '#\r\n':           # skip comments or blank lines
+                if line[pos] == '#':
+                    comment_token = line[pos:].rstrip('\r\n')
+                    nl_pos = pos + len(comment_token)
+                    yield (COMMENT, comment_token,
+                           (lnum, pos), (lnum, pos + len(comment_token)), line)
+                    yield (NL, line[nl_pos:],
+                           (lnum, nl_pos), (lnum, len(line)), line)
+                else:
+                    yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
+                           (lnum, pos), (lnum, len(line)), line)
+                continue
+
+            if column > indents[-1]:           # count indents or dedents
+                indents.append(column)
+                yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
+            while column < indents[-1]:
+                if column not in indents:
+                    raise IndentationError(
+                        "unindent does not match any outer indentation level",
+                        ("<tokenize>", lnum, pos, line))
+                indents = indents[:-1]
+                yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
+
+        else:                                  # continued statement
+            if not line:
+                raise TokenError("EOF in multi-line statement", (lnum, 0))
+            continued = 0
+
+        while pos < max:
+            pseudomatch = pseudoprog.match(line, pos)
+            if pseudomatch:                                # scan for tokens
+                start, end = pseudomatch.span(1)
+                spos, epos, pos = (lnum, start), (lnum, end), end
+                if start == end:
+                    continue
+                token, initial = line[start:end], line[start]
+
+                if initial in numchars or \
+                   (initial == '.' and token != '.'):      # ordinary number
+                    yield (NUMBER, token, spos, epos, line)
+                elif initial in '\r\n':
+                    if parenlev > 0:
+                        n = NL
+                    else:
+                        n = NEWLINE
+                    yield (n, token, spos, epos, line)
+                elif initial == '#':
+                    assert not token.endswith("\n")
+                    yield (COMMENT, token, spos, epos, line)
+                elif token in triple_quoted:
+                    endprog = endprogs[token]
+                    endmatch = endprog.match(line, pos)
+                    if endmatch:                           # all on one line
+                        pos = endmatch.end(0)
+                        token = line[start:pos]
+                        yield (STRING, token, spos, (lnum, pos), line)
+                    else:
+                        strstart = (lnum, start)           # multiple lines
+                        contstr = line[start:]
+                        contline = line
+                        break
+                elif initial in single_quoted or \
+                    token[:2] in single_quoted or \
+                    token[:3] in single_quoted:
+                    if token[-1] == '\n':                  # continued string
+                        strstart = (lnum, start)
+                        endprog = (endprogs[initial] or endprogs[token[1]] or
+                                   endprogs[token[2]])
+                        contstr, needcont = line[start:], 1
+                        contline = line
+                        break
+                    else:                                  # ordinary string
+                        yield (STRING, token, spos, epos, line)
+                elif initial in namechars:                 # ordinary name
+                    yield (NAME, token, spos, epos, line)
+                elif initial == '\\':                      # continued stmt
+                    continued = 1
+                else:
+                    if initial in '([{':
+                        parenlev += 1
+                    elif initial in ')]}':
+                        parenlev -= 1
+                    yield (OP, token, spos, epos, line)
+            else:
+                yield (ERRORTOKEN, line[pos],
+                           (lnum, pos), (lnum, pos+1), line)
+                pos += 1
+
+    for indent in indents[1:]:                 # pop remaining indent levels
+        yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
+    yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
+
+if __name__ == '__main__':                     # testing
+    import sys
+    if len(sys.argv) > 1:
+        tokenize(open(sys.argv[1]).readline)
+    else:
+        tokenize(sys.stdin.readline)

mitogen-0.2.7/mitogen/debug.py

@@ -0,0 +1,236 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+"""
+Basic signal handler for dumping thread stacks.
+"""
+
+import difflib
+import logging
+import os
+import gc
+import signal
+import sys
+import threading
+import time
+import traceback
+
+import mitogen.core
+import mitogen.parent
+
+
+LOG = logging.getLogger(__name__)
+_last = None
+
+
+def enable_evil_interrupts():
+    signal.signal(signal.SIGALRM, (lambda a, b: None))
+    signal.setitimer(signal.ITIMER_REAL, 0.01, 0.01)
+
+
+def disable_evil_interrupts():
+    signal.setitimer(signal.ITIMER_REAL, 0, 0)
+
+
+def _hex(n):
+    return '%08x' % n
+
+
+def get_subclasses(klass):
+    """
+    Rather than statically import every interesting subclass, forcing it all to
+    be transferred and potentially disrupting the debugged environment,
+    enumerate only those loaded in memory. Also returns the original class.
+    """
+    stack = [klass]
+    seen = set()
+    while stack:
+        klass = stack.pop()
+        seen.add(klass)
+        stack.extend(klass.__subclasses__())
+    return seen
+
+
+def get_routers():
+    return dict(
+        (_hex(id(router)), router)
+        for klass in get_subclasses(mitogen.core.Router)
+        for router in gc.get_referrers(klass)
+        if isinstance(router, mitogen.core.Router)
+    )
+
+
+def get_router_info():
+    return {
+        'routers': dict(
+            (id_, {
+                'id': id_,
+                'streams': len(set(router._stream_by_id.values())),
+                'contexts': len(set(router._context_by_id.values())),
+                'handles': len(router._handle_map),
+            })
+            for id_, router in get_routers().items()
+        )
+    }
+
+
+def get_stream_info(router_id):
+    router = get_routers().get(router_id)
+    return {
+        'streams': dict(
+            (_hex(id(stream)), ({
+                'name': stream.name,
+                'remote_id': stream.remote_id,
+                'sent_module_count': len(getattr(stream, 'sent_modules', [])),
+                'routes': sorted(getattr(stream, 'routes', [])),
+                'type': type(stream).__module__,
+            }))
+            for via_id, stream in router._stream_by_id.items()
+        )
+    }
+
+
+def format_stacks():
+    name_by_id = dict(
+        (t.ident, t.name)
+        for t in threading.enumerate()
+    )
+
+    l = ['', '']
+    for threadId, stack in sys._current_frames().items():
+        l += ["# PID %d ThreadID: (%s) %s; %r" % (
+            os.getpid(),
+            name_by_id.get(threadId, '<no name>'),
+            threadId,
+            stack,
+        )]
+        #stack = stack.f_back.f_back
+
+        for filename, lineno, name, line in traceback.extract_stack(stack):
+            l += [
+                'File: "%s", line %d, in %s' % (
+                    filename,
+                    lineno,
+                    name
+                )
+            ]
+            if line:
+                l += ['    ' + line.strip()]
+        l += ['']
+
+    l += ['', '']
+    return '\n'.join(l)
+
+
+def get_snapshot():
+    global _last
+
+    s = format_stacks()
+    snap = s
+    if _last:
+        snap += '\n'
+        diff = list(difflib.unified_diff(
+            a=_last.splitlines(),
+            b=s.splitlines(),
+            fromfile='then',
+            tofile='now'
+        ))
+
+        if diff:
+            snap += '\n'.join(diff) + '\n'
+        else:
+            snap += '(no change since last time)\n'
+    _last = s
+    return snap
+
+
+def _handler(*_):
+    fp = open('/dev/tty', 'w', 1)
+    fp.write(get_snapshot())
+    fp.close()
+
+
+def install_handler():
+    signal.signal(signal.SIGUSR2, _handler)
+
+
+def _logging_main(secs):
+    while True:
+        time.sleep(secs)
+        LOG.info('PERIODIC THREAD DUMP\n\n%s', get_snapshot())
+
+
+def dump_to_logger(secs=5):
+    th = threading.Thread(
+        target=_logging_main,
+        kwargs={'secs': secs},
+        name='mitogen.debug.dump_to_logger',
+    )
+    th.setDaemon(True)
+    th.start()
+
+
+class ContextDebugger(object):
+    @classmethod
+    @mitogen.core.takes_econtext
+    def _configure_context(cls, econtext):
+        mitogen.parent.upgrade_router(econtext)
+        econtext.debugger = cls(econtext.router)
+
+    def __init__(self, router):
+        self.router = router
+        self.router.add_handler(
+            func=self._on_debug_msg,
+            handle=mitogen.core.DEBUG,
+            persist=True,
+            policy=mitogen.core.has_parent_authority,
+        )
+        mitogen.core.listen(router, 'register', self._on_stream_register)
+        LOG.debug('Context debugging configured.')
+
+    def _on_stream_register(self, context, stream):
+        LOG.debug('_on_stream_register: sending configure() to %r', stream)
+        context.call_async(ContextDebugger._configure_context)
+
+    def _on_debug_msg(self, msg):
+        if msg != mitogen.core._DEAD:
+            threading.Thread(
+                target=self._handle_debug_msg,
+                name='ContextDebuggerHandler',
+                args=(msg,)
+            ).start()
+
+    def _handle_debug_msg(self, msg):
+        try:
+            method, args, kwargs = msg.unpickle()
+            msg.reply(getattr(cls, method)(*args, **kwargs))
+        except Exception:
+            e = sys.exc_info()[1]
+            msg.reply(mitogen.core.CallError(e))

mitogen-0.2.7/mitogen/doas.py

@@ -0,0 +1,113 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+import logging
+
+import mitogen.core
+import mitogen.parent
+from mitogen.core import b
+
+
+LOG = logging.getLogger(__name__)
+
+
+class PasswordError(mitogen.core.StreamError):
+    pass
+
+
+class Stream(mitogen.parent.Stream):
+    create_child = staticmethod(mitogen.parent.hybrid_tty_create_child)
+    child_is_immediate_subprocess = False
+
+    username = 'root'
+    password = None
+    doas_path = 'doas'
+    password_prompt = b('Password:')
+    incorrect_prompts = (
+        b('doas: authentication failed'),
+    )
+
+    def construct(self, username=None, password=None, doas_path=None,
+                  password_prompt=None, incorrect_prompts=None, **kwargs):
+        super(Stream, self).construct(**kwargs)
+        if username is not None:
+            self.username = username
+        if password is not None:
+            self.password = password
+        if doas_path is not None:
+            self.doas_path = doas_path
+        if password_prompt is not None:
+            self.password_prompt = password_prompt.lower()
+        if incorrect_prompts is not None:
+            self.incorrect_prompts = map(str.lower, incorrect_prompts)
+
+    def _get_name(self):
+        return u'doas.' + mitogen.core.to_text(self.username)
+
+    def get_boot_command(self):
+        bits = [self.doas_path, '-u', self.username, '--']
+        bits = bits + super(Stream, self).get_boot_command()
+        LOG.debug('doas command line: %r', bits)
+        return bits
+
+    password_incorrect_msg = 'doas password is incorrect'
+    password_required_msg = 'doas password is required'
+
+    def _connect_input_loop(self, it):
+        password_sent = False
+        for buf in it:
+            LOG.debug('%r: received %r', self, buf)
+            if buf.endswith(self.EC0_MARKER):
+                self._ec0_received()
+                return
+            if any(s in buf.lower() for s in self.incorrect_prompts):
+                if password_sent:
+                    raise PasswordError(self.password_incorrect_msg)
+            elif self.password_prompt in buf.lower():
+                if self.password is None:
+                    raise PasswordError(self.password_required_msg)
+                if password_sent:
+                    raise PasswordError(self.password_incorrect_msg)
+                LOG.debug('sending password')
+                self.diag_stream.transmit_side.write(
+                    mitogen.core.to_text(self.password + '\n').encode('utf-8')
+                )
+                password_sent = True
+        raise mitogen.core.StreamError('bootstrap failed')
+
+    def _connect_bootstrap(self):
+        it = mitogen.parent.iter_read(
+            fds=[self.receive_side.fd, self.diag_stream.receive_side.fd],
+            deadline=self.connect_deadline,
+        )
+        try:
+            self._connect_input_loop(it)
+        finally:
+            it.close()

mitogen-0.2.7/mitogen/docker.py

@@ -0,0 +1,81 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+import logging
+
+import mitogen.core
+import mitogen.parent
+
+
+LOG = logging.getLogger(__name__)
+
+
+class Stream(mitogen.parent.Stream):
+    child_is_immediate_subprocess = False
+
+    container = None
+    image = None
+    username = None
+    docker_path = 'docker'
+
+    # TODO: better way of capturing errors such as "No such container."
+    create_child_args = {
+        'merge_stdio': True
+    }
+
+    def construct(self, container=None, image=None,
+                  docker_path=None, username=None,
+                  **kwargs):
+        assert container or image
+        super(Stream, self).construct(**kwargs)
+        if container:
+            self.container = container
+        if image:
+            self.image = image
+        if docker_path:
+            self.docker_path = docker_path
+        if username:
+            self.username = username
+
+    def _get_name(self):
+        return u'docker.' + (self.container or self.image)
+
+    def get_boot_command(self):
+        args = ['--interactive']
+        if self.username:
+            args += ['--user=' + self.username]
+
+        bits = [self.docker_path]
+        if self.container:
+            bits += ['exec'] + args + [self.container]
+        elif self.image:
+            bits += ['run'] + args + ['--rm', self.image]
+
+        return bits + super(Stream, self).get_boot_command()

mitogen-0.2.7/mitogen/fakessh.py

@@ -0,0 +1,461 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+"""
+:mod:`mitogen.fakessh` is a stream implementation that starts a subprocess with
+its environment modified such that ``PATH`` searches for `ssh` return a Mitogen
+implementation of SSH. When invoked, this implementation arranges for the
+command line supplied by the caller to be executed in a remote context, reusing
+the parent context's (possibly proxied) connection to that remote context.
+
+This allows tools like `rsync` and `scp` to transparently reuse the connections
+and tunnels already established by the host program to connect to a target
+machine, without wasteful redundant SSH connection setup, 3-way handshakes, or
+firewall hopping configurations, and enables these tools to be used in
+impossible scenarios, such as over `sudo` with ``requiretty`` enabled.
+
+The fake `ssh` command source is written to a temporary file on disk, and
+consists of a copy of the :py:mod:`mitogen.core` source code (just like any
+other child context), with a line appended to cause it to connect back to the
+host process over an FD it inherits. As there is no reliance on an existing
+filesystem file, it is possible for child contexts to use fakessh.
+
+As a consequence of connecting back through an inherited FD, only one SSH
+invocation is possible, which is fine for tools like `rsync`, however in future
+this restriction will be lifted.
+
+Sequence:
+
+    1. ``fakessh`` Context and Stream created by parent context. The stream's
+       buffer has a :py:func:`_fakessh_main` :py:data:`CALL_FUNCTION
+       <mitogen.core.CALL_FUNCTION>` enqueued.
+    2. Target program (`rsync/scp/sftp`) invoked, which internally executes
+       `ssh` from ``PATH``.
+    3. :py:mod:`mitogen.core` bootstrap begins, recovers the stream FD
+       inherited via the target program, established itself as the fakessh
+       context.
+    4. :py:func:`_fakessh_main` :py:data:`CALL_FUNCTION
+       <mitogen.core.CALL_FUNCTION>` is read by fakessh context,
+
+        a. sets up :py:class:`IoPump` for stdio, registers
+           stdin_handle for local context.
+        b. Enqueues :py:data:`CALL_FUNCTION <mitogen.core.CALL_FUNCTION>` for
+           :py:func:`_start_slave` invoked in target context,
+
+            i. the program from the `ssh` command line is started
+            ii. sets up :py:class:`IoPump` for `ssh` command line process's
+                stdio pipes
+            iii. returns `(control_handle, stdin_handle)` to
+                 :py:func:`_fakessh_main`
+
+    5. :py:func:`_fakessh_main` receives control/stdin handles from from
+       :py:func:`_start_slave`,
+
+        a. registers remote's stdin_handle with local :py:class:`IoPump`.
+        b. sends `("start", local_stdin_handle)` to remote's control_handle
+        c. registers local :py:class:`IoPump` with
+           :py:class:`mitogen.core.Broker`.
+        d. loops waiting for `local stdout closed && remote stdout closed`
+
+    6. :py:func:`_start_slave` control channel receives `("start", stdin_handle)`,
+
+        a. registers remote's stdin_handle with local :py:class:`IoPump`
+        b. registers local :py:class:`IoPump` with
+           :py:class:`mitogen.core.Broker`.
+        c. loops waiting for `local stdout closed && remote stdout closed`
+"""
+
+import getopt
+import inspect
+import os
+import shutil
+import socket
+import subprocess
+import sys
+import tempfile
+import threading
+
+import mitogen.core
+import mitogen.master
+import mitogen.parent
+
+from mitogen.core import LOG, IOLOG
+
+
+SSH_GETOPTS = (
+    "1246ab:c:e:fgi:kl:m:no:p:qstvx"
+    "ACD:E:F:I:KL:MNO:PQ:R:S:TVw:W:XYy"
+)
+
+_mitogen = None
+
+
+class IoPump(mitogen.core.BasicStream):
+    _output_buf = ''
+    _closed = False
+
+    def __init__(self, broker, stdin_fd, stdout_fd):
+        self._broker = broker
+        self.receive_side = mitogen.core.Side(self, stdout_fd)
+        self.transmit_side = mitogen.core.Side(self, stdin_fd)
+
+    def write(self, s):
+        self._output_buf += s
+        self._broker._start_transmit(self)
+
+    def close(self):
+        self._closed = True
+        # If local process hasn't exitted yet, ensure its write buffer is
+        # drained before lazily triggering disconnect in on_transmit.
+        if self.transmit_side.fd is not None:
+            self._broker._start_transmit(self)
+
+    def on_shutdown(self, broker):
+        self.close()
+
+    def on_transmit(self, broker):
+        written = self.transmit_side.write(self._output_buf)
+        IOLOG.debug('%r.on_transmit() -> len %r', self, written)
+        if written is None:
+            self.on_disconnect(broker)
+        else:
+            self._output_buf = self._output_buf[written:]
+
+        if not self._output_buf:
+            broker._stop_transmit(self)
+            if self._closed:
+                self.on_disconnect(broker)
+
+    def on_receive(self, broker):
+        s = self.receive_side.read()
+        IOLOG.debug('%r.on_receive() -> len %r', self, len(s))
+        if s:
+            mitogen.core.fire(self, 'receive', s)
+        else:
+            self.on_disconnect(broker)
+
+    def __repr__(self):
+        return 'IoPump(%r, %r)' % (
+            self.receive_side.fd,
+            self.transmit_side.fd,
+        )
+
+
+class Process(object):
+    """
+    Manages the lifetime and pipe connections of the SSH command running in the
+    slave.
+    """
+    def __init__(self, router, stdin_fd, stdout_fd, proc=None):
+        self.router = router
+        self.stdin_fd = stdin_fd
+        self.stdout_fd = stdout_fd
+        self.proc = proc
+        self.control_handle = router.add_handler(self._on_control)
+        self.stdin_handle = router.add_handler(self._on_stdin)
+        self.pump = IoPump(router.broker, stdin_fd, stdout_fd)
+        self.stdin = None
+        self.control = None
+        self.wake_event = threading.Event()
+
+        mitogen.core.listen(self.pump, 'disconnect', self._on_pump_disconnect)
+        mitogen.core.listen(self.pump, 'receive', self._on_pump_receive)
+
+        if proc:
+            pmon = mitogen.parent.ProcessMonitor.instance()
+            pmon.add(proc.pid, self._on_proc_exit)
+
+    def __repr__(self):
+        return 'Process(%r, %r)' % (self.stdin_fd, self.stdout_fd)
+
+    def _on_proc_exit(self, status):
+        LOG.debug('%r._on_proc_exit(%r)', self, status)
+        self.control.put(('exit', status))
+
+    def _on_stdin(self, msg):
+        if msg.is_dead:
+            IOLOG.debug('%r._on_stdin() -> %r', self, data)
+            self.pump.close()
+            return
+
+        data = msg.unpickle()
+        IOLOG.debug('%r._on_stdin() -> len %d', self, len(data))
+        self.pump.write(data)
+
+    def _on_control(self, msg):
+        if not msg.is_dead:
+            command, arg = msg.unpickle(throw=False)
+            LOG.debug('%r._on_control(%r, %s)', self, command, arg)
+
+            func = getattr(self, '_on_%s' % (command,), None)
+            if func:
+                return func(msg, arg)
+
+            LOG.warning('%r: unknown command %r', self, command)
+
+    def _on_start(self, msg, arg):
+        dest = mitogen.core.Context(self.router, msg.src_id)
+        self.control = mitogen.core.Sender(dest, arg[0])
+        self.stdin = mitogen.core.Sender(dest, arg[1])
+        self.router.broker.start_receive(self.pump)
+
+    def _on_exit(self, msg, arg):
+        LOG.debug('on_exit: proc = %r', self.proc)
+        if self.proc:
+            self.proc.terminate()
+        else:
+            self.router.broker.shutdown()
+
+    def _on_pump_receive(self, s):
+        IOLOG.info('%r._on_pump_receive(len %d)', self, len(s))
+        self.stdin.put(s)
+
+    def _on_pump_disconnect(self):
+        LOG.debug('%r._on_pump_disconnect()', self)
+        mitogen.core.fire(self, 'disconnect')
+        self.stdin.close()
+        self.wake_event.set()
+
+    def start_master(self, stdin, control):
+        self.stdin = stdin
+        self.control = control
+        control.put(('start', (self.control_handle, self.stdin_handle)))
+        self.router.broker.start_receive(self.pump)
+
+    def wait(self):
+        while not self.wake_event.isSet():
+            # Timeout is used so that sleep is interruptible, as blocking
+            # variants of libc thread operations cannot be interrupted e.g. via
+            # KeyboardInterrupt. isSet() test and wait() are separate since in
+            # <2.7 wait() always returns None.
+            self.wake_event.wait(0.1)
+
+
+@mitogen.core.takes_router
+def _start_slave(src_id, cmdline, router):
+    """
+    This runs in the target context, it is invoked by _fakessh_main running in
+    the fakessh context immediately after startup. It starts the slave process
+    (the the point where it has a stdin_handle to target but not stdout_chan to
+    write to), and waits for main to.
+    """
+    LOG.debug('_start_slave(%r, %r)', router, cmdline)
+
+    proc = subprocess.Popen(
+        cmdline,
+        # SSH server always uses user's shell.
+        shell=True,
+        # SSH server always executes new commands in the user's HOME.
+        cwd=os.path.expanduser('~'),
+
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+    )
+
+    process = Process(
+        router,
+        proc.stdin.fileno(),
+        proc.stdout.fileno(),
+        proc,
+    )
+
+    return process.control_handle, process.stdin_handle
+
+
+#
+# SSH client interface.
+#
+
+
+def exit():
+    _mitogen.broker.shutdown()
+
+
+def die(msg, *args):
+    if args:
+        msg %= args
+    sys.stderr.write('%s\n' % (msg,))
+    exit()
+
+
+def parse_args():
+    hostname = None
+    remain = sys.argv[1:]
+    allopts = []
+    restarted = 0
+
+    while remain and restarted < 2:
+        opts, args = getopt.getopt(remain, SSH_GETOPTS)
+        remain = remain[:]  # getopt bug!
+        allopts += opts
+        if not args:
+            break
+
+        if not hostname:
+            hostname = args.pop(0)
+            remain = remain[remain.index(hostname) + 1:]
+
+        restarted += 1
+
+    return hostname, allopts, args
+
+
+@mitogen.core.takes_econtext
+def _fakessh_main(dest_context_id, econtext):
+    hostname, opts, args = parse_args()
+    if not hostname:
+        die('Missing hostname')
+
+    subsystem = False
+    for opt, optarg in opts:
+        if opt == '-s':
+            subsystem = True
+        else:
+            LOG.debug('Warning option %s %s is ignored.', opt, optarg)
+
+    LOG.debug('hostname: %r', hostname)
+    LOG.debug('opts: %r', opts)
+    LOG.debug('args: %r', args)
+
+    if subsystem:
+        die('-s <subsystem> is not yet supported')
+
+    if not args:
+        die('fakessh: login mode not supported and no command specified')
+
+    dest = mitogen.parent.Context(econtext.router, dest_context_id)
+
+    # Even though SSH receives an argument vector, it still cats the vector
+    # together before sending to the server, the server just uses /bin/sh -c to
+    # run the command. We must remain puke-for-puke compatible.
+    control_handle, stdin_handle = dest.call(_start_slave,
+        mitogen.context_id, ' '.join(args))
+
+    LOG.debug('_fakessh_main: received control_handle=%r, stdin_handle=%r',
+              control_handle, stdin_handle)
+
+    process = Process(econtext.router, 1, 0)
+    process.start_master(
+        stdin=mitogen.core.Sender(dest, stdin_handle),
+        control=mitogen.core.Sender(dest, control_handle),
+    )
+    process.wait()
+    process.control.put(('exit', None))
+
+
+def _get_econtext_config(context, sock2):
+    parent_ids = mitogen.parent_ids[:]
+    parent_ids.insert(0, mitogen.context_id)
+    return {
+        'context_id': context.context_id,
+        'core_src_fd': None,
+        'debug': getattr(context.router, 'debug', False),
+        'in_fd': sock2.fileno(),
+        'log_level': mitogen.parent.get_log_level(),
+        'max_message_size': context.router.max_message_size,
+        'out_fd': sock2.fileno(),
+        'parent_ids': parent_ids,
+        'profiling': getattr(context.router, 'profiling', False),
+        'unidirectional': getattr(context.router, 'unidirectional', False),
+        'setup_stdio': False,
+        'version': mitogen.__version__,
+    }
+
+
+#
+# Public API.
+#
+
+@mitogen.core.takes_econtext
+@mitogen.core.takes_router
+def run(dest, router, args, deadline=None, econtext=None):
+    """
+    Run the command specified by `args` such that ``PATH`` searches for SSH by
+    the command will cause its attempt to use SSH to execute a remote program
+    to be redirected to use mitogen to execute that program using the context
+    `dest` instead.
+
+    :param list args:
+        Argument vector.
+    :param mitogen.core.Context dest:
+        The destination context to execute the SSH command line in.
+
+    :param mitogen.core.Router router:
+
+    :param list[str] args:
+        Command line arguments for local program, e.g.
+        ``['rsync', '/tmp', 'remote:/tmp']``
+
+    :returns:
+        Exit status of the child process.
+    """
+    if econtext is not None:
+        mitogen.parent.upgrade_router(econtext)
+
+    context_id = router.allocate_id()
+    fakessh = mitogen.parent.Context(router, context_id)
+    fakessh.name = u'fakessh.%d' % (context_id,)
+
+    sock1, sock2 = socket.socketpair()
+
+    stream = mitogen.core.Stream(router, context_id)
+    stream.name = u'fakessh'
+    stream.accept(sock1.fileno(), sock1.fileno())
+    router.register(fakessh, stream)
+
+    # Held in socket buffer until process is booted.
+    fakessh.call_async(_fakessh_main, dest.context_id)
+
+    tmp_path = tempfile.mkdtemp(prefix='mitogen_fakessh')
+    try:
+        ssh_path = os.path.join(tmp_path, 'ssh')
+        fp = open(ssh_path, 'w')
+        try:
+            fp.write('#!%s\n' % (mitogen.parent.get_sys_executable(),))
+            fp.write(inspect.getsource(mitogen.core))
+            fp.write('\n')
+            fp.write('ExternalContext(%r).main()\n' % (
+                _get_econtext_config(context, sock2),
+            ))
+        finally:
+            fp.close()
+
+        os.chmod(ssh_path, int('0755', 8))
+        env = os.environ.copy()
+        env.update({
+            'PATH': '%s:%s' % (tmp_path, env.get('PATH', '')),
+            'ARGV0': mitogen.parent.get_sys_executable(),
+            'SSH_PATH': ssh_path,
+        })
+
+        proc = subprocess.Popen(args, env=env)
+        return proc.wait()
+    finally:
+        shutil.rmtree(tmp_path)

mitogen-0.2.7/mitogen/fork.py

@@ -0,0 +1,223 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+import logging
+import os
+import random
+import sys
+import threading
+import traceback
+
+import mitogen.core
+import mitogen.parent
+
+
+LOG = logging.getLogger('mitogen')
+
+# Python 2.4/2.5 cannot support fork+threads whatsoever, it doesn't even fix up
+# interpreter state. So 2.4/2.5 interpreters start .local() contexts for
+# isolation instead. Since we don't have any crazy memory sharing problems to
+# avoid, there is no virginal fork parent either. The child is started directly
+# from the login/become process. In future this will be default everywhere,
+# fork is brainwrong from the stone age.
+FORK_SUPPORTED = sys.version_info >= (2, 6)
+
+
+class Error(mitogen.core.StreamError):
+    pass
+
+
+def fixup_prngs():
+    """
+    Add 256 bits of /dev/urandom to OpenSSL's PRNG in the child, and re-seed
+    the random package with the same data.
+    """
+    s = os.urandom(256 // 8)
+    random.seed(s)
+    if 'ssl' in sys.modules:
+        sys.modules['ssl'].RAND_add(s, 75.0)
+
+
+def reset_logging_framework():
+    """
+    After fork, ensure any logging.Handler locks are recreated, as a variety of
+    threads in the parent may have been using the logging package at the moment
+    of fork.
+
+    It is not possible to solve this problem in general; see
+    https://github.com/dw/mitogen/issues/150 for a full discussion.
+    """
+    logging._lock = threading.RLock()
+
+    # The root logger does not appear in the loggerDict.
+    for name in [None] + list(logging.Logger.manager.loggerDict):
+        for handler in logging.getLogger(name).handlers:
+            handler.createLock()
+
+    root = logging.getLogger()
+    root.handlers = [
+        handler
+        for handler in root.handlers
+        if not isinstance(handler, mitogen.core.LogHandler)
+    ]
+
+
+def on_fork():
+    """
+    Should be called by any program integrating Mitogen each time the process
+    is forked, in the context of the new child.
+    """
+    reset_logging_framework()  # Must be first!
+    fixup_prngs()
+    mitogen.core.Latch._on_fork()
+    mitogen.core.Side._on_fork()
+    mitogen.core.ExternalContext.service_stub_lock = threading.Lock()
+
+    mitogen__service = sys.modules.get('mitogen.service')
+    if mitogen__service:
+        mitogen__service._pool_lock = threading.Lock()
+
+
+def handle_child_crash():
+    """
+    Respond to _child_main() crashing by ensuring the relevant exception is
+    logged to /dev/tty.
+    """
+    tty = open('/dev/tty', 'wb')
+    tty.write('\n\nFORKED CHILD PID %d CRASHED\n%s\n\n' % (
+        os.getpid(),
+        traceback.format_exc(),
+    ))
+    tty.close()
+    os._exit(1)
+
+
+class Stream(mitogen.parent.Stream):
+    child_is_immediate_subprocess = True
+
+    #: Reference to the importer, if any, recovered from the parent.
+    importer = None
+
+    #: User-supplied function for cleaning up child process state.
+    on_fork = None
+
+    python_version_msg = (
+        "The mitogen.fork method is not supported on Python versions "
+        "prior to 2.6, since those versions made no attempt to repair "
+        "critical interpreter state following a fork. Please use the "
+        "local() method instead."
+    )
+
+    def construct(self, old_router, max_message_size, on_fork=None,
+                  debug=False, profiling=False, unidirectional=False,
+                  on_start=None):
+        if not FORK_SUPPORTED:
+            raise Error(self.python_version_msg)
+
+        # fork method only supports a tiny subset of options.
+        super(Stream, self).construct(max_message_size=max_message_size,
+                                      debug=debug, profiling=profiling,
+                                      unidirectional=False)
+        self.on_fork = on_fork
+        self.on_start = on_start
+
+        responder = getattr(old_router, 'responder', None)
+        if isinstance(responder, mitogen.parent.ModuleForwarder):
+            self.importer = responder.importer
+
+    name_prefix = u'fork'
+
+    def start_child(self):
+        parentfp, childfp = mitogen.parent.create_socketpair()
+        self.pid = os.fork()
+        if self.pid:
+            childfp.close()
+            # Decouple the socket from the lifetime of the Python socket object.
+            fd = os.dup(parentfp.fileno())
+            parentfp.close()
+            return self.pid, fd, None
+        else:
+            parentfp.close()
+            self._wrap_child_main(childfp)
+
+    def _wrap_child_main(self, childfp):
+        try:
+            self._child_main(childfp)
+        except BaseException:
+            handle_child_crash()
+
+    def _child_main(self, childfp):
+        on_fork()
+        if self.on_fork:
+            self.on_fork()
+        mitogen.core.set_block(childfp.fileno())
+
+        # Expected by the ExternalContext.main().
+        os.dup2(childfp.fileno(), 1)
+        os.dup2(childfp.fileno(), 100)
+
+        # Overwritten by ExternalContext.main(); we must replace the
+        # parent-inherited descriptors that were closed by Side._on_fork() to
+        # avoid ExternalContext.main() accidentally allocating new files over
+        # the standard handles.
+        os.dup2(childfp.fileno(), 0)
+
+        # Avoid corrupting the stream on fork crash by dupping /dev/null over
+        # stderr. Instead, handle_child_crash() uses /dev/tty to log errors.
+        devnull = os.open('/dev/null', os.O_WRONLY)
+        if devnull != 2:
+            os.dup2(devnull, 2)
+            os.close(devnull)
+
+        # If we're unlucky, childfp.fileno() may coincidentally be one of our
+        # desired FDs. In that case closing it breaks ExternalContext.main().
+        if childfp.fileno() not in (0, 1, 100):
+            childfp.close()
+
+        config = self.get_econtext_config()
+        config['core_src_fd'] = None
+        config['importer'] = self.importer
+        config['setup_package'] = False
+        if self.on_start:
+            config['on_start'] = self.on_start
+
+        try:
+            try:
+                mitogen.core.ExternalContext(config).main()
+            except Exception:
+                # TODO: report exception somehow.
+                os._exit(72)
+        finally:
+            # Don't trigger atexit handlers, they were copied from the parent.
+            os._exit(0)
+
+    def _connect_bootstrap(self):
+        # None required.
+        pass

mitogen-0.2.7/mitogen/jail.py

@@ -0,0 +1,65 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+import logging
+
+import mitogen.core
+import mitogen.parent
+
+
+LOG = logging.getLogger(__name__)
+
+
+class Stream(mitogen.parent.Stream):
+    child_is_immediate_subprocess = False
+    create_child_args = {
+        'merge_stdio': True
+    }
+
+    container = None
+    username = None
+    jexec_path = '/usr/sbin/jexec'
+
+    def construct(self, container, jexec_path=None, username=None, **kwargs):
+        super(Stream, self).construct(**kwargs)
+        self.container = container
+        self.username = username
+        if jexec_path:
+            self.jexec_path = jexec_path
+
+    def _get_name(self):
+        return u'jail.' + self.container
+
+    def get_boot_command(self):
+        bits = [self.jexec_path]
+        if self.username:
+            bits += ['-U', self.username]
+        bits += [self.container]
+        return bits + super(Stream, self).get_boot_command()

mitogen-0.2.7/mitogen/kubectl.py

@@ -0,0 +1,65 @@
+# Copyright 2018, Yannig Perre
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+import logging
+
+import mitogen.core
+import mitogen.parent
+
+
+LOG = logging.getLogger(__name__)
+
+
+class Stream(mitogen.parent.Stream):
+    child_is_immediate_subprocess = True
+
+    pod = None
+    kubectl_path = 'kubectl'
+    kubectl_args = None
+
+    # TODO: better way of capturing errors such as "No such container."
+    create_child_args = {
+        'merge_stdio': True
+    }
+
+    def construct(self, pod, kubectl_path=None, kubectl_args=None, **kwargs):
+        super(Stream, self).construct(**kwargs)
+        assert pod
+        self.pod = pod
+        if kubectl_path:
+            self.kubectl_path = kubectl_path
+        self.kubectl_args = kubectl_args or []
+
+    def _get_name(self):
+        return u'kubectl.%s%s' % (self.pod, self.kubectl_args)
+
+    def get_boot_command(self):
+        bits = [self.kubectl_path] + self.kubectl_args + ['exec', '-it', self.pod]
+        return bits + ["--"] + super(Stream, self).get_boot_command()

mitogen-0.2.7/mitogen/lxc.py

@@ -0,0 +1,75 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+import logging
+
+import mitogen.core
+import mitogen.parent
+
+
+LOG = logging.getLogger(__name__)
+
+
+class Stream(mitogen.parent.Stream):
+    child_is_immediate_subprocess = False
+    create_child_args = {
+        # If lxc-attach finds any of stdin, stdout, stderr connected to a TTY,
+        # to prevent input injection it creates a proxy pty, forcing all IO to
+        # be buffered in <4KiB chunks. So ensure stderr is also routed to the
+        # socketpair.
+        'merge_stdio': True
+    }
+
+    container = None
+    lxc_attach_path = 'lxc-attach'
+
+    eof_error_hint = (
+        'Note: many versions of LXC do not report program execution failure '
+        'meaningfully. Please check the host logs (/var/log) for more '
+        'information.'
+    )
+
+    def construct(self, container, lxc_attach_path=None, **kwargs):
+        super(Stream, self).construct(**kwargs)
+        self.container = container
+        if lxc_attach_path:
+            self.lxc_attach_path = lxc_attach_path
+
+    def _get_name(self):
+        return u'lxc.' + self.container
+
+    def get_boot_command(self):
+        bits = [
+            self.lxc_attach_path,
+            '--clear-env',
+            '--name', self.container,
+            '--',
+        ]
+        return bits + super(Stream, self).get_boot_command()

mitogen-0.2.7/mitogen/lxd.py

@@ -0,0 +1,77 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+import logging
+
+import mitogen.core
+import mitogen.parent
+
+
+LOG = logging.getLogger(__name__)
+
+
+class Stream(mitogen.parent.Stream):
+    child_is_immediate_subprocess = False
+    create_child_args = {
+        # If lxc finds any of stdin, stdout, stderr connected to a TTY, to
+        # prevent input injection it creates a proxy pty, forcing all IO to be
+        # buffered in <4KiB chunks. So ensure stderr is also routed to the
+        # socketpair.
+        'merge_stdio': True
+    }
+
+    container = None
+    lxc_path = 'lxc'
+    python_path = 'python'
+
+    eof_error_hint = (
+        'Note: many versions of LXC do not report program execution failure '
+        'meaningfully. Please check the host logs (/var/log) for more '
+        'information.'
+    )
+
+    def construct(self, container, lxc_path=None, **kwargs):
+        super(Stream, self).construct(**kwargs)
+        self.container = container
+        if lxc_path:
+            self.lxc_path = lxc_path
+
+    def _get_name(self):
+        return u'lxd.' + self.container
+
+    def get_boot_command(self):
+        bits = [
+            self.lxc_path,
+            'exec',
+            '--mode=noninteractive',
+            self.container,
+            '--',
+        ]
+        return bits + super(Stream, self).get_boot_command()

mitogen-0.2.7/mitogen/minify.py

@@ -0,0 +1,139 @@
+# Copyright 2017, Alex Willmer
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+import sys
+
+try:
+    from io import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+import mitogen.core
+
+if sys.version_info < (2, 7, 11):
+    from mitogen.compat import tokenize
+else:
+    import tokenize
+
+
+def minimize_source(source):
+    """Remove comments and docstrings from Python `source`, preserving line
+    numbers and syntax of empty blocks.
+
+    :param str source:
+        The source to minimize.
+
+    :returns str:
+        The minimized source.
+    """
+    source = mitogen.core.to_text(source)
+    tokens = tokenize.generate_tokens(StringIO(source).readline)
+    tokens = strip_comments(tokens)
+    tokens = strip_docstrings(tokens)
+    tokens = reindent(tokens)
+    return tokenize.untokenize(tokens)
+
+
+def strip_comments(tokens):
+    """Drop comment tokens from a `tokenize` stream.
+
+    Comments on lines 1-2 are kept, to preserve hashbang and encoding.
+    Trailing whitespace is remove from all lines.
+    """
+    prev_typ = None
+    prev_end_col = 0
+    for typ, tok, (start_row, start_col), (end_row, end_col), line in tokens:
+        if typ in (tokenize.NL, tokenize.NEWLINE):
+            if prev_typ in (tokenize.NL, tokenize.NEWLINE):
+                start_col = 0
+            else:
+                start_col = prev_end_col
+            end_col = start_col + 1
+        elif typ == tokenize.COMMENT and start_row > 2:
+            continue
+        prev_typ = typ
+        prev_end_col = end_col
+        yield typ, tok, (start_row, start_col), (end_row, end_col), line
+
+
+def strip_docstrings(tokens):
+    """Replace docstring tokens with NL tokens in a `tokenize` stream.
+
+    Any STRING token not part of an expression is deemed a docstring.
+    Indented docstrings are not yet recognised.
+    """
+    stack = []
+    state = 'wait_string'
+    for t in tokens:
+        typ = t[0]
+        if state == 'wait_string':
+            if typ in (tokenize.NL, tokenize.COMMENT):
+                yield t
+            elif typ in (tokenize.DEDENT, tokenize.INDENT, tokenize.STRING):
+                stack.append(t)
+            elif typ == tokenize.NEWLINE:
+                stack.append(t)
+                start_line, end_line = stack[0][2][0], stack[-1][3][0]+1
+                for i in range(start_line, end_line):
+                    yield tokenize.NL, '\n', (i, 0), (i,1), '\n'
+                for t in stack:
+                    if t[0] in (tokenize.DEDENT, tokenize.INDENT):
+                        yield t[0], t[1], (i+1, t[2][1]), (i+1, t[3][1]), t[4]
+                del stack[:]
+            else:
+                stack.append(t)
+                for t in stack: yield t
+                del stack[:]
+                state = 'wait_newline'
+        elif state == 'wait_newline':
+            if typ == tokenize.NEWLINE:
+                state = 'wait_string'
+            yield t
+
+
+def reindent(tokens, indent=' '):
+    """Replace existing indentation in a token steam, with `indent`.
+    """
+    old_levels = []
+    old_level = 0
+    new_level = 0
+    for typ, tok, (start_row, start_col), (end_row, end_col), line in tokens:
+        if typ == tokenize.INDENT:
+            old_levels.append(old_level)
+            old_level = len(tok)
+            new_level += 1
+            tok = indent * new_level
+        elif typ == tokenize.DEDENT:
+            old_level = old_levels.pop()
+            new_level -= 1
+        start_col = max(0, start_col - old_level + new_level)
+        if start_row == end_row:
+            end_col = start_col + len(tok)
+        yield typ, tok, (start_row, start_col), (end_row, end_col), line

mitogen-0.2.7/mitogen/os_fork.py

@@ -0,0 +1,183 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+"""
+Support for operating in a mixed threading/forking environment.
+"""
+
+import os
+import socket
+import sys
+import weakref
+
+import mitogen.core
+
+
+# List of weakrefs. On Python 2.4, mitogen.core registers its Broker on this
+# list and mitogen.service registers its Pool too.
+_brokers = weakref.WeakKeyDictionary()
+_pools = weakref.WeakKeyDictionary()
+
+
+def _notice_broker_or_pool(obj):
+    """
+    Used by :mod:`mitogen.core` and :mod:`mitogen.service` to automatically
+    register every broker and pool on Python 2.4/2.5.
+    """
+    if isinstance(obj, mitogen.core.Broker):
+        _brokers[obj] = True
+    else:
+        _pools[obj] = True
+
+
+def wrap_os__fork():
+    corker = Corker(
+        brokers=list(_brokers),
+        pools=list(_pools),
+    )
+    try:
+        corker.cork()
+        return os__fork()
+    finally:
+        corker.uncork()
+
+
+# If Python 2.4/2.5 where threading state is not fixed up, subprocess.Popen()
+# may still deadlock due to the broker thread. In this case, pause os.fork() so
+# that all active threads are paused during fork.
+if sys.version_info < (2, 6):
+    os__fork = os.fork
+    os.fork = wrap_os__fork
+
+
+class Corker(object):
+    """
+    Arrange for :class:`mitogen.core.Broker` and optionally
+    :class:`mitogen.service.Pool` to be temporarily "corked" while fork
+    operations may occur.
+
+    In a mixed threading/forking environment, it is critical no threads are
+    active at the moment of fork, as they could hold mutexes whose state is
+    unrecoverably snapshotted in the locked state in the fork child, causing
+    deadlocks at random future moments.
+
+    To ensure a target thread has all locks dropped, it is made to write a
+    large string to a socket with a small buffer that has :data:`os.O_NONBLOCK`
+    disabled. CPython will drop the GIL and enter the ``write()`` system call,
+    where it will block until the socket buffer is drained, or the write side
+    is closed.
+
+    :class:`mitogen.core.Poller` is used to ensure the thread really has
+    blocked outside any Python locks, by checking if the socket buffer has
+    started to fill.
+
+    Since this necessarily involves posting a message to every existent thread
+    and verifying acknowledgement, it will never be a fast operation.
+
+    This does not yet handle the case of corking being initiated from within a
+    thread that is also a cork target.
+
+    :param brokers:
+        Sequence of :class:`mitogen.core.Broker` instances to cork.
+    :param pools:
+        Sequence of :class:`mitogen.core.Pool` instances to cork.
+    """
+    def __init__(self, brokers=(), pools=()):
+        self.brokers = brokers
+        self.pools = pools
+
+    def _do_cork(self, s, wsock):
+        try:
+            try:
+                while True:
+                    # at least EINTR is possible. Do our best to keep handling
+                    # outside the GIL in this case using sendall().
+                    wsock.sendall(s)
+            except socket.error:
+                pass
+        finally:
+            wsock.close()
+
+    def _cork_one(self, s, obj):
+        """
+        Construct a socketpair, saving one side of it, and passing the other to
+        `obj` to be written to by one of its threads.
+        """
+        rsock, wsock = mitogen.parent.create_socketpair(size=4096)
+        mitogen.core.set_cloexec(rsock.fileno())
+        mitogen.core.set_cloexec(wsock.fileno())
+        mitogen.core.set_block(wsock)  # gevent
+        self._rsocks.append(rsock)
+        obj.defer(self._do_cork, s, wsock)
+
+    def _verify_one(self, rsock):
+        """
+        Pause until the socket `rsock` indicates readability, due to
+        :meth:`_do_cork` triggering a blocking write on another thread.
+        """
+        poller = mitogen.core.Poller()
+        poller.start_receive(rsock.fileno())
+        try:
+            while True:
+                for fd in poller.poll():
+                    return
+        finally:
+            poller.close()
+
+    def cork(self):
+        """
+        Arrange for any associated brokers and pools to be paused with no locks
+        held. This will not return until each thread acknowledges it has ceased
+        execution.
+        """
+        s = mitogen.core.b('CORK') * ((128 // 4) * 1024)
+        self._rsocks = []
+
+        # Pools must be paused first, as existing work may require the
+        # participation of a broker in order to complete.
+        for pool in self.pools:
+            if not pool.closed:
+                for x in range(pool.size):
+                    self._cork_one(s, pool)
+
+        for broker in self.brokers:
+            if broker._alive:
+                self._cork_one(s, broker)
+
+        # Pause until we can detect every thread has entered write().
+        for rsock in self._rsocks:
+            self._verify_one(rsock)
+
+    def uncork(self):
+        """
+        Arrange for paused threads to resume operation.
+        """
+        for rsock in self._rsocks:
+            rsock.close()

mitogen-0.2.7/mitogen/profiler.py

@@ -0,0 +1,166 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+"""mitogen.profiler
+    Record and report cProfile statistics from a run. Creates one aggregated
+    output file, one aggregate containing only workers, and one for the
+    top-level process.
+
+Usage:
+    mitogen.profiler record <dest_path> <tool> [args ..]
+    mitogen.profiler report <dest_path> [sort_mode]
+    mitogen.profiler stat <sort_mode> <tool> [args ..]
+
+Mode:
+    record: Record a trace.
+    report: Report on a previously recorded trace.
+    stat: Record and report in a single step.
+
+Where:
+    dest_path: Filesystem prefix to write .pstats files to.
+    sort_mode: Sorting mode; defaults to "cumulative". See:
+        https://docs.python.org/2/library/profile.html#pstats.Stats.sort_stats
+
+Example:
+    mitogen.profiler record /tmp/mypatch ansible-playbook foo.yml
+    mitogen.profiler dump /tmp/mypatch-worker.pstats
+"""
+
+from __future__ import print_function
+import os
+import pstats
+import cProfile
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+
+import mitogen.core
+
+
+def try_merge(stats, path):
+    try:
+        stats.add(path)
+        return True
+    except Exception as e:
+        print('Failed. Race? Will retry. %s' % (e,))
+        return False
+
+
+def merge_stats(outpath, inpaths):
+    first, rest = inpaths[0], inpaths[1:]
+    for x in range(5):
+        try:
+            stats = pstats.Stats(first)
+        except EOFError:
+            time.sleep(0.2)
+            continue
+
+        print("Writing %r..." % (outpath,))
+        for path in rest:
+            #print("Merging %r into %r.." % (os.path.basename(path), outpath))
+            for x in range(5):
+                if try_merge(stats, path):
+                    break
+                time.sleep(0.2)
+
+    stats.dump_stats(outpath)
+
+
+def generate_stats(outpath, tmpdir):
+    print('Generating stats..')
+    all_paths = []
+    paths_by_ident = {}
+
+    for name in os.listdir(tmpdir):
+        if name.endswith('-dump.pstats'):
+            ident, _, pid = name.partition('-')
+            path = os.path.join(tmpdir, name)
+            all_paths.append(path)
+            paths_by_ident.setdefault(ident, []).append(path)
+
+    merge_stats('%s-all.pstat' % (outpath,), all_paths)
+    for ident, paths in paths_by_ident.items():
+        merge_stats('%s-%s.pstat' % (outpath, ident), paths)
+
+
+def do_record(tmpdir, path, *args):
+    env = os.environ.copy()
+    fmt = '%(identity)s-%(pid)s.%(now)s-dump.%(ext)s'
+    env['MITOGEN_PROFILING'] = '1'
+    env['MITOGEN_PROFILE_FMT'] = os.path.join(tmpdir, fmt)
+    rc = subprocess.call(args, env=env)
+    generate_stats(path, tmpdir)
+    return rc
+
+
+def do_report(tmpdir, path, sort='cumulative'):
+    stats = pstats.Stats(path).sort_stats(sort)
+    stats.print_stats(100)
+
+
+def do_stat(tmpdir, sort, *args):
+    valid_sorts = pstats.Stats.sort_arg_dict_default
+    if sort not in valid_sorts:
+        sys.stderr.write('Invalid sort %r, must be one of %s\n' %
+                         (sort, ', '.join(sorted(valid_sorts))))
+        sys.exit(1)
+
+    outfile = os.path.join(tmpdir, 'combined')
+    do_record(tmpdir, outfile, *args)
+    aggs = ('app.main', 'mitogen.broker', 'mitogen.child_main',
+            'mitogen.service.pool', 'Strategy', 'WorkerProcess',
+            'all')
+    for agg in aggs:
+        path = '%s-%s.pstat' % (outfile, agg)
+        if os.path.exists(path):
+            print()
+            print()
+            print('------ Aggregation %r ------' % (agg,))
+            print()
+            do_report(tmpdir, path, sort)
+            print()
+
+
+def main():
+    if len(sys.argv) < 2 or sys.argv[1] not in ('record', 'report', 'stat'):
+        sys.stderr.write(__doc__)
+        sys.exit(1)
+
+    func = globals()['do_' + sys.argv[1]]
+    tmpdir = tempfile.mkdtemp(prefix='mitogen.profiler')
+    try:
+        sys.exit(func(tmpdir, *sys.argv[2:]) or 0)
+    finally:
+        shutil.rmtree(tmpdir)
+
+if __name__ == '__main__':
+    main()

mitogen-0.2.7/mitogen/select.py

@@ -0,0 +1,333 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+import mitogen.core
+
+
+class Error(mitogen.core.Error):
+    pass
+
+
+class Event(object):
+    """
+    Represents one selected event.
+    """
+    #: The first Receiver or Latch the event traversed.
+    source = None
+
+    #: The :class:`mitogen.core.Message` delivered to a receiver, or the object
+    #: posted to a latch.
+    data = None
+
+
+class Select(object):
+    """
+    Support scatter/gather asynchronous calls and waiting on multiple
+    :class:`receivers <mitogen.core.Receiver>`,
+    :class:`channels <mitogen.core.Channel>`,
+    :class:`latches <mitogen.core.Latch>`, and
+    :class:`sub-selects <Select>`.
+
+    If `oneshot` is :data:`True`, then remove each receiver as it yields a
+    result; since :meth:`__iter__` terminates once the final receiver is
+    removed, this makes it convenient to respond to calls made in parallel:
+
+    .. code-block:: python
+
+        total = 0
+        recvs = [c.call_async(long_running_operation) for c in contexts]
+
+        for msg in mitogen.select.Select(recvs):
+            print('Got %s from %s' % (msg, msg.receiver))
+            total += msg.unpickle()
+
+        # Iteration ends when last Receiver yields a result.
+        print('Received total %s from %s receivers' % (total, len(recvs)))
+
+    :class:`Select` may drive a long-running scheduler:
+
+    .. code-block:: python
+
+        with mitogen.select.Select(oneshot=False) as select:
+            while running():
+                for msg in select:
+                    process_result(msg.receiver.context, msg.unpickle())
+                for context, workfunc in get_new_work():
+                    select.add(context.call_async(workfunc))
+
+    :class:`Select` may be nested:
+
+    .. code-block:: python
+
+        subselects = [
+            mitogen.select.Select(get_some_work()),
+            mitogen.select.Select(get_some_work()),
+            mitogen.select.Select([
+                mitogen.select.Select(get_some_work()),
+                mitogen.select.Select(get_some_work())
+            ])
+        ]
+
+        for msg in mitogen.select.Select(selects):
+            print(msg.unpickle())
+
+    :class:`Select` may be used to mix inter-thread and inter-process IO:
+
+        latch = mitogen.core.Latch()
+        start_thread(latch)
+        recv = remote_host.call_async(os.getuid)
+
+        sel = Select([latch, recv])
+        event = sel.get_event()
+        if event.source is latch:
+            # woken by a local thread
+        else:
+            # woken by function call result
+    """
+
+    notify = None
+
+    def __init__(self, receivers=(), oneshot=True):
+        self._receivers = []
+        self._oneshot = oneshot
+        self._latch = mitogen.core.Latch()
+        for recv in receivers:
+            self.add(recv)
+
+    @classmethod
+    def all(cls, receivers):
+        """
+        Take an iterable of receivers and retrieve a :class:`Message` from
+        each, returning the result of calling `msg.unpickle()` on each in turn.
+        Results are returned in the order they arrived.
+
+        This is sugar for handling batch :meth:`Context.call_async
+        <mitogen.parent.Context.call_async>` invocations:
+
+        .. code-block:: python
+
+            print('Total disk usage: %.02fMiB' % (sum(
+                mitogen.select.Select.all(
+                    context.call_async(get_disk_usage)
+                    for context in contexts
+                ) / 1048576.0
+            ),))
+
+        However, unlike in a naive comprehension such as:
+
+        .. code-block:: python
+
+            recvs = [c.call_async(get_disk_usage) for c in contexts]
+            sum(recv.get().unpickle() for recv in recvs)
+
+        Result processing happens in the order results arrive, rather than the
+        order requests were issued, so :meth:`all` should always be faster.
+        """
+        return list(msg.unpickle() for msg in cls(receivers))
+
+    def _put(self, value):
+        self._latch.put(value)
+        if self.notify:
+            self.notify(self)
+
+    def __bool__(self):
+        """
+        Return :data:`True` if any receivers are registered with this select.
+        """
+        return bool(self._receivers)
+
+    __nonzero__ = __bool__
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, e_type, e_val, e_tb):
+        self.close()
+
+    def iter_data(self):
+        """
+        Yield :attr:`Event.data` until no receivers remain in the select,
+        either because `oneshot` is :data:`True`, or each receiver was
+        explicitly removed via :meth:`remove`.
+
+        :meth:`__iter__` is an alias for :meth:`iter_data`, allowing loops
+        like::
+
+            for msg in Select([recv1, recv2]):
+                print msg.unpickle()
+        """
+        while self._receivers:
+            yield self.get_event().data
+
+    __iter__ = iter_data
+
+    def iter_events(self):
+        """
+        Yield :class:`Event` instances until no receivers remain in the select.
+        """
+        while self._receivers:
+            yield self.get_event()
+
+    loop_msg = 'Adding this Select instance would create a Select cycle'
+
+    def _check_no_loop(self, recv):
+        if recv is self:
+            raise Error(self.loop_msg)
+
+        for recv_ in self._receivers:
+            if recv_ == recv:
+                raise Error(self.loop_msg)
+            if isinstance(recv_, Select):
+                recv_._check_no_loop(recv)
+
+    owned_msg = 'Cannot add: Receiver is already owned by another Select'
+
+    def add(self, recv):
+        """
+        Add a :class:`mitogen.core.Receiver`, :class:`Select` or
+        :class:`mitogen.core.Latch` to the select.
+
+        :raises mitogen.select.Error:
+            An attempt was made to add a :class:`Select` to which this select
+            is indirectly a member of.
+        """
+        if isinstance(recv, Select):
+            recv._check_no_loop(self)
+
+        self._receivers.append(recv)
+        if recv.notify is not None:
+            raise Error(self.owned_msg)
+
+        recv.notify = self._put
+        # Avoid race by polling once after installation.
+        if not recv.empty():
+            self._put(recv)
+
+    not_present_msg = 'Instance is not a member of this Select'
+
+    def remove(self, recv):
+        """
+        Remove an object from from the select. Note that if the receiver has
+        notified prior to :meth:`remove`, it will still be returned by a
+        subsequent :meth:`get`. This may change in a future version.
+        """
+        try:
+            if recv.notify != self._put:
+                raise ValueError
+            self._receivers.remove(recv)
+            recv.notify = None
+        except (IndexError, ValueError):
+            raise Error(self.not_present_msg)
+
+    def close(self):
+        """
+        Remove the select's notifier function from each registered receiver,
+        mark the associated latch as closed, and cause any thread currently
+        sleeping in :meth:`get` to be woken with
+        :class:`mitogen.core.LatchError`.
+
+        This is necessary to prevent memory leaks in long-running receivers. It
+        is called automatically when the Python :keyword:`with` statement is
+        used.
+        """
+        for recv in self._receivers[:]:
+            self.remove(recv)
+        self._latch.close()
+
+    def empty(self):
+        """
+        Return :data:`True` if calling :meth:`get` would block.
+
+        As with :class:`Queue.Queue`, :data:`True` may be returned even though
+        a subsequent call to :meth:`get` will succeed, since a message may be
+        posted at any moment between :meth:`empty` and :meth:`get`.
+
+        :meth:`empty` may return :data:`False` even when :meth:`get` would
+        block if another thread has drained a receiver added to this select.
+        This can be avoided by only consuming each receiver from a single
+        thread.
+        """
+        return self._latch.empty()
+
+    empty_msg = 'Cannot get(), Select instance is empty'
+
+    def get(self, timeout=None, block=True):
+        """
+        Call `get_event(timeout, block)` returning :attr:`Event.data` of the
+        first available event.
+        """
+        return self.get_event(timeout, block).data
+
+    def get_event(self, timeout=None, block=True):
+        """
+        Fetch the next available :class:`Event` from any source, or raise
+        :class:`mitogen.core.TimeoutError` if no value is available within
+        `timeout` seconds.
+
+        On success, the message's :attr:`receiver
+        <mitogen.core.Message.receiver>` attribute is set to the receiver.
+
+        :param float timeout:
+            Timeout in seconds.
+        :param bool block:
+            If :data:`False`, immediately raise
+            :class:`mitogen.core.TimeoutError` if the select is empty.
+        :return:
+            :class:`Event`.
+        :raises mitogen.core.TimeoutError:
+            Timeout was reached.
+        :raises mitogen.core.LatchError:
+            :meth:`close` has been called, and the underlying latch is no
+            longer valid.
+        """
+        if not self._receivers:
+            raise Error(self.empty_msg)
+
+        event = Event()
+        while True:
+            recv = self._latch.get(timeout=timeout, block=block)
+            try:
+                if isinstance(recv, Select):
+                    event = recv.get_event(block=False)
+                else:
+                    event.source = recv
+                    event.data = recv.get(block=False)
+                if self._oneshot:
+                    self.remove(recv)
+                if isinstance(recv, mitogen.core.Receiver):
+                    # Remove in 0.3.x.
+                    event.data.receiver = recv
+                return event
+            except mitogen.core.TimeoutError:
+                # A receiver may have been queued with no result if another
+                # thread drained it before we woke up, or because another
+                # thread drained it between add() calling recv.empty() and
+                # self._put(). In this case just sleep again.
+                continue

mitogen-0.2.7/mitogen/setns.py

@@ -0,0 +1,238 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+import ctypes
+import grp
+import logging
+import os
+import pwd
+import subprocess
+import sys
+
+import mitogen.core
+import mitogen.parent
+
+
+LOG = logging.getLogger(__name__)
+LIBC = ctypes.CDLL(None, use_errno=True)
+LIBC__strerror = LIBC.strerror
+LIBC__strerror.restype = ctypes.c_char_p
+
+
+class Error(mitogen.core.StreamError):
+    pass
+
+
+def setns(kind, fd):
+    if LIBC.setns(int(fd), 0) == -1:
+        errno = ctypes.get_errno()
+        msg = 'setns(%s, %s): %s' % (fd, kind, LIBC__strerror(errno))
+        raise OSError(errno, msg)
+
+
+def _run_command(args):
+    argv = mitogen.parent.Argv(args)
+    try:
+        proc = subprocess.Popen(
+            args=args,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT
+        )
+    except OSError:
+        e = sys.exc_info()[1]
+        raise Error('could not execute %s: %s', argv, e)
+
+    output, _ = proc.communicate()
+    if not proc.returncode:
+        return output.decode('utf-8', 'replace')
+
+    raise Error("%s exitted with status %d: %s",
+                mitogen.parent.Argv(args), proc.returncode, output)
+
+
+def get_docker_pid(path, name):
+    args = [path, 'inspect', '--format={{.State.Pid}}', name]
+    output = _run_command(args)
+    try:
+        return int(output)
+    except ValueError:
+        raise Error("could not find PID from docker output.\n%s", output)
+
+
+def get_lxc_pid(path, name):
+    output = _run_command([path, '-n', name])
+    for line in output.splitlines():
+        bits = line.split()
+        if bits and bits[0] == 'PID:':
+            return int(bits[1])
+
+    raise Error("could not find PID from lxc-info output.\n%s", output)
+
+
+def get_lxd_pid(path, name):
+    output = _run_command([path, 'info', name])
+    for line in output.splitlines():
+        bits = line.split()
+        if bits and bits[0] == 'Pid:':
+            return int(bits[1])
+
+    raise Error("could not find PID from lxc output.\n%s", output)
+
+
+def get_machinectl_pid(path, name):
+    output = _run_command([path, 'status', name])
+    for line in output.splitlines():
+        bits = line.split()
+        if bits and bits[0] == 'Leader:':
+            return int(bits[1])
+
+    raise Error("could not find PID from machinectl output.\n%s", output)
+
+
+class Stream(mitogen.parent.Stream):
+    child_is_immediate_subprocess = False
+
+    container = None
+    username = 'root'
+    kind = None
+    python_path = 'python'
+    docker_path = 'docker'
+    lxc_path = 'lxc'
+    lxc_info_path = 'lxc-info'
+    machinectl_path = 'machinectl'
+
+    GET_LEADER_BY_KIND = {
+        'docker': ('docker_path', get_docker_pid),
+        'lxc': ('lxc_info_path', get_lxc_pid),
+        'lxd': ('lxc_path', get_lxd_pid),
+        'machinectl': ('machinectl_path', get_machinectl_pid),
+    }
+
+    def construct(self, container, kind, username=None, docker_path=None,
+                  lxc_path=None, lxc_info_path=None, machinectl_path=None,
+                  **kwargs):
+        super(Stream, self).construct(**kwargs)
+        if kind not in self.GET_LEADER_BY_KIND:
+            raise Error('unsupported container kind: %r', kind)
+
+        self.container = container
+        self.kind = kind
+        if username:
+            self.username = username
+        if docker_path:
+            self.docker_path = docker_path
+        if lxc_path:
+            self.lxc_path = lxc_path
+        if lxc_info_path:
+            self.lxc_info_path = lxc_info_path
+        if machinectl_path:
+            self.machinectl_path = machinectl_path
+
+    # Order matters. https://github.com/karelzak/util-linux/commit/854d0fe/
+    NS_ORDER = ('ipc', 'uts', 'net', 'pid', 'mnt', 'user')
+
+    def preexec_fn(self):
+        nspath = '/proc/%d/ns/' % (self.leader_pid,)
+        selfpath = '/proc/self/ns/'
+        try:
+            ns_fps = [
+                open(nspath + name)
+                for name in self.NS_ORDER
+                if os.path.exists(nspath + name) and (
+                    os.readlink(nspath + name) != os.readlink(selfpath + name)
+                )
+            ]
+        except Exception:
+            e = sys.exc_info()[1]
+            raise Error(str(e))
+
+        os.chdir('/proc/%s/root' % (self.leader_pid,))
+        os.chroot('.')
+        os.chdir('/')
+        for fp in ns_fps:
+            setns(fp.name, fp.fileno())
+            fp.close()
+
+        for sym in 'endpwent', 'endgrent', 'endspent', 'endsgent':
+            try:
+                getattr(LIBC, sym)()
+            except AttributeError:
+                pass
+
+        try:
+            os.setgroups([grent.gr_gid
+                          for grent in grp.getgrall()
+                          if self.username in grent.gr_mem])
+            pwent = pwd.getpwnam(self.username)
+            os.setreuid(pwent.pw_uid, pwent.pw_uid)
+            # shadow-4.4/libmisc/setupenv.c. Not done: MAIL, PATH
+            os.environ.update({
+                'HOME': pwent.pw_dir,
+                'SHELL': pwent.pw_shell or '/bin/sh',
+                'LOGNAME': self.username,
+                'USER': self.username,
+            })
+            if ((os.path.exists(pwent.pw_dir) and
+                 os.access(pwent.pw_dir, os.X_OK))):
+                os.chdir(pwent.pw_dir)
+        except Exception:
+            e = sys.exc_info()[1]
+            raise Error(self.username_msg, self.username, self.container,
+                        type(e).__name__, e)
+
+    username_msg = 'while transitioning to user %r in container %r: %s: %s'
+
+    def get_boot_command(self):
+        # With setns(CLONE_NEWPID), new children of the caller receive a new
+        # PID namespace, however the caller's namespace won't change. That
+        # causes subsequent calls to clone() specifying CLONE_THREAD to fail
+        # with EINVAL, as threads in the same process can't have varying PID
+        # namespaces, meaning starting new threads in the exec'd program will
+        # fail. The solution is forking, so inject a /bin/sh call to achieve
+        # this.
+        argv = super(Stream, self).get_boot_command()
+        # bash will exec() if a single command was specified and the shell has
+        # nothing left to do, so "; exit $?" gives bash a reason to live.
+        return ['/bin/sh', '-c', '%s; exit $?' % (mitogen.parent.Argv(argv),)]
+
+    def create_child(self, args):
+        return mitogen.parent.create_child(args, preexec_fn=self.preexec_fn)
+
+    def _get_name(self):
+        return u'setns.' + self.container
+
+    def connect(self):
+        self.name = self._get_name()
+        attr, func = self.GET_LEADER_BY_KIND[self.kind]
+        tool_path = getattr(self, attr)
+        self.leader_pid = func(tool_path, self.container)
+        LOG.debug('Leader PID for %s container %r: %d',
+                  self.kind, self.container, self.leader_pid)
+        super(Stream, self).connect()

mitogen-0.2.7/mitogen/ssh.py

@@ -0,0 +1,317 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+"""
+Functionality to allow establishing new slave contexts over an SSH connection.
+"""
+
+import logging
+import re
+
+try:
+    from shlex import quote as shlex_quote
+except ImportError:
+    from pipes import quote as shlex_quote
+
+import mitogen.parent
+from mitogen.core import b
+from mitogen.core import bytes_partition
+
+try:
+    any
+except NameError:
+    from mitogen.core import any
+
+
+LOG = logging.getLogger('mitogen')
+
+# sshpass uses 'assword' because it doesn't lowercase the input.
+PASSWORD_PROMPT = b('password')
+HOSTKEY_REQ_PROMPT = b('are you sure you want to continue connecting (yes/no)?')
+HOSTKEY_FAIL = b('host key verification failed.')
+
+# [user@host: ] permission denied
+PERMDENIED_RE = re.compile(
+    ('(?:[^@]+@[^:]+: )?'  # Absent in OpenSSH <7.5
+     'Permission denied').encode(),
+    re.I
+)
+
+
+DEBUG_PREFIXES = (b('debug1:'), b('debug2:'), b('debug3:'))
+
+
+def filter_debug(stream, it):
+    """
+    Read line chunks from it, either yielding them directly, or building up and
+    logging individual lines if they look like SSH debug output.
+
+    This contains the mess of dealing with both line-oriented input, and partial
+    lines such as the password prompt.
+
+    Yields `(line, partial)` tuples, where `line` is the line, `partial` is
+    :data:`True` if no terminating newline character was present and no more
+    data exists in the read buffer. Consuming code can use this to unreliably
+    detect the presence of an interactive prompt.
+    """
+    # The `partial` test is unreliable, but is only problematic when verbosity
+    # is enabled: it's possible for a combination of SSH banner, password
+    # prompt, verbose output, timing and OS buffering specifics to create a
+    # situation where an otherwise newline-terminated line appears to not be
+    # terminated, due to a partial read(). If something is broken when
+    # ssh_debug_level>0, this is the first place to look.
+    state = 'start_of_line'
+    buf = b('')
+    for chunk in it:
+        buf += chunk
+        while buf:
+            if state == 'start_of_line':
+                if len(buf) < 8:
+                    # short read near buffer limit, block awaiting at least 8
+                    # bytes so we can discern a debug line, or the minimum
+                    # interesting token from above or the bootstrap
+                    # ('password', 'MITO000\n').
+                    break
+                elif any(buf.startswith(p) for p in DEBUG_PREFIXES):
+                    state = 'in_debug'
+                else:
+                    state = 'in_plain'
+            elif state == 'in_debug':
+                if b('\n') not in buf:
+                    break
+                line, _, buf = bytes_partition(buf, b('\n'))
+                LOG.debug('%s: %s', stream.name,
+                          mitogen.core.to_text(line.rstrip()))
+                state = 'start_of_line'
+            elif state == 'in_plain':
+                line, nl, buf = bytes_partition(buf, b('\n'))
+                yield line + nl, not (nl or buf)
+                if nl:
+                    state = 'start_of_line'
+
+
+class PasswordError(mitogen.core.StreamError):
+    pass
+
+
+class HostKeyError(mitogen.core.StreamError):
+    pass
+
+
+class Stream(mitogen.parent.Stream):
+    child_is_immediate_subprocess = False
+
+    #: Default to whatever is available as 'python' on the remote machine,
+    #: overriding sys.executable use.
+    python_path = 'python'
+
+    #: Number of -v invocations to pass on command line.
+    ssh_debug_level = 0
+
+    #: The path to the SSH binary.
+    ssh_path = 'ssh'
+
+    hostname = None
+    username = None
+    port = None
+
+    identity_file = None
+    password = None
+    ssh_args = None
+
+    check_host_keys_msg = 'check_host_keys= must be set to accept, enforce or ignore'
+
+    def construct(self, hostname, username=None, ssh_path=None, port=None,
+                  check_host_keys='enforce', password=None, identity_file=None,
+                  compression=True, ssh_args=None, keepalive_enabled=True,
+                  keepalive_count=3, keepalive_interval=15,
+                  identities_only=True, ssh_debug_level=None, **kwargs):
+        super(Stream, self).construct(**kwargs)
+        if check_host_keys not in ('accept', 'enforce', 'ignore'):
+            raise ValueError(self.check_host_keys_msg)
+
+        self.hostname = hostname
+        self.username = username
+        self.port = port
+        self.check_host_keys = check_host_keys
+        self.password = password
+        self.identity_file = identity_file
+        self.identities_only = identities_only
+        self.compression = compression
+        self.keepalive_enabled = keepalive_enabled
+        self.keepalive_count = keepalive_count
+        self.keepalive_interval = keepalive_interval
+        if ssh_path:
+            self.ssh_path = ssh_path
+        if ssh_args:
+            self.ssh_args = ssh_args
+        if ssh_debug_level:
+            self.ssh_debug_level = ssh_debug_level
+
+        self._init_create_child()
+
+    def _requires_pty(self):
+        """
+        Return :data:`True` if the configuration requires a PTY to be
+        allocated. This is only true if we must interactively accept host keys,
+        or type a password.
+        """
+        return (self.check_host_keys == 'accept' or
+                self.password is not None)
+
+    def _init_create_child(self):
+        """
+        Initialize the base class :attr:`create_child` and
+        :attr:`create_child_args` according to whether we need a PTY or not.
+        """
+        if self._requires_pty():
+            self.create_child = mitogen.parent.hybrid_tty_create_child
+        else:
+            self.create_child = mitogen.parent.create_child
+            self.create_child_args = {
+                'stderr_pipe': True,
+            }
+
+    def get_boot_command(self):
+        bits = [self.ssh_path]
+        if self.ssh_debug_level:
+            bits += ['-' + ('v' * min(3, self.ssh_debug_level))]
+        else:
+            # issue #307: suppress any login banner, as it may contain the
+            # password prompt, and there is no robust way to tell the
+            # difference.
+            bits += ['-o', 'LogLevel ERROR']
+        if self.username:
+            bits += ['-l', self.username]
+        if self.port is not None:
+            bits += ['-p', str(self.port)]
+        if self.identities_only and (self.identity_file or self.password):
+            bits += ['-o', 'IdentitiesOnly yes']
+        if self.identity_file:
+            bits += ['-i', self.identity_file]
+        if self.compression:
+            bits += ['-o', 'Compression yes']
+        if self.keepalive_enabled:
+            bits += [
+                '-o', 'ServerAliveInterval %s' % (self.keepalive_interval,),
+                '-o', 'ServerAliveCountMax %s' % (self.keepalive_count,),
+            ]
+        if not self._requires_pty():
+            bits += ['-o', 'BatchMode yes']
+        if self.check_host_keys == 'enforce':
+            bits += ['-o', 'StrictHostKeyChecking yes']
+        if self.check_host_keys == 'accept':
+            bits += ['-o', 'StrictHostKeyChecking ask']
+        elif self.check_host_keys == 'ignore':
+            bits += [
+                '-o', 'StrictHostKeyChecking no',
+                '-o', 'UserKnownHostsFile /dev/null',
+                '-o', 'GlobalKnownHostsFile /dev/null',
+            ]
+        if self.ssh_args:
+            bits += self.ssh_args
+        bits.append(self.hostname)
+        base = super(Stream, self).get_boot_command()
+        return bits + [shlex_quote(s).strip() for s in base]
+
+    def _get_name(self):
+        s = u'ssh.' + mitogen.core.to_text(self.hostname)
+        if self.port:
+            s += u':%s' % (self.port,)
+        return s
+
+    auth_incorrect_msg = 'SSH authentication is incorrect'
+    password_incorrect_msg = 'SSH password is incorrect'
+    password_required_msg = 'SSH password was requested, but none specified'
+    hostkey_config_msg = (
+        'SSH requested permission to accept unknown host key, but '
+        'check_host_keys=ignore. This is likely due to ssh_args=  '
+        'conflicting with check_host_keys=. Please correct your '
+        'configuration.'
+    )
+    hostkey_failed_msg = (
+        'Host key checking is enabled, and SSH reported an unrecognized or '
+        'mismatching host key.'
+    )
+
+    def _host_key_prompt(self):
+        if self.check_host_keys == 'accept':
+            LOG.debug('%s: accepting host key', self.name)
+            self.diag_stream.transmit_side.write(b('yes\n'))
+            return
+
+        # _host_key_prompt() should never be reached with ignore or enforce
+        # mode, SSH should have handled that. User's ssh_args= is conflicting
+        # with ours.
+        raise HostKeyError(self.hostkey_config_msg)
+
+    def _connect_input_loop(self, it):
+        password_sent = False
+        for buf, partial in filter_debug(self, it):
+            LOG.debug('%s: stdout: %s', self.name, buf.rstrip())
+            if buf.endswith(self.EC0_MARKER):
+                self._ec0_received()
+                return
+            elif HOSTKEY_REQ_PROMPT in buf.lower():
+                self._host_key_prompt()
+            elif HOSTKEY_FAIL in buf.lower():
+                raise HostKeyError(self.hostkey_failed_msg)
+            elif PERMDENIED_RE.match(buf):
+                # issue #271: work around conflict with user shell reporting
+                # 'permission denied' e.g. during chdir($HOME) by only matching
+                # it at the start of the line.
+                if self.password is not None and password_sent:
+                    raise PasswordError(self.password_incorrect_msg)
+                elif PASSWORD_PROMPT in buf and self.password is None:
+                    # Permission denied (password,pubkey)
+                    raise PasswordError(self.password_required_msg)
+                else:
+                    raise PasswordError(self.auth_incorrect_msg)
+            elif partial and PASSWORD_PROMPT in buf.lower():
+                if self.password is None:
+                    raise PasswordError(self.password_required_msg)
+                LOG.debug('%s: sending password', self.name)
+                self.diag_stream.transmit_side.write(
+                    (self.password + '\n').encode()
+                )
+                password_sent = True
+
+        raise mitogen.core.StreamError('bootstrap failed')
+
+    def _connect_bootstrap(self):
+        fds = [self.receive_side.fd]
+        if self.diag_stream is not None:
+            fds.append(self.diag_stream.receive_side.fd)
+
+        it = mitogen.parent.iter_read(fds=fds, deadline=self.connect_deadline)
+        try:
+            self._connect_input_loop(it)
+        finally:
+            it.close()

mitogen-0.2.7/mitogen/su.py

@@ -0,0 +1,128 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+import logging
+
+import mitogen.core
+import mitogen.parent
+from mitogen.core import b
+
+try:
+    any
+except NameError:
+    from mitogen.core import any
+
+
+LOG = logging.getLogger(__name__)
+
+
+class PasswordError(mitogen.core.StreamError):
+    pass
+
+
+class Stream(mitogen.parent.Stream):
+    # TODO: BSD su cannot handle stdin being a socketpair, but it does let the
+    # child inherit fds from the parent. So we can still pass a socketpair in
+    # for hybrid_tty_create_child(), there just needs to be either a shell
+    # snippet or bootstrap support for fixing things up afterwards.
+    create_child = staticmethod(mitogen.parent.tty_create_child)
+    child_is_immediate_subprocess = False
+
+    #: Once connected, points to the corresponding DiagLogStream, allowing it to
+    #: be disconnected at the same time this stream is being torn down.
+
+    username = 'root'
+    password = None
+    su_path = 'su'
+    password_prompt = b('password:')
+    incorrect_prompts = (
+        b('su: sorry'),                    # BSD
+        b('su: authentication failure'),   # Linux
+        b('su: incorrect password'),       # CentOS 6
+        b('authentication is denied'),     # AIX
+    )
+
+    def construct(self, username=None, password=None, su_path=None,
+                  password_prompt=None, incorrect_prompts=None, **kwargs):
+        super(Stream, self).construct(**kwargs)
+        if username is not None:
+            self.username = username
+        if password is not None:
+            self.password = password
+        if su_path is not None:
+            self.su_path = su_path
+        if password_prompt is not None:
+            self.password_prompt = password_prompt.lower()
+        if incorrect_prompts is not None:
+            self.incorrect_prompts = map(str.lower, incorrect_prompts)
+
+    def _get_name(self):
+        return u'su.' + mitogen.core.to_text(self.username)
+
+    def get_boot_command(self):
+        argv = mitogen.parent.Argv(super(Stream, self).get_boot_command())
+        return [self.su_path, self.username, '-c', str(argv)]
+
+    password_incorrect_msg = 'su password is incorrect'
+    password_required_msg = 'su password is required'
+
+    def _connect_input_loop(self, it):
+        password_sent = False
+
+        for buf in it:
+            LOG.debug('%r: received %r', self, buf)
+            if buf.endswith(self.EC0_MARKER):
+                self._ec0_received()
+                return
+            if any(s in buf.lower() for s in self.incorrect_prompts):
+                if password_sent:
+                    raise PasswordError(self.password_incorrect_msg)
+            elif self.password_prompt in buf.lower():
+                if self.password is None:
+                    raise PasswordError(self.password_required_msg)
+                if password_sent:
+                    raise PasswordError(self.password_incorrect_msg)
+                LOG.debug('sending password')
+                self.transmit_side.write(
+                    mitogen.core.to_text(self.password + '\n').encode('utf-8')
+                )
+                password_sent = True
+
+        raise mitogen.core.StreamError('bootstrap failed')
+
+    def _connect_bootstrap(self):
+        it = mitogen.parent.iter_read(
+            fds=[self.receive_side.fd],
+            deadline=self.connect_deadline,
+        )
+        try:
+            self._connect_input_loop(it)
+        finally:
+            it.close()

mitogen-0.2.7/mitogen/sudo.py

@@ -0,0 +1,277 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+import base64
+import logging
+import optparse
+import re
+
+import mitogen.core
+import mitogen.parent
+from mitogen.core import b
+
+
+LOG = logging.getLogger(__name__)
+
+# These are base64-encoded UTF-8 as our existing minifier/module server
+# struggles with Unicode Python source in some (forgotten) circumstances.
+PASSWORD_PROMPTS = [
+    'cGFzc3dvcmQ=',                                              # english
+    'bG96aW5rYQ==',                                              # sr@latin.po
+    '44OR44K544Ov44O844OJ',                                      # ja.po
+    '4Kaq4Ka+4Ka44KaT4Kef4Ka+4Kaw4KeN4Kah',                      # bn.po
+    '2YPZhNmF2Kkg2KfZhNiz2LE=',                                  # ar.po
+    'cGFzYWhpdHph',                                              # eu.po
+    '0L/QsNGA0L7Qu9GM',                                          # uk.po
+    'cGFyb29s',                                                  # et.po
+    'c2FsYXNhbmE=',                                              # fi.po
+    '4Kiq4Ki+4Ki44Ki14Kiw4Kih',                                  # pa.po
+    'Y29udHJhc2lnbm8=',                                          # ia.po
+    'Zm9jYWwgZmFpcmU=',                                          # ga.po
+    '16HXodee15Q=',                                              # he.po
+    '4Kqq4Kq+4Kq44Kq14Kqw4KuN4Kqh',                              # gu.po
+    '0L/QsNGA0L7Qu9Cw',                                          # bg.po
+    '4Kyq4K2N4Kyw4Kys4K2H4Ky2IOCsuOCsmeCtjeCsleCth+CspA==',      # or.po
+    '4K6V4K6f4K614K+B4K6a4K+N4K6a4K+K4K6y4K+N',                  # ta.po
+    'cGFzc3dvcnQ=',                                              # de.po
+    '7JWU7Zi4',                                                  # ko.po
+    '0LvQvtC30LjQvdC60LA=',                                      # sr.po
+    'beG6rXQga2jhuql1',                                          # vi.po
+    'c2VuaGE=',                                                  # pt_BR.po
+    'cGFzc3dvcmQ=',                                              # it.po
+    'aGVzbG8=',                                                  # cs.po
+    '5a+G56K877ya',                                              # zh_TW.po
+    'aGVzbG8=',                                                  # sk.po
+    '4LC44LCC4LCV4LGH4LCk4LCq4LCm4LCu4LGB',                      # te.po
+    '0L/QsNGA0L7Qu9GM',                                          # kk.po
+    'aGFzxYJv',                                                  # pl.po
+    'Y29udHJhc2VueWE=',                                          # ca.po
+    'Y29udHJhc2XDsWE=',                                          # es.po
+    '4LSF4LSf4LSv4LS+4LSz4LS14LS+4LSV4LWN4LSV4LWN',              # ml.po
+    'c2VuaGE=',                                                  # pt.po
+    '5a+G56CB77ya',                                              # zh_CN.po
+    '4KSX4KWB4KSq4KWN4KSk4KS24KSs4KWN4KSm',                      # mr.po
+    'bMO2c2Vub3Jk',                                              # sv.po
+    '4YOe4YOQ4YOg4YOd4YOa4YOY',                                  # ka.po
+    '4KS24KSs4KWN4KSm4KSV4KWC4KSf',                              # hi.po
+    'YWRnYW5nc2tvZGU=',                                          # da.po
+    '4La74LeE4LeD4LeK4La04Lav4La6',                              # si.po
+    'cGFzc29yZA==',                                              # nb.po
+    'd2FjaHR3b29yZA==',                                          # nl.po
+    '4Kaq4Ka+4Ka44KaT4Kef4Ka+4Kaw4KeN4Kah',                      # bn_IN.po
+    'cGFyb2xh',                                                  # tr.po
+    '4LKX4LOB4LKq4LON4LKk4LKq4LKm',                              # kn.po
+    'c2FuZGk=',                                                  # id.po
+    '0L/QsNGA0L7Qu9GM',                                          # ru.po
+    'amVsc3rDsw==',                                              # hu.po
+    'bW90IGRlIHBhc3Nl',                                          # fr.po
+    'aXBoYXNpd2VkaQ==',                                          # zu.po
+    '4Z6W4Z624Z6A4Z+S4Z6Z4Z6f4Z6Y4Z+S4Z6E4Z624Z6P4Z+LwqDhn5Y=',  # km.po
+    '4KaX4KeB4Kaq4KeN4Kak4Ka24Kas4KeN4Kam',                      # as.po
+]
+
+
+PASSWORD_PROMPT_RE = re.compile(
+    u'|'.join(
+        base64.b64decode(s).decode('utf-8')
+        for s in PASSWORD_PROMPTS
+    )
+)
+
+
+PASSWORD_PROMPT = b('password')
+SUDO_OPTIONS = [
+    #(False, 'bool', '--askpass', '-A')
+    #(False, 'str', '--auth-type', '-a')
+    #(False, 'bool', '--background', '-b')
+    #(False, 'str', '--close-from', '-C')
+    #(False, 'str', '--login-class', 'c')
+    (True,  'bool', '--preserve-env', '-E'),
+    #(False, 'bool', '--edit', '-e')
+    #(False, 'str', '--group', '-g')
+    (True,  'bool', '--set-home', '-H'),
+    #(False, 'str', '--host', '-h')
+    (False, 'bool', '--login', '-i'),
+    #(False, 'bool', '--remove-timestamp', '-K')
+    #(False, 'bool', '--reset-timestamp', '-k')
+    #(False, 'bool', '--list', '-l')
+    #(False, 'bool', '--preserve-groups', '-P')
+    #(False, 'str', '--prompt', '-p')
+
+    # SELinux options. Passed through as-is.
+    (False, 'str', '--role', '-r'),
+    (False, 'str', '--type', '-t'),
+
+    # These options are supplied by default by Ansible, but are ignored, as
+    # sudo always runs under a TTY with Mitogen.
+    (True, 'bool', '--stdin', '-S'),
+    (True, 'bool', '--non-interactive', '-n'),
+
+    #(False, 'str', '--shell', '-s')
+    #(False, 'str', '--other-user', '-U')
+    (False, 'str', '--user', '-u'),
+    #(False, 'bool', '--version', '-V')
+    #(False, 'bool', '--validate', '-v')
+]
+
+
+class OptionParser(optparse.OptionParser):
+    def help(self):
+        self.exit()
+    def error(self, msg):
+        self.exit(msg=msg)
+    def exit(self, status=0, msg=None):
+        msg = 'sudo: ' + (msg or 'unsupported option')
+        raise mitogen.core.StreamError(msg)
+
+
+def make_sudo_parser():
+    parser = OptionParser()
+    for supported, kind, longopt, shortopt in SUDO_OPTIONS:
+        if kind == 'bool':
+            parser.add_option(longopt, shortopt, action='store_true')
+        else:
+            parser.add_option(longopt, shortopt)
+    return parser
+
+
+def parse_sudo_flags(args):
+    parser = make_sudo_parser()
+    opts, args = parser.parse_args(args)
+    if len(args):
+        raise mitogen.core.StreamError('unsupported sudo arguments:'+str(args))
+    return opts
+
+
+class PasswordError(mitogen.core.StreamError):
+    pass
+
+
+def option(default, *args):
+    for arg in args:
+        if arg is not None:
+            return arg
+    return default
+
+
+class Stream(mitogen.parent.Stream):
+    create_child = staticmethod(mitogen.parent.hybrid_tty_create_child)
+    child_is_immediate_subprocess = False
+
+    sudo_path = 'sudo'
+    username = 'root'
+    password = None
+    preserve_env = False
+    set_home = False
+    login = False
+
+    selinux_role = None
+    selinux_type = None
+
+    def construct(self, username=None, sudo_path=None, password=None,
+                  preserve_env=None, set_home=None, sudo_args=None,
+                  login=None, selinux_role=None, selinux_type=None, **kwargs):
+        super(Stream, self).construct(**kwargs)
+        opts = parse_sudo_flags(sudo_args or [])
+
+        self.username = option(self.username, username, opts.user)
+        self.sudo_path = option(self.sudo_path, sudo_path)
+        self.password = password or None
+        self.preserve_env = option(self.preserve_env,
+            preserve_env, opts.preserve_env)
+        self.set_home = option(self.set_home, set_home, opts.set_home)
+        self.login = option(self.login, login, opts.login)
+        self.selinux_role = option(self.selinux_role, selinux_role, opts.role)
+        self.selinux_type = option(self.selinux_type, selinux_type, opts.type)
+
+    def _get_name(self):
+        return u'sudo.' + mitogen.core.to_text(self.username)
+
+    def get_boot_command(self):
+        # Note: sudo did not introduce long-format option processing until July
+        # 2013, so even though we parse long-format options, supply short-form
+        # to the sudo command.
+        bits = [self.sudo_path, '-u', self.username]
+        if self.preserve_env:
+            bits += ['-E']
+        if self.set_home:
+            bits += ['-H']
+        if self.login:
+            bits += ['-i']
+        if self.selinux_role:
+            bits += ['-r', self.selinux_role]
+        if self.selinux_type:
+            bits += ['-t', self.selinux_type]
+
+        bits = bits + ['--'] + super(Stream, self).get_boot_command()
+        LOG.debug('sudo command line: %r', bits)
+        return bits
+
+    password_incorrect_msg = 'sudo password is incorrect'
+    password_required_msg = 'sudo password is required'
+
+    def _connect_input_loop(self, it):
+        password_sent = False
+
+        for buf in it:
+            LOG.debug('%s: received %r', self.name, buf)
+            if buf.endswith(self.EC0_MARKER):
+                self._ec0_received()
+                return
+
+            match = PASSWORD_PROMPT_RE.search(buf.decode('utf-8').lower())
+            if match is not None:
+                LOG.debug('%s: matched password prompt %r',
+                          self.name, match.group(0))
+                if self.password is None:
+                    raise PasswordError(self.password_required_msg)
+                if password_sent:
+                    raise PasswordError(self.password_incorrect_msg)
+                self.diag_stream.transmit_side.write(
+                    (mitogen.core.to_text(self.password) + '\n').encode('utf-8')
+                )
+                password_sent = True
+
+        raise mitogen.core.StreamError('bootstrap failed')
+
+    def _connect_bootstrap(self):
+        fds = [self.receive_side.fd]
+        if self.diag_stream is not None:
+            fds.append(self.diag_stream.receive_side.fd)
+
+        it = mitogen.parent.iter_read(
+            fds=fds,
+            deadline=self.connect_deadline,
+        )
+
+        try:
+            self._connect_input_loop(it)
+        finally:
+            it.close()

mitogen-0.2.7/mitogen/unix.py

@@ -0,0 +1,168 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+"""
+Permit connection of additional contexts that may act with the authority of
+this context. For now, the UNIX socket is always mode 0600, i.e. can only be
+accessed by root or the same UID. Therefore we can always trust connections to
+have the same privilege (auth_id) as the current process.
+"""
+
+import errno
+import os
+import socket
+import struct
+import sys
+import tempfile
+
+import mitogen.core
+import mitogen.master
+
+from mitogen.core import LOG
+
+
+def is_path_dead(path):
+    s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    try:
+        try:
+            s.connect(path)
+        except socket.error:
+            e = sys.exc_info()[1]
+            return e.args[0] in (errno.ECONNREFUSED, errno.ENOENT)
+    finally:
+        s.close()
+    return False
+
+
+def make_socket_path():
+    return tempfile.mktemp(prefix='mitogen_unix_', suffix='.sock')
+
+
+class Listener(mitogen.core.BasicStream):
+    keep_alive = True
+
+    def __repr__(self):
+        return '%s.%s(%r)' % (
+            __name__,
+            self.__class__.__name__,
+            self.path,
+        )
+
+    def __init__(self, router, path=None, backlog=100):
+        self._router = router
+        self.path = path or make_socket_path()
+        self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+
+        if os.path.exists(self.path) and is_path_dead(self.path):
+            LOG.debug('%r: deleting stale %r', self, self.path)
+            os.unlink(self.path)
+
+        self._sock.bind(self.path)
+        os.chmod(self.path, int('0600', 8))
+        self._sock.listen(backlog)
+        self.receive_side = mitogen.core.Side(self, self._sock.fileno())
+        router.broker.start_receive(self)
+
+    def _unlink_socket(self):
+        try:
+            os.unlink(self.path)
+        except OSError:
+            e = sys.exc_info()[1]
+            # Prevent a shutdown race with the parent process.
+            if e.args[0] != errno.ENOENT:
+                raise
+
+    def on_shutdown(self, broker):
+        broker.stop_receive(self)
+        self._unlink_socket()
+        self._sock.close()
+        self.receive_side.closed = True
+
+    def _accept_client(self, sock):
+        sock.setblocking(True)
+        try:
+            pid, = struct.unpack('>L', sock.recv(4))
+        except (struct.error, socket.error):
+            LOG.error('%r: failed to read remote identity: %s',
+                      self, sys.exc_info()[1])
+            return
+
+        context_id = self._router.id_allocator.allocate()
+        context = mitogen.parent.Context(self._router, context_id)
+        stream = mitogen.core.Stream(self._router, context_id)
+        stream.name = u'unix_client.%d' % (pid,)
+        stream.auth_id = mitogen.context_id
+        stream.is_privileged = True
+
+        try:
+            sock.send(struct.pack('>LLL', context_id, mitogen.context_id,
+                                  os.getpid()))
+        except socket.error:
+            LOG.error('%r: failed to assign identity to PID %d: %s',
+                      self, pid, sys.exc_info()[1])
+            return
+
+        LOG.debug('%r: accepted %r', self, stream)
+        stream.accept(sock.fileno(), sock.fileno())
+        self._router.register(context, stream)
+
+    def on_receive(self, broker):
+        sock, _ = self._sock.accept()
+        try:
+            self._accept_client(sock)
+        finally:
+            sock.close()
+
+
+def connect(path, broker=None):
+    LOG.debug('unix.connect(path=%r)', path)
+    sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+    sock.connect(path)
+    sock.send(struct.pack('>L', os.getpid()))
+    mitogen.context_id, remote_id, pid = struct.unpack('>LLL', sock.recv(12))
+    mitogen.parent_id = remote_id
+    mitogen.parent_ids = [remote_id]
+
+    LOG.debug('unix.connect(): local ID is %r, remote is %r',
+              mitogen.context_id, remote_id)
+
+    router = mitogen.master.Router(broker=broker)
+    stream = mitogen.core.Stream(router, remote_id)
+    stream.accept(sock.fileno(), sock.fileno())
+    stream.name = u'unix_listener.%d' % (pid,)
+
+    context = mitogen.parent.Context(router, remote_id)
+    router.register(context, stream)
+
+    mitogen.core.listen(router.broker, 'shutdown',
+                        lambda: router.disconnect_stream(stream))
+
+    sock.close()
+    return router, context

mitogen-0.2.7/mitogen/utils.py

@@ -0,0 +1,227 @@
+# Copyright 2019, David Wilson
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright notice,
+# this list of conditions and the following disclaimer in the documentation
+# and/or other materials provided with the distribution.
+#
+# 3. Neither the name of the copyright holder nor the names of its contributors
+# may be used to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+# POSSIBILITY OF SUCH DAMAGE.
+
+# !mitogen: minify_safe
+
+import datetime
+import logging
+import os
+import sys
+
+import mitogen
+import mitogen.core
+import mitogen.master
+import mitogen.parent
+
+
+LOG = logging.getLogger('mitogen')
+iteritems = getattr(dict, 'iteritems', dict.items)
+
+if mitogen.core.PY3:
+    iteritems = dict.items
+else:
+    iteritems = dict.iteritems
+
+
+def setup_gil():
+    """
+    Set extremely long GIL release interval to let threads naturally progress
+    through CPU-heavy sequences without forcing the wake of another thread that
+    may contend trying to run the same CPU-heavy code. For the new-style
+    Ansible work, this drops runtime ~33% and involuntary context switches by
+    >80%, essentially making threads cooperatively scheduled.
+    """
+    try:
+        # Python 2.
+        sys.setcheckinterval(100000)
+    except AttributeError:
+        pass
+
+    try:
+        # Python 3.
+        sys.setswitchinterval(10)
+    except AttributeError:
+        pass
+
+
+def disable_site_packages():
+    """
+    Remove all entries mentioning ``site-packages`` or ``Extras`` from
+    :attr:sys.path. Used primarily for testing on OS X within a virtualenv,
+    where OS X bundles some ancient version of the :mod:`six` module.
+    """
+    for entry in sys.path[:]:
+        if 'site-packages' in entry or 'Extras' in entry:
+            sys.path.remove(entry)
+
+
+def _formatTime(record, datefmt=None):
+    dt = datetime.datetime.fromtimestamp(record.created)
+    return dt.strftime(datefmt)
+
+
+def log_get_formatter():
+    datefmt = '%H:%M:%S'
+    if sys.version_info > (2, 6):
+        datefmt += '.%f'
+    fmt = '%(asctime)s %(levelname).1s %(name)s: %(message)s'
+    formatter = logging.Formatter(fmt, datefmt)
+    formatter.formatTime = _formatTime
+    return formatter
+
+
+def log_to_file(path=None, io=False, level='INFO'):
+    """
+    Install a new :class:`logging.Handler` writing applications logs to the
+    filesystem. Useful when debugging slave IO problems.
+
+    Parameters to this function may be overridden at runtime using environment
+    variables. See :ref:`logging-env-vars`.
+
+    :param str path:
+        If not :data:`None`, a filesystem path to write logs to. Otherwise,
+        logs are written to :data:`sys.stderr`.
+
+    :param bool io:
+        If :data:`True`, include extremely verbose IO logs in the output.
+        Useful for debugging hangs, less useful for debugging application code.
+
+    :param str level:
+        Name of the :mod:`logging` package constant that is the minimum level
+        to log at. Useful levels are ``DEBUG``, ``INFO``, ``WARNING``, and
+        ``ERROR``.
+    """
+    log = logging.getLogger('')
+    if path:
+        fp = open(path, 'w', 1)
+        mitogen.core.set_cloexec(fp.fileno())
+    else:
+        fp = sys.stderr
+
+    level = os.environ.get('MITOGEN_LOG_LEVEL', level).upper()
+    io = level == 'IO'
+    if io:
+        level = 'DEBUG'
+        logging.getLogger('mitogen.io').setLevel(level)
+
+    level = getattr(logging, level, logging.INFO)
+    log.setLevel(level)
+
+    # Prevent accidental duplicate log_to_file() calls from generating
+    # duplicate output.
+    for handler_ in reversed(log.handlers):
+        if getattr(handler_, 'is_mitogen', None):
+            log.handlers.remove(handler_)
+
+    handler = logging.StreamHandler(fp)
+    handler.is_mitogen = True
+    handler.formatter = log_get_formatter()
+    log.handlers.insert(0, handler)
+
+
+def run_with_router(func, *args, **kwargs):
+    """
+    Arrange for `func(router, *args, **kwargs)` to run with a temporary
+    :class:`mitogen.master.Router`, ensuring the Router and Broker are
+    correctly shut down during normal or exceptional return.
+
+    :returns:
+        `func`'s return value.
+    """
+    broker = mitogen.master.Broker()
+    router = mitogen.master.Router(broker)
+    try:
+        return func(router, *args, **kwargs)
+    finally:
+        broker.shutdown()
+        broker.join()
+
+
+def with_router(func):
+    """
+    Decorator version of :func:`run_with_router`. Example:
+
+    .. code-block:: python
+
+        @with_router
+        def do_stuff(router, arg):
+            pass
+
+        do_stuff(blah, 123)
+    """
+    def wrapper(*args, **kwargs):
+        return run_with_router(func, *args, **kwargs)
+    if mitogen.core.PY3:
+        wrapper.func_name = func.__name__
+    else:
+        wrapper.func_name = func.func_name
+    return wrapper
+
+
+PASSTHROUGH = (
+    int, float, bool,
+    type(None),
+    mitogen.core.Context,
+    mitogen.core.CallError,
+    mitogen.core.Blob,
+    mitogen.core.Secret,
+)
+
+
+def cast(obj):
+    """
+    Many tools love to subclass built-in types in order to implement useful
+    functionality, such as annotating the safety of a Unicode string, or adding
+    additional methods to a dict. However, cPickle loves to preserve those
+    subtypes during serialization, resulting in CallError during :meth:`call
+    <mitogen.parent.Context.call>` in the target when it tries to deserialize
+    the data.
+
+    This function walks the object graph `obj`, producing a copy with any
+    custom sub-types removed. The functionality is not default since the
+    resulting walk may be computationally expensive given a large enough graph.
+
+    See :ref:`serialization-rules` for a list of supported types.
+
+    :param obj:
+        Object to undecorate.
+    :returns:
+        Undecorated object.
+    """
+    if isinstance(obj, dict):
+        return dict((cast(k), cast(v)) for k, v in iteritems(obj))
+    if isinstance(obj, (list, tuple)):
+        return [cast(v) for v in obj]
+    if isinstance(obj, PASSTHROUGH):
+        return obj
+    if isinstance(obj, mitogen.core.UnicodeType):
+        return mitogen.core.UnicodeType(obj)
+    if isinstance(obj, mitogen.core.BytesType):
+        return mitogen.core.BytesType(obj)
+
+    raise TypeError("Cannot serialize: %r: %r" % (type(obj), obj))