diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 000000000..3518b85
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "mitogen"]
+ path = mitogen
+ url = https://github.com/dw/mitogen.git
diff --git a/ansible.cfg b/ansible.cfg
index 6749b25..2a43fac 100644
--- a/ansible.cfg
+++ b/ansible.cfg
@@ -1,8 +1,8 @@
[defaults]
host_key_checking = False
retry_files_enabled = False
-strategy_plugins = ./mitogen-0.2.7/ansible_mitogen/plugins/strategy
-strategy = mitogen_linear
+strategy_plugins = ./mitogen/ansible_mitogen/plugins/strategy
+#strategy = mitogen_linear
[ssh_connection]
pipelining = True
diff --git a/mitogen b/mitogen
new file mode 160000
index 000000000..4fedf88
--- /dev/null
+++ b/mitogen
@@ -0,0 +1 @@
+Subproject commit 4fedf88d3868830d02435ab2148ed6662bb3a434
diff --git a/mitogen-0.2.7/LICENSE b/mitogen-0.2.7/LICENSE
deleted file mode 100644
index 70e43a9..000000000
--- a/mitogen-0.2.7/LICENSE
+++ /dev/null
@@ -1,26 +0,0 @@
-Copyright 2019, David Wilson
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this
-list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice,
-this list of conditions and the following disclaimer in the documentation
-and/or other materials provided with the distribution.
-
-3. Neither the name of the copyright holder nor the names of its contributors
-may be used to endorse or promote products derived from this software without
-specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/mitogen-0.2.7/MANIFEST.in b/mitogen-0.2.7/MANIFEST.in
deleted file mode 100644
index 1aba38f..000000000
--- a/mitogen-0.2.7/MANIFEST.in
+++ /dev/null
@@ -1 +0,0 @@
-include LICENSE
diff --git a/mitogen-0.2.7/PKG-INFO b/mitogen-0.2.7/PKG-INFO
deleted file mode 100644
index f63ba91..000000000
--- a/mitogen-0.2.7/PKG-INFO
+++ /dev/null
@@ -1,23 +0,0 @@
-Metadata-Version: 1.1
-Name: mitogen
-Version: 0.2.7
-Summary: Library for writing distributed self-replicating programs.
-Home-page: https://github.com/dw/mitogen/
-Author: David Wilson
-Author-email: UNKNOWN
-License: New BSD
-Description: UNKNOWN
-Platform: UNKNOWN
-Classifier: Environment :: Console
-Classifier: Intended Audience :: System Administrators
-Classifier: License :: OSI Approved :: BSD License
-Classifier: Operating System :: POSIX
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 2.4
-Classifier: Programming Language :: Python :: 2.5
-Classifier: Programming Language :: Python :: 2.6
-Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: Implementation :: CPython
-Classifier: Topic :: System :: Distributed Computing
-Classifier: Topic :: System :: Systems Administration
diff --git a/mitogen-0.2.7/README.md b/mitogen-0.2.7/README.md
deleted file mode 100644
index 5ef2447..000000000
--- a/mitogen-0.2.7/README.md
+++ /dev/null
@@ -1,13 +0,0 @@
-
-# Mitogen
-
-
-Please see the documentation.
-
-
-
-[](https://lgtm.com/projects/g/dw/mitogen/alerts/)
-
-[](https://travis-ci.org/dw/mitogen)
-
-[](https://dev.azure.com/dw-mitogen/Mitogen/_build/latest?definitionId=1?branchName=master)
diff --git a/mitogen-0.2.7/ansible_mitogen/__init__.py b/mitogen-0.2.7/ansible_mitogen/__init__.py
deleted file mode 100644
index e69de29..000000000
diff --git a/mitogen-0.2.7/ansible_mitogen/affinity.py b/mitogen-0.2.7/ansible_mitogen/affinity.py
deleted file mode 100644
index 09a6ace..000000000
--- a/mitogen-0.2.7/ansible_mitogen/affinity.py
+++ /dev/null
@@ -1,269 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-"""
-As Mitogen separates asynchronous IO out to a broker thread, communication
-necessarily involves context switching and waking that thread. When application
-threads and the broker share a CPU, this can be almost invisibly fast - around
-25 microseconds for a full A->B->A round-trip.
-
-However when threads are scheduled on different CPUs, round-trip delays
-regularly vary wildly, and easily into milliseconds. Many contributing factors
-exist, not least scenarios like:
-
-1. A is preempted immediately after waking B, but before releasing the GIL.
-2. B wakes from IO wait only to immediately enter futex wait.
-3. A may wait 10ms or more for another timeslice, as the scheduler on its CPU
- runs threads unrelated to its transaction (i.e. not B), wake only to release
- its GIL, before entering IO sleep waiting for a reply from B, which cannot
- exist yet.
-4. B wakes, acquires GIL, performs work, and sends reply to A, causing it to
- wake. B is preempted before releasing GIL.
-5. A wakes from IO wait only to immediately enter futex wait.
-6. B may wait 10ms or more for another timeslice, wake only to release its GIL,
- before sleeping again.
-7. A wakes, acquires GIL, finally receives reply.
-
-Per above if we are unlucky, on an even moderately busy machine it is possible
-to lose milliseconds just in scheduling delay, and the effect is compounded
-when pairs of threads in process A are communicating with pairs of threads in
-process B using the same scheme, such as when Ansible WorkerProcess is
-communicating with ContextService in the connection multiplexer. In the worst
-case it could involve 4 threads working in lockstep spread across 4 busy CPUs.
-
-Since multithreading in Python is essentially useless except for waiting on IO
-due to the presence of the GIL, at least in Ansible there is no good reason for
-threads in the same process to run on distinct CPUs - they always operate in
-lockstep due to the GIL, and are thus vulnerable to issues like above.
-
-Linux lacks any natural API to describe what we want, it only permits
-individual threads to be constrained to run on specific CPUs, and for that
-constraint to be inherited by new threads and forks of the constrained thread.
-
-This module therefore implements a CPU pinning policy for Ansible processes,
-providing methods that should be called early in any new process, either to
-rebalance which CPU it is pinned to, or in the case of subprocesses, to remove
-the pinning entirely. It is likely to require ongoing tweaking, since pinning
-necessarily involves preventing the scheduler from making load balancing
-decisions.
-"""
-
-import ctypes
-import mmap
-import multiprocessing
-import os
-import struct
-
-import mitogen.core
-import mitogen.parent
-
-
-try:
- _libc = ctypes.CDLL(None, use_errno=True)
- _strerror = _libc.strerror
- _strerror.restype = ctypes.c_char_p
- _pthread_mutex_init = _libc.pthread_mutex_init
- _pthread_mutex_lock = _libc.pthread_mutex_lock
- _pthread_mutex_unlock = _libc.pthread_mutex_unlock
- _sched_setaffinity = _libc.sched_setaffinity
-except (OSError, AttributeError):
- _libc = None
- _strerror = None
- _pthread_mutex_init = None
- _pthread_mutex_lock = None
- _pthread_mutex_unlock = None
- _sched_setaffinity = None
-
-
-class pthread_mutex_t(ctypes.Structure):
- """
- Wrap pthread_mutex_t to allow storing a lock in shared memory.
- """
- _fields_ = [
- ('data', ctypes.c_uint8 * 512),
- ]
-
- def init(self):
- if _pthread_mutex_init(self.data, 0):
- raise Exception(_strerror(ctypes.get_errno()))
-
- def acquire(self):
- if _pthread_mutex_lock(self.data):
- raise Exception(_strerror(ctypes.get_errno()))
-
- def release(self):
- if _pthread_mutex_unlock(self.data):
- raise Exception(_strerror(ctypes.get_errno()))
-
-
-class State(ctypes.Structure):
- """
- Contents of shared memory segment. This allows :meth:`Manager.assign` to be
- called from any child, since affinity assignment must happen from within
- the context of the new child process.
- """
- _fields_ = [
- ('lock', pthread_mutex_t),
- ('counter', ctypes.c_uint8),
- ]
-
-
-class Policy(object):
- """
- Process affinity policy.
- """
- def assign_controller(self):
- """
- Assign the Ansible top-level policy to this process.
- """
-
- def assign_muxprocess(self):
- """
- Assign the MuxProcess policy to this process.
- """
-
- def assign_worker(self):
- """
- Assign the WorkerProcess policy to this process.
- """
-
- def assign_subprocess(self):
- """
- Assign the helper subprocess policy to this process.
- """
-
-class FixedPolicy(Policy):
- """
- :class:`Policy` for machines where the only control method available is
- fixed CPU placement. The scheme here was tested on an otherwise idle 16
- thread machine.
-
- - The connection multiplexer is pinned to CPU 0.
- - The Ansible top-level (strategy) is pinned to CPU 1.
- - WorkerProcesses are pinned sequentually to 2..N, wrapping around when no
- more CPUs exist.
- - Children such as SSH may be scheduled on any CPU except 0/1.
-
- If the machine has less than 4 cores available, the top-level and workers
- are pinned between CPU 2..N, i.e. no CPU is reserved for the top-level
- process.
-
- This could at least be improved by having workers pinned to independent
- cores, before reusing the second hyperthread of an existing core.
-
- A hook is installed that causes :meth:`reset` to run in the child of any
- process created with :func:`mitogen.parent.detach_popen`, ensuring
- CPU-intensive children like SSH are not forced to share the same core as
- the (otherwise potentially very busy) parent.
- """
- def __init__(self, cpu_count=None):
- #: For tests.
- self.cpu_count = cpu_count or multiprocessing.cpu_count()
- self.mem = mmap.mmap(-1, 4096)
- self.state = State.from_buffer(self.mem)
- self.state.lock.init()
-
- if self.cpu_count < 2:
- # uniprocessor
- self._reserve_mux = False
- self._reserve_controller = False
- self._reserve_mask = 0
- self._reserve_shift = 0
- elif self.cpu_count < 4:
- # small SMP
- self._reserve_mux = True
- self._reserve_controller = False
- self._reserve_mask = 1
- self._reserve_shift = 1
- else:
- # big SMP
- self._reserve_mux = True
- self._reserve_controller = True
- self._reserve_mask = 3
- self._reserve_shift = 2
-
- def _set_affinity(self, mask):
- mitogen.parent._preexec_hook = self._clear
- self._set_cpu_mask(mask)
-
- def _balance(self):
- self.state.lock.acquire()
- try:
- n = self.state.counter
- self.state.counter += 1
- finally:
- self.state.lock.release()
-
- self._set_cpu(self._reserve_shift + (
- (n % (self.cpu_count - self._reserve_shift))
- ))
-
- def _set_cpu(self, cpu):
- self._set_affinity(1 << cpu)
-
- def _clear(self):
- all_cpus = (1 << self.cpu_count) - 1
- self._set_affinity(all_cpus & ~self._reserve_mask)
-
- def assign_controller(self):
- if self._reserve_controller:
- self._set_cpu(1)
- else:
- self._balance()
-
- def assign_muxprocess(self):
- self._set_cpu(0)
-
- def assign_worker(self):
- self._balance()
-
- def assign_subprocess(self):
- self._clear()
-
-
-class LinuxPolicy(FixedPolicy):
- def _mask_to_bytes(self, mask):
- """
- Convert the (type long) mask to a cpu_set_t.
- """
- chunks = []
- shiftmask = (2 ** 64) - 1
- for x in range(16):
- chunks.append(struct.pack('>= 64
- return mitogen.core.b('').join(chunks)
-
- def _set_cpu_mask(self, mask):
- s = self._mask_to_bytes(mask)
- _sched_setaffinity(os.getpid(), len(s), s)
-
-
-if _sched_setaffinity is not None:
- policy = LinuxPolicy()
-else:
- policy = Policy()
diff --git a/mitogen-0.2.7/ansible_mitogen/compat/__init__.py b/mitogen-0.2.7/ansible_mitogen/compat/__init__.py
deleted file mode 100644
index e69de29..000000000
diff --git a/mitogen-0.2.7/ansible_mitogen/compat/simplejson/__init__.py b/mitogen-0.2.7/ansible_mitogen/compat/simplejson/__init__.py
deleted file mode 100644
index d5b4d39..000000000
--- a/mitogen-0.2.7/ansible_mitogen/compat/simplejson/__init__.py
+++ /dev/null
@@ -1,318 +0,0 @@
-r"""JSON (JavaScript Object Notation) is a subset of
-JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data
-interchange format.
-
-:mod:`simplejson` exposes an API familiar to users of the standard library
-:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained
-version of the :mod:`json` library contained in Python 2.6, but maintains
-compatibility with Python 2.4 and Python 2.5 and (currently) has
-significant performance advantages, even without using the optional C
-extension for speedups.
-
-Encoding basic Python object hierarchies::
-
- >>> import simplejson as json
- >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}])
- '["foo", {"bar": ["baz", null, 1.0, 2]}]'
- >>> print json.dumps("\"foo\bar")
- "\"foo\bar"
- >>> print json.dumps(u'\u1234')
- "\u1234"
- >>> print json.dumps('\\')
- "\\"
- >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)
- {"a": 0, "b": 0, "c": 0}
- >>> from StringIO import StringIO
- >>> io = StringIO()
- >>> json.dump(['streaming API'], io)
- >>> io.getvalue()
- '["streaming API"]'
-
-Compact encoding::
-
- >>> import simplejson as json
- >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':'))
- '[1,2,3,{"4":5,"6":7}]'
-
-Pretty printing::
-
- >>> import simplejson as json
- >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)
- >>> print '\n'.join([l.rstrip() for l in s.splitlines()])
- {
- "4": 5,
- "6": 7
- }
-
-Decoding JSON::
-
- >>> import simplejson as json
- >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}]
- >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj
- True
- >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar'
- True
- >>> from StringIO import StringIO
- >>> io = StringIO('["streaming API"]')
- >>> json.load(io)[0] == 'streaming API'
- True
-
-Specializing JSON object decoding::
-
- >>> import simplejson as json
- >>> def as_complex(dct):
- ... if '__complex__' in dct:
- ... return complex(dct['real'], dct['imag'])
- ... return dct
- ...
- >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}',
- ... object_hook=as_complex)
- (1+2j)
- >>> import decimal
- >>> json.loads('1.1', parse_float=decimal.Decimal) == decimal.Decimal('1.1')
- True
-
-Specializing JSON object encoding::
-
- >>> import simplejson as json
- >>> def encode_complex(obj):
- ... if isinstance(obj, complex):
- ... return [obj.real, obj.imag]
- ... raise TypeError(repr(o) + " is not JSON serializable")
- ...
- >>> json.dumps(2 + 1j, default=encode_complex)
- '[2.0, 1.0]'
- >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j)
- '[2.0, 1.0]'
- >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j))
- '[2.0, 1.0]'
-
-
-Using simplejson.tool from the shell to validate and pretty-print::
-
- $ echo '{"json":"obj"}' | python -m simplejson.tool
- {
- "json": "obj"
- }
- $ echo '{ 1.2:3.4}' | python -m simplejson.tool
- Expecting property name: line 1 column 2 (char 2)
-"""
-__version__ = '2.0.9'
-__all__ = [
- 'dump', 'dumps', 'load', 'loads',
- 'JSONDecoder', 'JSONEncoder',
-]
-
-__author__ = 'Bob Ippolito '
-
-from decoder import JSONDecoder
-from encoder import JSONEncoder
-
-_default_encoder = JSONEncoder(
- skipkeys=False,
- ensure_ascii=True,
- check_circular=True,
- allow_nan=True,
- indent=None,
- separators=None,
- encoding='utf-8',
- default=None,
-)
-
-def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True,
- allow_nan=True, cls=None, indent=None, separators=None,
- encoding='utf-8', default=None, **kw):
- """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a
- ``.write()``-supporting file-like object).
-
- If ``skipkeys`` is true then ``dict`` keys that are not basic types
- (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
- will be skipped instead of raising a ``TypeError``.
-
- If ``ensure_ascii`` is false, then the some chunks written to ``fp``
- may be ``unicode`` instances, subject to normal Python ``str`` to
- ``unicode`` coercion rules. Unless ``fp.write()`` explicitly
- understands ``unicode`` (as in ``codecs.getwriter()``) this is likely
- to cause an error.
-
- If ``check_circular`` is false, then the circular reference check
- for container types will be skipped and a circular reference will
- result in an ``OverflowError`` (or worse).
-
- If ``allow_nan`` is false, then it will be a ``ValueError`` to
- serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``)
- in strict compliance of the JSON specification, instead of using the
- JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
-
- If ``indent`` is a non-negative integer, then JSON array elements and object
- members will be pretty-printed with that indent level. An indent level
- of 0 will only insert newlines. ``None`` is the most compact representation.
-
- If ``separators`` is an ``(item_separator, dict_separator)`` tuple
- then it will be used instead of the default ``(', ', ': ')`` separators.
- ``(',', ':')`` is the most compact JSON representation.
-
- ``encoding`` is the character encoding for str instances, default is UTF-8.
-
- ``default(obj)`` is a function that should return a serializable version
- of obj or raise TypeError. The default simply raises TypeError.
-
- To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
- ``.default()`` method to serialize additional types), specify it with
- the ``cls`` kwarg.
-
- """
- # cached encoder
- if (not skipkeys and ensure_ascii and
- check_circular and allow_nan and
- cls is None and indent is None and separators is None and
- encoding == 'utf-8' and default is None and not kw):
- iterable = _default_encoder.iterencode(obj)
- else:
- if cls is None:
- cls = JSONEncoder
- iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii,
- check_circular=check_circular, allow_nan=allow_nan, indent=indent,
- separators=separators, encoding=encoding,
- default=default, **kw).iterencode(obj)
- # could accelerate with writelines in some versions of Python, at
- # a debuggability cost
- for chunk in iterable:
- fp.write(chunk)
-
-
-def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True,
- allow_nan=True, cls=None, indent=None, separators=None,
- encoding='utf-8', default=None, **kw):
- """Serialize ``obj`` to a JSON formatted ``str``.
-
- If ``skipkeys`` is false then ``dict`` keys that are not basic types
- (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``)
- will be skipped instead of raising a ``TypeError``.
-
- If ``ensure_ascii`` is false, then the return value will be a
- ``unicode`` instance subject to normal Python ``str`` to ``unicode``
- coercion rules instead of being escaped to an ASCII ``str``.
-
- If ``check_circular`` is false, then the circular reference check
- for container types will be skipped and a circular reference will
- result in an ``OverflowError`` (or worse).
-
- If ``allow_nan`` is false, then it will be a ``ValueError`` to
- serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in
- strict compliance of the JSON specification, instead of using the
- JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``).
-
- If ``indent`` is a non-negative integer, then JSON array elements and
- object members will be pretty-printed with that indent level. An indent
- level of 0 will only insert newlines. ``None`` is the most compact
- representation.
-
- If ``separators`` is an ``(item_separator, dict_separator)`` tuple
- then it will be used instead of the default ``(', ', ': ')`` separators.
- ``(',', ':')`` is the most compact JSON representation.
-
- ``encoding`` is the character encoding for str instances, default is UTF-8.
-
- ``default(obj)`` is a function that should return a serializable version
- of obj or raise TypeError. The default simply raises TypeError.
-
- To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the
- ``.default()`` method to serialize additional types), specify it with
- the ``cls`` kwarg.
-
- """
- # cached encoder
- if (not skipkeys and ensure_ascii and
- check_circular and allow_nan and
- cls is None and indent is None and separators is None and
- encoding == 'utf-8' and default is None and not kw):
- return _default_encoder.encode(obj)
- if cls is None:
- cls = JSONEncoder
- return cls(
- skipkeys=skipkeys, ensure_ascii=ensure_ascii,
- check_circular=check_circular, allow_nan=allow_nan, indent=indent,
- separators=separators, encoding=encoding, default=default,
- **kw).encode(obj)
-
-
-_default_decoder = JSONDecoder(encoding=None, object_hook=None)
-
-
-def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None,
- parse_int=None, parse_constant=None, **kw):
- """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing
- a JSON document) to a Python object.
-
- If the contents of ``fp`` is encoded with an ASCII based encoding other
- than utf-8 (e.g. latin-1), then an appropriate ``encoding`` name must
- be specified. Encodings that are not ASCII based (such as UCS-2) are
- not allowed, and should be wrapped with
- ``codecs.getreader(fp)(encoding)``, or simply decoded to a ``unicode``
- object and passed to ``loads()``
-
- ``object_hook`` is an optional function that will be called with the
- result of any object literal decode (a ``dict``). The return value of
- ``object_hook`` will be used instead of the ``dict``. This feature
- can be used to implement custom decoders (e.g. JSON-RPC class hinting).
-
- To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
- kwarg.
-
- """
- return loads(fp.read(),
- encoding=encoding, cls=cls, object_hook=object_hook,
- parse_float=parse_float, parse_int=parse_int,
- parse_constant=parse_constant, **kw)
-
-
-def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None,
- parse_int=None, parse_constant=None, **kw):
- """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON
- document) to a Python object.
-
- If ``s`` is a ``str`` instance and is encoded with an ASCII based encoding
- other than utf-8 (e.g. latin-1) then an appropriate ``encoding`` name
- must be specified. Encodings that are not ASCII based (such as UCS-2)
- are not allowed and should be decoded to ``unicode`` first.
-
- ``object_hook`` is an optional function that will be called with the
- result of any object literal decode (a ``dict``). The return value of
- ``object_hook`` will be used instead of the ``dict``. This feature
- can be used to implement custom decoders (e.g. JSON-RPC class hinting).
-
- ``parse_float``, if specified, will be called with the string
- of every JSON float to be decoded. By default this is equivalent to
- float(num_str). This can be used to use another datatype or parser
- for JSON floats (e.g. decimal.Decimal).
-
- ``parse_int``, if specified, will be called with the string
- of every JSON int to be decoded. By default this is equivalent to
- int(num_str). This can be used to use another datatype or parser
- for JSON integers (e.g. float).
-
- ``parse_constant``, if specified, will be called with one of the
- following strings: -Infinity, Infinity, NaN, null, true, false.
- This can be used to raise an exception if invalid JSON numbers
- are encountered.
-
- To use a custom ``JSONDecoder`` subclass, specify it with the ``cls``
- kwarg.
-
- """
- if (cls is None and encoding is None and object_hook is None and
- parse_int is None and parse_float is None and
- parse_constant is None and not kw):
- return _default_decoder.decode(s)
- if cls is None:
- cls = JSONDecoder
- if object_hook is not None:
- kw['object_hook'] = object_hook
- if parse_float is not None:
- kw['parse_float'] = parse_float
- if parse_int is not None:
- kw['parse_int'] = parse_int
- if parse_constant is not None:
- kw['parse_constant'] = parse_constant
- return cls(encoding=encoding, **kw).decode(s)
diff --git a/mitogen-0.2.7/ansible_mitogen/compat/simplejson/decoder.py b/mitogen-0.2.7/ansible_mitogen/compat/simplejson/decoder.py
deleted file mode 100644
index b769ea4..000000000
--- a/mitogen-0.2.7/ansible_mitogen/compat/simplejson/decoder.py
+++ /dev/null
@@ -1,354 +0,0 @@
-"""Implementation of JSONDecoder
-"""
-import re
-import sys
-import struct
-
-from simplejson.scanner import make_scanner
-try:
- from simplejson._speedups import scanstring as c_scanstring
-except ImportError:
- c_scanstring = None
-
-__all__ = ['JSONDecoder']
-
-FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
-
-def _floatconstants():
- _BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
- if sys.byteorder != 'big':
- _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
- nan, inf = struct.unpack('dd', _BYTES)
- return nan, inf, -inf
-
-NaN, PosInf, NegInf = _floatconstants()
-
-
-def linecol(doc, pos):
- lineno = doc.count('\n', 0, pos) + 1
- if lineno == 1:
- colno = pos
- else:
- colno = pos - doc.rindex('\n', 0, pos)
- return lineno, colno
-
-
-def errmsg(msg, doc, pos, end=None):
- # Note that this function is called from _speedups
- lineno, colno = linecol(doc, pos)
- if end is None:
- #fmt = '{0}: line {1} column {2} (char {3})'
- #return fmt.format(msg, lineno, colno, pos)
- fmt = '%s: line %d column %d (char %d)'
- return fmt % (msg, lineno, colno, pos)
- endlineno, endcolno = linecol(doc, end)
- #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})'
- #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end)
- fmt = '%s: line %d column %d - line %d column %d (char %d - %d)'
- return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end)
-
-
-_CONSTANTS = {
- '-Infinity': NegInf,
- 'Infinity': PosInf,
- 'NaN': NaN,
-}
-
-STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS)
-BACKSLASH = {
- '"': u'"', '\\': u'\\', '/': u'/',
- 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
-}
-
-DEFAULT_ENCODING = "utf-8"
-
-def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match):
- """Scan the string s for a JSON string. End is the index of the
- character in s after the quote that started the JSON string.
- Unescapes all valid JSON string escape sequences and raises ValueError
- on attempt to decode an invalid string. If strict is False then literal
- control characters are allowed in the string.
-
- Returns a tuple of the decoded string and the index of the character in s
- after the end quote."""
- if encoding is None:
- encoding = DEFAULT_ENCODING
- chunks = []
- _append = chunks.append
- begin = end - 1
- while 1:
- chunk = _m(s, end)
- if chunk is None:
- raise ValueError(
- errmsg("Unterminated string starting at", s, begin))
- end = chunk.end()
- content, terminator = chunk.groups()
- # Content is contains zero or more unescaped string characters
- if content:
- if not isinstance(content, unicode):
- content = unicode(content, encoding)
- _append(content)
- # Terminator is the end of string, a literal control character,
- # or a backslash denoting that an escape sequence follows
- if terminator == '"':
- break
- elif terminator != '\\':
- if strict:
- msg = "Invalid control character %r at" % (terminator,)
- #msg = "Invalid control character {0!r} at".format(terminator)
- raise ValueError(errmsg(msg, s, end))
- else:
- _append(terminator)
- continue
- try:
- esc = s[end]
- except IndexError:
- raise ValueError(
- errmsg("Unterminated string starting at", s, begin))
- # If not a unicode escape sequence, must be in the lookup table
- if esc != 'u':
- try:
- char = _b[esc]
- except KeyError:
- msg = "Invalid \\escape: " + repr(esc)
- raise ValueError(errmsg(msg, s, end))
- end += 1
- else:
- # Unicode escape sequence
- esc = s[end + 1:end + 5]
- next_end = end + 5
- if len(esc) != 4:
- msg = "Invalid \\uXXXX escape"
- raise ValueError(errmsg(msg, s, end))
- uni = int(esc, 16)
- # Check for surrogate pair on UCS-4 systems
- if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535:
- msg = "Invalid \\uXXXX\\uXXXX surrogate pair"
- if not s[end + 5:end + 7] == '\\u':
- raise ValueError(errmsg(msg, s, end))
- esc2 = s[end + 7:end + 11]
- if len(esc2) != 4:
- raise ValueError(errmsg(msg, s, end))
- uni2 = int(esc2, 16)
- uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00))
- next_end += 6
- char = unichr(uni)
- end = next_end
- # Append the unescaped character
- _append(char)
- return u''.join(chunks), end
-
-
-# Use speedup if available
-scanstring = c_scanstring or py_scanstring
-
-WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS)
-WHITESPACE_STR = ' \t\n\r'
-
-def JSONObject((s, end), encoding, strict, scan_once, object_hook, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
- pairs = {}
- # Use a slice to prevent IndexError from being raised, the following
- # check will raise a more specific ValueError if the string is empty
- nextchar = s[end:end + 1]
- # Normally we expect nextchar == '"'
- if nextchar != '"':
- if nextchar in _ws:
- end = _w(s, end).end()
- nextchar = s[end:end + 1]
- # Trivial empty object
- if nextchar == '}':
- return pairs, end + 1
- elif nextchar != '"':
- raise ValueError(errmsg("Expecting property name", s, end))
- end += 1
- while True:
- key, end = scanstring(s, end, encoding, strict)
-
- # To skip some function call overhead we optimize the fast paths where
- # the JSON key separator is ": " or just ":".
- if s[end:end + 1] != ':':
- end = _w(s, end).end()
- if s[end:end + 1] != ':':
- raise ValueError(errmsg("Expecting : delimiter", s, end))
-
- end += 1
-
- try:
- if s[end] in _ws:
- end += 1
- if s[end] in _ws:
- end = _w(s, end + 1).end()
- except IndexError:
- pass
-
- try:
- value, end = scan_once(s, end)
- except StopIteration:
- raise ValueError(errmsg("Expecting object", s, end))
- pairs[key] = value
-
- try:
- nextchar = s[end]
- if nextchar in _ws:
- end = _w(s, end + 1).end()
- nextchar = s[end]
- except IndexError:
- nextchar = ''
- end += 1
-
- if nextchar == '}':
- break
- elif nextchar != ',':
- raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
-
- try:
- nextchar = s[end]
- if nextchar in _ws:
- end += 1
- nextchar = s[end]
- if nextchar in _ws:
- end = _w(s, end + 1).end()
- nextchar = s[end]
- except IndexError:
- nextchar = ''
-
- end += 1
- if nextchar != '"':
- raise ValueError(errmsg("Expecting property name", s, end - 1))
-
- if object_hook is not None:
- pairs = object_hook(pairs)
- return pairs, end
-
-def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR):
- values = []
- nextchar = s[end:end + 1]
- if nextchar in _ws:
- end = _w(s, end + 1).end()
- nextchar = s[end:end + 1]
- # Look-ahead for trivial empty array
- if nextchar == ']':
- return values, end + 1
- _append = values.append
- while True:
- try:
- value, end = scan_once(s, end)
- except StopIteration:
- raise ValueError(errmsg("Expecting object", s, end))
- _append(value)
- nextchar = s[end:end + 1]
- if nextchar in _ws:
- end = _w(s, end + 1).end()
- nextchar = s[end:end + 1]
- end += 1
- if nextchar == ']':
- break
- elif nextchar != ',':
- raise ValueError(errmsg("Expecting , delimiter", s, end))
-
- try:
- if s[end] in _ws:
- end += 1
- if s[end] in _ws:
- end = _w(s, end + 1).end()
- except IndexError:
- pass
-
- return values, end
-
-class JSONDecoder(object):
- """Simple JSON decoder
-
- Performs the following translations in decoding by default:
-
- +---------------+-------------------+
- | JSON | Python |
- +===============+===================+
- | object | dict |
- +---------------+-------------------+
- | array | list |
- +---------------+-------------------+
- | string | unicode |
- +---------------+-------------------+
- | number (int) | int, long |
- +---------------+-------------------+
- | number (real) | float |
- +---------------+-------------------+
- | true | True |
- +---------------+-------------------+
- | false | False |
- +---------------+-------------------+
- | null | None |
- +---------------+-------------------+
-
- It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
- their corresponding ``float`` values, which is outside the JSON spec.
-
- """
-
- def __init__(self, encoding=None, object_hook=None, parse_float=None,
- parse_int=None, parse_constant=None, strict=True):
- """``encoding`` determines the encoding used to interpret any ``str``
- objects decoded by this instance (utf-8 by default). It has no
- effect when decoding ``unicode`` objects.
-
- Note that currently only encodings that are a superset of ASCII work,
- strings of other encodings should be passed in as ``unicode``.
-
- ``object_hook``, if specified, will be called with the result
- of every JSON object decoded and its return value will be used in
- place of the given ``dict``. This can be used to provide custom
- deserializations (e.g. to support JSON-RPC class hinting).
-
- ``parse_float``, if specified, will be called with the string
- of every JSON float to be decoded. By default this is equivalent to
- float(num_str). This can be used to use another datatype or parser
- for JSON floats (e.g. decimal.Decimal).
-
- ``parse_int``, if specified, will be called with the string
- of every JSON int to be decoded. By default this is equivalent to
- int(num_str). This can be used to use another datatype or parser
- for JSON integers (e.g. float).
-
- ``parse_constant``, if specified, will be called with one of the
- following strings: -Infinity, Infinity, NaN.
- This can be used to raise an exception if invalid JSON numbers
- are encountered.
-
- """
- self.encoding = encoding
- self.object_hook = object_hook
- self.parse_float = parse_float or float
- self.parse_int = parse_int or int
- self.parse_constant = parse_constant or _CONSTANTS.__getitem__
- self.strict = strict
- self.parse_object = JSONObject
- self.parse_array = JSONArray
- self.parse_string = scanstring
- self.scan_once = make_scanner(self)
-
- def decode(self, s, _w=WHITESPACE.match):
- """Return the Python representation of ``s`` (a ``str`` or ``unicode``
- instance containing a JSON document)
-
- """
- obj, end = self.raw_decode(s, idx=_w(s, 0).end())
- end = _w(s, end).end()
- if end != len(s):
- raise ValueError(errmsg("Extra data", s, end, len(s)))
- return obj
-
- def raw_decode(self, s, idx=0):
- """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning
- with a JSON document) and return a 2-tuple of the Python
- representation and the index in ``s`` where the document ended.
-
- This can be used to decode a JSON document from a string that may
- have extraneous data at the end.
-
- """
- try:
- obj, end = self.scan_once(s, idx)
- except StopIteration:
- raise ValueError("No JSON object could be decoded")
- return obj, end
diff --git a/mitogen-0.2.7/ansible_mitogen/compat/simplejson/encoder.py b/mitogen-0.2.7/ansible_mitogen/compat/simplejson/encoder.py
deleted file mode 100644
index cf58290..000000000
--- a/mitogen-0.2.7/ansible_mitogen/compat/simplejson/encoder.py
+++ /dev/null
@@ -1,440 +0,0 @@
-"""Implementation of JSONEncoder
-"""
-import re
-
-try:
- from simplejson._speedups import encode_basestring_ascii as c_encode_basestring_ascii
-except ImportError:
- c_encode_basestring_ascii = None
-try:
- from simplejson._speedups import make_encoder as c_make_encoder
-except ImportError:
- c_make_encoder = None
-
-ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]')
-ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])')
-HAS_UTF8 = re.compile(r'[\x80-\xff]')
-ESCAPE_DCT = {
- '\\': '\\\\',
- '"': '\\"',
- '\b': '\\b',
- '\f': '\\f',
- '\n': '\\n',
- '\r': '\\r',
- '\t': '\\t',
-}
-for i in range(0x20):
- #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i))
- ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,))
-
-# Assume this produces an infinity on all machines (probably not guaranteed)
-INFINITY = float('1e66666')
-FLOAT_REPR = repr
-
-def encode_basestring(s):
- """Return a JSON representation of a Python string
-
- """
- def replace(match):
- return ESCAPE_DCT[match.group(0)]
- return '"' + ESCAPE.sub(replace, s) + '"'
-
-
-def py_encode_basestring_ascii(s):
- """Return an ASCII-only JSON representation of a Python string
-
- """
- if isinstance(s, str) and HAS_UTF8.search(s) is not None:
- s = s.decode('utf-8')
- def replace(match):
- s = match.group(0)
- try:
- return ESCAPE_DCT[s]
- except KeyError:
- n = ord(s)
- if n < 0x10000:
- #return '\\u{0:04x}'.format(n)
- return '\\u%04x' % (n,)
- else:
- # surrogate pair
- n -= 0x10000
- s1 = 0xd800 | ((n >> 10) & 0x3ff)
- s2 = 0xdc00 | (n & 0x3ff)
- #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2)
- return '\\u%04x\\u%04x' % (s1, s2)
- return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"'
-
-
-encode_basestring_ascii = c_encode_basestring_ascii or py_encode_basestring_ascii
-
-class JSONEncoder(object):
- """Extensible JSON encoder for Python data structures.
-
- Supports the following objects and types by default:
-
- +-------------------+---------------+
- | Python | JSON |
- +===================+===============+
- | dict | object |
- +-------------------+---------------+
- | list, tuple | array |
- +-------------------+---------------+
- | str, unicode | string |
- +-------------------+---------------+
- | int, long, float | number |
- +-------------------+---------------+
- | True | true |
- +-------------------+---------------+
- | False | false |
- +-------------------+---------------+
- | None | null |
- +-------------------+---------------+
-
- To extend this to recognize other objects, subclass and implement a
- ``.default()`` method with another method that returns a serializable
- object for ``o`` if possible, otherwise it should call the superclass
- implementation (to raise ``TypeError``).
-
- """
- item_separator = ', '
- key_separator = ': '
- def __init__(self, skipkeys=False, ensure_ascii=True,
- check_circular=True, allow_nan=True, sort_keys=False,
- indent=None, separators=None, encoding='utf-8', default=None):
- """Constructor for JSONEncoder, with sensible defaults.
-
- If skipkeys is false, then it is a TypeError to attempt
- encoding of keys that are not str, int, long, float or None. If
- skipkeys is True, such items are simply skipped.
-
- If ensure_ascii is true, the output is guaranteed to be str
- objects with all incoming unicode characters escaped. If
- ensure_ascii is false, the output will be unicode object.
-
- If check_circular is true, then lists, dicts, and custom encoded
- objects will be checked for circular references during encoding to
- prevent an infinite recursion (which would cause an OverflowError).
- Otherwise, no such check takes place.
-
- If allow_nan is true, then NaN, Infinity, and -Infinity will be
- encoded as such. This behavior is not JSON specification compliant,
- but is consistent with most JavaScript based encoders and decoders.
- Otherwise, it will be a ValueError to encode such floats.
-
- If sort_keys is true, then the output of dictionaries will be
- sorted by key; this is useful for regression tests to ensure
- that JSON serializations can be compared on a day-to-day basis.
-
- If indent is a non-negative integer, then JSON array
- elements and object members will be pretty-printed with that
- indent level. An indent level of 0 will only insert newlines.
- None is the most compact representation.
-
- If specified, separators should be a (item_separator, key_separator)
- tuple. The default is (', ', ': '). To get the most compact JSON
- representation you should specify (',', ':') to eliminate whitespace.
-
- If specified, default is a function that gets called for objects
- that can't otherwise be serialized. It should return a JSON encodable
- version of the object or raise a ``TypeError``.
-
- If encoding is not None, then all input strings will be
- transformed into unicode using that encoding prior to JSON-encoding.
- The default is UTF-8.
-
- """
-
- self.skipkeys = skipkeys
- self.ensure_ascii = ensure_ascii
- self.check_circular = check_circular
- self.allow_nan = allow_nan
- self.sort_keys = sort_keys
- self.indent = indent
- if separators is not None:
- self.item_separator, self.key_separator = separators
- if default is not None:
- self.default = default
- self.encoding = encoding
-
- def default(self, o):
- """Implement this method in a subclass such that it returns
- a serializable object for ``o``, or calls the base implementation
- (to raise a ``TypeError``).
-
- For example, to support arbitrary iterators, you could
- implement default like this::
-
- def default(self, o):
- try:
- iterable = iter(o)
- except TypeError:
- pass
- else:
- return list(iterable)
- return JSONEncoder.default(self, o)
-
- """
- raise TypeError(repr(o) + " is not JSON serializable")
-
- def encode(self, o):
- """Return a JSON string representation of a Python data structure.
-
- >>> JSONEncoder().encode({"foo": ["bar", "baz"]})
- '{"foo": ["bar", "baz"]}'
-
- """
- # This is for extremely simple cases and benchmarks.
- if isinstance(o, basestring):
- if isinstance(o, str):
- _encoding = self.encoding
- if (_encoding is not None
- and not (_encoding == 'utf-8')):
- o = o.decode(_encoding)
- if self.ensure_ascii:
- return encode_basestring_ascii(o)
- else:
- return encode_basestring(o)
- # This doesn't pass the iterator directly to ''.join() because the
- # exceptions aren't as detailed. The list call should be roughly
- # equivalent to the PySequence_Fast that ''.join() would do.
- chunks = self.iterencode(o, _one_shot=True)
- if not isinstance(chunks, (list, tuple)):
- chunks = list(chunks)
- return ''.join(chunks)
-
- def iterencode(self, o, _one_shot=False):
- """Encode the given object and yield each string
- representation as available.
-
- For example::
-
- for chunk in JSONEncoder().iterencode(bigobject):
- mysocket.write(chunk)
-
- """
- if self.check_circular:
- markers = {}
- else:
- markers = None
- if self.ensure_ascii:
- _encoder = encode_basestring_ascii
- else:
- _encoder = encode_basestring
- if self.encoding != 'utf-8':
- def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding):
- if isinstance(o, str):
- o = o.decode(_encoding)
- return _orig_encoder(o)
-
- def floatstr(o, allow_nan=self.allow_nan, _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY):
- # Check for specials. Note that this type of test is processor- and/or
- # platform-specific, so do tests which don't depend on the internals.
-
- if o != o:
- text = 'NaN'
- elif o == _inf:
- text = 'Infinity'
- elif o == _neginf:
- text = '-Infinity'
- else:
- return _repr(o)
-
- if not allow_nan:
- raise ValueError(
- "Out of range float values are not JSON compliant: " +
- repr(o))
-
- return text
-
-
- if _one_shot and c_make_encoder is not None and not self.indent and not self.sort_keys:
- _iterencode = c_make_encoder(
- markers, self.default, _encoder, self.indent,
- self.key_separator, self.item_separator, self.sort_keys,
- self.skipkeys, self.allow_nan)
- else:
- _iterencode = _make_iterencode(
- markers, self.default, _encoder, self.indent, floatstr,
- self.key_separator, self.item_separator, self.sort_keys,
- self.skipkeys, _one_shot)
- return _iterencode(o, 0)
-
-def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot,
- ## HACK: hand-optimized bytecode; turn globals into locals
- False=False,
- True=True,
- ValueError=ValueError,
- basestring=basestring,
- dict=dict,
- float=float,
- id=id,
- int=int,
- isinstance=isinstance,
- list=list,
- long=long,
- str=str,
- tuple=tuple,
- ):
-
- def _iterencode_list(lst, _current_indent_level):
- if not lst:
- yield '[]'
- return
- if markers is not None:
- markerid = id(lst)
- if markerid in markers:
- raise ValueError("Circular reference detected")
- markers[markerid] = lst
- buf = '['
- if _indent is not None:
- _current_indent_level += 1
- newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
- separator = _item_separator + newline_indent
- buf += newline_indent
- else:
- newline_indent = None
- separator = _item_separator
- first = True
- for value in lst:
- if first:
- first = False
- else:
- buf = separator
- if isinstance(value, basestring):
- yield buf + _encoder(value)
- elif value is None:
- yield buf + 'null'
- elif value is True:
- yield buf + 'true'
- elif value is False:
- yield buf + 'false'
- elif isinstance(value, (int, long)):
- yield buf + str(value)
- elif isinstance(value, float):
- yield buf + _floatstr(value)
- else:
- yield buf
- if isinstance(value, (list, tuple)):
- chunks = _iterencode_list(value, _current_indent_level)
- elif isinstance(value, dict):
- chunks = _iterencode_dict(value, _current_indent_level)
- else:
- chunks = _iterencode(value, _current_indent_level)
- for chunk in chunks:
- yield chunk
- if newline_indent is not None:
- _current_indent_level -= 1
- yield '\n' + (' ' * (_indent * _current_indent_level))
- yield ']'
- if markers is not None:
- del markers[markerid]
-
- def _iterencode_dict(dct, _current_indent_level):
- if not dct:
- yield '{}'
- return
- if markers is not None:
- markerid = id(dct)
- if markerid in markers:
- raise ValueError("Circular reference detected")
- markers[markerid] = dct
- yield '{'
- if _indent is not None:
- _current_indent_level += 1
- newline_indent = '\n' + (' ' * (_indent * _current_indent_level))
- item_separator = _item_separator + newline_indent
- yield newline_indent
- else:
- newline_indent = None
- item_separator = _item_separator
- first = True
- if _sort_keys:
- items = dct.items()
- items.sort(key=lambda kv: kv[0])
- else:
- items = dct.iteritems()
- for key, value in items:
- if isinstance(key, basestring):
- pass
- # JavaScript is weakly typed for these, so it makes sense to
- # also allow them. Many encoders seem to do something like this.
- elif isinstance(key, float):
- key = _floatstr(key)
- elif key is True:
- key = 'true'
- elif key is False:
- key = 'false'
- elif key is None:
- key = 'null'
- elif isinstance(key, (int, long)):
- key = str(key)
- elif _skipkeys:
- continue
- else:
- raise TypeError("key " + repr(key) + " is not a string")
- if first:
- first = False
- else:
- yield item_separator
- yield _encoder(key)
- yield _key_separator
- if isinstance(value, basestring):
- yield _encoder(value)
- elif value is None:
- yield 'null'
- elif value is True:
- yield 'true'
- elif value is False:
- yield 'false'
- elif isinstance(value, (int, long)):
- yield str(value)
- elif isinstance(value, float):
- yield _floatstr(value)
- else:
- if isinstance(value, (list, tuple)):
- chunks = _iterencode_list(value, _current_indent_level)
- elif isinstance(value, dict):
- chunks = _iterencode_dict(value, _current_indent_level)
- else:
- chunks = _iterencode(value, _current_indent_level)
- for chunk in chunks:
- yield chunk
- if newline_indent is not None:
- _current_indent_level -= 1
- yield '\n' + (' ' * (_indent * _current_indent_level))
- yield '}'
- if markers is not None:
- del markers[markerid]
-
- def _iterencode(o, _current_indent_level):
- if isinstance(o, basestring):
- yield _encoder(o)
- elif o is None:
- yield 'null'
- elif o is True:
- yield 'true'
- elif o is False:
- yield 'false'
- elif isinstance(o, (int, long)):
- yield str(o)
- elif isinstance(o, float):
- yield _floatstr(o)
- elif isinstance(o, (list, tuple)):
- for chunk in _iterencode_list(o, _current_indent_level):
- yield chunk
- elif isinstance(o, dict):
- for chunk in _iterencode_dict(o, _current_indent_level):
- yield chunk
- else:
- if markers is not None:
- markerid = id(o)
- if markerid in markers:
- raise ValueError("Circular reference detected")
- markers[markerid] = o
- o = _default(o)
- for chunk in _iterencode(o, _current_indent_level):
- yield chunk
- if markers is not None:
- del markers[markerid]
-
- return _iterencode
diff --git a/mitogen-0.2.7/ansible_mitogen/compat/simplejson/scanner.py b/mitogen-0.2.7/ansible_mitogen/compat/simplejson/scanner.py
deleted file mode 100644
index adbc6ec..000000000
--- a/mitogen-0.2.7/ansible_mitogen/compat/simplejson/scanner.py
+++ /dev/null
@@ -1,65 +0,0 @@
-"""JSON token scanner
-"""
-import re
-try:
- from simplejson._speedups import make_scanner as c_make_scanner
-except ImportError:
- c_make_scanner = None
-
-__all__ = ['make_scanner']
-
-NUMBER_RE = re.compile(
- r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?',
- (re.VERBOSE | re.MULTILINE | re.DOTALL))
-
-def py_make_scanner(context):
- parse_object = context.parse_object
- parse_array = context.parse_array
- parse_string = context.parse_string
- match_number = NUMBER_RE.match
- encoding = context.encoding
- strict = context.strict
- parse_float = context.parse_float
- parse_int = context.parse_int
- parse_constant = context.parse_constant
- object_hook = context.object_hook
-
- def _scan_once(string, idx):
- try:
- nextchar = string[idx]
- except IndexError:
- raise StopIteration
-
- if nextchar == '"':
- return parse_string(string, idx + 1, encoding, strict)
- elif nextchar == '{':
- return parse_object((string, idx + 1), encoding, strict, _scan_once, object_hook)
- elif nextchar == '[':
- return parse_array((string, idx + 1), _scan_once)
- elif nextchar == 'n' and string[idx:idx + 4] == 'null':
- return None, idx + 4
- elif nextchar == 't' and string[idx:idx + 4] == 'true':
- return True, idx + 4
- elif nextchar == 'f' and string[idx:idx + 5] == 'false':
- return False, idx + 5
-
- m = match_number(string, idx)
- if m is not None:
- integer, frac, exp = m.groups()
- if frac or exp:
- res = parse_float(integer + (frac or '') + (exp or ''))
- else:
- res = parse_int(integer)
- return res, m.end()
- elif nextchar == 'N' and string[idx:idx + 3] == 'NaN':
- return parse_constant('NaN'), idx + 3
- elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity':
- return parse_constant('Infinity'), idx + 8
- elif nextchar == '-' and string[idx:idx + 9] == '-Infinity':
- return parse_constant('-Infinity'), idx + 9
- else:
- raise StopIteration
-
- return _scan_once
-
-make_scanner = c_make_scanner or py_make_scanner
diff --git a/mitogen-0.2.7/ansible_mitogen/connection.py b/mitogen-0.2.7/ansible_mitogen/connection.py
deleted file mode 100644
index b5f28d3..000000000
--- a/mitogen-0.2.7/ansible_mitogen/connection.py
+++ /dev/null
@@ -1,1021 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
-import errno
-import logging
-import os
-import pprint
-import stat
-import sys
-import time
-
-import jinja2.runtime
-import ansible.constants as C
-import ansible.errors
-import ansible.plugins.connection
-import ansible.utils.shlex
-
-import mitogen.core
-import mitogen.fork
-import mitogen.unix
-import mitogen.utils
-
-import ansible_mitogen.parsing
-import ansible_mitogen.process
-import ansible_mitogen.services
-import ansible_mitogen.target
-import ansible_mitogen.transport_config
-
-
-LOG = logging.getLogger(__name__)
-
-
-def get_remote_name(spec):
- """
- Return the value to use for the "remote_name" parameter.
- """
- if spec.mitogen_mask_remote_name():
- return 'ansible'
- return None
-
-
-def optional_int(value):
- """
- Convert `value` to an integer if it is not :data:`None`, otherwise return
- :data:`None`.
- """
- try:
- return int(value)
- except (TypeError, ValueError):
- return None
-
-
-def convert_bool(obj):
- if isinstance(obj, bool):
- return obj
- if str(obj).lower() in ('no', 'false', '0'):
- return False
- if str(obj).lower() not in ('yes', 'true', '1'):
- raise ansible.errors.AnsibleConnectionFailure(
- 'expected yes/no/true/false/0/1, got %r' % (obj,)
- )
- return True
-
-
-def default(value, default):
- """
- Return `default` is `value` is :data:`None`, otherwise return `value`.
- """
- if value is None:
- return default
- return value
-
-
-def _connect_local(spec):
- """
- Return ContextService arguments for a local connection.
- """
- return {
- 'method': 'local',
- 'kwargs': {
- 'python_path': spec.python_path(),
- }
- }
-
-
-def _connect_ssh(spec):
- """
- Return ContextService arguments for an SSH connection.
- """
- if C.HOST_KEY_CHECKING:
- check_host_keys = 'enforce'
- else:
- check_host_keys = 'ignore'
-
- # #334: tilde-expand private_key_file to avoid implementation difference
- # between Python and OpenSSH.
- private_key_file = spec.private_key_file()
- if private_key_file is not None:
- private_key_file = os.path.expanduser(private_key_file)
-
- return {
- 'method': 'ssh',
- 'kwargs': {
- 'check_host_keys': check_host_keys,
- 'hostname': spec.remote_addr(),
- 'username': spec.remote_user(),
- 'compression': convert_bool(
- default(spec.mitogen_ssh_compression(), True)
- ),
- 'password': spec.password(),
- 'port': spec.port(),
- 'python_path': spec.python_path(),
- 'identity_file': private_key_file,
- 'identities_only': False,
- 'ssh_path': spec.ssh_executable(),
- 'connect_timeout': spec.ansible_ssh_timeout(),
- 'ssh_args': spec.ssh_args(),
- 'ssh_debug_level': spec.mitogen_ssh_debug_level(),
- 'remote_name': get_remote_name(spec),
- }
- }
-
-
-def _connect_docker(spec):
- """
- Return ContextService arguments for a Docker connection.
- """
- return {
- 'method': 'docker',
- 'kwargs': {
- 'username': spec.remote_user(),
- 'container': spec.remote_addr(),
- 'python_path': spec.python_path(),
- 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
- 'remote_name': get_remote_name(spec),
- }
- }
-
-
-def _connect_kubectl(spec):
- """
- Return ContextService arguments for a Kubernetes connection.
- """
- return {
- 'method': 'kubectl',
- 'kwargs': {
- 'pod': spec.remote_addr(),
- 'python_path': spec.python_path(),
- 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
- 'kubectl_path': spec.mitogen_kubectl_path(),
- 'kubectl_args': spec.extra_args(),
- 'remote_name': get_remote_name(spec),
- }
- }
-
-
-def _connect_jail(spec):
- """
- Return ContextService arguments for a FreeBSD jail connection.
- """
- return {
- 'method': 'jail',
- 'kwargs': {
- 'username': spec.remote_user(),
- 'container': spec.remote_addr(),
- 'python_path': spec.python_path(),
- 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
- 'remote_name': get_remote_name(spec),
- }
- }
-
-
-def _connect_lxc(spec):
- """
- Return ContextService arguments for an LXC Classic container connection.
- """
- return {
- 'method': 'lxc',
- 'kwargs': {
- 'container': spec.remote_addr(),
- 'python_path': spec.python_path(),
- 'lxc_attach_path': spec.mitogen_lxc_attach_path(),
- 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
- 'remote_name': get_remote_name(spec),
- }
- }
-
-
-def _connect_lxd(spec):
- """
- Return ContextService arguments for an LXD container connection.
- """
- return {
- 'method': 'lxd',
- 'kwargs': {
- 'container': spec.remote_addr(),
- 'python_path': spec.python_path(),
- 'lxc_path': spec.mitogen_lxc_path(),
- 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(),
- 'remote_name': get_remote_name(spec),
- }
- }
-
-
-def _connect_machinectl(spec):
- """
- Return ContextService arguments for a machinectl connection.
- """
- return _connect_setns(spec, kind='machinectl')
-
-
-def _connect_setns(spec, kind=None):
- """
- Return ContextService arguments for a mitogen_setns connection.
- """
- return {
- 'method': 'setns',
- 'kwargs': {
- 'container': spec.remote_addr(),
- 'username': spec.remote_user(),
- 'python_path': spec.python_path(),
- 'kind': kind or spec.mitogen_kind(),
- 'docker_path': spec.mitogen_docker_path(),
- 'lxc_path': spec.mitogen_lxc_path(),
- 'lxc_info_path': spec.mitogen_lxc_info_path(),
- 'machinectl_path': spec.mitogen_machinectl_path(),
- }
- }
-
-
-def _connect_su(spec):
- """
- Return ContextService arguments for su as a become method.
- """
- return {
- 'method': 'su',
- 'enable_lru': True,
- 'kwargs': {
- 'username': spec.become_user(),
- 'password': spec.become_pass(),
- 'python_path': spec.python_path(),
- 'su_path': spec.become_exe(),
- 'connect_timeout': spec.timeout(),
- 'remote_name': get_remote_name(spec),
- }
- }
-
-
-def _connect_sudo(spec):
- """
- Return ContextService arguments for sudo as a become method.
- """
- return {
- 'method': 'sudo',
- 'enable_lru': True,
- 'kwargs': {
- 'username': spec.become_user(),
- 'password': spec.become_pass(),
- 'python_path': spec.python_path(),
- 'sudo_path': spec.become_exe(),
- 'connect_timeout': spec.timeout(),
- 'sudo_args': spec.sudo_args(),
- 'remote_name': get_remote_name(spec),
- }
- }
-
-
-def _connect_doas(spec):
- """
- Return ContextService arguments for doas as a become method.
- """
- return {
- 'method': 'doas',
- 'enable_lru': True,
- 'kwargs': {
- 'username': spec.become_user(),
- 'password': spec.become_pass(),
- 'python_path': spec.python_path(),
- 'doas_path': spec.become_exe(),
- 'connect_timeout': spec.timeout(),
- 'remote_name': get_remote_name(spec),
- }
- }
-
-
-def _connect_mitogen_su(spec):
- """
- Return ContextService arguments for su as a first class connection.
- """
- return {
- 'method': 'su',
- 'kwargs': {
- 'username': spec.remote_user(),
- 'password': spec.password(),
- 'python_path': spec.python_path(),
- 'su_path': spec.become_exe(),
- 'connect_timeout': spec.timeout(),
- 'remote_name': get_remote_name(spec),
- }
- }
-
-
-def _connect_mitogen_sudo(spec):
- """
- Return ContextService arguments for sudo as a first class connection.
- """
- return {
- 'method': 'sudo',
- 'kwargs': {
- 'username': spec.remote_user(),
- 'password': spec.password(),
- 'python_path': spec.python_path(),
- 'sudo_path': spec.become_exe(),
- 'connect_timeout': spec.timeout(),
- 'sudo_args': spec.sudo_args(),
- 'remote_name': get_remote_name(spec),
- }
- }
-
-
-def _connect_mitogen_doas(spec):
- """
- Return ContextService arguments for doas as a first class connection.
- """
- return {
- 'method': 'doas',
- 'kwargs': {
- 'username': spec.remote_user(),
- 'password': spec.password(),
- 'python_path': spec.python_path(),
- 'doas_path': spec.become_exe(),
- 'connect_timeout': spec.timeout(),
- 'remote_name': get_remote_name(spec),
- }
- }
-
-
-#: Mapping of connection method names to functions invoked as `func(spec)`
-#: generating ContextService keyword arguments matching a connection
-#: specification.
-CONNECTION_METHOD = {
- 'docker': _connect_docker,
- 'kubectl': _connect_kubectl,
- 'jail': _connect_jail,
- 'local': _connect_local,
- 'lxc': _connect_lxc,
- 'lxd': _connect_lxd,
- 'machinectl': _connect_machinectl,
- 'setns': _connect_setns,
- 'ssh': _connect_ssh,
- 'smart': _connect_ssh, # issue #548.
- 'su': _connect_su,
- 'sudo': _connect_sudo,
- 'doas': _connect_doas,
- 'mitogen_su': _connect_mitogen_su,
- 'mitogen_sudo': _connect_mitogen_sudo,
- 'mitogen_doas': _connect_mitogen_doas,
-}
-
-
-class Broker(mitogen.master.Broker):
- """
- WorkerProcess maintains at most 2 file descriptors, therefore does not need
- the exuberant syscall expense of EpollPoller, so override it and restore
- the poll() poller.
- """
- poller_class = mitogen.core.Poller
-
-
-class CallChain(mitogen.parent.CallChain):
- """
- Extend :class:`mitogen.parent.CallChain` to additionally cause the
- associated :class:`Connection` to be reset if a ChannelError occurs.
-
- This only catches failures that occur while a call is pending, it is a
- stop-gap until a more general method is available to notice connection in
- every situation.
- """
- call_aborted_msg = (
- 'Mitogen was disconnected from the remote environment while a call '
- 'was in-progress. If you feel this is in error, please file a bug. '
- 'Original error was: %s'
- )
-
- def __init__(self, connection, context, pipelined=False):
- super(CallChain, self).__init__(context, pipelined)
- #: The connection to reset on CallError.
- self._connection = connection
-
- def _rethrow(self, recv):
- try:
- return recv.get().unpickle()
- except mitogen.core.ChannelError as e:
- self._connection.reset()
- raise ansible.errors.AnsibleConnectionFailure(
- self.call_aborted_msg % (e,)
- )
-
- def call(self, func, *args, **kwargs):
- """
- Like :meth:`mitogen.parent.CallChain.call`, but log timings.
- """
- t0 = time.time()
- try:
- recv = self.call_async(func, *args, **kwargs)
- return self._rethrow(recv)
- finally:
- LOG.debug('Call took %d ms: %r', 1000 * (time.time() - t0),
- mitogen.parent.CallSpec(func, args, kwargs))
-
-
-class Connection(ansible.plugins.connection.ConnectionBase):
- #: mitogen.master.Broker for this worker.
- broker = None
-
- #: mitogen.master.Router for this worker.
- router = None
-
- #: mitogen.parent.Context representing the parent Context, which is
- #: presently always the connection multiplexer process.
- parent = None
-
- #: mitogen.parent.Context for the target account on the target, possibly
- #: reached via become.
- context = None
-
- #: Context for the login account on the target. This is always the login
- #: account, even when become=True.
- login_context = None
-
- #: Only sudo, su, and doas are supported for now.
- become_methods = ['sudo', 'su', 'doas']
-
- #: Dict containing init_child() return value as recorded at startup by
- #: ContextService. Contains:
- #:
- #: fork_context: Context connected to the fork parent : process in the
- #: target account.
- #: home_dir: Target context's home directory.
- #: good_temp_dir: A writeable directory where new temporary directories
- #: can be created.
- init_child_result = None
-
- #: A :class:`mitogen.parent.CallChain` for calls made to the target
- #: account, to ensure subsequent calls fail with the original exception if
- #: pipelined directory creation or file transfer fails.
- chain = None
-
- #
- # Note: any of the attributes below may be :data:`None` if the connection
- # plugin was constructed directly by a non-cooperative action, such as in
- # the case of the synchronize module.
- #
-
- #: Set to the host name as it appears in inventory by on_action_run().
- inventory_hostname = None
-
- #: Set to task_vars by on_action_run().
- _task_vars = None
-
- #: Set to 'hostvars' by on_action_run()
- host_vars = None
-
- #: Set by on_action_run()
- delegate_to_hostname = None
-
- #: Set to '_loader.get_basedir()' by on_action_run(). Used by mitogen_local
- #: to change the working directory to that of the current playbook,
- #: matching vanilla Ansible behaviour.
- loader_basedir = None
-
- def __init__(self, play_context, new_stdin, **kwargs):
- assert ansible_mitogen.process.MuxProcess.unix_listener_path, (
- 'Mitogen connection types may only be instantiated '
- 'while the "mitogen" strategy is active.'
- )
- super(Connection, self).__init__(play_context, new_stdin)
-
- def __del__(self):
- """
- Ansible cannot be trusted to always call close() e.g. the synchronize
- action constructs a local connection like this. So provide a destructor
- in the hopes of catching these cases.
- """
- # https://github.com/dw/mitogen/issues/140
- self.close()
-
- def on_action_run(self, task_vars, delegate_to_hostname, loader_basedir):
- """
- Invoked by ActionModuleMixin to indicate a new task is about to start
- executing. We use the opportunity to grab relevant bits from the
- task-specific data.
-
- :param dict task_vars:
- Task variable dictionary.
- :param str delegate_to_hostname:
- :data:`None`, or the template-expanded inventory hostname this task
- is being delegated to. A similar variable exists on PlayContext
- when ``delegate_to:`` is active, however it is unexpanded.
- :param str loader_basedir:
- Loader base directory; see :attr:`loader_basedir`.
- """
- self.inventory_hostname = task_vars['inventory_hostname']
- self._task_vars = task_vars
- self.host_vars = task_vars['hostvars']
- self.delegate_to_hostname = delegate_to_hostname
- self.loader_basedir = loader_basedir
- self._mitogen_reset(mode='put')
-
- def get_task_var(self, key, default=None):
- """
- Fetch the value of a task variable related to connection configuration,
- or, if delegate_to is active, fetch the same variable via HostVars for
- the delegated-to machine.
-
- When running with delegate_to, Ansible tasks have variables associated
- with the original machine, not the delegated-to machine, therefore it
- does not make sense to extract connection-related configuration for the
- delegated-to machine from them.
- """
- if self._task_vars:
- if self.delegate_to_hostname is None:
- if key in self._task_vars:
- return self._task_vars[key]
- else:
- delegated_vars = self._task_vars['ansible_delegated_vars']
- if self.delegate_to_hostname in delegated_vars:
- task_vars = delegated_vars[self.delegate_to_hostname]
- if key in task_vars:
- return task_vars[key]
-
- return default
-
- @property
- def homedir(self):
- self._connect()
- return self.init_child_result['home_dir']
-
- @property
- def connected(self):
- return self.context is not None
-
- def _spec_from_via(self, proxied_inventory_name, via_spec):
- """
- Produce a dict connection specifiction given a string `via_spec`, of
- the form `[[become_method:]become_user@]inventory_hostname`.
- """
- become_user, _, inventory_name = via_spec.rpartition('@')
- become_method, _, become_user = become_user.rpartition(':')
-
- # must use __contains__ to avoid a TypeError for a missing host on
- # Ansible 2.3.
- if self.host_vars is None or inventory_name not in self.host_vars:
- raise ansible.errors.AnsibleConnectionFailure(
- self.unknown_via_msg % (
- via_spec,
- proxied_inventory_name,
- )
- )
-
- via_vars = self.host_vars[inventory_name]
- return ansible_mitogen.transport_config.MitogenViaSpec(
- inventory_name=inventory_name,
- play_context=self._play_context,
- host_vars=dict(via_vars), # TODO: make it lazy
- become_method=become_method or None,
- become_user=become_user or None,
- )
-
- unknown_via_msg = 'mitogen_via=%s of %s specifies an unknown hostname'
- via_cycle_msg = 'mitogen_via=%s of %s creates a cycle (%s)'
-
- def _stack_from_spec(self, spec, stack=(), seen_names=()):
- """
- Return a tuple of ContextService parameter dictionaries corresponding
- to the connection described by `spec`, and any connection referenced by
- its `mitogen_via` or `become` fields. Each element is a dict of the
- form::
-
- {
- # Optional. If present and `True`, this hop is elegible for
- # interpreter recycling.
- "enable_lru": True,
- # mitogen.master.Router method name.
- "method": "ssh",
- # mitogen.master.Router method kwargs.
- "kwargs": {
- "hostname": "..."
- }
- }
-
- :param ansible_mitogen.transport_config.Spec spec:
- Connection specification.
- :param tuple stack:
- Stack elements from parent call (used for recursion).
- :param tuple seen_names:
- Inventory hostnames from parent call (cycle detection).
- :returns:
- Tuple `(stack, seen_names)`.
- """
- if spec.inventory_name() in seen_names:
- raise ansible.errors.AnsibleConnectionFailure(
- self.via_cycle_msg % (
- spec.mitogen_via(),
- spec.inventory_name(),
- ' -> '.join(reversed(
- seen_names + (spec.inventory_name(),)
- )),
- )
- )
-
- if spec.mitogen_via():
- stack = self._stack_from_spec(
- self._spec_from_via(spec.inventory_name(), spec.mitogen_via()),
- stack=stack,
- seen_names=seen_names + (spec.inventory_name(),),
- )
-
- stack += (CONNECTION_METHOD[spec.transport()](spec),)
- if spec.become() and ((spec.become_user() != spec.remote_user()) or
- C.BECOME_ALLOW_SAME_USER):
- stack += (CONNECTION_METHOD[spec.become_method()](spec),)
-
- return stack
-
- def _connect_broker(self):
- """
- Establish a reference to the Broker, Router and parent context used for
- connections.
- """
- if not self.broker:
- self.broker = mitogen.master.Broker()
- self.router, self.parent = mitogen.unix.connect(
- path=ansible_mitogen.process.MuxProcess.unix_listener_path,
- broker=self.broker,
- )
-
- def _build_stack(self):
- """
- Construct a list of dictionaries representing the connection
- configuration between the controller and the target. This is
- additionally used by the integration tests "mitogen_get_stack" action
- to fetch the would-be connection configuration.
- """
- return self._stack_from_spec(
- ansible_mitogen.transport_config.PlayContextSpec(
- connection=self,
- play_context=self._play_context,
- transport=self.transport,
- inventory_name=self.inventory_hostname,
- )
- )
-
- def _connect_stack(self, stack):
- """
- Pass `stack` to ContextService, requesting a copy of the context object
- representing the last tuple element. If no connection exists yet,
- ContextService will recursively establish it before returning it or
- throwing an error.
-
- See :meth:`ansible_mitogen.services.ContextService.get` docstring for
- description of the returned dictionary.
- """
- try:
- dct = self.parent.call_service(
- service_name='ansible_mitogen.services.ContextService',
- method_name='get',
- stack=mitogen.utils.cast(list(stack)),
- )
- except mitogen.core.CallError:
- LOG.warning('Connection failed; stack configuration was:\n%s',
- pprint.pformat(stack))
- raise
-
- if dct['msg']:
- if dct['method_name'] in self.become_methods:
- raise ansible.errors.AnsibleModuleError(dct['msg'])
- raise ansible.errors.AnsibleConnectionFailure(dct['msg'])
-
- self.context = dct['context']
- self.chain = CallChain(self, self.context, pipelined=True)
- if self._play_context.become:
- self.login_context = dct['via']
- else:
- self.login_context = self.context
-
- self.init_child_result = dct['init_child_result']
-
- def get_good_temp_dir(self):
- """
- Return the 'good temporary directory' as discovered by
- :func:`ansible_mitogen.target.init_child` immediately after
- ContextService constructed the target context.
- """
- self._connect()
- return self.init_child_result['good_temp_dir']
-
- def _connect(self):
- """
- Establish a connection to the master process's UNIX listener socket,
- constructing a mitogen.master.Router to communicate with the master,
- and a mitogen.parent.Context to represent it.
-
- Depending on the original transport we should emulate, trigger one of
- the _connect_*() service calls defined above to cause the master
- process to establish the real connection on our behalf, or return a
- reference to the existing one.
- """
- if self.connected:
- return
-
- self._connect_broker()
- stack = self._build_stack()
- self._connect_stack(stack)
-
- def _mitogen_reset(self, mode):
- """
- Forget everything we know about the connected context. This function
- cannot be called _reset() since that name is used as a public API by
- Ansible 2.4 wait_for_connection plug-in.
-
- :param str mode:
- Name of ContextService method to use to discard the context, either
- 'put' or 'reset'.
- """
- if not self.context:
- return
-
- self.chain.reset()
- self.parent.call_service(
- service_name='ansible_mitogen.services.ContextService',
- method_name=mode,
- context=self.context
- )
-
- self.context = None
- self.login_context = None
- self.init_child_result = None
- self.chain = None
-
- def _shutdown_broker(self):
- """
- Shutdown the broker thread during :meth:`close` or :meth:`reset`.
- """
- if self.broker:
- self.broker.shutdown()
- self.broker.join()
- self.broker = None
- self.router = None
-
- # #420: Ansible executes "meta" actions in the top-level process,
- # meaning "reset_connection" will cause :class:`mitogen.core.Latch`
- # FDs to be cached and erroneously shared by children on subsequent
- # WorkerProcess forks. To handle that, call on_fork() to ensure any
- # shared state is discarded.
- # #490: only attempt to clean up when it's known that some
- # resources exist to cleanup, otherwise later __del__ double-call
- # to close() due to GC at random moment may obliterate an unrelated
- # Connection's resources.
- mitogen.fork.on_fork()
-
- def close(self):
- """
- Arrange for the mitogen.master.Router running in the worker to
- gracefully shut down, and wait for shutdown to complete. Safe to call
- multiple times.
- """
- self._mitogen_reset(mode='put')
- self._shutdown_broker()
-
- def _reset_find_task_vars(self):
- """
- Monsterous hack: since "meta: reset_connection" does not run from an
- action, we cannot capture task variables via :meth:`on_action_run`.
- Instead walk the parent frames searching for the `all_vars` local from
- StrategyBase._execute_meta(). If this fails, just leave task_vars
- unset, likely causing a subtly wrong configuration to be selected.
- """
- frame = sys._getframe()
- while frame and not self._task_vars:
- self._task_vars = frame.f_locals.get('all_vars')
- frame = frame.f_back
-
- reset_compat_msg = (
- 'Mitogen only supports "reset_connection" on Ansible 2.5.6 or later'
- )
-
- def reset(self):
- """
- Explicitly terminate the connection to the remote host. This discards
- any local state we hold for the connection, returns the Connection to
- the 'disconnected' state, and informs ContextService the connection is
- bad somehow, and should be shut down and discarded.
- """
- if self._task_vars is None:
- self._reset_find_task_vars()
-
- if self._play_context.remote_addr is None:
- # <2.5.6 incorrectly populate PlayContext for reset_connection
- # https://github.com/ansible/ansible/issues/27520
- raise ansible.errors.AnsibleConnectionFailure(
- self.reset_compat_msg
- )
-
- self._connect()
- self._mitogen_reset(mode='reset')
- self._shutdown_broker()
-
- # Compatibility with Ansible 2.4 wait_for_connection plug-in.
- _reset = reset
-
- def get_chain(self, use_login=False, use_fork=False):
- """
- Return the :class:`mitogen.parent.CallChain` to use for executing
- function calls.
-
- :param bool use_login:
- If :data:`True`, always return the chain for the login account
- rather than any active become user.
- :param bool use_fork:
- If :data:`True`, return the chain for the fork parent.
- :returns mitogen.parent.CallChain:
- """
- self._connect()
- if use_login:
- return self.login_context.default_call_chain
- # See FORK_SUPPORTED comments in target.py.
- if use_fork and self.init_child_result['fork_context'] is not None:
- return self.init_child_result['fork_context'].default_call_chain
- return self.chain
-
- def spawn_isolated_child(self):
- """
- Fork or launch a new child off the target context.
-
- :returns:
- mitogen.core.Context of the new child.
- """
- return self.get_chain(use_fork=True).call(
- ansible_mitogen.target.spawn_isolated_child
- )
-
- def get_extra_args(self):
- """
- Overridden by connections/mitogen_kubectl.py to a list of additional
- arguments for the command.
- """
- # TODO: maybe use this for SSH too.
- return []
-
- def get_default_cwd(self):
- """
- Overridden by connections/mitogen_local.py to emulate behaviour of CWD
- being fixed to that of ActionBase._loader.get_basedir().
- """
- return None
-
- def get_default_env(self):
- """
- Overridden by connections/mitogen_local.py to emulate behaviour of
- WorkProcess environment inherited from WorkerProcess.
- """
- return None
-
- def exec_command(self, cmd, in_data='', sudoable=True, mitogen_chdir=None):
- """
- Implement exec_command() by calling the corresponding
- ansible_mitogen.target function in the target.
-
- :param str cmd:
- Shell command to execute.
- :param bytes in_data:
- Data to supply on ``stdin`` of the process.
- :returns:
- (return code, stdout bytes, stderr bytes)
- """
- emulate_tty = (not in_data and sudoable)
- rc, stdout, stderr = self.get_chain().call(
- ansible_mitogen.target.exec_command,
- cmd=mitogen.utils.cast(cmd),
- in_data=mitogen.utils.cast(in_data),
- chdir=mitogen_chdir or self.get_default_cwd(),
- emulate_tty=emulate_tty,
- )
-
- stderr += b'Shared connection to %s closed.%s' % (
- self._play_context.remote_addr.encode(),
- (b'\r\n' if emulate_tty else b'\n'),
- )
- return rc, stdout, stderr
-
- def fetch_file(self, in_path, out_path):
- """
- Implement fetch_file() by calling the corresponding
- ansible_mitogen.target function in the target.
-
- :param str in_path:
- Remote filesystem path to read.
- :param str out_path:
- Local filesystem path to write.
- """
- output = self.get_chain().call(
- ansible_mitogen.target.read_path,
- mitogen.utils.cast(in_path),
- )
- ansible_mitogen.target.write_path(out_path, output)
-
- def put_data(self, out_path, data, mode=None, utimes=None):
- """
- Implement put_file() by caling the corresponding ansible_mitogen.target
- function in the target, transferring small files inline. This is
- pipelined and will return immediately; failed transfers are reported as
- exceptions in subsequent functon calls.
-
- :param str out_path:
- Remote filesystem path to write.
- :param byte data:
- File contents to put.
- """
- self.get_chain().call_no_reply(
- ansible_mitogen.target.write_path,
- mitogen.utils.cast(out_path),
- mitogen.core.Blob(data),
- mode=mode,
- utimes=utimes,
- )
-
- #: Maximum size of a small file before switching to streaming
- #: transfer. This should really be the same as
- #: mitogen.services.FileService.IO_SIZE, however the message format has
- #: slightly more overhead, so just randomly subtract 4KiB.
- SMALL_FILE_LIMIT = mitogen.core.CHUNK_SIZE - 4096
-
- def _throw_io_error(self, e, path):
- if e.args[0] == errno.ENOENT:
- s = 'file or module does not exist: ' + path
- raise ansible.errors.AnsibleFileNotFound(s)
-
- def put_file(self, in_path, out_path):
- """
- Implement put_file() by streamily transferring the file via
- FileService.
-
- :param str in_path:
- Local filesystem path to read.
- :param str out_path:
- Remote filesystem path to write.
- """
- try:
- st = os.stat(in_path)
- except OSError as e:
- self._throw_io_error(e, in_path)
- raise
-
- if not stat.S_ISREG(st.st_mode):
- raise IOError('%r is not a regular file.' % (in_path,))
-
- # If the file is sufficiently small, just ship it in the argument list
- # rather than introducing an extra RTT for the child to request it from
- # FileService.
- if st.st_size <= self.SMALL_FILE_LIMIT:
- try:
- fp = open(in_path, 'rb')
- try:
- s = fp.read(self.SMALL_FILE_LIMIT + 1)
- finally:
- fp.close()
- except OSError:
- self._throw_io_error(e, in_path)
- raise
-
- # Ensure did not grow during read.
- if len(s) == st.st_size:
- return self.put_data(out_path, s, mode=st.st_mode,
- utimes=(st.st_atime, st.st_mtime))
-
- self._connect()
- self.parent.call_service(
- service_name='mitogen.service.FileService',
- method_name='register',
- path=mitogen.utils.cast(in_path)
- )
-
- # For now this must remain synchronous, as the action plug-in may have
- # passed us a temporary file to transfer. A future FileService could
- # maintain an LRU list of open file descriptors to keep the temporary
- # file alive, but that requires more work.
- self.get_chain().call(
- ansible_mitogen.target.transfer_file,
- context=self.parent,
- in_path=in_path,
- out_path=out_path
- )
diff --git a/mitogen-0.2.7/ansible_mitogen/loaders.py b/mitogen-0.2.7/ansible_mitogen/loaders.py
deleted file mode 100644
index ff06c0c..000000000
--- a/mitogen-0.2.7/ansible_mitogen/loaders.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-"""
-Stable names for PluginLoader instances across Ansible versions.
-"""
-
-from __future__ import absolute_import
-
-try:
- from ansible.plugins.loader import action_loader
- from ansible.plugins.loader import connection_loader
- from ansible.plugins.loader import module_loader
- from ansible.plugins.loader import module_utils_loader
- from ansible.plugins.loader import shell_loader
- from ansible.plugins.loader import strategy_loader
-except ImportError: # Ansible <2.4
- from ansible.plugins import action_loader
- from ansible.plugins import connection_loader
- from ansible.plugins import module_loader
- from ansible.plugins import module_utils_loader
- from ansible.plugins import shell_loader
- from ansible.plugins import strategy_loader
diff --git a/mitogen-0.2.7/ansible_mitogen/logging.py b/mitogen-0.2.7/ansible_mitogen/logging.py
deleted file mode 100644
index ce6f165..000000000
--- a/mitogen-0.2.7/ansible_mitogen/logging.py
+++ /dev/null
@@ -1,127 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import logging
-import os
-
-import mitogen.core
-import mitogen.utils
-
-try:
- from __main__ import display
-except ImportError:
- from ansible.utils.display import Display
- display = Display()
-
-
-#: The process name set via :func:`set_process_name`.
-_process_name = None
-
-#: The PID of the process that last called :func:`set_process_name`, so its
-#: value can be ignored in unknown fork children.
-_process_pid = None
-
-
-def set_process_name(name):
- """
- Set a name to adorn log messages with.
- """
- global _process_name
- _process_name = name
-
- global _process_pid
- _process_pid = os.getpid()
-
-
-class Handler(logging.Handler):
- """
- Use Mitogen's log format, but send the result to a Display method.
- """
- def __init__(self, normal_method):
- logging.Handler.__init__(self)
- self.formatter = mitogen.utils.log_get_formatter()
- self.normal_method = normal_method
-
- #: Set of target loggers that produce warnings and errors that spam the
- #: console needlessly. Their log level is forced to INFO. A better strategy
- #: may simply be to bury all target logs in DEBUG output, but not by
- #: overriding their log level as done here.
- NOISY_LOGGERS = frozenset([
- 'dnf', # issue #272; warns when a package is already installed.
- 'boto', # issue #541; normal boto retry logic can cause ERROR logs.
- ])
-
- def emit(self, record):
- mitogen_name = getattr(record, 'mitogen_name', '')
- if mitogen_name == 'stderr':
- record.levelno = logging.ERROR
- if mitogen_name in self.NOISY_LOGGERS and record.levelno >= logging.WARNING:
- record.levelno = logging.DEBUG
-
- if _process_pid == os.getpid():
- process_name = _process_name
- else:
- process_name = '?'
-
- s = '[%-4s %d] %s' % (process_name, os.getpid(), self.format(record))
- if record.levelno >= logging.ERROR:
- display.error(s, wrap_text=False)
- elif record.levelno >= logging.WARNING:
- display.warning(s, formatted=True)
- else:
- self.normal_method(s)
-
-
-def setup():
- """
- Install handlers for Mitogen loggers to redirect them into the Ansible
- display framework. Ansible installs its own logging framework handlers when
- C.DEFAULT_LOG_PATH is set, therefore disable propagation for our handlers.
- """
- l_mitogen = logging.getLogger('mitogen')
- l_mitogen_io = logging.getLogger('mitogen.io')
- l_ansible_mitogen = logging.getLogger('ansible_mitogen')
-
- for logger in l_mitogen, l_mitogen_io, l_ansible_mitogen:
- logger.handlers = [Handler(display.vvv)]
- logger.propagate = False
-
- if display.verbosity > 2:
- l_ansible_mitogen.setLevel(logging.DEBUG)
- l_mitogen.setLevel(logging.DEBUG)
- else:
- # Mitogen copies the active log level into new children, allowing them
- # to filter tiny messages before they hit the network, and therefore
- # before they wake the IO loop. Explicitly setting INFO saves ~4%
- # running against just the local machine.
- l_mitogen.setLevel(logging.ERROR)
- l_ansible_mitogen.setLevel(logging.ERROR)
-
- if display.verbosity > 3:
- l_mitogen_io.setLevel(logging.DEBUG)
diff --git a/mitogen-0.2.7/ansible_mitogen/mixins.py b/mitogen-0.2.7/ansible_mitogen/mixins.py
deleted file mode 100644
index 890467f..000000000
--- a/mitogen-0.2.7/ansible_mitogen/mixins.py
+++ /dev/null
@@ -1,432 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import logging
-import os
-import pwd
-import random
-import traceback
-
-try:
- from shlex import quote as shlex_quote
-except ImportError:
- from pipes import quote as shlex_quote
-
-from ansible.module_utils._text import to_bytes
-from ansible.parsing.utils.jsonify import jsonify
-
-import ansible
-import ansible.constants
-import ansible.plugins
-import ansible.plugins.action
-
-import mitogen.core
-import mitogen.select
-import mitogen.utils
-
-import ansible_mitogen.connection
-import ansible_mitogen.planner
-import ansible_mitogen.target
-from ansible.module_utils._text import to_text
-
-
-LOG = logging.getLogger(__name__)
-
-
-class ActionModuleMixin(ansible.plugins.action.ActionBase):
- """
- The Mitogen-patched PluginLoader dynamically mixes this into every action
- class that Ansible attempts to load. It exists to override all the
- assumptions built into the base action class that should really belong in
- some middle layer, or at least in the connection layer.
-
- Functionality is defined here for:
-
- * Capturing the final set of task variables and giving Connection a chance
- to update its idea of the correct execution environment, before any
- attempt is made to call a Connection method. While it's not expected for
- the interpreter to change on a per-task basis, Ansible permits this, and
- so it must be supported.
-
- * Overriding lots of methods that try to call out to shell for mundane
- reasons, such as copying files around, changing file permissions,
- creating temporary directories and suchlike.
-
- * Short-circuiting any use of Ansiballz or related code for executing a
- module remotely using shell commands and SSH.
-
- * Short-circuiting most of the logic in dealing with the fact that Ansible
- always runs become: tasks across at least the SSH user account and the
- destination user account, and handling the security permission issues
- that crop up due to this. Mitogen always runs a task completely within
- the target user account, so it's not a problem for us.
- """
- def __init__(self, task, connection, *args, **kwargs):
- """
- Verify the received connection is really a Mitogen connection. If not,
- transmute this instance back into the original unadorned base class.
-
- This allows running the Mitogen strategy in mixed-target playbooks,
- where some targets use SSH while others use WinRM or some fancier UNIX
- connection plug-in. That's because when the Mitogen strategy is active,
- ActionModuleMixin is unconditionally mixed into any action module that
- is instantiated, and there is no direct way for the monkey-patch to
- know what kind of connection will be used upfront.
- """
- super(ActionModuleMixin, self).__init__(task, connection, *args, **kwargs)
- if not isinstance(connection, ansible_mitogen.connection.Connection):
- _, self.__class__ = type(self).__bases__
-
- def run(self, tmp=None, task_vars=None):
- """
- Override run() to notify Connection of task-specific data, so it has a
- chance to know e.g. the Python interpreter in use.
- """
- self._connection.on_action_run(
- task_vars=task_vars,
- delegate_to_hostname=self._task.delegate_to,
- loader_basedir=self._loader.get_basedir(),
- )
- return super(ActionModuleMixin, self).run(tmp, task_vars)
-
- COMMAND_RESULT = {
- 'rc': 0,
- 'stdout': '',
- 'stdout_lines': [],
- 'stderr': ''
- }
-
- def fake_shell(self, func, stdout=False):
- """
- Execute a function and decorate its return value in the style of
- _low_level_execute_command(). This produces a return value that looks
- like some shell command was run, when really func() was implemented
- entirely in Python.
-
- If the function raises :py:class:`mitogen.core.CallError`, this will be
- translated into a failed shell command with a non-zero exit status.
-
- :param func:
- Function invoked as `func()`.
- :returns:
- See :py:attr:`COMMAND_RESULT`.
- """
- dct = self.COMMAND_RESULT.copy()
- try:
- rc = func()
- if stdout:
- dct['stdout'] = repr(rc)
- except mitogen.core.CallError:
- LOG.exception('While emulating a shell command')
- dct['rc'] = 1
- dct['stderr'] = traceback.format_exc()
-
- return dct
-
- def _remote_file_exists(self, path):
- """
- Determine if `path` exists by directly invoking os.path.exists() in the
- target user account.
- """
- LOG.debug('_remote_file_exists(%r)', path)
- return self._connection.get_chain().call(
- ansible_mitogen.target.file_exists,
- mitogen.utils.cast(path)
- )
-
- def _configure_module(self, module_name, module_args, task_vars=None):
- """
- Mitogen does not use the Ansiballz framework. This call should never
- happen when ActionMixin is active, so crash if it does.
- """
- assert False, "_configure_module() should never be called."
-
- def _is_pipelining_enabled(self, module_style, wrap_async=False):
- """
- Mitogen does not use SSH pipelining. This call should never happen when
- ActionMixin is active, so crash if it does.
- """
- assert False, "_is_pipelining_enabled() should never be called."
-
- def _generate_tmp_path(self):
- return os.path.join(
- self._connection.get_good_temp_dir(),
- 'ansible_mitogen_action_%016x' % (
- random.getrandbits(8*8),
- )
- )
-
- def _generate_tmp_path(self):
- return os.path.join(
- self._connection.get_good_temp_dir(),
- 'ansible_mitogen_action_%016x' % (
- random.getrandbits(8*8),
- )
- )
-
- def _make_tmp_path(self, remote_user=None):
- """
- Create a temporary subdirectory as a child of the temporary directory
- managed by the remote interpreter.
- """
- LOG.debug('_make_tmp_path(remote_user=%r)', remote_user)
- path = self._generate_tmp_path()
- LOG.debug('Temporary directory: %r', path)
- self._connection.get_chain().call_no_reply(os.mkdir, path)
- self._connection._shell.tmpdir = path
- return path
-
- def _remove_tmp_path(self, tmp_path):
- """
- Replace the base implementation's invocation of rm -rf, replacing it
- with a pipelined call to :func:`ansible_mitogen.target.prune_tree`.
- """
- LOG.debug('_remove_tmp_path(%r)', tmp_path)
- if tmp_path is None and ansible.__version__ > '2.6':
- tmp_path = self._connection._shell.tmpdir # 06f73ad578d
- if tmp_path is not None:
- self._connection.get_chain().call_no_reply(
- ansible_mitogen.target.prune_tree,
- tmp_path,
- )
- self._connection._shell.tmpdir = None
-
- def _transfer_data(self, remote_path, data):
- """
- Used by the base _execute_module(), and in <2.4 also by the template
- action module, and probably others.
- """
- if isinstance(data, dict):
- data = jsonify(data)
- if not isinstance(data, bytes):
- data = to_bytes(data, errors='surrogate_or_strict')
-
- LOG.debug('_transfer_data(%r, %s ..%d bytes)',
- remote_path, type(data), len(data))
- self._connection.put_data(remote_path, data)
- return remote_path
-
- #: Actions listed here cause :func:`_fixup_perms2` to avoid a needless
- #: roundtrip, as they modify file modes separately afterwards. This is due
- #: to the method prototype having a default of `execute=True`.
- FIXUP_PERMS_RED_HERRING = set(['copy'])
-
- def _fixup_perms2(self, remote_paths, remote_user=None, execute=True):
- """
- Mitogen always executes ActionBase helper methods in the context of the
- target user account, so it is never necessary to modify permissions
- except to ensure the execute bit is set if requested.
- """
- LOG.debug('_fixup_perms2(%r, remote_user=%r, execute=%r)',
- remote_paths, remote_user, execute)
- if execute and self._task.action not in self.FIXUP_PERMS_RED_HERRING:
- return self._remote_chmod(remote_paths, mode='u+x')
- return self.COMMAND_RESULT.copy()
-
- def _remote_chmod(self, paths, mode, sudoable=False):
- """
- Issue an asynchronous set_file_mode() call for every path in `paths`,
- then format the resulting return value list with fake_shell().
- """
- LOG.debug('_remote_chmod(%r, mode=%r, sudoable=%r)',
- paths, mode, sudoable)
- return self.fake_shell(lambda: mitogen.select.Select.all(
- self._connection.get_chain().call_async(
- ansible_mitogen.target.set_file_mode, path, mode
- )
- for path in paths
- ))
-
- def _remote_chown(self, paths, user, sudoable=False):
- """
- Issue an asynchronous os.chown() call for every path in `paths`, then
- format the resulting return value list with fake_shell().
- """
- LOG.debug('_remote_chown(%r, user=%r, sudoable=%r)',
- paths, user, sudoable)
- ent = self._connection.get_chain().call(pwd.getpwnam, user)
- return self.fake_shell(lambda: mitogen.select.Select.all(
- self._connection.get_chain().call_async(
- os.chown, path, ent.pw_uid, ent.pw_gid
- )
- for path in paths
- ))
-
- def _remote_expand_user(self, path, sudoable=True):
- """
- Replace the base implementation's attempt to emulate
- os.path.expanduser() with an actual call to os.path.expanduser().
-
- :param bool sudoable:
- If :data:`True`, indicate unqualified tilde ("~" with no username)
- should be evaluated in the context of the login account, not any
- become_user.
- """
- LOG.debug('_remote_expand_user(%r, sudoable=%r)', path, sudoable)
- if not path.startswith('~'):
- # /home/foo -> /home/foo
- return path
- if sudoable or not self._play_context.become:
- if path == '~':
- # ~ -> /home/dmw
- return self._connection.homedir
- if path.startswith('~/'):
- # ~/.ansible -> /home/dmw/.ansible
- return os.path.join(self._connection.homedir, path[2:])
- # ~root/.ansible -> /root/.ansible
- return self._connection.get_chain(use_login=(not sudoable)).call(
- os.path.expanduser,
- mitogen.utils.cast(path),
- )
-
- def get_task_timeout_secs(self):
- """
- Return the task "async:" value, portable across 2.4-2.5.
- """
- try:
- return self._task.async_val
- except AttributeError:
- return getattr(self._task, 'async')
-
- def _temp_file_gibberish(self, module_args, wrap_async):
- # Ansible>2.5 module_utils reuses the action's temporary directory if
- # one exists. Older versions error if this key is present.
- if ansible.__version__ > '2.5':
- if wrap_async:
- # Sharing is not possible with async tasks, as in that case,
- # the directory must outlive the action plug-in.
- module_args['_ansible_tmpdir'] = None
- else:
- module_args['_ansible_tmpdir'] = self._connection._shell.tmpdir
-
- # If _ansible_tmpdir is unset, Ansible>2.6 module_utils will use
- # _ansible_remote_tmp as the location to create the module's temporary
- # directory. Older versions error if this key is present.
- if ansible.__version__ > '2.6':
- module_args['_ansible_remote_tmp'] = (
- self._connection.get_good_temp_dir()
- )
-
- def _execute_module(self, module_name=None, module_args=None, tmp=None,
- task_vars=None, persist_files=False,
- delete_remote_tmp=True, wrap_async=False):
- """
- Collect up a module's execution environment then use it to invoke
- target.run_module() or helpers.run_module_async() in the target
- context.
- """
- if module_name is None:
- module_name = self._task.action
- if module_args is None:
- module_args = self._task.args
- if task_vars is None:
- task_vars = {}
-
- self._update_module_args(module_name, module_args, task_vars)
- env = {}
- self._compute_environment_string(env)
- self._temp_file_gibberish(module_args, wrap_async)
-
- self._connection._connect()
- result = ansible_mitogen.planner.invoke(
- ansible_mitogen.planner.Invocation(
- action=self,
- connection=self._connection,
- module_name=mitogen.core.to_text(module_name),
- module_args=mitogen.utils.cast(module_args),
- task_vars=task_vars,
- templar=self._templar,
- env=mitogen.utils.cast(env),
- wrap_async=wrap_async,
- timeout_secs=self.get_task_timeout_secs(),
- )
- )
-
- if ansible.__version__ < '2.5' and delete_remote_tmp and \
- getattr(self._connection._shell, 'tmpdir', None) is not None:
- # Built-in actions expected tmpdir to be cleaned up automatically
- # on _execute_module().
- self._remove_tmp_path(self._connection._shell.tmpdir)
-
- return result
-
- def _postprocess_response(self, result):
- """
- Apply fixups mimicking ActionBase._execute_module(); this is copied
- verbatim from action/__init__.py, the guts of _parse_returned_data are
- garbage and should be removed or reimplemented once tests exist.
-
- :param dict result:
- Dictionary with format::
-
- {
- "rc": int,
- "stdout": "stdout data",
- "stderr": "stderr data"
- }
- """
- data = self._parse_returned_data(result)
-
- # Cutpasted from the base implementation.
- if 'stdout' in data and 'stdout_lines' not in data:
- data['stdout_lines'] = (data['stdout'] or u'').splitlines()
- if 'stderr' in data and 'stderr_lines' not in data:
- data['stderr_lines'] = (data['stderr'] or u'').splitlines()
-
- return data
-
- def _low_level_execute_command(self, cmd, sudoable=True, in_data=None,
- executable=None,
- encoding_errors='surrogate_then_replace',
- chdir=None):
- """
- Override the base implementation by simply calling
- target.exec_command() in the target context.
- """
- LOG.debug('_low_level_execute_command(%r, in_data=%r, exe=%r, dir=%r)',
- cmd, type(in_data), executable, chdir)
- if executable is None: # executable defaults to False
- executable = self._play_context.executable
- if executable:
- cmd = executable + ' -c ' + shlex_quote(cmd)
-
- rc, stdout, stderr = self._connection.exec_command(
- cmd=cmd,
- in_data=in_data,
- sudoable=sudoable,
- mitogen_chdir=chdir,
- )
- stdout_text = to_text(stdout, errors=encoding_errors)
-
- return {
- 'rc': rc,
- 'stdout': stdout_text,
- 'stdout_lines': stdout_text.splitlines(),
- 'stderr': stderr,
- }
diff --git a/mitogen-0.2.7/ansible_mitogen/module_finder.py b/mitogen-0.2.7/ansible_mitogen/module_finder.py
deleted file mode 100644
index 633e3ca..000000000
--- a/mitogen-0.2.7/ansible_mitogen/module_finder.py
+++ /dev/null
@@ -1,157 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
-import collections
-import imp
-import os
-
-import mitogen.master
-
-
-PREFIX = 'ansible.module_utils.'
-
-
-Module = collections.namedtuple('Module', 'name path kind parent')
-
-
-def get_fullname(module):
- """
- Reconstruct a Module's canonical path by recursing through its parents.
- """
- bits = [str(module.name)]
- while module.parent:
- bits.append(str(module.parent.name))
- module = module.parent
- return '.'.join(reversed(bits))
-
-
-def get_code(module):
- """
- Compile and return a Module's code object.
- """
- fp = open(module.path)
- try:
- return compile(fp.read(), str(module.name), 'exec')
- finally:
- fp.close()
-
-
-def is_pkg(module):
- """
- Return :data:`True` if a Module represents a package.
- """
- return module.kind == imp.PKG_DIRECTORY
-
-
-def find(name, path=(), parent=None):
- """
- Return a Module instance describing the first matching module found on the
- search path.
-
- :param str name:
- Module name.
- :param list path:
- List of directory names to search for the module.
- :param Module parent:
- Optional module parent.
- """
- assert isinstance(path, tuple)
- head, _, tail = name.partition('.')
- try:
- tup = imp.find_module(head, list(path))
- except ImportError:
- return parent
-
- fp, modpath, (suffix, mode, kind) = tup
- if fp:
- fp.close()
-
- if parent and modpath == parent.path:
- # 'from timeout import timeout', where 'timeout' is a function but also
- # the name of the module being imported.
- return None
-
- if kind == imp.PKG_DIRECTORY:
- modpath = os.path.join(modpath, '__init__.py')
-
- module = Module(head, modpath, kind, parent)
- # TODO: this code is entirely wrong on Python 3.x, but works well enough
- # for Ansible. We need a new find_child() that only looks in the package
- # directory, never falling back to the parent search path.
- if tail and kind == imp.PKG_DIRECTORY:
- return find_relative(module, tail, path)
- return module
-
-
-def find_relative(parent, name, path=()):
- if parent.kind == imp.PKG_DIRECTORY:
- path = (os.path.dirname(parent.path),) + path
- return find(name, path, parent=parent)
-
-
-def scan_fromlist(code):
- for level, modname_s, fromlist in mitogen.master.scan_code_imports(code):
- for name in fromlist:
- yield level, '%s.%s' % (modname_s, name)
- if not fromlist:
- yield level, modname_s
-
-
-def scan(module_name, module_path, search_path):
- module = Module(module_name, module_path, imp.PY_SOURCE, None)
- stack = [module]
- seen = set()
-
- while stack:
- module = stack.pop(0)
- for level, fromname in scan_fromlist(get_code(module)):
- if not fromname.startswith(PREFIX):
- continue
-
- imported = find(fromname[len(PREFIX):], search_path)
- if imported is None or imported in seen:
- continue
-
- seen.add(imported)
- stack.append(imported)
- parent = imported.parent
- while parent:
- fullname = get_fullname(parent)
- module = Module(fullname, parent.path, parent.kind, None)
- if module not in seen:
- seen.add(module)
- stack.append(module)
- parent = parent.parent
-
- return sorted(
- (PREFIX + get_fullname(module), module.path, is_pkg(module))
- for module in seen
- )
diff --git a/mitogen-0.2.7/ansible_mitogen/parsing.py b/mitogen-0.2.7/ansible_mitogen/parsing.py
deleted file mode 100644
index 525e60c..000000000
--- a/mitogen-0.2.7/ansible_mitogen/parsing.py
+++ /dev/null
@@ -1,84 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-"""
-Classes to detect each case from [0] and prepare arguments necessary for the
-corresponding Runner class within the target, including preloading requisite
-files/modules known missing.
-
-[0] "Ansible Module Architecture", developing_program_flow_modules.html
-"""
-
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
-import mitogen.core
-
-
-def parse_script_interpreter(source):
- """
- Parse the script interpreter portion of a UNIX hashbang using the rules
- Linux uses.
-
- :param str source: String like "/usr/bin/env python".
-
- :returns:
- Tuple of `(interpreter, arg)`, where `intepreter` is the script
- interpreter and `arg` is its sole argument if present, otherwise
- :py:data:`None`.
- """
- # Find terminating newline. Assume last byte of binprm_buf if absent.
- nl = source.find(b'\n', 0, 128)
- if nl == -1:
- nl = min(128, len(source))
-
- # Split once on the first run of whitespace. If no whitespace exists,
- # bits just contains the interpreter filename.
- bits = source[0:nl].strip().split(None, 1)
- if len(bits) == 1:
- return mitogen.core.to_text(bits[0]), None
- return mitogen.core.to_text(bits[0]), mitogen.core.to_text(bits[1])
-
-
-def parse_hashbang(source):
- """
- Parse a UNIX "hashbang line" using the syntax supported by Linux.
-
- :param str source: String like "#!/usr/bin/env python".
-
- :returns:
- Tuple of `(interpreter, arg)`, where `intepreter` is the script
- interpreter and `arg` is its sole argument if present, otherwise
- :py:data:`None`.
- """
- # Linux requires first 2 bytes with no whitespace, pretty sure it's the
- # same everywhere. See binfmt_script.c.
- if not source.startswith(b'#!'):
- return None, None
-
- return parse_script_interpreter(source[2:])
diff --git a/mitogen-0.2.7/ansible_mitogen/planner.py b/mitogen-0.2.7/ansible_mitogen/planner.py
deleted file mode 100644
index 2eebd36..000000000
--- a/mitogen-0.2.7/ansible_mitogen/planner.py
+++ /dev/null
@@ -1,499 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-"""
-Classes to detect each case from [0] and prepare arguments necessary for the
-corresponding Runner class within the target, including preloading requisite
-files/modules known missing.
-
-[0] "Ansible Module Architecture", developing_program_flow_modules.html
-"""
-
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
-import json
-import logging
-import os
-import random
-
-from ansible.executor import module_common
-import ansible.errors
-import ansible.module_utils
-import mitogen.core
-import mitogen.select
-
-import ansible_mitogen.loaders
-import ansible_mitogen.parsing
-import ansible_mitogen.target
-
-
-LOG = logging.getLogger(__name__)
-NO_METHOD_MSG = 'Mitogen: no invocation method found for: '
-NO_INTERPRETER_MSG = 'module (%s) is missing interpreter line'
-NO_MODULE_MSG = 'The module %s was not found in configured module paths.'
-
-
-class Invocation(object):
- """
- Collect up a module's execution environment then use it to invoke
- target.run_module() or helpers.run_module_async() in the target context.
- """
- def __init__(self, action, connection, module_name, module_args,
- task_vars, templar, env, wrap_async, timeout_secs):
- #: ActionBase instance invoking the module. Required to access some
- #: output postprocessing methods that don't belong in ActionBase at
- #: all.
- self.action = action
- #: Ansible connection to use to contact the target. Must be an
- #: ansible_mitogen connection.
- self.connection = connection
- #: Name of the module ('command', 'shell', etc.) to execute.
- self.module_name = module_name
- #: Final module arguments.
- self.module_args = module_args
- #: Task variables, needed to extract ansible_*_interpreter.
- self.task_vars = task_vars
- #: Templar, needed to extract ansible_*_interpreter.
- self.templar = templar
- #: Final module environment.
- self.env = env
- #: Boolean, if :py:data:`True`, launch the module asynchronously.
- self.wrap_async = wrap_async
- #: Integer, if >0, limit the time an asynchronous job may run for.
- self.timeout_secs = timeout_secs
- #: Initially ``None``, but set by :func:`invoke`. The path on the
- #: master to the module's implementation file.
- self.module_path = None
- #: Initially ``None``, but set by :func:`invoke`. The raw source or
- #: binary contents of the module.
- self.module_source = None
-
- def __repr__(self):
- return 'Invocation(module_name=%s)' % (self.module_name,)
-
-
-class Planner(object):
- """
- A Planner receives a module name and the contents of its implementation
- file, indicates whether or not it understands how to run the module, and
- exports a method to run the module.
- """
- def __init__(self, invocation):
- self._inv = invocation
-
- def detect(self):
- """
- Return true if the supplied `invocation` matches the module type
- implemented by this planner.
- """
- raise NotImplementedError()
-
- def should_fork(self):
- """
- Asynchronous tasks must always be forked.
- """
- return self._inv.wrap_async
-
- def get_push_files(self):
- """
- Return a list of files that should be propagated to the target context
- using PushFileService. The default implementation pushes nothing.
- """
- return []
-
- def get_module_deps(self):
- """
- Return a list of the Python module names imported by the module.
- """
- return []
-
- def get_kwargs(self, **kwargs):
- """
- If :meth:`detect` returned :data:`True`, plan for the module's
- execution, including granting access to or delivering any files to it
- that are known to be absent, and finally return a dict::
-
- {
- # Name of the class from runners.py that implements the
- # target-side execution of this module type.
- "runner_name": "...",
-
- # Remaining keys are passed to the constructor of the class
- # named by `runner_name`.
- }
- """
- new = dict((mitogen.core.UnicodeType(k), kwargs[k])
- for k in kwargs)
- new.setdefault('good_temp_dir',
- self._inv.connection.get_good_temp_dir())
- new.setdefault('cwd', self._inv.connection.get_default_cwd())
- new.setdefault('extra_env', self._inv.connection.get_default_env())
- new.setdefault('emulate_tty', True)
- new.setdefault('service_context', self._inv.connection.parent)
- return new
-
- def __repr__(self):
- return '%s()' % (type(self).__name__,)
-
-
-class BinaryPlanner(Planner):
- """
- Binary modules take their arguments and will return data to Ansible in the
- same way as want JSON modules.
- """
- runner_name = 'BinaryRunner'
-
- def detect(self):
- return module_common._is_binary(self._inv.module_source)
-
- def get_push_files(self):
- return [mitogen.core.to_text(self._inv.module_path)]
-
- def get_kwargs(self, **kwargs):
- return super(BinaryPlanner, self).get_kwargs(
- runner_name=self.runner_name,
- module=self._inv.module_name,
- path=self._inv.module_path,
- json_args=json.dumps(self._inv.module_args),
- env=self._inv.env,
- **kwargs
- )
-
-
-class ScriptPlanner(BinaryPlanner):
- """
- Common functionality for script module planners -- handle interpreter
- detection and rewrite.
- """
- def _rewrite_interpreter(self, path):
- """
- Given the original interpreter binary extracted from the script's
- interpreter line, look up the associated `ansible_*_interpreter`
- variable, render it and return it.
-
- :param str path:
- Absolute UNIX path to original interpreter.
-
- :returns:
- Shell fragment prefix used to execute the script via "/bin/sh -c".
- While `ansible_*_interpreter` documentation suggests shell isn't
- involved here, the vanilla implementation uses it and that use is
- exploited in common playbooks.
- """
- key = u'ansible_%s_interpreter' % os.path.basename(path).strip()
- try:
- template = self._inv.task_vars[key]
- except KeyError:
- return path
-
- return mitogen.utils.cast(self._inv.templar.template(template))
-
- def _get_interpreter(self):
- path, arg = ansible_mitogen.parsing.parse_hashbang(
- self._inv.module_source
- )
- if path is None:
- raise ansible.errors.AnsibleError(NO_INTERPRETER_MSG % (
- self._inv.module_name,
- ))
-
- fragment = self._rewrite_interpreter(path)
- if arg:
- fragment += ' ' + arg
-
- return fragment, path.startswith('python')
-
- def get_kwargs(self, **kwargs):
- interpreter_fragment, is_python = self._get_interpreter()
- return super(ScriptPlanner, self).get_kwargs(
- interpreter_fragment=interpreter_fragment,
- is_python=is_python,
- **kwargs
- )
-
-
-class JsonArgsPlanner(ScriptPlanner):
- """
- Script that has its interpreter directive and the task arguments
- substituted into its source as a JSON string.
- """
- runner_name = 'JsonArgsRunner'
-
- def detect(self):
- return module_common.REPLACER_JSONARGS in self._inv.module_source
-
-
-class WantJsonPlanner(ScriptPlanner):
- """
- If a module has the string WANT_JSON in it anywhere, Ansible treats it as a
- non-native module that accepts a filename as its only command line
- parameter. The filename is for a temporary file containing a JSON string
- containing the module's parameters. The module needs to open the file, read
- and parse the parameters, operate on the data, and print its return data as
- a JSON encoded dictionary to stdout before exiting.
-
- These types of modules are self-contained entities. As of Ansible 2.1,
- Ansible only modifies them to change a shebang line if present.
- """
- runner_name = 'WantJsonRunner'
-
- def detect(self):
- return b'WANT_JSON' in self._inv.module_source
-
-
-class NewStylePlanner(ScriptPlanner):
- """
- The Ansiballz framework differs from module replacer in that it uses real
- Python imports of things in ansible/module_utils instead of merely
- preprocessing the module.
- """
- runner_name = 'NewStyleRunner'
- marker = b'from ansible.module_utils.'
-
- def detect(self):
- return self.marker in self._inv.module_source
-
- def _get_interpreter(self):
- return None, None
-
- def get_push_files(self):
- return super(NewStylePlanner, self).get_push_files() + [
- mitogen.core.to_text(path)
- for fullname, path, is_pkg in self.get_module_map()['custom']
- ]
-
- def get_module_deps(self):
- return self.get_module_map()['builtin']
-
- #: Module names appearing in this set always require forking, usually due
- #: to some terminal leakage that cannot be worked around in any sane
- #: manner.
- ALWAYS_FORK_MODULES = frozenset([
- 'dnf', # issue #280; py-dnf/hawkey need therapy
- 'firewalld', # issue #570: ansible module_utils caches dbus conn
- ])
-
- def should_fork(self):
- """
- In addition to asynchronous tasks, new-style modules should be forked
- if:
-
- * the user specifies mitogen_task_isolation=fork, or
- * the new-style module has a custom module search path, or
- * the module is known to leak like a sieve.
- """
- return (
- super(NewStylePlanner, self).should_fork() or
- (self._inv.task_vars.get('mitogen_task_isolation') == 'fork') or
- (self._inv.module_name in self.ALWAYS_FORK_MODULES) or
- (len(self.get_module_map()['custom']) > 0)
- )
-
- def get_search_path(self):
- return tuple(
- path
- for path in ansible_mitogen.loaders.module_utils_loader._get_paths(
- subdirs=False
- )
- if os.path.isdir(path)
- )
-
- _module_map = None
-
- def get_module_map(self):
- if self._module_map is None:
- self._module_map = self._inv.connection.parent.call_service(
- service_name='ansible_mitogen.services.ModuleDepService',
- method_name='scan',
-
- module_name='ansible_module_%s' % (self._inv.module_name,),
- module_path=self._inv.module_path,
- search_path=self.get_search_path(),
- builtin_path=module_common._MODULE_UTILS_PATH,
- context=self._inv.connection.context,
- )
- return self._module_map
-
- def get_kwargs(self):
- return super(NewStylePlanner, self).get_kwargs(
- module_map=self.get_module_map(),
- )
-
-
-class ReplacerPlanner(NewStylePlanner):
- """
- The Module Replacer framework is the original framework implementing
- new-style modules. It is essentially a preprocessor (like the C
- Preprocessor for those familiar with that programming language). It does
- straight substitutions of specific substring patterns in the module file.
- There are two types of substitutions.
-
- * Replacements that only happen in the module file. These are public
- replacement strings that modules can utilize to get helpful boilerplate
- or access to arguments.
-
- "from ansible.module_utils.MOD_LIB_NAME import *" is replaced with the
- contents of the ansible/module_utils/MOD_LIB_NAME.py. These should only
- be used with new-style Python modules.
-
- "#<>" is equivalent to
- "from ansible.module_utils.basic import *" and should also only apply to
- new-style Python modules.
-
- "# POWERSHELL_COMMON" substitutes the contents of
- "ansible/module_utils/powershell.ps1". It should only be used with
- new-style Powershell modules.
- """
- runner_name = 'ReplacerRunner'
-
- def detect(self):
- return module_common.REPLACER in self._inv.module_source
-
-
-class OldStylePlanner(ScriptPlanner):
- runner_name = 'OldStyleRunner'
-
- def detect(self):
- # Everything else.
- return True
-
-
-_planners = [
- BinaryPlanner,
- # ReplacerPlanner,
- NewStylePlanner,
- JsonArgsPlanner,
- WantJsonPlanner,
- OldStylePlanner,
-]
-
-
-def get_module_data(name):
- path = ansible_mitogen.loaders.module_loader.find_plugin(name, '')
- if path is None:
- raise ansible.errors.AnsibleError(NO_MODULE_MSG % (name,))
-
- with open(path, 'rb') as fp:
- source = fp.read()
- return mitogen.core.to_text(path), source
-
-
-def _propagate_deps(invocation, planner, context):
- invocation.connection.parent.call_service(
- service_name='mitogen.service.PushFileService',
- method_name='propagate_paths_and_modules',
- context=context,
- paths=planner.get_push_files(),
- modules=planner.get_module_deps(),
- )
-
-
-def _invoke_async_task(invocation, planner):
- job_id = '%016x' % random.randint(0, 2**64)
- context = invocation.connection.spawn_isolated_child()
- _propagate_deps(invocation, planner, context)
-
- with mitogen.core.Receiver(context.router) as started_recv:
- call_recv = context.call_async(
- ansible_mitogen.target.run_module_async,
- job_id=job_id,
- timeout_secs=invocation.timeout_secs,
- started_sender=started_recv.to_sender(),
- kwargs=planner.get_kwargs(),
- )
-
- # Wait for run_module_async() to crash, or for AsyncRunner to indicate
- # the job file has been written.
- for msg in mitogen.select.Select([started_recv, call_recv]):
- if msg.receiver is call_recv:
- # It can only be an exception.
- raise msg.unpickle()
- break
-
- return {
- 'stdout': json.dumps({
- # modules/utilities/logic/async_wrapper.py::_run_module().
- 'changed': True,
- 'started': 1,
- 'finished': 0,
- 'ansible_job_id': job_id,
- })
- }
-
-
-def _invoke_isolated_task(invocation, planner):
- context = invocation.connection.spawn_isolated_child()
- _propagate_deps(invocation, planner, context)
- try:
- return context.call(
- ansible_mitogen.target.run_module,
- kwargs=planner.get_kwargs(),
- )
- finally:
- context.shutdown()
-
-
-def _get_planner(invocation):
- for klass in _planners:
- planner = klass(invocation)
- if planner.detect():
- LOG.debug('%r accepted %r (filename %r)', planner,
- invocation.module_name, invocation.module_path)
- return planner
- LOG.debug('%r rejected %r', planner, invocation.module_name)
- raise ansible.errors.AnsibleError(NO_METHOD_MSG + repr(invocation))
-
-
-def invoke(invocation):
- """
- Find a Planner subclass corresnding to `invocation` and use it to invoke
- the module.
-
- :param Invocation invocation:
- :returns:
- Module return dict.
- :raises ansible.errors.AnsibleError:
- Unrecognized/unsupported module type.
- """
- (invocation.module_path,
- invocation.module_source) = get_module_data(invocation.module_name)
- planner = _get_planner(invocation)
-
- if invocation.wrap_async:
- response = _invoke_async_task(invocation, planner)
- elif planner.should_fork():
- response = _invoke_isolated_task(invocation, planner)
- else:
- _propagate_deps(invocation, planner, invocation.connection.context)
- response = invocation.connection.get_chain().call(
- ansible_mitogen.target.run_module,
- kwargs=planner.get_kwargs(),
- )
-
- return invocation.action._postprocess_response(response)
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/__init__.py b/mitogen-0.2.7/ansible_mitogen/plugins/__init__.py
deleted file mode 100644
index e69de29..000000000
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/action/__init__.py b/mitogen-0.2.7/ansible_mitogen/plugins/action/__init__.py
deleted file mode 100644
index e69de29..000000000
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/action/mitogen_get_stack.py b/mitogen-0.2.7/ansible_mitogen/plugins/action/mitogen_get_stack.py
deleted file mode 100644
index 12afbfb..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/action/mitogen_get_stack.py
+++ /dev/null
@@ -1,54 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
-"""
-Fetch the connection configuration stack that would be used to connect to a
-target, without actually connecting to it.
-"""
-
-import ansible_mitogen.connection
-
-from ansible.plugins.action import ActionBase
-
-
-class ActionModule(ActionBase):
- def run(self, tmp=None, task_vars=None):
- if not isinstance(self._connection,
- ansible_mitogen.connection.Connection):
- return {
- 'skipped': True,
- }
-
- return {
- 'changed': True,
- 'result': self._connection._build_stack(),
- '_ansible_verbose_always': True,
- }
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/__init__.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/__init__.py
deleted file mode 100644
index e69de29..000000000
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_doas.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_doas.py
deleted file mode 100644
index 1113d7c..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_doas.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os.path
-import sys
-
-try:
- import ansible_mitogen.connection
-except ImportError:
- base_dir = os.path.dirname(__file__)
- sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
- del base_dir
-
-import ansible_mitogen.connection
-
-
-class Connection(ansible_mitogen.connection.Connection):
- transport = 'mitogen_doas'
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_docker.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_docker.py
deleted file mode 100644
index b71ef5f..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_docker.py
+++ /dev/null
@@ -1,51 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os.path
-import sys
-
-try:
- import ansible_mitogen
-except ImportError:
- base_dir = os.path.dirname(__file__)
- sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
- del base_dir
-
-import ansible_mitogen.connection
-
-
-class Connection(ansible_mitogen.connection.Connection):
- transport = 'docker'
-
- @property
- def docker_cmd(self):
- """
- Ansible 2.3 synchronize module wants to know how we run Docker.
- """
- return 'docker'
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_jail.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_jail.py
deleted file mode 100644
index c7475fb..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_jail.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os.path
-import sys
-
-try:
- import ansible_mitogen
-except ImportError:
- base_dir = os.path.dirname(__file__)
- sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
- del base_dir
-
-import ansible_mitogen.connection
-
-
-class Connection(ansible_mitogen.connection.Connection):
- transport = 'jail'
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_kubectl.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_kubectl.py
deleted file mode 100644
index 2dab131..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_kubectl.py
+++ /dev/null
@@ -1,71 +0,0 @@
-# coding: utf-8
-# Copyright 2018, Yannig Perré
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os.path
-import sys
-
-try:
- from ansible.plugins.connection import kubectl
-except ImportError:
- kubectl = None
-
-from ansible.errors import AnsibleConnectionFailure
-from ansible.module_utils.six import iteritems
-
-try:
- import ansible_mitogen
-except ImportError:
- base_dir = os.path.dirname(__file__)
- sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
- del base_dir
-
-import ansible_mitogen.connection
-
-
-class Connection(ansible_mitogen.connection.Connection):
- transport = 'kubectl'
-
- not_supported_msg = (
- 'The "mitogen_kubectl" plug-in requires a version of Ansible '
- 'that ships with the "kubectl" connection plug-in.'
- )
-
- def __init__(self, *args, **kwargs):
- if kubectl is None:
- raise AnsibleConnectionFailure(self.not_supported_msg)
- super(Connection, self).__init__(*args, **kwargs)
-
- def get_extra_args(self):
- parameters = []
- for key, option in iteritems(kubectl.CONNECTION_OPTIONS):
- if self.get_task_var('ansible_' + key) is not None:
- parameters += [ option, self.get_task_var('ansible_' + key) ]
-
- return parameters
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_local.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_local.py
deleted file mode 100644
index 24b84a0..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_local.py
+++ /dev/null
@@ -1,86 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os.path
-import sys
-
-try:
- import ansible_mitogen.connection
-except ImportError:
- base_dir = os.path.dirname(__file__)
- sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
- del base_dir
-
-import ansible_mitogen.connection
-import ansible_mitogen.process
-
-
-if sys.version_info > (3,):
- viewkeys = dict.keys
-elif sys.version_info > (2, 7):
- viewkeys = dict.viewkeys
-else:
- viewkeys = lambda dct: set(dct)
-
-
-def dict_diff(old, new):
- """
- Return a dict representing the differences between the dicts `old` and
- `new`. Deleted keys appear as a key with the value :data:`None`, added and
- changed keys appear as a key with the new value.
- """
- old_keys = viewkeys(old)
- new_keys = viewkeys(dict(new))
- out = {}
- for key in new_keys - old_keys:
- out[key] = new[key]
- for key in old_keys - new_keys:
- out[key] = None
- for key in old_keys & new_keys:
- if old[key] != new[key]:
- out[key] = new[key]
- return out
-
-
-class Connection(ansible_mitogen.connection.Connection):
- transport = 'local'
-
- def get_default_cwd(self):
- # https://github.com/ansible/ansible/issues/14489
- return self.loader_basedir
-
- def get_default_env(self):
- """
- Vanilla Ansible local commands execute with an environment inherited
- from WorkerProcess, we must emulate that.
- """
- return dict_diff(
- old=ansible_mitogen.process.MuxProcess.original_env,
- new=os.environ,
- )
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_lxc.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_lxc.py
deleted file mode 100644
index 696c9ab..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_lxc.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os.path
-import sys
-
-try:
- import ansible_mitogen
-except ImportError:
- base_dir = os.path.dirname(__file__)
- sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
- del base_dir
-
-import ansible_mitogen.connection
-
-
-class Connection(ansible_mitogen.connection.Connection):
- transport = 'lxc'
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_lxd.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_lxd.py
deleted file mode 100644
index 95e692a..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_lxd.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os.path
-import sys
-
-try:
- import ansible_mitogen
-except ImportError:
- base_dir = os.path.dirname(__file__)
- sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
- del base_dir
-
-import ansible_mitogen.connection
-
-
-class Connection(ansible_mitogen.connection.Connection):
- transport = 'lxd'
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_machinectl.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_machinectl.py
deleted file mode 100644
index 0f5a0d2..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_machinectl.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os.path
-import sys
-
-try:
- import ansible_mitogen.connection
-except ImportError:
- base_dir = os.path.dirname(__file__)
- sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
- del base_dir
-
-import ansible_mitogen.connection
-
-
-class Connection(ansible_mitogen.connection.Connection):
- transport = 'machinectl'
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_setns.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_setns.py
deleted file mode 100644
index 20c6f13..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_setns.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os.path
-import sys
-
-try:
- import ansible_mitogen.connection
-except ImportError:
- base_dir = os.path.dirname(__file__)
- sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
- del base_dir
-
-import ansible_mitogen.connection
-
-
-class Connection(ansible_mitogen.connection.Connection):
- transport = 'setns'
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_ssh.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_ssh.py
deleted file mode 100644
index df0e87c..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_ssh.py
+++ /dev/null
@@ -1,65 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os.path
-import sys
-
-DOCUMENTATION = """
- author: David Wilson
- connection: mitogen_ssh
- short_description: Connect over SSH via Mitogen
- description:
- - This connects using an OpenSSH client controlled by the Mitogen for
- Ansible extension. It accepts every option the vanilla ssh plugin
- accepts.
- version_added: "2.5"
- options:
-"""
-
-import ansible.plugins.connection.ssh
-
-try:
- import ansible_mitogen.connection
-except ImportError:
- base_dir = os.path.dirname(__file__)
- sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
- del base_dir
-
-import ansible_mitogen.connection
-
-
-class Connection(ansible_mitogen.connection.Connection):
- transport = 'ssh'
- vanilla_class = ansible.plugins.connection.ssh.Connection
-
- @staticmethod
- def _create_control_path(*args, **kwargs):
- """Forward _create_control_path() to the implementation in ssh.py."""
- # https://github.com/dw/mitogen/issues/342
- return Connection.vanilla_class._create_control_path(*args, **kwargs)
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_su.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_su.py
deleted file mode 100644
index 4ab2711..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_su.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os.path
-import sys
-
-try:
- import ansible_mitogen.connection
-except ImportError:
- base_dir = os.path.dirname(__file__)
- sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
- del base_dir
-
-import ansible_mitogen.connection
-
-
-class Connection(ansible_mitogen.connection.Connection):
- transport = 'mitogen_su'
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_sudo.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_sudo.py
deleted file mode 100644
index 130f544..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_sudo.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os.path
-import sys
-
-try:
- import ansible_mitogen.connection
-except ImportError:
- base_dir = os.path.dirname(__file__)
- sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..')))
- del base_dir
-
-import ansible_mitogen.connection
-
-
-class Connection(ansible_mitogen.connection.Connection):
- transport = 'mitogen_sudo'
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/strategy/__init__.py b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/__init__.py
deleted file mode 100644
index e69de29..000000000
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen.py b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen.py
deleted file mode 100644
index 6687266..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen.py
+++ /dev/null
@@ -1,61 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os.path
-import sys
-
-#
-# This is not the real Strategy implementation module, it simply exists as a
-# proxy to the real module, which is loaded using Python's regular import
-# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
-# results in ansible_mitogen plugin modules being loaded twice: once by
-# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is
-# stuffed into sys.modules even though attempting to import it will trigger an
-# ImportError, and once under its canonical name, "ansible_mitogen.strategy".
-#
-# Therefore we have a proxy module that imports it under the real name, and
-# sets up the duff PluginLoader-imported module to just contain objects from
-# the real module, so duplicate types don't exist in memory, and things like
-# debuggers and isinstance() work predictably.
-#
-
-BASE_DIR = os.path.abspath(
- os.path.join(os.path.dirname(__file__), '../../..')
-)
-
-if BASE_DIR not in sys.path:
- sys.path.insert(0, BASE_DIR)
-
-import ansible_mitogen.strategy
-import ansible.plugins.strategy.linear
-
-
-class StrategyModule(ansible_mitogen.strategy.StrategyMixin,
- ansible.plugins.strategy.linear.StrategyModule):
- pass
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_free.py b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_free.py
deleted file mode 100644
index ffe2fbd..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_free.py
+++ /dev/null
@@ -1,62 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os.path
-import sys
-
-#
-# This is not the real Strategy implementation module, it simply exists as a
-# proxy to the real module, which is loaded using Python's regular import
-# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
-# results in ansible_mitogen plugin modules being loaded twice: once by
-# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is
-# stuffed into sys.modules even though attempting to import it will trigger an
-# ImportError, and once under its canonical name, "ansible_mitogen.strategy".
-#
-# Therefore we have a proxy module that imports it under the real name, and
-# sets up the duff PluginLoader-imported module to just contain objects from
-# the real module, so duplicate types don't exist in memory, and things like
-# debuggers and isinstance() work predictably.
-#
-
-BASE_DIR = os.path.abspath(
- os.path.join(os.path.dirname(__file__), '../../..')
-)
-
-if BASE_DIR not in sys.path:
- sys.path.insert(0, BASE_DIR)
-
-import ansible_mitogen.loaders
-import ansible_mitogen.strategy
-
-
-Base = ansible_mitogen.loaders.strategy_loader.get('free', class_only=True)
-
-class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base):
- pass
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py
deleted file mode 100644
index 23eccd3..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py
+++ /dev/null
@@ -1,67 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os.path
-import sys
-
-#
-# This is not the real Strategy implementation module, it simply exists as a
-# proxy to the real module, which is loaded using Python's regular import
-# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
-# results in ansible_mitogen plugin modules being loaded twice: once by
-# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is
-# stuffed into sys.modules even though attempting to import it will trigger an
-# ImportError, and once under its canonical name, "ansible_mitogen.strategy".
-#
-# Therefore we have a proxy module that imports it under the real name, and
-# sets up the duff PluginLoader-imported module to just contain objects from
-# the real module, so duplicate types don't exist in memory, and things like
-# debuggers and isinstance() work predictably.
-#
-
-BASE_DIR = os.path.abspath(
- os.path.join(os.path.dirname(__file__), '../../..')
-)
-
-if BASE_DIR not in sys.path:
- sys.path.insert(0, BASE_DIR)
-
-import ansible_mitogen.loaders
-import ansible_mitogen.strategy
-
-
-Base = ansible_mitogen.loaders.strategy_loader.get('host_pinned', class_only=True)
-
-if Base is None:
- raise ImportError(
- 'The host_pinned strategy is only available in Ansible 2.7 or newer.'
- )
-
-class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base):
- pass
diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_linear.py b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_linear.py
deleted file mode 100644
index 1b198e6..000000000
--- a/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_linear.py
+++ /dev/null
@@ -1,62 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os.path
-import sys
-
-#
-# This is not the real Strategy implementation module, it simply exists as a
-# proxy to the real module, which is loaded using Python's regular import
-# mechanism, to prevent Ansible's PluginLoader from making up a fake name that
-# results in ansible_mitogen plugin modules being loaded twice: once by
-# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is
-# stuffed into sys.modules even though attempting to import it will trigger an
-# ImportError, and once under its canonical name, "ansible_mitogen.strategy".
-#
-# Therefore we have a proxy module that imports it under the real name, and
-# sets up the duff PluginLoader-imported module to just contain objects from
-# the real module, so duplicate types don't exist in memory, and things like
-# debuggers and isinstance() work predictably.
-#
-
-BASE_DIR = os.path.abspath(
- os.path.join(os.path.dirname(__file__), '../../..')
-)
-
-if BASE_DIR not in sys.path:
- sys.path.insert(0, BASE_DIR)
-
-import ansible_mitogen.loaders
-import ansible_mitogen.strategy
-
-
-Base = ansible_mitogen.loaders.strategy_loader.get('linear', class_only=True)
-
-class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base):
- pass
diff --git a/mitogen-0.2.7/ansible_mitogen/process.py b/mitogen-0.2.7/ansible_mitogen/process.py
deleted file mode 100644
index e4e61e8..000000000
--- a/mitogen-0.2.7/ansible_mitogen/process.py
+++ /dev/null
@@ -1,358 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import atexit
-import errno
-import logging
-import os
-import signal
-import socket
-import sys
-import time
-
-try:
- import faulthandler
-except ImportError:
- faulthandler = None
-
-import mitogen
-import mitogen.core
-import mitogen.debug
-import mitogen.master
-import mitogen.parent
-import mitogen.service
-import mitogen.unix
-import mitogen.utils
-
-import ansible
-import ansible.constants as C
-import ansible_mitogen.logging
-import ansible_mitogen.services
-
-from mitogen.core import b
-import ansible_mitogen.affinity
-
-
-LOG = logging.getLogger(__name__)
-
-ANSIBLE_PKG_OVERRIDE = (
- u"__version__ = %r\n"
- u"__author__ = %r\n"
-)
-
-
-def clean_shutdown(sock):
- """
- Shut the write end of `sock`, causing `recv` in the worker process to wake
- up with a 0-byte read and initiate mux process exit, then wait for a 0-byte
- read from the read end, which will occur after the the child closes the
- descriptor on exit.
-
- This is done using :mod:`atexit` since Ansible lacks any more sensible hook
- to run code during exit, and unless some synchronization exists with
- MuxProcess, debug logs may appear on the user's terminal *after* the prompt
- has been printed.
- """
- sock.shutdown(socket.SHUT_WR)
- sock.recv(1)
-
-
-def getenv_int(key, default=0):
- """
- Get an integer-valued environment variable `key`, if it exists and parses
- as an integer, otherwise return `default`.
- """
- try:
- return int(os.environ.get(key, str(default)))
- except ValueError:
- return default
-
-
-def save_pid(name):
- """
- When debugging and profiling, it is very annoying to poke through the
- process list to discover the currently running Ansible and MuxProcess IDs,
- especially when trying to catch an issue during early startup. So here, if
- a magic environment variable set, stash them in hidden files in the CWD::
-
- alias muxpid="cat .ansible-mux.pid"
- alias anspid="cat .ansible-controller.pid"
-
- gdb -p $(muxpid)
- perf top -p $(anspid)
- """
- if os.environ.get('MITOGEN_SAVE_PIDS'):
- with open('.ansible-%s.pid' % (name,), 'w') as fp:
- fp.write(str(os.getpid()))
-
-
-class MuxProcess(object):
- """
- Implement a subprocess forked from the Ansible top-level, as a safe place
- to contain the Mitogen IO multiplexer thread, keeping its use of the
- logging package (and the logging package's heavy use of locks) far away
- from the clutches of os.fork(), which is used continuously by the
- multiprocessing package in the top-level process.
-
- The problem with running the multiplexer in that process is that should the
- multiplexer thread be in the process of emitting a log entry (and holding
- its lock) at the point of fork, in the child, the first attempt to log any
- log entry using the same handler will deadlock the child, as in the memory
- image the child received, the lock will always be marked held.
-
- See https://bugs.python.org/issue6721 for a thorough description of the
- class of problems this worker is intended to avoid.
- """
-
- #: In the top-level process, this references one end of a socketpair(),
- #: which the MuxProcess blocks reading from in order to determine when
- #: the master process dies. Once the read returns, the MuxProcess will
- #: begin shutting itself down.
- worker_sock = None
-
- #: In the worker process, this references the other end of
- #: :py:attr:`worker_sock`.
- child_sock = None
-
- #: In the top-level process, this is the PID of the single MuxProcess
- #: that was spawned.
- worker_pid = None
-
- #: A copy of :data:`os.environ` at the time the multiplexer process was
- #: started. It's used by mitogen_local.py to find changes made to the
- #: top-level environment (e.g. vars plugins -- issue #297) that must be
- #: applied to locally executed commands and modules.
- original_env = None
-
- #: In both processes, this is the temporary UNIX socket used for
- #: forked WorkerProcesses to contact the MuxProcess
- unix_listener_path = None
-
- #: Singleton.
- _instance = None
-
- @classmethod
- def start(cls, _init_logging=True):
- """
- Arrange for the subprocess to be started, if it is not already running.
-
- The parent process picks a UNIX socket path the child will use prior to
- fork, creates a socketpair used essentially as a semaphore, then blocks
- waiting for the child to indicate the UNIX socket is ready for use.
-
- :param bool _init_logging:
- For testing, if :data:`False`, don't initialize logging.
- """
- if cls.worker_sock is not None:
- return
-
- if faulthandler is not None:
- faulthandler.enable()
-
- mitogen.utils.setup_gil()
- cls.unix_listener_path = mitogen.unix.make_socket_path()
- cls.worker_sock, cls.child_sock = socket.socketpair()
- atexit.register(lambda: clean_shutdown(cls.worker_sock))
- mitogen.core.set_cloexec(cls.worker_sock.fileno())
- mitogen.core.set_cloexec(cls.child_sock.fileno())
-
- cls.profiling = os.environ.get('MITOGEN_PROFILING') is not None
- if cls.profiling:
- mitogen.core.enable_profiling()
- if _init_logging:
- ansible_mitogen.logging.setup()
-
- cls.original_env = dict(os.environ)
- cls.child_pid = os.fork()
- if cls.child_pid:
- save_pid('controller')
- ansible_mitogen.logging.set_process_name('top')
- ansible_mitogen.affinity.policy.assign_controller()
- cls.child_sock.close()
- cls.child_sock = None
- mitogen.core.io_op(cls.worker_sock.recv, 1)
- else:
- save_pid('mux')
- ansible_mitogen.logging.set_process_name('mux')
- ansible_mitogen.affinity.policy.assign_muxprocess()
- cls.worker_sock.close()
- cls.worker_sock = None
- self = cls()
- self.worker_main()
-
- def worker_main(self):
- """
- The main function of for the mux process: setup the Mitogen broker
- thread and ansible_mitogen services, then sleep waiting for the socket
- connected to the parent to be closed (indicating the parent has died).
- """
- self._setup_master()
- self._setup_services()
-
- try:
- # Let the parent know our listening socket is ready.
- mitogen.core.io_op(self.child_sock.send, b('1'))
- # Block until the socket is closed, which happens on parent exit.
- mitogen.core.io_op(self.child_sock.recv, 1)
- finally:
- self.broker.shutdown()
- self.broker.join()
-
- # Test frameworks living somewhere higher on the stack of the
- # original parent process may try to catch sys.exit(), so do a C
- # level exit instead.
- os._exit(0)
-
- def _enable_router_debug(self):
- if 'MITOGEN_ROUTER_DEBUG' in os.environ:
- self.router.enable_debug()
-
- def _enable_stack_dumps(self):
- secs = getenv_int('MITOGEN_DUMP_THREAD_STACKS', default=0)
- if secs:
- mitogen.debug.dump_to_logger(secs=secs)
-
- def _setup_simplejson(self, responder):
- """
- We support serving simplejson for Python 2.4 targets on Ansible 2.3, at
- least so the package's own CI Docker scripts can run without external
- help, however newer versions of simplejson no longer support Python
- 2.4. Therefore override any installed/loaded version with a
- 2.4-compatible version we ship in the compat/ directory.
- """
- responder.whitelist_prefix('simplejson')
-
- # issue #536: must be at end of sys.path, in case existing newer
- # version is already loaded.
- compat_path = os.path.join(os.path.dirname(__file__), 'compat')
- sys.path.append(compat_path)
-
- for fullname, is_pkg, suffix in (
- (u'simplejson', True, '__init__.py'),
- (u'simplejson.decoder', False, 'decoder.py'),
- (u'simplejson.encoder', False, 'encoder.py'),
- (u'simplejson.scanner', False, 'scanner.py'),
- ):
- path = os.path.join(compat_path, 'simplejson', suffix)
- fp = open(path, 'rb')
- try:
- source = fp.read()
- finally:
- fp.close()
-
- responder.add_source_override(
- fullname=fullname,
- path=path,
- source=source,
- is_pkg=is_pkg,
- )
-
- def _setup_responder(self, responder):
- """
- Configure :class:`mitogen.master.ModuleResponder` to only permit
- certain packages, and to generate custom responses for certain modules.
- """
- responder.whitelist_prefix('ansible')
- responder.whitelist_prefix('ansible_mitogen')
- self._setup_simplejson(responder)
-
- # Ansible 2.3 is compatible with Python 2.4 targets, however
- # ansible/__init__.py is not. Instead, executor/module_common.py writes
- # out a 2.4-compatible namespace package for unknown reasons. So we
- # copy it here.
- responder.add_source_override(
- fullname='ansible',
- path=ansible.__file__,
- source=(ANSIBLE_PKG_OVERRIDE % (
- ansible.__version__,
- ansible.__author__,
- )).encode(),
- is_pkg=True,
- )
-
- def _setup_master(self):
- """
- Construct a Router, Broker, and mitogen.unix listener
- """
- self.broker = mitogen.master.Broker(install_watcher=False)
- self.router = mitogen.master.Router(
- broker=self.broker,
- max_message_size=4096 * 1048576,
- )
- self._setup_responder(self.router.responder)
- mitogen.core.listen(self.broker, 'shutdown', self.on_broker_shutdown)
- mitogen.core.listen(self.broker, 'exit', self.on_broker_exit)
- self.listener = mitogen.unix.Listener(
- router=self.router,
- path=self.unix_listener_path,
- backlog=C.DEFAULT_FORKS,
- )
- self._enable_router_debug()
- self._enable_stack_dumps()
-
- def _setup_services(self):
- """
- Construct a ContextService and a thread to service requests for it
- arriving from worker processes.
- """
- self.pool = mitogen.service.Pool(
- router=self.router,
- services=[
- mitogen.service.FileService(router=self.router),
- mitogen.service.PushFileService(router=self.router),
- ansible_mitogen.services.ContextService(self.router),
- ansible_mitogen.services.ModuleDepService(self.router),
- ],
- size=getenv_int('MITOGEN_POOL_SIZE', default=32),
- )
- LOG.debug('Service pool configured: size=%d', self.pool.size)
-
- def on_broker_shutdown(self):
- """
- Respond to broker shutdown by beginning service pool shutdown. Do not
- join on the pool yet, since that would block the broker thread which
- then cannot clean up pending handlers, which is required for the
- threads to exit gracefully.
- """
- # In normal operation we presently kill the process because there is
- # not yet any way to cancel connect().
- self.pool.stop(join=self.profiling)
-
- def on_broker_exit(self):
- """
- Respond to the broker thread about to exit by sending SIGTERM to
- ourself. In future this should gracefully join the pool, but TERM is
- fine for now.
- """
- if not self.profiling:
- # In normal operation we presently kill the process because there is
- # not yet any way to cancel connect(). When profiling, threads
- # including the broker must shut down gracefully, otherwise pstats
- # won't be written.
- os.kill(os.getpid(), signal.SIGTERM)
diff --git a/mitogen-0.2.7/ansible_mitogen/runner.py b/mitogen-0.2.7/ansible_mitogen/runner.py
deleted file mode 100644
index 30c36be..000000000
--- a/mitogen-0.2.7/ansible_mitogen/runner.py
+++ /dev/null
@@ -1,928 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-"""
-These classes implement execution for each style of Ansible module. They are
-instantiated in the target context by way of target.py::run_module().
-
-Each class in here has a corresponding Planner class in planners.py that knows
-how to build arguments for it, preseed related data, etc.
-"""
-
-import atexit
-import codecs
-import imp
-import os
-import re
-import shlex
-import shutil
-import sys
-import tempfile
-import traceback
-import types
-
-import mitogen.core
-import ansible_mitogen.target # TODO: circular import
-from mitogen.core import b
-from mitogen.core import bytes_partition
-from mitogen.core import str_partition
-from mitogen.core import str_rpartition
-from mitogen.core import to_text
-
-try:
- import ctypes
-except ImportError:
- # Python 2.4
- ctypes = None
-
-try:
- import json
-except ImportError:
- # Python 2.4
- import simplejson as json
-
-try:
- # Cannot use cStringIO as it does not support Unicode.
- from StringIO import StringIO
-except ImportError:
- from io import StringIO
-
-try:
- from shlex import quote as shlex_quote
-except ImportError:
- from pipes import quote as shlex_quote
-
-# Absolute imports for <2.5.
-logging = __import__('logging')
-
-
-# Prevent accidental import of an Ansible module from hanging on stdin read.
-import ansible.module_utils.basic
-ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
-
-# For tasks that modify /etc/resolv.conf, non-Debian derivative glibcs cache
-# resolv.conf at startup and never implicitly reload it. Cope with that via an
-# explicit call to res_init() on each task invocation. BSD-alikes export it
-# directly, Linux #defines it as "__res_init".
-libc__res_init = None
-if ctypes:
- libc = ctypes.CDLL(None)
- for symbol in 'res_init', '__res_init':
- try:
- libc__res_init = getattr(libc, symbol)
- except AttributeError:
- pass
-
-iteritems = getattr(dict, 'iteritems', dict.items)
-LOG = logging.getLogger(__name__)
-
-
-if mitogen.core.PY3:
- shlex_split = shlex.split
-else:
- def shlex_split(s, comments=False):
- return [mitogen.core.to_text(token)
- for token in shlex.split(str(s), comments=comments)]
-
-
-class EnvironmentFileWatcher(object):
- """
- Usually Ansible edits to /etc/environment and ~/.pam_environment are
- reflected in subsequent tasks if become:true or SSH multiplexing is
- disabled, due to sudo and/or SSH reinvoking pam_env. Rather than emulate
- existing semantics, do our best to ensure edits are always reflected.
-
- This can't perfectly replicate the existing behaviour, but it can safely
- update and remove keys that appear to originate in `path`, and that do not
- conflict with any existing environment key inherited from elsewhere.
-
- A more robust future approach may simply be to arrange for the persistent
- interpreter to restart when a change is detected.
- """
- def __init__(self, path):
- self.path = os.path.expanduser(path)
- #: Inode data at time of last check.
- self._st = self._stat()
- #: List of inherited keys appearing to originated from this file.
- self._keys = [key for key, value in self._load()
- if value == os.environ.get(key)]
- LOG.debug('%r installed; existing keys: %r', self, self._keys)
-
- def __repr__(self):
- return 'EnvironmentFileWatcher(%r)' % (self.path,)
-
- def _stat(self):
- try:
- return os.stat(self.path)
- except OSError:
- return None
-
- def _load(self):
- try:
- fp = codecs.open(self.path, 'r', encoding='utf-8')
- try:
- return list(self._parse(fp))
- finally:
- fp.close()
- except IOError:
- return []
-
- def _parse(self, fp):
- """
- linux-pam-1.3.1/modules/pam_env/pam_env.c#L207
- """
- for line in fp:
- # ' #export foo=some var ' -> ['#export', 'foo=some var ']
- bits = shlex_split(line, comments=True)
- if (not bits) or bits[0].startswith('#'):
- continue
-
- if bits[0] == u'export':
- bits.pop(0)
-
- key, sep, value = str_partition(u' '.join(bits), u'=')
- if key and sep:
- yield key, value
-
- def _on_file_changed(self):
- LOG.debug('%r: file changed, reloading', self)
- for key, value in self._load():
- if key in os.environ:
- LOG.debug('%r: existing key %r=%r exists, not setting %r',
- self, key, os.environ[key], value)
- else:
- LOG.debug('%r: setting key %r to %r', self, key, value)
- self._keys.append(key)
- os.environ[key] = value
-
- def _remove_existing(self):
- """
- When a change is detected, remove keys that existed in the old file.
- """
- for key in self._keys:
- if key in os.environ:
- LOG.debug('%r: removing old key %r', self, key)
- del os.environ[key]
- self._keys = []
-
- def check(self):
- """
- Compare the :func:`os.stat` for the pam_env style environmnt file
- `path` with the previous result `old_st`, which may be :data:`None` if
- the previous stat attempt failed. Reload its contents if the file has
- changed or appeared since last attempt.
-
- :returns:
- New :func:`os.stat` result. The new call to :func:`reload_env` should
- pass it as the value of `old_st`.
- """
- st = self._stat()
- if self._st == st:
- return
-
- self._st = st
- self._remove_existing()
-
- if st is None:
- LOG.debug('%r: file has disappeared', self)
- else:
- self._on_file_changed()
-
-_pam_env_watcher = EnvironmentFileWatcher('~/.pam_environment')
-_etc_env_watcher = EnvironmentFileWatcher('/etc/environment')
-
-
-def utf8(s):
- """
- Coerce an object to bytes if it is Unicode.
- """
- if isinstance(s, mitogen.core.UnicodeType):
- s = s.encode('utf-8')
- return s
-
-
-def reopen_readonly(fp):
- """
- Replace the file descriptor belonging to the file object `fp` with one
- open on the same file (`fp.name`), but opened with :py:data:`os.O_RDONLY`.
- This enables temporary files to be executed on Linux, which usually throws
- ``ETXTBUSY`` if any writeable handle exists pointing to a file passed to
- `execve()`.
- """
- fd = os.open(fp.name, os.O_RDONLY)
- os.dup2(fd, fp.fileno())
- os.close(fd)
-
-
-class Runner(object):
- """
- Ansible module runner. After instantiation (with kwargs supplied by the
- corresponding Planner), `.run()` is invoked, upon which `setup()`,
- `_run()`, and `revert()` are invoked, with the return value of `_run()`
- returned by `run()`.
-
- Subclasses may override `_run`()` and extend `setup()` and `revert()`.
-
- :param str module:
- Name of the module to execute, e.g. "shell"
- :param mitogen.core.Context service_context:
- Context to which we should direct FileService calls. For now, always
- the connection multiplexer process on the controller.
- :param str json_args:
- Ansible module arguments. A mixture of user and internal keys created
- by :meth:`ansible.plugins.action.ActionBase._execute_module`.
-
- This is passed as a string rather than a dict in order to mimic the
- implicit bytes/str conversion behaviour of a 2.x controller running
- against a 3.x target.
- :param str good_temp_dir:
- The writeable temporary directory for this user account reported by
- :func:`ansible_mitogen.target.init_child` passed via the controller.
- This is specified explicitly to remain compatible with Ansible<2.5, and
- for forked tasks where init_child never runs.
- :param dict env:
- Additional environment variables to set during the run. Keys with
- :data:`None` are unset if present.
- :param str cwd:
- If not :data:`None`, change to this directory before executing.
- :param mitogen.core.ExternalContext econtext:
- When `detach` is :data:`True`, a reference to the ExternalContext the
- runner is executing in.
- :param bool detach:
- When :data:`True`, indicate the runner should detach the context from
- its parent after setup has completed successfully.
- """
- def __init__(self, module, service_context, json_args, good_temp_dir,
- extra_env=None, cwd=None, env=None, econtext=None,
- detach=False):
- self.module = module
- self.service_context = service_context
- self.econtext = econtext
- self.detach = detach
- self.args = json.loads(mitogen.core.to_text(json_args))
- self.good_temp_dir = good_temp_dir
- self.extra_env = extra_env
- self.env = env
- self.cwd = cwd
- #: If not :data:`None`, :meth:`get_temp_dir` had to create a temporary
- #: directory for this run, because we're in an asynchronous task, or
- #: because the originating action did not create a directory.
- self._temp_dir = None
-
- def get_temp_dir(self):
- path = self.args.get('_ansible_tmpdir')
- if path is not None:
- return path
-
- if self._temp_dir is None:
- self._temp_dir = tempfile.mkdtemp(
- prefix='ansible_mitogen_runner_',
- dir=self.good_temp_dir,
- )
-
- return self._temp_dir
-
- def revert_temp_dir(self):
- if self._temp_dir is not None:
- ansible_mitogen.target.prune_tree(self._temp_dir)
- self._temp_dir = None
-
- def setup(self):
- """
- Prepare for running a module, including fetching necessary dependencies
- from the parent, as :meth:`run` may detach prior to beginning
- execution. The base implementation simply prepares the environment.
- """
- self._setup_cwd()
- self._setup_environ()
-
- def _setup_cwd(self):
- """
- For situations like sudo to a non-privileged account, CWD could be
- $HOME of the old account, which could have mode go=, which means it is
- impossible to restore the old directory, so don't even try.
- """
- if self.cwd:
- os.chdir(self.cwd)
-
- def _setup_environ(self):
- """
- Apply changes from /etc/environment files before creating a
- TemporaryEnvironment to snapshot environment state prior to module run.
- """
- _pam_env_watcher.check()
- _etc_env_watcher.check()
- env = dict(self.extra_env or {})
- if self.env:
- env.update(self.env)
- self._env = TemporaryEnvironment(env)
-
- def revert(self):
- """
- Revert any changes made to the process after running a module. The base
- implementation simply restores the original environment.
- """
- self._env.revert()
- self.revert_temp_dir()
-
- def _run(self):
- """
- The _run() method is expected to return a dictionary in the form of
- ActionBase._low_level_execute_command() output, i.e. having::
-
- {
- "rc": int,
- "stdout": "stdout data",
- "stderr": "stderr data"
- }
- """
- raise NotImplementedError()
-
- def run(self):
- """
- Set up the process environment in preparation for running an Ansible
- module. This monkey-patches the Ansible libraries in various places to
- prevent it from trying to kill the process on completion, and to
- prevent it from reading sys.stdin.
-
- :returns:
- Module result dictionary.
- """
- self.setup()
- if self.detach:
- self.econtext.detach()
-
- try:
- return self._run()
- finally:
- self.revert()
-
-
-class AtExitWrapper(object):
- """
- issue #397, #454: Newer Ansibles use :func:`atexit.register` to trigger
- tmpdir cleanup when AnsibleModule.tmpdir is responsible for creating its
- own temporary directory, however with Mitogen processes are preserved
- across tasks, meaning cleanup must happen earlier.
-
- Patch :func:`atexit.register`, catching :func:`shutil.rmtree` calls so they
- can be executed on task completion, rather than on process shutdown.
- """
- # Wrapped in a dict to avoid instance method decoration.
- original = {
- 'register': atexit.register
- }
-
- def __init__(self):
- assert atexit.register == self.original['register'], \
- "AtExitWrapper installed twice."
- atexit.register = self._atexit__register
- self.deferred = []
-
- def revert(self):
- """
- Restore the original :func:`atexit.register`.
- """
- assert atexit.register == self._atexit__register, \
- "AtExitWrapper not installed."
- atexit.register = self.original['register']
-
- def run_callbacks(self):
- while self.deferred:
- func, targs, kwargs = self.deferred.pop()
- try:
- func(*targs, **kwargs)
- except Exception:
- LOG.exception('While running atexit callbacks')
-
- def _atexit__register(self, func, *targs, **kwargs):
- """
- Intercept :func:`atexit.register` calls, diverting any to
- :func:`shutil.rmtree` into a private list.
- """
- if func == shutil.rmtree:
- self.deferred.append((func, targs, kwargs))
- return
-
- self.original['register'](func, *targs, **kwargs)
-
-
-class ModuleUtilsImporter(object):
- """
- :param list module_utils:
- List of `(fullname, path, is_pkg)` tuples.
- """
- def __init__(self, context, module_utils):
- self._context = context
- self._by_fullname = dict(
- (fullname, (path, is_pkg))
- for fullname, path, is_pkg in module_utils
- )
- self._loaded = set()
- sys.meta_path.insert(0, self)
-
- def revert(self):
- sys.meta_path.remove(self)
- for fullname in self._loaded:
- sys.modules.pop(fullname, None)
-
- def find_module(self, fullname, path=None):
- if fullname in self._by_fullname:
- return self
-
- def load_module(self, fullname):
- path, is_pkg = self._by_fullname[fullname]
- source = ansible_mitogen.target.get_small_file(self._context, path)
- code = compile(source, path, 'exec', 0, 1)
- mod = sys.modules.setdefault(fullname, imp.new_module(fullname))
- mod.__file__ = "master:%s" % (path,)
- mod.__loader__ = self
- if is_pkg:
- mod.__path__ = []
- mod.__package__ = str(fullname)
- else:
- mod.__package__ = str(str_rpartition(to_text(fullname), '.')[0])
- exec(code, mod.__dict__)
- self._loaded.add(fullname)
- return mod
-
-
-class TemporaryEnvironment(object):
- """
- Apply environment changes from `env` until :meth:`revert` is called. Values
- in the dict may be :data:`None` to indicate the relevant key should be
- deleted.
- """
- def __init__(self, env=None):
- self.original = dict(os.environ)
- self.env = env or {}
- for key, value in iteritems(self.env):
- key = mitogen.core.to_text(key)
- value = mitogen.core.to_text(value)
- if value is None:
- os.environ.pop(key, None)
- else:
- os.environ[key] = str(value)
-
- def revert(self):
- """
- Revert changes made by the module to the process environment. This must
- always run, as some modules (e.g. git.py) set variables like GIT_SSH
- that must be cleared out between runs.
- """
- os.environ.clear()
- os.environ.update(self.original)
-
-
-class TemporaryArgv(object):
- def __init__(self, argv):
- self.original = sys.argv[:]
- sys.argv[:] = map(str, argv)
-
- def revert(self):
- sys.argv[:] = self.original
-
-
-class NewStyleStdio(object):
- """
- Patch ansible.module_utils.basic argument globals.
- """
- def __init__(self, args, temp_dir):
- self.temp_dir = temp_dir
- self.original_stdout = sys.stdout
- self.original_stderr = sys.stderr
- self.original_stdin = sys.stdin
- sys.stdout = StringIO()
- sys.stderr = StringIO()
- encoded = json.dumps({'ANSIBLE_MODULE_ARGS': args})
- ansible.module_utils.basic._ANSIBLE_ARGS = utf8(encoded)
- sys.stdin = StringIO(mitogen.core.to_text(encoded))
-
- self.original_get_path = getattr(ansible.module_utils.basic,
- 'get_module_path', None)
- ansible.module_utils.basic.get_module_path = self._get_path
-
- def _get_path(self):
- return self.temp_dir
-
- def revert(self):
- ansible.module_utils.basic.get_module_path = self.original_get_path
- sys.stdout = self.original_stdout
- sys.stderr = self.original_stderr
- sys.stdin = self.original_stdin
- ansible.module_utils.basic._ANSIBLE_ARGS = '{}'
-
-
-class ProgramRunner(Runner):
- """
- Base class for runners that run external programs.
-
- :param str path:
- Absolute path to the program file on the master, as it can be retrieved
- via :class:`mitogen.service.FileService`.
- :param bool emulate_tty:
- If :data:`True`, execute the program with `stdout` and `stderr` merged
- into a single pipe, emulating Ansible behaviour when an SSH TTY is in
- use.
- """
- def __init__(self, path, emulate_tty=None, **kwargs):
- super(ProgramRunner, self).__init__(**kwargs)
- self.emulate_tty = emulate_tty
- self.path = path
-
- def setup(self):
- super(ProgramRunner, self).setup()
- self._setup_program()
-
- def _get_program_filename(self):
- """
- Return the filename used for program on disk. Ansible uses the original
- filename for non-Ansiballz runs, and "ansible_module_+filename for
- Ansiballz runs.
- """
- return os.path.basename(self.path)
-
- program_fp = None
-
- def _setup_program(self):
- """
- Create a temporary file containing the program code. The code is
- fetched via :meth:`_get_program`.
- """
- filename = self._get_program_filename()
- path = os.path.join(self.get_temp_dir(), filename)
- self.program_fp = open(path, 'wb')
- self.program_fp.write(self._get_program())
- self.program_fp.flush()
- os.chmod(self.program_fp.name, int('0700', 8))
- reopen_readonly(self.program_fp)
-
- def _get_program(self):
- """
- Fetch the module binary from the master if necessary.
- """
- return ansible_mitogen.target.get_small_file(
- context=self.service_context,
- path=self.path,
- )
-
- def _get_program_args(self):
- """
- Return any arguments to pass to the program.
- """
- return []
-
- def revert(self):
- """
- Delete the temporary program file.
- """
- if self.program_fp:
- self.program_fp.close()
- super(ProgramRunner, self).revert()
-
- def _get_argv(self):
- """
- Return the final argument vector used to execute the program.
- """
- return [
- self.args.get('_ansible_shell_executable', '/bin/sh'),
- '-c',
- self._get_shell_fragment(),
- ]
-
- def _get_shell_fragment(self):
- return "%s %s" % (
- shlex_quote(self.program_fp.name),
- ' '.join(map(shlex_quote, self._get_program_args())),
- )
-
- def _run(self):
- try:
- rc, stdout, stderr = ansible_mitogen.target.exec_args(
- args=self._get_argv(),
- emulate_tty=self.emulate_tty,
- )
- except Exception:
- LOG.exception('While running %s', self._get_argv())
- e = sys.exc_info()[1]
- return {
- u'rc': 1,
- u'stdout': u'',
- u'stderr': u'%s: %s' % (type(e), e),
- }
-
- return {
- u'rc': rc,
- u'stdout': mitogen.core.to_text(stdout),
- u'stderr': mitogen.core.to_text(stderr),
- }
-
-
-class ArgsFileRunner(Runner):
- def setup(self):
- super(ArgsFileRunner, self).setup()
- self._setup_args()
-
- def _setup_args(self):
- """
- Create a temporary file containing the module's arguments. The
- arguments are formatted via :meth:`_get_args`.
- """
- self.args_fp = tempfile.NamedTemporaryFile(
- prefix='ansible_mitogen',
- suffix='-args',
- dir=self.get_temp_dir(),
- )
- self.args_fp.write(utf8(self._get_args_contents()))
- self.args_fp.flush()
- reopen_readonly(self.program_fp)
-
- def _get_args_contents(self):
- """
- Return the module arguments formatted as JSON.
- """
- return json.dumps(self.args)
-
- def _get_program_args(self):
- return [self.args_fp.name]
-
- def revert(self):
- """
- Delete the temporary argument file.
- """
- self.args_fp.close()
- super(ArgsFileRunner, self).revert()
-
-
-class BinaryRunner(ArgsFileRunner, ProgramRunner):
- pass
-
-
-class ScriptRunner(ProgramRunner):
- def __init__(self, interpreter_fragment, is_python, **kwargs):
- super(ScriptRunner, self).__init__(**kwargs)
- self.interpreter_fragment = interpreter_fragment
- self.is_python = is_python
-
- b_ENCODING_STRING = b('# -*- coding: utf-8 -*-')
-
- def _get_program(self):
- return self._rewrite_source(
- super(ScriptRunner, self)._get_program()
- )
-
- def _get_argv(self):
- return [
- self.args.get('_ansible_shell_executable', '/bin/sh'),
- '-c',
- self._get_shell_fragment(),
- ]
-
- def _get_shell_fragment(self):
- """
- Scripts are eligible for having their hashbang line rewritten, and to
- be executed via /bin/sh using the ansible_*_interpreter value used as a
- shell fragment prefixing to the invocation.
- """
- return "%s %s %s" % (
- self.interpreter_fragment,
- shlex_quote(self.program_fp.name),
- ' '.join(map(shlex_quote, self._get_program_args())),
- )
-
- def _rewrite_source(self, s):
- """
- Mutate the source according to the per-task parameters.
- """
- # While Ansible rewrites the #! using ansible_*_interpreter, it is
- # never actually used to execute the script, instead it is a shell
- # fragment consumed by shell/__init__.py::build_module_command().
- new = [b('#!') + utf8(self.interpreter_fragment)]
- if self.is_python:
- new.append(self.b_ENCODING_STRING)
-
- _, _, rest = bytes_partition(s, b('\n'))
- new.append(rest)
- return b('\n').join(new)
-
-
-class NewStyleRunner(ScriptRunner):
- """
- Execute a new-style Ansible module, where Module Replacer-related tricks
- aren't required.
- """
- #: path => new-style module bytecode.
- _code_by_path = {}
-
- def __init__(self, module_map, **kwargs):
- super(NewStyleRunner, self).__init__(**kwargs)
- self.module_map = module_map
-
- def _setup_imports(self):
- """
- Ensure the local importer and PushFileService has everything for the
- Ansible module before setup() completes, but before detach() is called
- in an asynchronous task.
-
- The master automatically streams modules towards us concurrent to the
- runner invocation, however there is no public API to synchronize on the
- completion of those preloads. Instead simply reuse the importer's
- synchronization mechanism by importing everything the module will need
- prior to detaching.
- """
- for fullname, _, _ in self.module_map['custom']:
- mitogen.core.import_module(fullname)
- for fullname in self.module_map['builtin']:
- mitogen.core.import_module(fullname)
-
- def _setup_excepthook(self):
- """
- Starting with Ansible 2.6, some modules (file.py) install a
- sys.excepthook and never clean it up. So we must preserve the original
- excepthook and restore it after the run completes.
- """
- self.original_excepthook = sys.excepthook
-
- def setup(self):
- super(NewStyleRunner, self).setup()
-
- self._stdio = NewStyleStdio(self.args, self.get_temp_dir())
- # It is possible that not supplying the script filename will break some
- # module, but this has never been a bug report. Instead act like an
- # interpreter that had its script piped on stdin.
- self._argv = TemporaryArgv([''])
- self._importer = ModuleUtilsImporter(
- context=self.service_context,
- module_utils=self.module_map['custom'],
- )
- self._setup_imports()
- self._setup_excepthook()
- self.atexit_wrapper = AtExitWrapper()
- if libc__res_init:
- libc__res_init()
-
- def _revert_excepthook(self):
- sys.excepthook = self.original_excepthook
-
- def revert(self):
- self.atexit_wrapper.revert()
- self._argv.revert()
- self._stdio.revert()
- self._revert_excepthook()
- super(NewStyleRunner, self).revert()
-
- def _get_program_filename(self):
- """
- See ProgramRunner._get_program_filename().
- """
- return 'ansible_module_' + os.path.basename(self.path)
-
- def _setup_args(self):
- pass
-
- # issue #555: in old times it was considered good form to reload sys and
- # change the default encoding. This hack was removed from Ansible long ago,
- # but not before permeating into many third party modules.
- PREHISTORIC_HACK_RE = re.compile(
- b(r'reload\s*\(\s*sys\s*\)\s*'
- r'sys\s*\.\s*setdefaultencoding\([^)]+\)')
- )
-
- def _setup_program(self):
- source = ansible_mitogen.target.get_small_file(
- context=self.service_context,
- path=self.path,
- )
- self.source = self.PREHISTORIC_HACK_RE.sub(b(''), source)
-
- def _get_code(self):
- try:
- return self._code_by_path[self.path]
- except KeyError:
- return self._code_by_path.setdefault(self.path, compile(
- # Py2.4 doesn't support kwargs.
- self.source, # source
- "master:" + self.path, # filename
- 'exec', # mode
- 0, # flags
- True, # dont_inherit
- ))
-
- if mitogen.core.PY3:
- main_module_name = '__main__'
- else:
- main_module_name = b('__main__')
-
- def _handle_magic_exception(self, mod, exc):
- """
- Beginning with Ansible >2.6, some modules (file.py) install a
- sys.excepthook which is a closure over AnsibleModule, redirecting the
- magical exception to AnsibleModule.fail_json().
-
- For extra special needs bonus points, the class is not defined in
- module_utils, but is defined in the module itself, meaning there is no
- type for isinstance() that outlasts the invocation.
- """
- klass = getattr(mod, 'AnsibleModuleError', None)
- if klass and isinstance(exc, klass):
- mod.module.fail_json(**exc.results)
-
- def _run_code(self, code, mod):
- try:
- if mitogen.core.PY3:
- exec(code, vars(mod))
- else:
- exec('exec code in vars(mod)')
- except Exception:
- self._handle_magic_exception(mod, sys.exc_info()[1])
- raise
-
- def _run(self):
- mod = types.ModuleType(self.main_module_name)
- mod.__package__ = None
- # Some Ansible modules use __file__ to find the Ansiballz temporary
- # directory. We must provide some temporary path in __file__, but we
- # don't want to pointlessly write the module to disk when it never
- # actually needs to exist. So just pass the filename as it would exist.
- mod.__file__ = os.path.join(
- self.get_temp_dir(),
- 'ansible_module_' + os.path.basename(self.path),
- )
-
- code = self._get_code()
- rc = 2
- try:
- try:
- self._run_code(code, mod)
- except SystemExit:
- exc = sys.exc_info()[1]
- rc = exc.args[0]
- except Exception:
- # This writes to stderr by default.
- traceback.print_exc()
- rc = 1
-
- finally:
- self.atexit_wrapper.run_callbacks()
-
- return {
- u'rc': rc,
- u'stdout': mitogen.core.to_text(sys.stdout.getvalue()),
- u'stderr': mitogen.core.to_text(sys.stderr.getvalue()),
- }
-
-
-class JsonArgsRunner(ScriptRunner):
- JSON_ARGS = b('<>')
-
- def _get_args_contents(self):
- return json.dumps(self.args).encode()
-
- def _rewrite_source(self, s):
- return (
- super(JsonArgsRunner, self)._rewrite_source(s)
- .replace(self.JSON_ARGS, self._get_args_contents())
- )
-
-
-class WantJsonRunner(ArgsFileRunner, ScriptRunner):
- pass
-
-
-class OldStyleRunner(ArgsFileRunner, ScriptRunner):
- def _get_args_contents(self):
- """
- Mimic the argument formatting behaviour of
- ActionBase._execute_module().
- """
- return ' '.join(
- '%s=%s' % (key, shlex_quote(str(self.args[key])))
- for key in self.args
- ) + ' ' # Bug-for-bug :(
diff --git a/mitogen-0.2.7/ansible_mitogen/services.py b/mitogen-0.2.7/ansible_mitogen/services.py
deleted file mode 100644
index a7c0e46..000000000
--- a/mitogen-0.2.7/ansible_mitogen/services.py
+++ /dev/null
@@ -1,537 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-"""
-Classes in this file define Mitogen 'services' that run (initially) within the
-connection multiplexer process that is forked off the top-level controller
-process.
-
-Once a worker process connects to a multiplexer process
-(Connection._connect()), it communicates with these services to establish new
-connections, grant access to files by children, and register for notification
-when a child has completed a job.
-"""
-
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
-import logging
-import os
-import os.path
-import sys
-import threading
-
-import ansible.constants
-
-import mitogen
-import mitogen.service
-import mitogen.utils
-import ansible_mitogen.loaders
-import ansible_mitogen.module_finder
-import ansible_mitogen.target
-
-
-LOG = logging.getLogger(__name__)
-
-# Force load of plugin to ensure ConfigManager has definitions loaded. Done
-# during module import to ensure a single-threaded environment; PluginLoader
-# is not thread-safe.
-ansible_mitogen.loaders.shell_loader.get('sh')
-
-
-if sys.version_info[0] == 3:
- def reraise(tp, value, tb):
- if value is None:
- value = tp()
- if value.__traceback__ is not tb:
- raise value.with_traceback(tb)
- raise value
-else:
- exec(
- "def reraise(tp, value, tb=None):\n"
- " raise tp, value, tb\n"
- )
-
-
-def _get_candidate_temp_dirs():
- try:
- # >=2.5
- options = ansible.constants.config.get_plugin_options('shell', 'sh')
- remote_tmp = options.get('remote_tmp') or ansible.constants.DEFAULT_REMOTE_TMP
- system_tmpdirs = options.get('system_tmpdirs', ('/var/tmp', '/tmp'))
- except AttributeError:
- # 2.3
- remote_tmp = ansible.constants.DEFAULT_REMOTE_TMP
- system_tmpdirs = ('/var/tmp', '/tmp')
-
- return mitogen.utils.cast([remote_tmp] + list(system_tmpdirs))
-
-
-def key_from_dict(**kwargs):
- """
- Return a unique string representation of a dict as quickly as possible.
- Used to generated deduplication keys from a request.
- """
- out = []
- stack = [kwargs]
- while stack:
- obj = stack.pop()
- if isinstance(obj, dict):
- stack.extend(sorted(obj.items()))
- elif isinstance(obj, (list, tuple)):
- stack.extend(obj)
- else:
- out.append(str(obj))
- return ''.join(out)
-
-
-class Error(Exception):
- pass
-
-
-class ContextService(mitogen.service.Service):
- """
- Used by workers to fetch the single Context instance corresponding to a
- connection configuration, creating the matching connection if it does not
- exist.
-
- For connection methods and their parameters, see:
- https://mitogen.readthedocs.io/en/latest/api.html#context-factories
-
- This concentrates connections in the top-level process, which may become a
- bottleneck. The bottleneck can be removed using per-CPU connection
- processes and arranging for the worker to select one according to a hash of
- the connection parameters (sharding).
- """
- max_interpreters = int(os.getenv('MITOGEN_MAX_INTERPRETERS', '20'))
-
- def __init__(self, *args, **kwargs):
- super(ContextService, self).__init__(*args, **kwargs)
- self._lock = threading.Lock()
- #: Records the :meth:`get` result dict for successful calls, returned
- #: for identical subsequent calls. Keyed by :meth:`key_from_dict`.
- self._response_by_key = {}
- #: List of :class:`mitogen.core.Latch` awaiting the result for a
- #: particular key.
- self._latches_by_key = {}
- #: Mapping of :class:`mitogen.core.Context` -> reference count. Each
- #: call to :meth:`get` increases this by one. Calls to :meth:`put`
- #: decrease it by one.
- self._refs_by_context = {}
- #: List of contexts in creation order by via= parameter. When
- #: :attr:`max_interpreters` is reached, the most recently used context
- #: is destroyed to make room for any additional context.
- self._lru_by_via = {}
- #: :func:`key_from_dict` result by Context.
- self._key_by_context = {}
- #: Mapping of Context -> parent Context
- self._via_by_context = {}
-
- @mitogen.service.expose(mitogen.service.AllowParents())
- @mitogen.service.arg_spec({
- 'context': mitogen.core.Context
- })
- def reset(self, context):
- """
- Return a reference, forcing close and discard of the underlying
- connection. Used for 'meta: reset_connection' or when some other error
- is detected.
- """
- LOG.debug('%r.reset(%r)', self, context)
- self._lock.acquire()
- try:
- self._shutdown_unlocked(context)
- finally:
- self._lock.release()
-
- @mitogen.service.expose(mitogen.service.AllowParents())
- @mitogen.service.arg_spec({
- 'context': mitogen.core.Context
- })
- def put(self, context):
- """
- Return a reference, making it eligable for recycling once its reference
- count reaches zero.
- """
- LOG.debug('%r.put(%r)', self, context)
- self._lock.acquire()
- try:
- if self._refs_by_context.get(context, 0) == 0:
- LOG.warning('%r.put(%r): refcount was 0. shutdown_all called?',
- self, context)
- return
- self._refs_by_context[context] -= 1
- finally:
- self._lock.release()
-
- def _produce_response(self, key, response):
- """
- Reply to every waiting request matching a configuration key with a
- response dictionary, deleting the list of waiters when done.
-
- :param str key:
- Result of :meth:`key_from_dict`
- :param dict response:
- Response dictionary
- :returns:
- Number of waiters that were replied to.
- """
- self._lock.acquire()
- try:
- latches = self._latches_by_key.pop(key)
- count = len(latches)
- for latch in latches:
- latch.put(response)
- finally:
- self._lock.release()
- return count
-
- def _forget_context_unlocked(self, context):
- key = self._key_by_context.get(context)
- if key is None:
- LOG.debug('%r: attempt to forget unknown %r', self, context)
- return
-
- self._response_by_key.pop(key, None)
- self._latches_by_key.pop(key, None)
- self._key_by_context.pop(context, None)
- self._refs_by_context.pop(context, None)
- self._via_by_context.pop(context, None)
- self._lru_by_via.pop(context, None)
-
- def _shutdown_unlocked(self, context, lru=None, new_context=None):
- """
- Arrange for `context` to be shut down, and optionally add `new_context`
- to the LRU list while holding the lock.
- """
- LOG.info('%r._shutdown_unlocked(): shutting down %r', self, context)
- context.shutdown()
- via = self._via_by_context.get(context)
- if via:
- lru = self._lru_by_via.get(via)
- if lru:
- if context in lru:
- lru.remove(context)
- if new_context:
- lru.append(new_context)
- self._forget_context_unlocked(context)
-
- def _update_lru_unlocked(self, new_context, spec, via):
- """
- Update the LRU ("MRU"?) list associated with the connection described
- by `kwargs`, destroying the most recently created context if the list
- is full. Finally add `new_context` to the list.
- """
- self._via_by_context[new_context] = via
-
- lru = self._lru_by_via.setdefault(via, [])
- if len(lru) < self.max_interpreters:
- lru.append(new_context)
- return
-
- for context in reversed(lru):
- if self._refs_by_context[context] == 0:
- break
- else:
- LOG.warning('via=%r reached maximum number of interpreters, '
- 'but they are all marked as in-use.', via)
- return
-
- self._shutdown_unlocked(context, lru=lru, new_context=new_context)
-
- def _update_lru(self, new_context, spec, via):
- self._lock.acquire()
- try:
- self._update_lru_unlocked(new_context, spec, via)
- finally:
- self._lock.release()
-
- @mitogen.service.expose(mitogen.service.AllowParents())
- def dump(self):
- """
- For testing, return a list of dicts describing every currently
- connected context.
- """
- return [
- {
- 'context_name': context.name,
- 'via': getattr(self._via_by_context.get(context),
- 'name', None),
- 'refs': self._refs_by_context.get(context),
- }
- for context, key in sorted(self._key_by_context.items(),
- key=lambda c_k: c_k[0].context_id)
- ]
-
- @mitogen.service.expose(mitogen.service.AllowParents())
- def shutdown_all(self):
- """
- For testing use, arrange for all connections to be shut down.
- """
- self._lock.acquire()
- try:
- for context in list(self._key_by_context):
- self._shutdown_unlocked(context)
- finally:
- self._lock.release()
-
- def _on_context_disconnect(self, context):
- """
- Respond to Context disconnect event by deleting any record of the no
- longer reachable context. This method runs in the Broker thread and
- must not to block.
- """
- self._lock.acquire()
- try:
- LOG.info('%r: Forgetting %r due to stream disconnect', self, context)
- self._forget_context_unlocked(context)
- finally:
- self._lock.release()
-
- ALWAYS_PRELOAD = (
- 'ansible.module_utils.basic',
- 'ansible.module_utils.json_utils',
- 'ansible.release',
- 'ansible_mitogen.runner',
- 'ansible_mitogen.target',
- 'mitogen.fork',
- 'mitogen.service',
- )
-
- def _send_module_forwards(self, context):
- self.router.responder.forward_modules(context, self.ALWAYS_PRELOAD)
-
- _candidate_temp_dirs = None
-
- def _get_candidate_temp_dirs(self):
- """
- Return a list of locations to try to create the single temporary
- directory used by the run. This simply caches the (expensive) plugin
- load of :func:`_get_candidate_temp_dirs`.
- """
- if self._candidate_temp_dirs is None:
- self._candidate_temp_dirs = _get_candidate_temp_dirs()
- return self._candidate_temp_dirs
-
- def _connect(self, key, spec, via=None):
- """
- Actual connect implementation. Arranges for the Mitogen connection to
- be created and enqueues an asynchronous call to start the forked task
- parent in the remote context.
-
- :param key:
- Deduplication key representing the connection configuration.
- :param spec:
- Connection specification.
- :returns:
- Dict like::
-
- {
- 'context': mitogen.core.Context or None,
- 'via': mitogen.core.Context or None,
- 'init_child_result': {
- 'fork_context': mitogen.core.Context,
- 'home_dir': str or None,
- },
- 'msg': str or None
- }
-
- Where `context` is a reference to the newly constructed context,
- `init_child_result` is the result of executing
- :func:`ansible_mitogen.target.init_child` in that context, `msg` is
- an error message and the remaining fields are :data:`None`, or
- `msg` is :data:`None` and the remaining fields are set.
- """
- try:
- method = getattr(self.router, spec['method'])
- except AttributeError:
- raise Error('unsupported method: %(transport)s' % spec)
-
- context = method(via=via, unidirectional=True, **spec['kwargs'])
- if via and spec.get('enable_lru'):
- self._update_lru(context, spec, via)
-
- # Forget the context when its disconnect event fires.
- mitogen.core.listen(context, 'disconnect',
- lambda: self._on_context_disconnect(context))
-
- self._send_module_forwards(context)
- init_child_result = context.call(
- ansible_mitogen.target.init_child,
- log_level=LOG.getEffectiveLevel(),
- candidate_temp_dirs=self._get_candidate_temp_dirs(),
- )
-
- if os.environ.get('MITOGEN_DUMP_THREAD_STACKS'):
- from mitogen import debug
- context.call(debug.dump_to_logger)
-
- self._key_by_context[context] = key
- self._refs_by_context[context] = 0
- return {
- 'context': context,
- 'via': via,
- 'init_child_result': init_child_result,
- 'msg': None,
- }
-
- def _wait_or_start(self, spec, via=None):
- latch = mitogen.core.Latch()
- key = key_from_dict(via=via, **spec)
- self._lock.acquire()
- try:
- response = self._response_by_key.get(key)
- if response is not None:
- self._refs_by_context[response['context']] += 1
- latch.put(response)
- return latch
-
- latches = self._latches_by_key.setdefault(key, [])
- first = len(latches) == 0
- latches.append(latch)
- finally:
- self._lock.release()
-
- if first:
- # I'm the first requestee, so I will create the connection.
- try:
- response = self._connect(key, spec, via=via)
- count = self._produce_response(key, response)
- # Only record the response for non-error results.
- self._response_by_key[key] = response
- # Set the reference count to the number of waiters.
- self._refs_by_context[response['context']] += count
- except Exception:
- self._produce_response(key, sys.exc_info())
-
- return latch
-
- disconnect_msg = (
- 'Channel was disconnected while connection attempt was in progress; '
- 'this may be caused by an abnormal Ansible exit, or due to an '
- 'unreliable target.'
- )
-
- @mitogen.service.expose(mitogen.service.AllowParents())
- @mitogen.service.arg_spec({
- 'stack': list
- })
- def get(self, msg, stack):
- """
- Return a Context referring to an established connection with the given
- configuration, establishing new connections as necessary.
-
- :param list stack:
- Connection descriptions. Each element is a dict containing 'method'
- and 'kwargs' keys describing the Router method and arguments.
- Subsequent elements are proxied via the previous.
-
- :returns dict:
- * context: mitogen.parent.Context or None.
- * init_child_result: Result of :func:`init_child`.
- * msg: StreamError exception text or None.
- * method_name: string failing method name.
- """
- via = None
- for spec in stack:
- try:
- result = self._wait_or_start(spec, via=via).get()
- if isinstance(result, tuple): # exc_info()
- reraise(*result)
- via = result['context']
- except mitogen.core.ChannelError:
- return {
- 'context': None,
- 'init_child_result': None,
- 'method_name': spec['method'],
- 'msg': self.disconnect_msg,
- }
- except mitogen.core.StreamError as e:
- return {
- 'context': None,
- 'init_child_result': None,
- 'method_name': spec['method'],
- 'msg': str(e),
- }
-
- return result
-
-
-class ModuleDepService(mitogen.service.Service):
- """
- Scan a new-style module and produce a cached mapping of module_utils names
- to their resolved filesystem paths.
- """
- invoker_class = mitogen.service.SerializedInvoker
-
- def __init__(self, *args, **kwargs):
- super(ModuleDepService, self).__init__(*args, **kwargs)
- self._cache = {}
-
- def _get_builtin_names(self, builtin_path, resolved):
- return [
- mitogen.core.to_text(fullname)
- for fullname, path, is_pkg in resolved
- if os.path.abspath(path).startswith(builtin_path)
- ]
-
- def _get_custom_tups(self, builtin_path, resolved):
- return [
- (mitogen.core.to_text(fullname),
- mitogen.core.to_text(path),
- is_pkg)
- for fullname, path, is_pkg in resolved
- if not os.path.abspath(path).startswith(builtin_path)
- ]
-
- @mitogen.service.expose(policy=mitogen.service.AllowParents())
- @mitogen.service.arg_spec({
- 'module_name': mitogen.core.UnicodeType,
- 'module_path': mitogen.core.FsPathTypes,
- 'search_path': tuple,
- 'builtin_path': mitogen.core.FsPathTypes,
- 'context': mitogen.core.Context,
- })
- def scan(self, module_name, module_path, search_path, builtin_path, context):
- key = (module_name, search_path)
- if key not in self._cache:
- resolved = ansible_mitogen.module_finder.scan(
- module_name=module_name,
- module_path=module_path,
- search_path=tuple(search_path) + (builtin_path,),
- )
- builtin_path = os.path.abspath(builtin_path)
- builtin = self._get_builtin_names(builtin_path, resolved)
- custom = self._get_custom_tups(builtin_path, resolved)
- self._cache[key] = {
- 'builtin': builtin,
- 'custom': custom,
- }
- return self._cache[key]
diff --git a/mitogen-0.2.7/ansible_mitogen/strategy.py b/mitogen-0.2.7/ansible_mitogen/strategy.py
deleted file mode 100644
index b9211fc..000000000
--- a/mitogen-0.2.7/ansible_mitogen/strategy.py
+++ /dev/null
@@ -1,296 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-import os
-import signal
-import threading
-
-import mitogen.core
-import ansible_mitogen.affinity
-import ansible_mitogen.loaders
-import ansible_mitogen.mixins
-import ansible_mitogen.process
-
-import ansible
-import ansible.executor.process.worker
-
-
-ANSIBLE_VERSION_MIN = '2.3'
-ANSIBLE_VERSION_MAX = '2.7'
-NEW_VERSION_MSG = (
- "Your Ansible version (%s) is too recent. The most recent version\n"
- "supported by Mitogen for Ansible is %s.x. Please check the Mitogen\n"
- "release notes to see if a new version is available, otherwise\n"
- "subscribe to the corresponding GitHub issue to be notified when\n"
- "support becomes available.\n"
- "\n"
- " https://mitogen.rtfd.io/en/latest/changelog.html\n"
- " https://github.com/dw/mitogen/issues/\n"
-)
-OLD_VERSION_MSG = (
- "Your version of Ansible (%s) is too old. The oldest version supported by "
- "Mitogen for Ansible is %s."
-)
-
-
-def _assert_supported_release():
- """
- Throw AnsibleError with a descriptive message in case of being loaded into
- an unsupported Ansible release.
- """
- v = ansible.__version__
-
- if v[:len(ANSIBLE_VERSION_MIN)] < ANSIBLE_VERSION_MIN:
- raise ansible.errors.AnsibleError(
- OLD_VERSION_MSG % (v, ANSIBLE_VERSION_MIN)
- )
-
- if v[:len(ANSIBLE_VERSION_MAX)] > ANSIBLE_VERSION_MAX:
- raise ansible.errors.AnsibleError(
- NEW_VERSION_MSG % (ansible.__version__, ANSIBLE_VERSION_MAX)
- )
-
-
-def _patch_awx_callback():
- """
- issue #400: AWX loads a display callback that suffers from thread-safety
- issues. Detect the presence of older AWX versions and patch the bug.
- """
- # AWX uses sitecustomize.py to force-load this package. If it exists, we're
- # running under AWX.
- try:
- from awx_display_callback.events import EventContext
- from awx_display_callback.events import event_context
- except ImportError:
- return
-
- if hasattr(EventContext(), '_local'):
- # Patched version.
- return
-
- def patch_add_local(self, **kwargs):
- tls = vars(self._local)
- ctx = tls.setdefault('_ctx', {})
- ctx.update(kwargs)
-
- EventContext._local = threading.local()
- EventContext.add_local = patch_add_local
-
-_patch_awx_callback()
-
-
-def wrap_action_loader__get(name, *args, **kwargs):
- """
- While the mitogen strategy is active, trap action_loader.get() calls,
- augmenting any fetched class with ActionModuleMixin, which replaces various
- helper methods inherited from ActionBase with implementations that avoid
- the use of shell fragments wherever possible.
-
- This is used instead of static subclassing as it generalizes to third party
- action modules outside the Ansible tree.
- """
- klass = action_loader__get(name, class_only=True)
- if klass:
- bases = (ansible_mitogen.mixins.ActionModuleMixin, klass)
- adorned_klass = type(str(name), bases, {})
- if kwargs.get('class_only'):
- return adorned_klass
- return adorned_klass(*args, **kwargs)
-
-
-def wrap_connection_loader__get(name, *args, **kwargs):
- """
- While the strategy is active, rewrite connection_loader.get() calls for
- some transports into requests for a compatible Mitogen transport.
- """
- if name in ('docker', 'kubectl', 'jail', 'local', 'lxc',
- 'lxd', 'machinectl', 'setns', 'ssh'):
- name = 'mitogen_' + name
- return connection_loader__get(name, *args, **kwargs)
-
-
-def wrap_worker__run(*args, **kwargs):
- """
- While the strategy is active, rewrite connection_loader.get() calls for
- some transports into requests for a compatible Mitogen transport.
- """
- # Ignore parent's attempts to murder us when we still need to write
- # profiling output.
- if mitogen.core._profile_hook.__name__ != '_profile_hook':
- signal.signal(signal.SIGTERM, signal.SIG_IGN)
-
- ansible_mitogen.logging.set_process_name('task')
- ansible_mitogen.affinity.policy.assign_worker()
- return mitogen.core._profile_hook('WorkerProcess',
- lambda: worker__run(*args, **kwargs)
- )
-
-
-class StrategyMixin(object):
- """
- This mix-in enhances any built-in strategy by arranging for various Mitogen
- services to be initialized in the Ansible top-level process, and for worker
- processes to grow support for using those top-level services to communicate
- with and execute modules on remote hosts.
-
- Mitogen:
-
- A private Broker IO multiplexer thread is created to dispatch IO
- between the local Router and any connected streams, including streams
- connected to Ansible WorkerProcesses, and SSH commands implementing
- connections to remote machines.
-
- A Router is created that implements message dispatch to any locally
- registered handlers, and message routing for remote streams. Router is
- the junction point through which WorkerProceses and remote SSH contexts
- can communicate.
-
- Router additionally adds message handlers for a variety of base
- services, review the Standard Handles section of the How It Works guide
- in the documentation.
-
- A ContextService is installed as a message handler in the master
- process and run on a private thread. It is responsible for accepting
- requests to establish new SSH connections from worker processes, and
- ensuring precisely one connection exists and is reused for subsequent
- playbook steps. The service presently runs in a single thread, so to
- begin with, new SSH connections are serialized.
-
- Finally a mitogen.unix listener is created through which WorkerProcess
- can establish a connection back into the master process, in order to
- avail of ContextService. A UNIX listener socket is necessary as there
- is no more sane mechanism to arrange for IPC between the Router in the
- master process, and the corresponding Router in the worker process.
-
- Ansible:
-
- PluginLoader monkey patches are installed to catch attempts to create
- connection and action plug-ins.
-
- For connection plug-ins, if the desired method is "local" or "ssh", it
- is redirected to the "mitogen" connection plug-in. That plug-in
- implements communication via a UNIX socket connection to the top-level
- Ansible process, and uses ContextService running in the top-level
- process to actually establish and manage the connection.
-
- For action plug-ins, the original class is looked up as usual, but a
- new subclass is created dynamically in order to mix-in
- ansible_mitogen.target.ActionModuleMixin, which overrides many of the
- methods usually inherited from ActionBase in order to replace them with
- pure-Python equivalents that avoid the use of shell.
-
- In particular, _execute_module() is overridden with an implementation
- that uses ansible_mitogen.target.run_module() executed in the target
- Context. run_module() implements module execution by importing the
- module as if it were a normal Python module, and capturing its output
- in the remote process. Since the Mitogen module loader is active in the
- remote process, all the heavy lifting of transferring the action module
- and its dependencies are automatically handled by Mitogen.
- """
- def _install_wrappers(self):
- """
- Install our PluginLoader monkey patches and update global variables
- with references to the real functions.
- """
- global action_loader__get
- action_loader__get = ansible_mitogen.loaders.action_loader.get
- ansible_mitogen.loaders.action_loader.get = wrap_action_loader__get
-
- global connection_loader__get
- connection_loader__get = ansible_mitogen.loaders.connection_loader.get
- ansible_mitogen.loaders.connection_loader.get = wrap_connection_loader__get
-
- global worker__run
- worker__run = ansible.executor.process.worker.WorkerProcess.run
- ansible.executor.process.worker.WorkerProcess.run = wrap_worker__run
-
- def _remove_wrappers(self):
- """
- Uninstall the PluginLoader monkey patches.
- """
- ansible_mitogen.loaders.action_loader.get = action_loader__get
- ansible_mitogen.loaders.connection_loader.get = connection_loader__get
- ansible.executor.process.worker.WorkerProcess.run = worker__run
-
- def _add_plugin_paths(self):
- """
- Add the Mitogen plug-in directories to the ModuleLoader path, avoiding
- the need for manual configuration.
- """
- base_dir = os.path.join(os.path.dirname(__file__), 'plugins')
- ansible_mitogen.loaders.connection_loader.add_directory(
- os.path.join(base_dir, 'connection')
- )
- ansible_mitogen.loaders.action_loader.add_directory(
- os.path.join(base_dir, 'action')
- )
-
- def _queue_task(self, host, task, task_vars, play_context):
- """
- Many PluginLoader caches are defective as they are only populated in
- the ephemeral WorkerProcess. Touch each plug-in path before forking to
- ensure all workers receive a hot cache.
- """
- ansible_mitogen.loaders.module_loader.find_plugin(
- name=task.action,
- mod_type='',
- )
- ansible_mitogen.loaders.connection_loader.get(
- name=play_context.connection,
- class_only=True,
- )
- ansible_mitogen.loaders.action_loader.get(
- name=task.action,
- class_only=True,
- )
-
- return super(StrategyMixin, self)._queue_task(
- host=host,
- task=task,
- task_vars=task_vars,
- play_context=play_context,
- )
-
- def run(self, iterator, play_context, result=0):
- """
- Arrange for a mitogen.master.Router to be available for the duration of
- the strategy's real run() method.
- """
- _assert_supported_release()
-
- ansible_mitogen.process.MuxProcess.start()
- run = super(StrategyMixin, self).run
- self._add_plugin_paths()
- self._install_wrappers()
- try:
- return mitogen.core._profile_hook('Strategy',
- lambda: run(iterator, play_context)
- )
- finally:
- self._remove_wrappers()
diff --git a/mitogen-0.2.7/ansible_mitogen/target.py b/mitogen-0.2.7/ansible_mitogen/target.py
deleted file mode 100644
index 652b5ad..000000000
--- a/mitogen-0.2.7/ansible_mitogen/target.py
+++ /dev/null
@@ -1,777 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-"""
-Helper functions intended to be executed on the target. These are entrypoints
-for file transfer, module execution and sundry bits like changing file modes.
-"""
-
-import errno
-import grp
-import operator
-import os
-import pwd
-import re
-import signal
-import stat
-import subprocess
-import sys
-import tempfile
-import traceback
-import types
-
-# Absolute imports for <2.5.
-logging = __import__('logging')
-
-import mitogen.core
-import mitogen.fork
-import mitogen.parent
-import mitogen.service
-from mitogen.core import b
-
-try:
- import json
-except ImportError:
- import simplejson as json
-
-try:
- reduce
-except NameError:
- # Python 3.x.
- from functools import reduce
-
-try:
- BaseException
-except NameError:
- # Python 2.4
- BaseException = Exception
-
-
-# Ansible since PR #41749 inserts "import __main__" into
-# ansible.module_utils.basic. Mitogen's importer will refuse such an import, so
-# we must setup a fake "__main__" before that module is ever imported. The
-# str() is to cast Unicode to bytes on Python 2.6.
-if not sys.modules.get(str('__main__')):
- sys.modules[str('__main__')] = types.ModuleType(str('__main__'))
-
-import ansible.module_utils.json_utils
-import ansible_mitogen.runner
-
-
-LOG = logging.getLogger(__name__)
-
-MAKE_TEMP_FAILED_MSG = (
- u"Unable to find a useable temporary directory. This likely means no\n"
- u"system-supplied TMP directory can be written to, or all directories\n"
- u"were mounted on 'noexec' filesystems.\n"
- u"\n"
- u"The following paths were tried:\n"
- u" %(paths)s\n"
- u"\n"
- u"Please check '-vvv' output for a log of individual path errors."
-)
-
-# Python 2.4/2.5 cannot support fork+threads whatsoever, it doesn't even fix up
-# interpreter state. So 2.4/2.5 interpreters start .local() contexts for
-# isolation instead. Since we don't have any crazy memory sharing problems to
-# avoid, there is no virginal fork parent either. The child is started directly
-# from the login/become process. In future this will be default everywhere,
-# fork is brainwrong from the stone age.
-FORK_SUPPORTED = sys.version_info >= (2, 6)
-
-#: Initialized to an econtext.parent.Context pointing at a pristine fork of
-#: the target Python interpreter before it executes any code or imports.
-_fork_parent = None
-
-#: Set by :func:`init_child` to the name of a writeable and executable
-#: temporary directory accessible by the active user account.
-good_temp_dir = None
-
-
-def subprocess__Popen__close_fds(self, but):
- """
- issue #362, #435: subprocess.Popen(close_fds=True) aka.
- AnsibleModule.run_command() loops the entire FD space on Python<3.2.
- CentOS>5 ships with 1,048,576 FDs by default, resulting in huge (>500ms)
- latency starting children. Therefore replace Popen._close_fds on Linux with
- a version that is O(fds) rather than O(_SC_OPEN_MAX).
- """
- try:
- names = os.listdir(u'/proc/self/fd')
- except OSError:
- # May fail if acting on a container that does not have /proc mounted.
- self._original_close_fds(but)
- return
-
- for name in names:
- if not name.isdigit():
- continue
-
- fd = int(name, 10)
- if fd > 2 and fd != but:
- try:
- os.close(fd)
- except OSError:
- pass
-
-
-if (
- sys.platform.startswith(u'linux') and
- sys.version < u'3.0' and
- hasattr(subprocess.Popen, u'_close_fds') and
- not mitogen.is_master
-):
- subprocess.Popen._original_close_fds = subprocess.Popen._close_fds
- subprocess.Popen._close_fds = subprocess__Popen__close_fds
-
-
-def get_small_file(context, path):
- """
- Basic in-memory caching module fetcher. This generates one roundtrip for
- every previously unseen file, so it is only a temporary solution.
-
- :param context:
- Context we should direct FileService requests to. For now (and probably
- forever) this is just the top-level Mitogen connection manager process.
- :param path:
- Path to fetch from FileService, must previously have been registered by
- a privileged context using the `register` command.
- :returns:
- Bytestring file data.
- """
- pool = mitogen.service.get_or_create_pool(router=context.router)
- service = pool.get_service(u'mitogen.service.PushFileService')
- return service.get(path)
-
-
-def transfer_file(context, in_path, out_path, sync=False, set_owner=False):
- """
- Streamily download a file from the connection multiplexer process in the
- controller.
-
- :param mitogen.core.Context context:
- Reference to the context hosting the FileService that will transmit the
- file.
- :param bytes in_path:
- FileService registered name of the input file.
- :param bytes out_path:
- Name of the output path on the local disk.
- :param bool sync:
- If :data:`True`, ensure the file content and metadat are fully on disk
- before renaming the temporary file over the existing file. This should
- ensure in the case of system crash, either the entire old or new file
- are visible post-reboot.
- :param bool set_owner:
- If :data:`True`, look up the metadata username and group on the local
- system and file the file owner using :func:`os.fchmod`.
- """
- out_path = os.path.abspath(out_path)
- fd, tmp_path = tempfile.mkstemp(suffix='.tmp',
- prefix='.ansible_mitogen_transfer-',
- dir=os.path.dirname(out_path))
- fp = os.fdopen(fd, 'wb', mitogen.core.CHUNK_SIZE)
- LOG.debug('transfer_file(%r) temporary file: %s', out_path, tmp_path)
-
- try:
- try:
- ok, metadata = mitogen.service.FileService.get(
- context=context,
- path=in_path,
- out_fp=fp,
- )
- if not ok:
- raise IOError('transfer of %r was interrupted.' % (in_path,))
-
- set_file_mode(tmp_path, metadata['mode'], fd=fp.fileno())
- if set_owner:
- set_file_owner(tmp_path, metadata['owner'], metadata['group'],
- fd=fp.fileno())
- finally:
- fp.close()
-
- if sync:
- os.fsync(fp.fileno())
- os.rename(tmp_path, out_path)
- except BaseException:
- os.unlink(tmp_path)
- raise
-
- os.utime(out_path, (metadata['atime'], metadata['mtime']))
-
-
-def prune_tree(path):
- """
- Like shutil.rmtree(), but log errors rather than discard them, and do not
- waste multiple os.stat() calls discovering whether the object can be
- deleted, just try deleting it instead.
- """
- try:
- os.unlink(path)
- return
- except OSError:
- e = sys.exc_info()[1]
- if not (os.path.isdir(path) and
- e.args[0] in (errno.EPERM, errno.EISDIR)):
- LOG.error('prune_tree(%r): %s', path, e)
- return
-
- try:
- # Ensure write access for readonly directories. Ignore error in case
- # path is on a weird filesystem (e.g. vfat).
- os.chmod(path, int('0700', 8))
- except OSError:
- e = sys.exc_info()[1]
- LOG.warning('prune_tree(%r): %s', path, e)
-
- try:
- for name in os.listdir(path):
- if name not in ('.', '..'):
- prune_tree(os.path.join(path, name))
- os.rmdir(path)
- except OSError:
- e = sys.exc_info()[1]
- LOG.error('prune_tree(%r): %s', path, e)
-
-
-def is_good_temp_dir(path):
- """
- Return :data:`True` if `path` can be used as a temporary directory, logging
- any failures that may cause it to be unsuitable. If the directory doesn't
- exist, we attempt to create it using :func:`os.makedirs`.
- """
- if not os.path.exists(path):
- try:
- os.makedirs(path, mode=int('0700', 8))
- except OSError:
- e = sys.exc_info()[1]
- LOG.debug('temp dir %r unusable: did not exist and attempting '
- 'to create it failed: %s', path, e)
- return False
-
- try:
- tmp = tempfile.NamedTemporaryFile(
- prefix='ansible_mitogen_is_good_temp_dir',
- dir=path,
- )
- except (OSError, IOError):
- e = sys.exc_info()[1]
- LOG.debug('temp dir %r unusable: %s', path, e)
- return False
-
- try:
- try:
- os.chmod(tmp.name, int('0700', 8))
- except OSError:
- e = sys.exc_info()[1]
- LOG.debug('temp dir %r unusable: chmod failed: %s', path, e)
- return False
-
- try:
- # access(.., X_OK) is sufficient to detect noexec.
- if not os.access(tmp.name, os.X_OK):
- raise OSError('filesystem appears to be mounted noexec')
- except OSError:
- e = sys.exc_info()[1]
- LOG.debug('temp dir %r unusable: %s', path, e)
- return False
- finally:
- tmp.close()
-
- return True
-
-
-def find_good_temp_dir(candidate_temp_dirs):
- """
- Given a list of candidate temp directories extracted from ``ansible.cfg``,
- combine it with the Python-builtin list of candidate directories used by
- :mod:`tempfile`, then iteratively try each until one is found that is both
- writeable and executable.
-
- :param list candidate_temp_dirs:
- List of candidate $variable-expanded and tilde-expanded directory paths
- that may be usable as a temporary directory.
- """
- paths = [os.path.expandvars(os.path.expanduser(p))
- for p in candidate_temp_dirs]
- paths.extend(tempfile._candidate_tempdir_list())
-
- for path in paths:
- if is_good_temp_dir(path):
- LOG.debug('Selected temp directory: %r (from %r)', path, paths)
- return path
-
- raise IOError(MAKE_TEMP_FAILED_MSG % {
- 'paths': '\n '.join(paths),
- })
-
-
-@mitogen.core.takes_econtext
-def init_child(econtext, log_level, candidate_temp_dirs):
- """
- Called by ContextService immediately after connection; arranges for the
- (presently) spotless Python interpreter to be forked, where the newly
- forked interpreter becomes the parent of any newly forked future
- interpreters.
-
- This is necessary to prevent modules that are executed in-process from
- polluting the global interpreter state in a way that effects explicitly
- isolated modules.
-
- :param int log_level:
- Logging package level active in the master.
- :param list[str] candidate_temp_dirs:
- List of $variable-expanded and tilde-expanded directory names to add to
- candidate list of temporary directories.
-
- :returns:
- Dict like::
-
- {
- 'fork_context': mitogen.core.Context or None,
- 'good_temp_dir': ...
- 'home_dir': str
- }
-
- Where `fork_context` refers to the newly forked 'fork parent' context
- the controller will use to start forked jobs, and `home_dir` is the
- home directory for the active user account.
- """
- # Copying the master's log level causes log messages to be filtered before
- # they reach LogForwarder, thus reducing an influx of tiny messges waking
- # the connection multiplexer process in the master.
- LOG.setLevel(log_level)
- logging.getLogger('ansible_mitogen').setLevel(log_level)
-
- # issue #536: if the json module is available, remove simplejson from the
- # importer whitelist to avoid confusing certain Ansible modules.
- if json.__name__ == 'json':
- econtext.importer.whitelist.remove('simplejson')
-
- global _fork_parent
- if FORK_SUPPORTED:
- mitogen.parent.upgrade_router(econtext)
- _fork_parent = econtext.router.fork()
-
- global good_temp_dir
- good_temp_dir = find_good_temp_dir(candidate_temp_dirs)
-
- return {
- u'fork_context': _fork_parent,
- u'home_dir': mitogen.core.to_text(os.path.expanduser('~')),
- u'good_temp_dir': good_temp_dir,
- }
-
-
-@mitogen.core.takes_econtext
-def spawn_isolated_child(econtext):
- """
- For helper functions executed in the fork parent context, arrange for
- the context's router to be upgraded as necessary and for a new child to be
- prepared.
-
- The actual fork occurs from the 'virginal fork parent', which does not have
- any Ansible modules loaded prior to fork, to avoid conflicts resulting from
- custom module_utils paths.
- """
- mitogen.parent.upgrade_router(econtext)
- if FORK_SUPPORTED:
- context = econtext.router.fork()
- else:
- context = econtext.router.local()
- LOG.debug('create_fork_child() -> %r', context)
- return context
-
-
-def run_module(kwargs):
- """
- Set up the process environment in preparation for running an Ansible
- module. This monkey-patches the Ansible libraries in various places to
- prevent it from trying to kill the process on completion, and to prevent it
- from reading sys.stdin.
- """
- runner_name = kwargs.pop('runner_name')
- klass = getattr(ansible_mitogen.runner, runner_name)
- impl = klass(**mitogen.core.Kwargs(kwargs))
- return impl.run()
-
-
-def _get_async_dir():
- return os.path.expanduser(
- os.environ.get('ANSIBLE_ASYNC_DIR', '~/.ansible_async')
- )
-
-
-class AsyncRunner(object):
- def __init__(self, job_id, timeout_secs, started_sender, econtext, kwargs):
- self.job_id = job_id
- self.timeout_secs = timeout_secs
- self.started_sender = started_sender
- self.econtext = econtext
- self.kwargs = kwargs
- self._timed_out = False
- self._init_path()
-
- def _init_path(self):
- async_dir = _get_async_dir()
- if not os.path.exists(async_dir):
- os.makedirs(async_dir)
- self.path = os.path.join(async_dir, self.job_id)
-
- def _update(self, dct):
- """
- Update an async job status file.
- """
- LOG.info('%r._update(%r, %r)', self, self.job_id, dct)
- dct.setdefault('ansible_job_id', self.job_id)
- dct.setdefault('data', '')
-
- fp = open(self.path + '.tmp', 'w')
- try:
- fp.write(json.dumps(dct))
- finally:
- fp.close()
- os.rename(self.path + '.tmp', self.path)
-
- def _on_sigalrm(self, signum, frame):
- """
- Respond to SIGALRM (job timeout) by updating the job file and killing
- the process.
- """
- msg = "Job reached maximum time limit of %d seconds." % (
- self.timeout_secs,
- )
- self._update({
- "failed": 1,
- "finished": 1,
- "msg": msg,
- })
- self._timed_out = True
- self.econtext.broker.shutdown()
-
- def _install_alarm(self):
- signal.signal(signal.SIGALRM, self._on_sigalrm)
- signal.alarm(self.timeout_secs)
-
- def _run_module(self):
- kwargs = dict(self.kwargs, **{
- 'detach': True,
- 'econtext': self.econtext,
- 'emulate_tty': False,
- })
- return run_module(kwargs)
-
- def _parse_result(self, dct):
- filtered, warnings = (
- ansible.module_utils.json_utils.
- _filter_non_json_lines(dct['stdout'])
- )
- result = json.loads(filtered)
- result.setdefault('warnings', []).extend(warnings)
- result['stderr'] = dct['stderr'] or result.get('stderr', '')
- self._update(result)
-
- def _run(self):
- """
- 1. Immediately updates the status file to mark the job as started.
- 2. Installs a timer/signal handler to implement the time limit.
- 3. Runs as with run_module(), writing the result to the status file.
-
- :param dict kwargs:
- Runner keyword arguments.
- :param str job_id:
- String job ID.
- :param int timeout_secs:
- If >0, limit the task's maximum run time.
- """
- self._update({
- 'started': 1,
- 'finished': 0,
- 'pid': os.getpid()
- })
- self.started_sender.send(True)
-
- if self.timeout_secs > 0:
- self._install_alarm()
-
- dct = self._run_module()
- if not self._timed_out:
- # After SIGALRM fires, there is a window between broker responding
- # to shutdown() by killing the process, and work continuing on the
- # main thread. If main thread was asleep in at least
- # basic.py/select.select(), an EINTR will be raised. We want to
- # discard that exception.
- try:
- self._parse_result(dct)
- except Exception:
- self._update({
- "failed": 1,
- "msg": traceback.format_exc(),
- "data": dct['stdout'], # temporary notice only
- "stderr": dct['stderr']
- })
-
- def run(self):
- try:
- try:
- self._run()
- except Exception:
- self._update({
- "failed": 1,
- "msg": traceback.format_exc(),
- })
- finally:
- self.econtext.broker.shutdown()
-
-
-@mitogen.core.takes_econtext
-def run_module_async(kwargs, job_id, timeout_secs, started_sender, econtext):
- """
- Execute a module with its run status and result written to a file,
- terminating on the process on completion. This function must run in a child
- forked using :func:`create_fork_child`.
-
- @param mitogen.core.Sender started_sender:
- A sender that will receive :data:`True` once the job has reached a
- point where its initial job file has been written. This is required to
- avoid a race where an overly eager controller can check for a task
- before it has reached that point in execution, which is possible at
- least on Python 2.4, where forking is not available for async tasks.
- """
- arunner = AsyncRunner(
- job_id,
- timeout_secs,
- started_sender,
- econtext,
- kwargs
- )
- arunner.run()
-
-
-def get_user_shell():
- """
- For commands executed directly via an SSH command-line, SSH looks up the
- user's shell via getpwuid() and only defaults to /bin/sh if that field is
- missing or empty.
- """
- try:
- pw_shell = pwd.getpwuid(os.geteuid()).pw_shell
- except KeyError:
- pw_shell = None
-
- return pw_shell or '/bin/sh'
-
-
-def exec_args(args, in_data='', chdir=None, shell=None, emulate_tty=False):
- """
- Run a command in a subprocess, emulating the argument handling behaviour of
- SSH.
-
- :param list[str]:
- Argument vector.
- :param bytes in_data:
- Optional standard input for the command.
- :param bool emulate_tty:
- If :data:`True`, arrange for stdout and stderr to be merged into the
- stdout pipe and for LF to be translated into CRLF, emulating the
- behaviour of a TTY.
- :return:
- (return code, stdout bytes, stderr bytes)
- """
- LOG.debug('exec_args(%r, ..., chdir=%r)', args, chdir)
- assert isinstance(args, list)
-
- if emulate_tty:
- stderr = subprocess.STDOUT
- else:
- stderr = subprocess.PIPE
-
- proc = subprocess.Popen(
- args=args,
- stdout=subprocess.PIPE,
- stderr=stderr,
- stdin=subprocess.PIPE,
- cwd=chdir,
- )
- stdout, stderr = proc.communicate(in_data)
-
- if emulate_tty:
- stdout = stdout.replace(b('\n'), b('\r\n'))
- return proc.returncode, stdout, stderr or b('')
-
-
-def exec_command(cmd, in_data='', chdir=None, shell=None, emulate_tty=False):
- """
- Run a command in a subprocess, emulating the argument handling behaviour of
- SSH.
-
- :param bytes cmd:
- String command line, passed to user's shell.
- :param bytes in_data:
- Optional standard input for the command.
- :return:
- (return code, stdout bytes, stderr bytes)
- """
- assert isinstance(cmd, mitogen.core.UnicodeType)
- return exec_args(
- args=[get_user_shell(), '-c', cmd],
- in_data=in_data,
- chdir=chdir,
- shell=shell,
- emulate_tty=emulate_tty,
- )
-
-
-def read_path(path):
- """
- Fetch the contents of a filesystem `path` as bytes.
- """
- return open(path, 'rb').read()
-
-
-def set_file_owner(path, owner, group=None, fd=None):
- if owner:
- uid = pwd.getpwnam(owner).pw_uid
- else:
- uid = os.geteuid()
-
- if group:
- gid = grp.getgrnam(group).gr_gid
- else:
- gid = os.getegid()
-
- if fd is not None and hasattr(os, 'fchown'):
- os.fchown(fd, (uid, gid))
- else:
- # Python<2.6
- os.chown(path, (uid, gid))
-
-
-def write_path(path, s, owner=None, group=None, mode=None,
- utimes=None, sync=False):
- """
- Writes bytes `s` to a filesystem `path`.
- """
- path = os.path.abspath(path)
- fd, tmp_path = tempfile.mkstemp(suffix='.tmp',
- prefix='.ansible_mitogen_transfer-',
- dir=os.path.dirname(path))
- fp = os.fdopen(fd, 'wb', mitogen.core.CHUNK_SIZE)
- LOG.debug('write_path(path=%r) temporary file: %s', path, tmp_path)
-
- try:
- try:
- if mode:
- set_file_mode(tmp_path, mode, fd=fp.fileno())
- if owner or group:
- set_file_owner(tmp_path, owner, group, fd=fp.fileno())
- fp.write(s)
- finally:
- fp.close()
-
- if sync:
- os.fsync(fp.fileno())
- os.rename(tmp_path, path)
- except BaseException:
- os.unlink(tmp_path)
- raise
-
- if utimes:
- os.utime(path, utimes)
-
-
-CHMOD_CLAUSE_PAT = re.compile(r'([uoga]*)([+\-=])([ugo]|[rwx]*)')
-CHMOD_MASKS = {
- 'u': stat.S_IRWXU,
- 'g': stat.S_IRWXG,
- 'o': stat.S_IRWXO,
- 'a': (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO),
-}
-CHMOD_BITS = {
- 'u': {'r': stat.S_IRUSR, 'w': stat.S_IWUSR, 'x': stat.S_IXUSR},
- 'g': {'r': stat.S_IRGRP, 'w': stat.S_IWGRP, 'x': stat.S_IXGRP},
- 'o': {'r': stat.S_IROTH, 'w': stat.S_IWOTH, 'x': stat.S_IXOTH},
- 'a': {
- 'r': (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH),
- 'w': (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH),
- 'x': (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
- }
-}
-
-
-def apply_mode_spec(spec, mode):
- """
- Given a symbolic file mode change specification in the style of chmod(1)
- `spec`, apply changes in the specification to the numeric file mode `mode`.
- """
- for clause in mitogen.core.to_text(spec).split(','):
- match = CHMOD_CLAUSE_PAT.match(clause)
- who, op, perms = match.groups()
- for ch in who or 'a':
- mask = CHMOD_MASKS[ch]
- bits = CHMOD_BITS[ch]
- cur_perm_bits = mode & mask
- new_perm_bits = reduce(operator.or_, (bits[p] for p in perms), 0)
- mode &= ~mask
- if op == '=':
- mode |= new_perm_bits
- elif op == '+':
- mode |= new_perm_bits | cur_perm_bits
- else:
- mode |= cur_perm_bits & ~new_perm_bits
- return mode
-
-
-def set_file_mode(path, spec, fd=None):
- """
- Update the permissions of a file using the same syntax as chmod(1).
- """
- if isinstance(spec, int):
- new_mode = spec
- elif not mitogen.core.PY3 and isinstance(spec, long):
- new_mode = spec
- elif spec.isdigit():
- new_mode = int(spec, 8)
- else:
- mode = os.stat(path).st_mode
- new_mode = apply_mode_spec(spec, mode)
-
- if fd is not None and hasattr(os, 'fchmod'):
- os.fchmod(fd, new_mode)
- else:
- os.chmod(path, new_mode)
-
-
-def file_exists(path):
- """
- Return :data:`True` if `path` exists. This is a wrapper function over
- :func:`os.path.exists`, since its implementation module varies across
- Python versions.
- """
- return os.path.exists(path)
diff --git a/mitogen-0.2.7/ansible_mitogen/transport_config.py b/mitogen-0.2.7/ansible_mitogen/transport_config.py
deleted file mode 100644
index ad1cab3..000000000
--- a/mitogen-0.2.7/ansible_mitogen/transport_config.py
+++ /dev/null
@@ -1,636 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
-"""
-Mitogen extends Ansible's target configuration mechanism in several ways that
-require some care:
-
-* Per-task configurables in Ansible like ansible_python_interpreter are
- connection-layer configurables in Mitogen. They must be extracted during each
- task execution to form the complete connection-layer configuration.
-
-* Mitogen has extra configurables not supported by Ansible at all, such as
- mitogen_ssh_debug_level. These are extracted the same way as
- ansible_python_interpreter.
-
-* Mitogen allows connections to be delegated to other machines. Ansible has no
- internal framework for this, and so Mitogen must figure out a delegated
- connection configuration all on its own. It cannot reuse much of the Ansible
- machinery for building a connection configuration, as that machinery is
- deeply spread out and hard-wired to expect Ansible's usual mode of operation.
-
-For normal and delegate_to connections, Ansible's PlayContext is reused where
-possible to maximize compatibility, but for proxy hops, configurations are
-built up using the HostVars magic class to call VariableManager.get_vars()
-behind the scenes on our behalf. Where Ansible has multiple sources of a
-configuration item, for example, ansible_ssh_extra_args, Mitogen must (ideally
-perfectly) reproduce how Ansible arrives at its value, without using mechanisms
-that are hard-wired or change across Ansible versions.
-
-That is what this file is for. It exports two spec classes, one that takes all
-information from PlayContext, and another that takes (almost) all information
-from HostVars.
-"""
-
-import abc
-import os
-import ansible.utils.shlex
-import ansible.constants as C
-
-from ansible.module_utils.six import with_metaclass
-
-
-import mitogen.core
-
-
-def parse_python_path(s):
- """
- Given the string set for ansible_python_interpeter, parse it using shell
- syntax and return an appropriate argument vector.
- """
- if s:
- return ansible.utils.shlex.shlex_split(s)
-
-
-def optional_secret(value):
- """
- Wrap `value` in :class:`mitogen.core.Secret` if it is not :data:`None`,
- otherwise return :data:`None`.
- """
- if value is not None:
- return mitogen.core.Secret(value)
-
-
-def first_true(it, default=None):
- """
- Return the first truthy element from `it`.
- """
- for elem in it:
- if elem:
- return elem
- return default
-
-
-class Spec(with_metaclass(abc.ABCMeta, object)):
- """
- A source for variables that comprise a connection configuration.
- """
-
- @abc.abstractmethod
- def transport(self):
- """
- The name of the Ansible plug-in implementing the connection.
- """
-
- @abc.abstractmethod
- def inventory_name(self):
- """
- The name of the target being connected to as it appears in Ansible's
- inventory.
- """
-
- @abc.abstractmethod
- def remote_addr(self):
- """
- The network address of the target, or for container and other special
- targets, some other unique identifier.
- """
-
- @abc.abstractmethod
- def remote_user(self):
- """
- The username of the login account on the target.
- """
-
- @abc.abstractmethod
- def password(self):
- """
- The password of the login account on the target.
- """
-
- @abc.abstractmethod
- def become(self):
- """
- :data:`True` if privilege escalation should be active.
- """
-
- @abc.abstractmethod
- def become_method(self):
- """
- The name of the Ansible become method to use.
- """
-
- @abc.abstractmethod
- def become_user(self):
- """
- The username of the target account for become.
- """
-
- @abc.abstractmethod
- def become_pass(self):
- """
- The password of the target account for become.
- """
-
- @abc.abstractmethod
- def port(self):
- """
- The port of the login service on the target machine.
- """
-
- @abc.abstractmethod
- def python_path(self):
- """
- Path to the Python interpreter on the target machine.
- """
-
- @abc.abstractmethod
- def private_key_file(self):
- """
- Path to the SSH private key file to use to login.
- """
-
- @abc.abstractmethod
- def ssh_executable(self):
- """
- Path to the SSH executable.
- """
-
- @abc.abstractmethod
- def timeout(self):
- """
- The generic timeout for all connections.
- """
-
- @abc.abstractmethod
- def ansible_ssh_timeout(self):
- """
- The SSH-specific timeout for a connection.
- """
-
- @abc.abstractmethod
- def ssh_args(self):
- """
- The list of additional arguments that should be included in an SSH
- invocation.
- """
-
- @abc.abstractmethod
- def become_exe(self):
- """
- The path to the executable implementing the become method on the remote
- machine.
- """
-
- @abc.abstractmethod
- def sudo_args(self):
- """
- The list of additional arguments that should be included in a become
- invocation.
- """
- # TODO: split out into sudo_args/become_args.
-
- @abc.abstractmethod
- def mitogen_via(self):
- """
- The value of the mitogen_via= variable for this connection. Indicates
- the connection should be established via an intermediary.
- """
-
- @abc.abstractmethod
- def mitogen_kind(self):
- """
- The type of container to use with the "setns" transport.
- """
-
- @abc.abstractmethod
- def mitogen_mask_remote_name(self):
- """
- Specifies whether to set a fixed "remote_name" field. The remote_name
- is the suffix of `argv[0]` for remote interpreters. By default it
- includes identifying information from the local process, which may be
- undesirable in some circumstances.
- """
-
- @abc.abstractmethod
- def mitogen_docker_path(self):
- """
- The path to the "docker" program for the 'docker' transport.
- """
-
- @abc.abstractmethod
- def mitogen_kubectl_path(self):
- """
- The path to the "kubectl" program for the 'docker' transport.
- """
-
- @abc.abstractmethod
- def mitogen_lxc_path(self):
- """
- The path to the "lxc" program for the 'lxd' transport.
- """
-
- @abc.abstractmethod
- def mitogen_lxc_attach_path(self):
- """
- The path to the "lxc-attach" program for the 'lxc' transport.
- """
-
- @abc.abstractmethod
- def mitogen_lxc_info_path(self):
- """
- The path to the "lxc-info" program for the 'lxc' transport.
- """
-
- @abc.abstractmethod
- def mitogen_machinectl_path(self):
- """
- The path to the "machinectl" program for the 'setns' transport.
- """
-
- @abc.abstractmethod
- def mitogen_ssh_debug_level(self):
- """
- The SSH debug level.
- """
-
- @abc.abstractmethod
- def mitogen_ssh_compression(self):
- """
- Whether SSH compression is enabled.
- """
-
- @abc.abstractmethod
- def extra_args(self):
- """
- Connection-specific arguments.
- """
-
-
-class PlayContextSpec(Spec):
- """
- PlayContextSpec takes almost all its information as-is from Ansible's
- PlayContext. It is used for normal connections and delegate_to connections,
- and should always be accurate.
- """
- def __init__(self, connection, play_context, transport, inventory_name):
- self._connection = connection
- self._play_context = play_context
- self._transport = transport
- self._inventory_name = inventory_name
-
- def transport(self):
- return self._transport
-
- def inventory_name(self):
- return self._inventory_name
-
- def remote_addr(self):
- return self._play_context.remote_addr
-
- def remote_user(self):
- return self._play_context.remote_user
-
- def become(self):
- return self._play_context.become
-
- def become_method(self):
- return self._play_context.become_method
-
- def become_user(self):
- return self._play_context.become_user
-
- def become_pass(self):
- return optional_secret(self._play_context.become_pass)
-
- def password(self):
- return optional_secret(self._play_context.password)
-
- def port(self):
- return self._play_context.port
-
- def python_path(self):
- s = self._connection.get_task_var('ansible_python_interpreter')
- # #511, #536: executor/module_common.py::_get_shebang() hard-wires
- # "/usr/bin/python" as the default interpreter path if no other
- # interpreter is specified.
- return parse_python_path(s or '/usr/bin/python')
-
- def private_key_file(self):
- return self._play_context.private_key_file
-
- def ssh_executable(self):
- return self._play_context.ssh_executable
-
- def timeout(self):
- return self._play_context.timeout
-
- def ansible_ssh_timeout(self):
- return (
- self._connection.get_task_var('ansible_timeout') or
- self._connection.get_task_var('ansible_ssh_timeout') or
- self.timeout()
- )
-
- def ssh_args(self):
- return [
- mitogen.core.to_text(term)
- for s in (
- getattr(self._play_context, 'ssh_args', ''),
- getattr(self._play_context, 'ssh_common_args', ''),
- getattr(self._play_context, 'ssh_extra_args', '')
- )
- for term in ansible.utils.shlex.shlex_split(s or '')
- ]
-
- def become_exe(self):
- return self._play_context.become_exe
-
- def sudo_args(self):
- return [
- mitogen.core.to_text(term)
- for term in ansible.utils.shlex.shlex_split(
- first_true((
- self._play_context.become_flags,
- self._play_context.sudo_flags,
- # Ansible 2.3.
- getattr(C, 'DEFAULT_BECOME_FLAGS', ''),
- getattr(C, 'DEFAULT_SUDO_FLAGS', '')
- ), default='')
- )
- ]
-
- def mitogen_via(self):
- return self._connection.get_task_var('mitogen_via')
-
- def mitogen_kind(self):
- return self._connection.get_task_var('mitogen_kind')
-
- def mitogen_mask_remote_name(self):
- return self._connection.get_task_var('mitogen_mask_remote_name')
-
- def mitogen_docker_path(self):
- return self._connection.get_task_var('mitogen_docker_path')
-
- def mitogen_kubectl_path(self):
- return self._connection.get_task_var('mitogen_kubectl_path')
-
- def mitogen_lxc_path(self):
- return self._connection.get_task_var('mitogen_lxc_path')
-
- def mitogen_lxc_attach_path(self):
- return self._connection.get_task_var('mitogen_lxc_attach_path')
-
- def mitogen_lxc_info_path(self):
- return self._connection.get_task_var('mitogen_lxc_info_path')
-
- def mitogen_machinectl_path(self):
- return self._connection.get_task_var('mitogen_machinectl_path')
-
- def mitogen_ssh_debug_level(self):
- return self._connection.get_task_var('mitogen_ssh_debug_level')
-
- def mitogen_ssh_compression(self):
- return self._connection.get_task_var('mitogen_ssh_compression')
-
- def extra_args(self):
- return self._connection.get_extra_args()
-
-
-class MitogenViaSpec(Spec):
- """
- MitogenViaSpec takes most of its information from the HostVars of the
- running task. HostVars is a lightweight wrapper around VariableManager, so
- it is better to say that VariableManager.get_vars() is the ultimate source
- of MitogenViaSpec's information.
-
- Due to this, mitogen_via= hosts must have all their configuration
- information represented as host and group variables. We cannot use any
- per-task configuration, as all that data belongs to the real target host.
-
- Ansible uses all kinds of strange historical logic for calculating
- variables, including making their precedence configurable. MitogenViaSpec
- must ultimately reimplement all of that logic. It is likely that if you are
- having a configruation problem with connection delegation, the answer to
- your problem lies in the method implementations below!
- """
- def __init__(self, inventory_name, host_vars, become_method, become_user,
- play_context):
- """
- :param str inventory_name:
- The inventory name of the intermediary machine, i.e. not the target
- machine.
- :param dict host_vars:
- The HostVars magic dictionary provided by Ansible in task_vars.
- :param str become_method:
- If the mitogen_via= spec included a become method, the method it
- specifies.
- :param str become_user:
- If the mitogen_via= spec included a become user, the user it
- specifies.
- :param PlayContext play_context:
- For some global values **only**, the PlayContext used to describe
- the real target machine. Values from this object are **strictly
- restricted** to values that are Ansible-global, e.g. the passwords
- specified interactively.
- """
- self._inventory_name = inventory_name
- self._host_vars = host_vars
- self._become_method = become_method
- self._become_user = become_user
- # Dangerous! You may find a variable you want in this object, but it's
- # almost certainly for the wrong machine!
- self._dangerous_play_context = play_context
-
- def transport(self):
- return (
- self._host_vars.get('ansible_connection') or
- C.DEFAULT_TRANSPORT
- )
-
- def inventory_name(self):
- return self._inventory_name
-
- def remote_addr(self):
- # play_context.py::MAGIC_VARIABLE_MAPPING
- return (
- self._host_vars.get('ansible_ssh_host') or
- self._host_vars.get('ansible_host') or
- self._inventory_name
- )
-
- def remote_user(self):
- return (
- self._host_vars.get('ansible_ssh_user') or
- self._host_vars.get('ansible_user') or
- C.DEFAULT_REMOTE_USER
- )
-
- def become(self):
- return bool(self._become_user)
-
- def become_method(self):
- return (
- self._become_method or
- self._host_vars.get('ansible_become_method') or
- C.DEFAULT_BECOME_METHOD
- )
-
- def become_user(self):
- return self._become_user
-
- def become_pass(self):
- return optional_secret(
- self._host_vars.get('ansible_become_password') or
- self._host_vars.get('ansible_become_pass')
- )
-
- def password(self):
- return optional_secret(
- self._host_vars.get('ansible_ssh_pass') or
- self._host_vars.get('ansible_password')
- )
-
- def port(self):
- return (
- self._host_vars.get('ansible_ssh_port') or
- self._host_vars.get('ansible_port') or
- C.DEFAULT_REMOTE_PORT
- )
-
- def python_path(self):
- s = self._host_vars.get('ansible_python_interpreter')
- # #511, #536: executor/module_common.py::_get_shebang() hard-wires
- # "/usr/bin/python" as the default interpreter path if no other
- # interpreter is specified.
- return parse_python_path(s or '/usr/bin/python')
-
- def private_key_file(self):
- # TODO: must come from PlayContext too.
- return (
- self._host_vars.get('ansible_ssh_private_key_file') or
- self._host_vars.get('ansible_private_key_file') or
- C.DEFAULT_PRIVATE_KEY_FILE
- )
-
- def ssh_executable(self):
- return (
- self._host_vars.get('ansible_ssh_executable') or
- C.ANSIBLE_SSH_EXECUTABLE
- )
-
- def timeout(self):
- # TODO: must come from PlayContext too.
- return C.DEFAULT_TIMEOUT
-
- def ansible_ssh_timeout(self):
- return (
- self._host_vars.get('ansible_timeout') or
- self._host_vars.get('ansible_ssh_timeout') or
- self.timeout()
- )
-
- def ssh_args(self):
- return [
- mitogen.core.to_text(term)
- for s in (
- (
- self._host_vars.get('ansible_ssh_args') or
- getattr(C, 'ANSIBLE_SSH_ARGS', None) or
- os.environ.get('ANSIBLE_SSH_ARGS')
- # TODO: ini entry. older versions.
- ),
- (
- self._host_vars.get('ansible_ssh_common_args') or
- os.environ.get('ANSIBLE_SSH_COMMON_ARGS')
- # TODO: ini entry.
- ),
- (
- self._host_vars.get('ansible_ssh_extra_args') or
- os.environ.get('ANSIBLE_SSH_EXTRA_ARGS')
- # TODO: ini entry.
- ),
- )
- for term in ansible.utils.shlex.shlex_split(s)
- if s
- ]
-
- def become_exe(self):
- return (
- self._host_vars.get('ansible_become_exe') or
- C.DEFAULT_BECOME_EXE
- )
-
- def sudo_args(self):
- return [
- mitogen.core.to_text(term)
- for s in (
- self._host_vars.get('ansible_sudo_flags') or '',
- self._host_vars.get('ansible_become_flags') or '',
- )
- for term in ansible.utils.shlex.shlex_split(s)
- ]
-
- def mitogen_via(self):
- return self._host_vars.get('mitogen_via')
-
- def mitogen_kind(self):
- return self._host_vars.get('mitogen_kind')
-
- def mitogen_mask_remote_name(self):
- return self._host_vars.get('mitogen_mask_remote_name')
-
- def mitogen_docker_path(self):
- return self._host_vars.get('mitogen_docker_path')
-
- def mitogen_kubectl_path(self):
- return self._host_vars.get('mitogen_kubectl_path')
-
- def mitogen_lxc_path(self):
- return self.host_vars.get('mitogen_lxc_path')
-
- def mitogen_lxc_attach_path(self):
- return self._host_vars.get('mitogen_lxc_attach_path')
-
- def mitogen_lxc_info_path(self):
- return self._host_vars.get('mitogen_lxc_info_path')
-
- def mitogen_machinectl_path(self):
- return self._host_vars.get('mitogen_machinectl_path')
-
- def mitogen_ssh_debug_level(self):
- return self._host_vars.get('mitogen_ssh_debug_level')
-
- def mitogen_ssh_compression(self):
- return self._host_vars.get('mitogen_ssh_compression')
-
- def extra_args(self):
- return [] # TODO
diff --git a/mitogen-0.2.7/mitogen.egg-info/PKG-INFO b/mitogen-0.2.7/mitogen.egg-info/PKG-INFO
deleted file mode 100644
index f63ba91..000000000
--- a/mitogen-0.2.7/mitogen.egg-info/PKG-INFO
+++ /dev/null
@@ -1,23 +0,0 @@
-Metadata-Version: 1.1
-Name: mitogen
-Version: 0.2.7
-Summary: Library for writing distributed self-replicating programs.
-Home-page: https://github.com/dw/mitogen/
-Author: David Wilson
-Author-email: UNKNOWN
-License: New BSD
-Description: UNKNOWN
-Platform: UNKNOWN
-Classifier: Environment :: Console
-Classifier: Intended Audience :: System Administrators
-Classifier: License :: OSI Approved :: BSD License
-Classifier: Operating System :: POSIX
-Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 2.4
-Classifier: Programming Language :: Python :: 2.5
-Classifier: Programming Language :: Python :: 2.6
-Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: Implementation :: CPython
-Classifier: Topic :: System :: Distributed Computing
-Classifier: Topic :: System :: Systems Administration
diff --git a/mitogen-0.2.7/mitogen.egg-info/SOURCES.txt b/mitogen-0.2.7/mitogen.egg-info/SOURCES.txt
deleted file mode 100644
index c2abd33..000000000
--- a/mitogen-0.2.7/mitogen.egg-info/SOURCES.txt
+++ /dev/null
@@ -1,78 +0,0 @@
-LICENSE
-MANIFEST.in
-README.md
-setup.cfg
-setup.py
-ansible_mitogen/__init__.py
-ansible_mitogen/affinity.py
-ansible_mitogen/connection.py
-ansible_mitogen/loaders.py
-ansible_mitogen/logging.py
-ansible_mitogen/mixins.py
-ansible_mitogen/module_finder.py
-ansible_mitogen/parsing.py
-ansible_mitogen/planner.py
-ansible_mitogen/process.py
-ansible_mitogen/runner.py
-ansible_mitogen/services.py
-ansible_mitogen/strategy.py
-ansible_mitogen/target.py
-ansible_mitogen/transport_config.py
-ansible_mitogen/compat/__init__.py
-ansible_mitogen/compat/simplejson/__init__.py
-ansible_mitogen/compat/simplejson/decoder.py
-ansible_mitogen/compat/simplejson/encoder.py
-ansible_mitogen/compat/simplejson/scanner.py
-ansible_mitogen/plugins/__init__.py
-ansible_mitogen/plugins/action/__init__.py
-ansible_mitogen/plugins/action/mitogen_get_stack.py
-ansible_mitogen/plugins/connection/__init__.py
-ansible_mitogen/plugins/connection/mitogen_doas.py
-ansible_mitogen/plugins/connection/mitogen_docker.py
-ansible_mitogen/plugins/connection/mitogen_jail.py
-ansible_mitogen/plugins/connection/mitogen_kubectl.py
-ansible_mitogen/plugins/connection/mitogen_local.py
-ansible_mitogen/plugins/connection/mitogen_lxc.py
-ansible_mitogen/plugins/connection/mitogen_lxd.py
-ansible_mitogen/plugins/connection/mitogen_machinectl.py
-ansible_mitogen/plugins/connection/mitogen_setns.py
-ansible_mitogen/plugins/connection/mitogen_ssh.py
-ansible_mitogen/plugins/connection/mitogen_su.py
-ansible_mitogen/plugins/connection/mitogen_sudo.py
-ansible_mitogen/plugins/strategy/__init__.py
-ansible_mitogen/plugins/strategy/mitogen.py
-ansible_mitogen/plugins/strategy/mitogen_free.py
-ansible_mitogen/plugins/strategy/mitogen_host_pinned.py
-ansible_mitogen/plugins/strategy/mitogen_linear.py
-mitogen/__init__.py
-mitogen/core.py
-mitogen/debug.py
-mitogen/doas.py
-mitogen/docker.py
-mitogen/fakessh.py
-mitogen/fork.py
-mitogen/jail.py
-mitogen/kubectl.py
-mitogen/lxc.py
-mitogen/lxd.py
-mitogen/master.py
-mitogen/minify.py
-mitogen/os_fork.py
-mitogen/parent.py
-mitogen/profiler.py
-mitogen/select.py
-mitogen/service.py
-mitogen/setns.py
-mitogen/ssh.py
-mitogen/su.py
-mitogen/sudo.py
-mitogen/unix.py
-mitogen/utils.py
-mitogen.egg-info/PKG-INFO
-mitogen.egg-info/SOURCES.txt
-mitogen.egg-info/dependency_links.txt
-mitogen.egg-info/not-zip-safe
-mitogen.egg-info/top_level.txt
-mitogen/compat/__init__.py
-mitogen/compat/pkgutil.py
-mitogen/compat/tokenize.py
\ No newline at end of file
diff --git a/mitogen-0.2.7/mitogen.egg-info/dependency_links.txt b/mitogen-0.2.7/mitogen.egg-info/dependency_links.txt
deleted file mode 100644
index 8b13789..000000000
--- a/mitogen-0.2.7/mitogen.egg-info/dependency_links.txt
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mitogen-0.2.7/mitogen.egg-info/not-zip-safe b/mitogen-0.2.7/mitogen.egg-info/not-zip-safe
deleted file mode 100644
index 8b13789..000000000
--- a/mitogen-0.2.7/mitogen.egg-info/not-zip-safe
+++ /dev/null
@@ -1 +0,0 @@
-
diff --git a/mitogen-0.2.7/mitogen.egg-info/top_level.txt b/mitogen-0.2.7/mitogen.egg-info/top_level.txt
deleted file mode 100644
index 2360b3f..000000000
--- a/mitogen-0.2.7/mitogen.egg-info/top_level.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-ansible_mitogen
-mitogen
diff --git a/mitogen-0.2.7/mitogen/__init__.py b/mitogen-0.2.7/mitogen/__init__.py
deleted file mode 100644
index 47fe4d3..000000000
--- a/mitogen-0.2.7/mitogen/__init__.py
+++ /dev/null
@@ -1,120 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-"""
-On the Mitogen master, this is imported from ``mitogen/__init__.py`` as would
-be expected. On the slave, it is built dynamically during startup.
-"""
-
-
-#: Library version as a tuple.
-__version__ = (0, 2, 7)
-
-
-#: This is :data:`False` in slave contexts. Previously it was used to prevent
-#: re-execution of :mod:`__main__` in single file programs, however that now
-#: happens automatically.
-is_master = True
-
-
-#: This is `0` in a master, otherwise it is the master-assigned ID unique to
-#: the slave context used for message routing.
-context_id = 0
-
-
-#: This is :data:`None` in a master, otherwise it is the master-assigned ID
-#: unique to the slave's parent context.
-parent_id = None
-
-
-#: This is an empty list in a master, otherwise it is a list of parent context
-#: IDs ordered from most direct to least direct.
-parent_ids = []
-
-
-import os
-_default_profiling = os.environ.get('MITOGEN_PROFILING') is not None
-del os
-
-
-def main(log_level='INFO', profiling=_default_profiling):
- """
- Convenience decorator primarily useful for writing discardable test
- scripts.
-
- In the master process, when `func` is defined in the :mod:`__main__`
- module, arranges for `func(router)` to be invoked immediately, with
- :py:class:`mitogen.master.Router` construction and destruction handled just
- as in :py:func:`mitogen.utils.run_with_router`. In slaves, this function
- does nothing.
-
- :param str log_level:
- Logging package level to configure via
- :py:func:`mitogen.utils.log_to_file`.
-
- :param bool profiling:
- If :py:data:`True`, equivalent to setting
- :py:attr:`mitogen.master.Router.profiling` prior to router
- construction. This causes ``/tmp`` files to be created everywhere at
- the end of a successful run with :py:mod:`cProfile` output for every
- thread.
-
- Example:
-
- ::
-
- import mitogen
- import requests
-
- def get_url(url):
- return requests.get(url).text
-
- @mitogen.main()
- def main(router):
- z = router.ssh(hostname='k3')
- print(z.call(get_url, 'https://example.org/')))))
-
- """
-
- def wrapper(func):
- if func.__module__ != '__main__':
- return func
- import mitogen.parent
- import mitogen.utils
- if profiling:
- mitogen.core.enable_profiling()
- mitogen.master.Router.profiling = profiling
- utils.log_to_file(level=log_level)
- return mitogen.core._profile_hook(
- 'app.main',
- utils.run_with_router,
- func,
- )
- return wrapper
diff --git a/mitogen-0.2.7/mitogen/compat/__init__.py b/mitogen-0.2.7/mitogen/compat/__init__.py
deleted file mode 100644
index e69de29..000000000
diff --git a/mitogen-0.2.7/mitogen/compat/pkgutil.py b/mitogen-0.2.7/mitogen/compat/pkgutil.py
deleted file mode 100644
index 28e2aea..000000000
--- a/mitogen-0.2.7/mitogen/compat/pkgutil.py
+++ /dev/null
@@ -1,593 +0,0 @@
-"""Utilities to support packages."""
-
-# !mitogen: minify_safe
-
-# NOTE: This module must remain compatible with Python 2.3, as it is shared
-# by setuptools for distribution with Python 2.3 and up.
-
-import os
-import sys
-import imp
-import os.path
-from types import ModuleType
-
-__all__ = [
- 'get_importer', 'iter_importers', 'get_loader', 'find_loader',
- 'walk_packages', 'iter_modules', 'get_data',
- 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path',
-]
-
-def read_code(stream):
- # This helper is needed in order for the PEP 302 emulation to
- # correctly handle compiled files
- import marshal
-
- magic = stream.read(4)
- if magic != imp.get_magic():
- return None
-
- stream.read(4) # Skip timestamp
- return marshal.load(stream)
-
-
-def simplegeneric(func):
- """Make a trivial single-dispatch generic function"""
- registry = {}
- def wrapper(*args, **kw):
- ob = args[0]
- try:
- cls = ob.__class__
- except AttributeError:
- cls = type(ob)
- try:
- mro = cls.__mro__
- except AttributeError:
- try:
- class cls(cls, object):
- pass
- mro = cls.__mro__[1:]
- except TypeError:
- mro = object, # must be an ExtensionClass or some such :(
- for t in mro:
- if t in registry:
- return registry[t](*args, **kw)
- else:
- return func(*args, **kw)
- try:
- wrapper.__name__ = func.__name__
- except (TypeError, AttributeError):
- pass # Python 2.3 doesn't allow functions to be renamed
-
- def register(typ, func=None):
- if func is None:
- return lambda f: register(typ, f)
- registry[typ] = func
- return func
-
- wrapper.__dict__ = func.__dict__
- wrapper.__doc__ = func.__doc__
- wrapper.register = register
- return wrapper
-
-
-def walk_packages(path=None, prefix='', onerror=None):
- """Yields (module_loader, name, ispkg) for all modules recursively
- on path, or, if path is None, all accessible modules.
-
- 'path' should be either None or a list of paths to look for
- modules in.
-
- 'prefix' is a string to output on the front of every module name
- on output.
-
- Note that this function must import all *packages* (NOT all
- modules!) on the given path, in order to access the __path__
- attribute to find submodules.
-
- 'onerror' is a function which gets called with one argument (the
- name of the package which was being imported) if any exception
- occurs while trying to import a package. If no onerror function is
- supplied, ImportErrors are caught and ignored, while all other
- exceptions are propagated, terminating the search.
-
- Examples:
-
- # list all modules python can access
- walk_packages()
-
- # list all submodules of ctypes
- walk_packages(ctypes.__path__, ctypes.__name__+'.')
- """
-
- def seen(p, m={}):
- if p in m:
- return True
- m[p] = True
-
- for importer, name, ispkg in iter_modules(path, prefix):
- yield importer, name, ispkg
-
- if ispkg:
- try:
- __import__(name)
- except ImportError:
- if onerror is not None:
- onerror(name)
- except Exception:
- if onerror is not None:
- onerror(name)
- else:
- raise
- else:
- path = getattr(sys.modules[name], '__path__', None) or []
-
- # don't traverse path items we've seen before
- path = [p for p in path if not seen(p)]
-
- for item in walk_packages(path, name+'.', onerror):
- yield item
-
-
-def iter_modules(path=None, prefix=''):
- """Yields (module_loader, name, ispkg) for all submodules on path,
- or, if path is None, all top-level modules on sys.path.
-
- 'path' should be either None or a list of paths to look for
- modules in.
-
- 'prefix' is a string to output on the front of every module name
- on output.
- """
-
- if path is None:
- importers = iter_importers()
- else:
- importers = map(get_importer, path)
-
- yielded = {}
- for i in importers:
- for name, ispkg in iter_importer_modules(i, prefix):
- if name not in yielded:
- yielded[name] = 1
- yield i, name, ispkg
-
-
-#@simplegeneric
-def iter_importer_modules(importer, prefix=''):
- if not hasattr(importer, 'iter_modules'):
- return []
- return importer.iter_modules(prefix)
-
-iter_importer_modules = simplegeneric(iter_importer_modules)
-
-
-class ImpImporter:
- """PEP 302 Importer that wraps Python's "classic" import algorithm
-
- ImpImporter(dirname) produces a PEP 302 importer that searches that
- directory. ImpImporter(None) produces a PEP 302 importer that searches
- the current sys.path, plus any modules that are frozen or built-in.
-
- Note that ImpImporter does not currently support being used by placement
- on sys.meta_path.
- """
-
- def __init__(self, path=None):
- self.path = path
-
- def find_module(self, fullname, path=None):
- # Note: we ignore 'path' argument since it is only used via meta_path
- subname = fullname.split(".")[-1]
- if subname != fullname and self.path is None:
- return None
- if self.path is None:
- path = None
- else:
- path = [os.path.realpath(self.path)]
- try:
- file, filename, etc = imp.find_module(subname, path)
- except ImportError:
- return None
- return ImpLoader(fullname, file, filename, etc)
-
- def iter_modules(self, prefix=''):
- if self.path is None or not os.path.isdir(self.path):
- return
-
- yielded = {}
- import inspect
- try:
- filenames = os.listdir(self.path)
- except OSError:
- # ignore unreadable directories like import does
- filenames = []
- filenames.sort() # handle packages before same-named modules
-
- for fn in filenames:
- modname = inspect.getmodulename(fn)
- if modname=='__init__' or modname in yielded:
- continue
-
- path = os.path.join(self.path, fn)
- ispkg = False
-
- if not modname and os.path.isdir(path) and '.' not in fn:
- modname = fn
- try:
- dircontents = os.listdir(path)
- except OSError:
- # ignore unreadable directories like import does
- dircontents = []
- for fn in dircontents:
- subname = inspect.getmodulename(fn)
- if subname=='__init__':
- ispkg = True
- break
- else:
- continue # not a package
-
- if modname and '.' not in modname:
- yielded[modname] = 1
- yield prefix + modname, ispkg
-
-
-class ImpLoader:
- """PEP 302 Loader that wraps Python's "classic" import algorithm
- """
- code = source = None
-
- def __init__(self, fullname, file, filename, etc):
- self.file = file
- self.filename = filename
- self.fullname = fullname
- self.etc = etc
-
- def load_module(self, fullname):
- self._reopen()
- try:
- mod = imp.load_module(fullname, self.file, self.filename, self.etc)
- finally:
- if self.file:
- self.file.close()
- # Note: we don't set __loader__ because we want the module to look
- # normal; i.e. this is just a wrapper for standard import machinery
- return mod
-
- def get_data(self, pathname):
- return open(pathname, "rb").read()
-
- def _reopen(self):
- if self.file and self.file.closed:
- mod_type = self.etc[2]
- if mod_type==imp.PY_SOURCE:
- self.file = open(self.filename, 'rU')
- elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION):
- self.file = open(self.filename, 'rb')
-
- def _fix_name(self, fullname):
- if fullname is None:
- fullname = self.fullname
- elif fullname != self.fullname:
- raise ImportError("Loader for module %s cannot handle "
- "module %s" % (self.fullname, fullname))
- return fullname
-
- def is_package(self, fullname):
- fullname = self._fix_name(fullname)
- return self.etc[2]==imp.PKG_DIRECTORY
-
- def get_code(self, fullname=None):
- fullname = self._fix_name(fullname)
- if self.code is None:
- mod_type = self.etc[2]
- if mod_type==imp.PY_SOURCE:
- source = self.get_source(fullname)
- self.code = compile(source, self.filename, 'exec')
- elif mod_type==imp.PY_COMPILED:
- self._reopen()
- try:
- self.code = read_code(self.file)
- finally:
- self.file.close()
- elif mod_type==imp.PKG_DIRECTORY:
- self.code = self._get_delegate().get_code()
- return self.code
-
- def get_source(self, fullname=None):
- fullname = self._fix_name(fullname)
- if self.source is None:
- mod_type = self.etc[2]
- if mod_type==imp.PY_SOURCE:
- self._reopen()
- try:
- self.source = self.file.read()
- finally:
- self.file.close()
- elif mod_type==imp.PY_COMPILED:
- if os.path.exists(self.filename[:-1]):
- f = open(self.filename[:-1], 'rU')
- self.source = f.read()
- f.close()
- elif mod_type==imp.PKG_DIRECTORY:
- self.source = self._get_delegate().get_source()
- return self.source
-
-
- def _get_delegate(self):
- return ImpImporter(self.filename).find_module('__init__')
-
- def get_filename(self, fullname=None):
- fullname = self._fix_name(fullname)
- mod_type = self.etc[2]
- if self.etc[2]==imp.PKG_DIRECTORY:
- return self._get_delegate().get_filename()
- elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION):
- return self.filename
- return None
-
-
-try:
- import zipimport
- from zipimport import zipimporter
-
- def iter_zipimport_modules(importer, prefix=''):
- dirlist = zipimport._zip_directory_cache[importer.archive].keys()
- dirlist.sort()
- _prefix = importer.prefix
- plen = len(_prefix)
- yielded = {}
- import inspect
- for fn in dirlist:
- if not fn.startswith(_prefix):
- continue
-
- fn = fn[plen:].split(os.sep)
-
- if len(fn)==2 and fn[1].startswith('__init__.py'):
- if fn[0] not in yielded:
- yielded[fn[0]] = 1
- yield fn[0], True
-
- if len(fn)!=1:
- continue
-
- modname = inspect.getmodulename(fn[0])
- if modname=='__init__':
- continue
-
- if modname and '.' not in modname and modname not in yielded:
- yielded[modname] = 1
- yield prefix + modname, False
-
- iter_importer_modules.register(zipimporter, iter_zipimport_modules)
-
-except ImportError:
- pass
-
-
-def get_importer(path_item):
- """Retrieve a PEP 302 importer for the given path item
-
- The returned importer is cached in sys.path_importer_cache
- if it was newly created by a path hook.
-
- If there is no importer, a wrapper around the basic import
- machinery is returned. This wrapper is never inserted into
- the importer cache (None is inserted instead).
-
- The cache (or part of it) can be cleared manually if a
- rescan of sys.path_hooks is necessary.
- """
- try:
- importer = sys.path_importer_cache[path_item]
- except KeyError:
- for path_hook in sys.path_hooks:
- try:
- importer = path_hook(path_item)
- break
- except ImportError:
- pass
- else:
- importer = None
- sys.path_importer_cache.setdefault(path_item, importer)
-
- if importer is None:
- try:
- importer = ImpImporter(path_item)
- except ImportError:
- importer = None
- return importer
-
-
-def iter_importers(fullname=""):
- """Yield PEP 302 importers for the given module name
-
- If fullname contains a '.', the importers will be for the package
- containing fullname, otherwise they will be importers for sys.meta_path,
- sys.path, and Python's "classic" import machinery, in that order. If
- the named module is in a package, that package is imported as a side
- effect of invoking this function.
-
- Non PEP 302 mechanisms (e.g. the Windows registry) used by the
- standard import machinery to find files in alternative locations
- are partially supported, but are searched AFTER sys.path. Normally,
- these locations are searched BEFORE sys.path, preventing sys.path
- entries from shadowing them.
-
- For this to cause a visible difference in behaviour, there must
- be a module or package name that is accessible via both sys.path
- and one of the non PEP 302 file system mechanisms. In this case,
- the emulation will find the former version, while the builtin
- import mechanism will find the latter.
-
- Items of the following types can be affected by this discrepancy:
- imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY
- """
- if fullname.startswith('.'):
- raise ImportError("Relative module names not supported")
- if '.' in fullname:
- # Get the containing package's __path__
- pkg = '.'.join(fullname.split('.')[:-1])
- if pkg not in sys.modules:
- __import__(pkg)
- path = getattr(sys.modules[pkg], '__path__', None) or []
- else:
- for importer in sys.meta_path:
- yield importer
- path = sys.path
- for item in path:
- yield get_importer(item)
- if '.' not in fullname:
- yield ImpImporter()
-
-def get_loader(module_or_name):
- """Get a PEP 302 "loader" object for module_or_name
-
- If the module or package is accessible via the normal import
- mechanism, a wrapper around the relevant part of that machinery
- is returned. Returns None if the module cannot be found or imported.
- If the named module is not already imported, its containing package
- (if any) is imported, in order to establish the package __path__.
-
- This function uses iter_importers(), and is thus subject to the same
- limitations regarding platform-specific special import locations such
- as the Windows registry.
- """
- if module_or_name in sys.modules:
- module_or_name = sys.modules[module_or_name]
- if isinstance(module_or_name, ModuleType):
- module = module_or_name
- loader = getattr(module, '__loader__', None)
- if loader is not None:
- return loader
- fullname = module.__name__
- else:
- fullname = module_or_name
- return find_loader(fullname)
-
-def find_loader(fullname):
- """Find a PEP 302 "loader" object for fullname
-
- If fullname contains dots, path must be the containing package's __path__.
- Returns None if the module cannot be found or imported. This function uses
- iter_importers(), and is thus subject to the same limitations regarding
- platform-specific special import locations such as the Windows registry.
- """
- for importer in iter_importers(fullname):
- loader = importer.find_module(fullname)
- if loader is not None:
- return loader
-
- return None
-
-
-def extend_path(path, name):
- """Extend a package's path.
-
- Intended use is to place the following code in a package's __init__.py:
-
- from pkgutil import extend_path
- __path__ = extend_path(__path__, __name__)
-
- This will add to the package's __path__ all subdirectories of
- directories on sys.path named after the package. This is useful
- if one wants to distribute different parts of a single logical
- package as multiple directories.
-
- It also looks for *.pkg files beginning where * matches the name
- argument. This feature is similar to *.pth files (see site.py),
- except that it doesn't special-case lines starting with 'import'.
- A *.pkg file is trusted at face value: apart from checking for
- duplicates, all entries found in a *.pkg file are added to the
- path, regardless of whether they are exist the filesystem. (This
- is a feature.)
-
- If the input path is not a list (as is the case for frozen
- packages) it is returned unchanged. The input path is not
- modified; an extended copy is returned. Items are only appended
- to the copy at the end.
-
- It is assumed that sys.path is a sequence. Items of sys.path that
- are not (unicode or 8-bit) strings referring to existing
- directories are ignored. Unicode items of sys.path that cause
- errors when used as filenames may cause this function to raise an
- exception (in line with os.path.isdir() behavior).
- """
-
- if not isinstance(path, list):
- # This could happen e.g. when this is called from inside a
- # frozen package. Return the path unchanged in that case.
- return path
-
- pname = os.path.join(*name.split('.')) # Reconstitute as relative path
- # Just in case os.extsep != '.'
- sname = os.extsep.join(name.split('.'))
- sname_pkg = sname + os.extsep + "pkg"
- init_py = "__init__" + os.extsep + "py"
-
- path = path[:] # Start with a copy of the existing path
-
- for dir in sys.path:
- if not isinstance(dir, basestring) or not os.path.isdir(dir):
- continue
- subdir = os.path.join(dir, pname)
- # XXX This may still add duplicate entries to path on
- # case-insensitive filesystems
- initfile = os.path.join(subdir, init_py)
- if subdir not in path and os.path.isfile(initfile):
- path.append(subdir)
- # XXX Is this the right thing for subpackages like zope.app?
- # It looks for a file named "zope.app.pkg"
- pkgfile = os.path.join(dir, sname_pkg)
- if os.path.isfile(pkgfile):
- try:
- f = open(pkgfile)
- except IOError, msg:
- sys.stderr.write("Can't open %s: %s\n" %
- (pkgfile, msg))
- else:
- for line in f:
- line = line.rstrip('\n')
- if not line or line.startswith('#'):
- continue
- path.append(line) # Don't check for existence!
- f.close()
-
- return path
-
-def get_data(package, resource):
- """Get a resource from a package.
-
- This is a wrapper round the PEP 302 loader get_data API. The package
- argument should be the name of a package, in standard module format
- (foo.bar). The resource argument should be in the form of a relative
- filename, using '/' as the path separator. The parent directory name '..'
- is not allowed, and nor is a rooted name (starting with a '/').
-
- The function returns a binary string, which is the contents of the
- specified resource.
-
- For packages located in the filesystem, which have already been imported,
- this is the rough equivalent of
-
- d = os.path.dirname(sys.modules[package].__file__)
- data = open(os.path.join(d, resource), 'rb').read()
-
- If the package cannot be located or loaded, or it uses a PEP 302 loader
- which does not support get_data(), then None is returned.
- """
-
- loader = get_loader(package)
- if loader is None or not hasattr(loader, 'get_data'):
- return None
- mod = sys.modules.get(package) or loader.load_module(package)
- if mod is None or not hasattr(mod, '__file__'):
- return None
-
- # Modify the resource name to be compatible with the loader.get_data
- # signature - an os.path format "filename" starting with the dirname of
- # the package's __file__
- parts = resource.split('/')
- parts.insert(0, os.path.dirname(mod.__file__))
- resource_name = os.path.join(*parts)
- return loader.get_data(resource_name)
diff --git a/mitogen-0.2.7/mitogen/compat/tokenize.py b/mitogen-0.2.7/mitogen/compat/tokenize.py
deleted file mode 100644
index 0473c6a..000000000
--- a/mitogen-0.2.7/mitogen/compat/tokenize.py
+++ /dev/null
@@ -1,453 +0,0 @@
-"""Tokenization help for Python programs.
-
-generate_tokens(readline) is a generator that breaks a stream of
-text into Python tokens. It accepts a readline-like method which is called
-repeatedly to get the next line of input (or "" for EOF). It generates
-5-tuples with these members:
-
- the token type (see token.py)
- the token (a string)
- the starting (row, column) indices of the token (a 2-tuple of ints)
- the ending (row, column) indices of the token (a 2-tuple of ints)
- the original line (string)
-
-It is designed to match the working of the Python tokenizer exactly, except
-that it produces COMMENT tokens for comments and gives type OP for all
-operators
-
-Older entry points
- tokenize_loop(readline, tokeneater)
- tokenize(readline, tokeneater=printtoken)
-are the same, except instead of generating tokens, tokeneater is a callback
-function to which the 5 fields described above are passed as 5 arguments,
-each time a new token is found."""
-
-# !mitogen: minify_safe
-
-__author__ = 'Ka-Ping Yee '
-__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
- 'Skip Montanaro, Raymond Hettinger')
-
-from itertools import chain
-import string, re
-from token import *
-
-import token
-__all__ = [x for x in dir(token) if not x.startswith("_")]
-__all__ += ["COMMENT", "tokenize", "generate_tokens", "NL", "untokenize"]
-del token
-
-COMMENT = N_TOKENS
-tok_name[COMMENT] = 'COMMENT'
-NL = N_TOKENS + 1
-tok_name[NL] = 'NL'
-N_TOKENS += 2
-
-def group(*choices): return '(' + '|'.join(choices) + ')'
-def any(*choices): return group(*choices) + '*'
-def maybe(*choices): return group(*choices) + '?'
-
-Whitespace = r'[ \f\t]*'
-Comment = r'#[^\r\n]*'
-Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
-Name = r'[a-zA-Z_]\w*'
-
-Hexnumber = r'0[xX][\da-fA-F]+[lL]?'
-Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?'
-Binnumber = r'0[bB][01]+[lL]?'
-Decnumber = r'[1-9]\d*[lL]?'
-Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
-Exponent = r'[eE][-+]?\d+'
-Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent)
-Expfloat = r'\d+' + Exponent
-Floatnumber = group(Pointfloat, Expfloat)
-Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]')
-Number = group(Imagnumber, Floatnumber, Intnumber)
-
-# Tail end of ' string.
-Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
-# Tail end of " string.
-Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
-# Tail end of ''' string.
-Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
-# Tail end of """ string.
-Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
-Triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""')
-# Single-line ' or " string.
-String = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
- r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
-
-# Because of leftmost-then-longest match semantics, be sure to put the
-# longest operators first (e.g., if = came before ==, == would get
-# recognized as two instances of =).
-Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=",
- r"//=?",
- r"[+\-*/%&|^=<>]=?",
- r"~")
-
-Bracket = '[][(){}]'
-Special = group(r'\r?\n', r'[:;.,`@]')
-Funny = group(Operator, Bracket, Special)
-
-PlainToken = group(Number, Funny, String, Name)
-Token = Ignore + PlainToken
-
-# First (or only) line of ' or " string.
-ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
- group("'", r'\\\r?\n'),
- r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
- group('"', r'\\\r?\n'))
-PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple)
-PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
-
-tokenprog, pseudoprog, single3prog, double3prog = map(
- re.compile, (Token, PseudoToken, Single3, Double3))
-endprogs = {"'": re.compile(Single), '"': re.compile(Double),
- "'''": single3prog, '"""': double3prog,
- "r'''": single3prog, 'r"""': double3prog,
- "u'''": single3prog, 'u"""': double3prog,
- "ur'''": single3prog, 'ur"""': double3prog,
- "R'''": single3prog, 'R"""': double3prog,
- "U'''": single3prog, 'U"""': double3prog,
- "uR'''": single3prog, 'uR"""': double3prog,
- "Ur'''": single3prog, 'Ur"""': double3prog,
- "UR'''": single3prog, 'UR"""': double3prog,
- "b'''": single3prog, 'b"""': double3prog,
- "br'''": single3prog, 'br"""': double3prog,
- "B'''": single3prog, 'B"""': double3prog,
- "bR'''": single3prog, 'bR"""': double3prog,
- "Br'''": single3prog, 'Br"""': double3prog,
- "BR'''": single3prog, 'BR"""': double3prog,
- 'r': None, 'R': None, 'u': None, 'U': None,
- 'b': None, 'B': None}
-
-triple_quoted = {}
-for t in ("'''", '"""',
- "r'''", 'r"""', "R'''", 'R"""',
- "u'''", 'u"""', "U'''", 'U"""',
- "ur'''", 'ur"""', "Ur'''", 'Ur"""',
- "uR'''", 'uR"""', "UR'''", 'UR"""',
- "b'''", 'b"""', "B'''", 'B"""',
- "br'''", 'br"""', "Br'''", 'Br"""',
- "bR'''", 'bR"""', "BR'''", 'BR"""'):
- triple_quoted[t] = t
-single_quoted = {}
-for t in ("'", '"',
- "r'", 'r"', "R'", 'R"',
- "u'", 'u"', "U'", 'U"',
- "ur'", 'ur"', "Ur'", 'Ur"',
- "uR'", 'uR"', "UR'", 'UR"',
- "b'", 'b"', "B'", 'B"',
- "br'", 'br"', "Br'", 'Br"',
- "bR'", 'bR"', "BR'", 'BR"' ):
- single_quoted[t] = t
-
-tabsize = 8
-
-class TokenError(Exception): pass
-
-class StopTokenizing(Exception): pass
-
-def printtoken(type, token, srow_scol, erow_ecol, line): # for testing
- srow, scol = srow_scol
- erow, ecol = erow_ecol
- print("%d,%d-%d,%d:\t%s\t%s" % \
- (srow, scol, erow, ecol, tok_name[type], repr(token)))
-
-def tokenize(readline, tokeneater=printtoken):
- """
- The tokenize() function accepts two parameters: one representing the
- input stream, and one providing an output mechanism for tokenize().
-
- The first parameter, readline, must be a callable object which provides
- the same interface as the readline() method of built-in file objects.
- Each call to the function should return one line of input as a string.
-
- The second parameter, tokeneater, must also be a callable object. It is
- called once for each token, with five arguments, corresponding to the
- tuples generated by generate_tokens().
- """
- try:
- tokenize_loop(readline, tokeneater)
- except StopTokenizing:
- pass
-
-# backwards compatible interface
-def tokenize_loop(readline, tokeneater):
- for token_info in generate_tokens(readline):
- tokeneater(*token_info)
-
-class Untokenizer:
-
- def __init__(self):
- self.tokens = []
- self.prev_row = 1
- self.prev_col = 0
-
- def add_whitespace(self, start):
- row, col = start
- if row < self.prev_row or row == self.prev_row and col < self.prev_col:
- raise ValueError("start ({},{}) precedes previous end ({},{})"
- .format(row, col, self.prev_row, self.prev_col))
- row_offset = row - self.prev_row
- if row_offset:
- self.tokens.append("\\\n" * row_offset)
- self.prev_col = 0
- col_offset = col - self.prev_col
- if col_offset:
- self.tokens.append(" " * col_offset)
-
- def untokenize(self, iterable):
- it = iter(iterable)
- indents = []
- startline = False
- for t in it:
- if len(t) == 2:
- self.compat(t, it)
- break
- tok_type, token, start, end, line = t
- if tok_type == ENDMARKER:
- break
- if tok_type == INDENT:
- indents.append(token)
- continue
- elif tok_type == DEDENT:
- indents.pop()
- self.prev_row, self.prev_col = end
- continue
- elif tok_type in (NEWLINE, NL):
- startline = True
- elif startline and indents:
- indent = indents[-1]
- if start[1] >= len(indent):
- self.tokens.append(indent)
- self.prev_col = len(indent)
- startline = False
- self.add_whitespace(start)
- self.tokens.append(token)
- self.prev_row, self.prev_col = end
- if tok_type in (NEWLINE, NL):
- self.prev_row += 1
- self.prev_col = 0
- return "".join(self.tokens)
-
- def compat(self, token, iterable):
- indents = []
- toks_append = self.tokens.append
- startline = token[0] in (NEWLINE, NL)
- prevstring = False
-
- for tok in chain([token], iterable):
- toknum, tokval = tok[:2]
-
- if toknum in (NAME, NUMBER):
- tokval += ' '
-
- # Insert a space between two consecutive strings
- if toknum == STRING:
- if prevstring:
- tokval = ' ' + tokval
- prevstring = True
- else:
- prevstring = False
-
- if toknum == INDENT:
- indents.append(tokval)
- continue
- elif toknum == DEDENT:
- indents.pop()
- continue
- elif toknum in (NEWLINE, NL):
- startline = True
- elif startline and indents:
- toks_append(indents[-1])
- startline = False
- toks_append(tokval)
-
-def untokenize(iterable):
- """Transform tokens back into Python source code.
-
- Each element returned by the iterable must be a token sequence
- with at least two elements, a token number and token value. If
- only two tokens are passed, the resulting output is poor.
-
- Round-trip invariant for full input:
- Untokenized source will match input source exactly
-
- Round-trip invariant for limited intput:
- # Output text will tokenize the back to the input
- t1 = [tok[:2] for tok in generate_tokens(f.readline)]
- newcode = untokenize(t1)
- readline = iter(newcode.splitlines(1)).next
- t2 = [tok[:2] for tok in generate_tokens(readline)]
- assert t1 == t2
- """
- ut = Untokenizer()
- return ut.untokenize(iterable)
-
-def generate_tokens(readline):
- """
- The generate_tokens() generator requires one argument, readline, which
- must be a callable object which provides the same interface as the
- readline() method of built-in file objects. Each call to the function
- should return one line of input as a string. Alternately, readline
- can be a callable function terminating with StopIteration:
- readline = open(myfile).next # Example of alternate readline
-
- The generator produces 5-tuples with these members: the token type; the
- token string; a 2-tuple (srow, scol) of ints specifying the row and
- column where the token begins in the source; a 2-tuple (erow, ecol) of
- ints specifying the row and column where the token ends in the source;
- and the line on which the token was found. The line passed is the
- logical line; continuation lines are included.
- """
- lnum = parenlev = continued = 0
- namechars, numchars = string.ascii_letters + '_', '0123456789'
- contstr, needcont = '', 0
- contline = None
- indents = [0]
-
- while 1: # loop over lines in stream
- try:
- line = readline()
- except StopIteration:
- line = ''
- lnum += 1
- pos, max = 0, len(line)
-
- if contstr: # continued string
- if not line:
- raise TokenError("EOF in multi-line string", strstart)
- endmatch = endprog.match(line)
- if endmatch:
- pos = end = endmatch.end(0)
- yield (STRING, contstr + line[:end],
- strstart, (lnum, end), contline + line)
- contstr, needcont = '', 0
- contline = None
- elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
- yield (ERRORTOKEN, contstr + line,
- strstart, (lnum, len(line)), contline)
- contstr = ''
- contline = None
- continue
- else:
- contstr = contstr + line
- contline = contline + line
- continue
-
- elif parenlev == 0 and not continued: # new statement
- if not line: break
- column = 0
- while pos < max: # measure leading whitespace
- if line[pos] == ' ':
- column += 1
- elif line[pos] == '\t':
- column = (column//tabsize + 1)*tabsize
- elif line[pos] == '\f':
- column = 0
- else:
- break
- pos += 1
- if pos == max:
- break
-
- if line[pos] in '#\r\n': # skip comments or blank lines
- if line[pos] == '#':
- comment_token = line[pos:].rstrip('\r\n')
- nl_pos = pos + len(comment_token)
- yield (COMMENT, comment_token,
- (lnum, pos), (lnum, pos + len(comment_token)), line)
- yield (NL, line[nl_pos:],
- (lnum, nl_pos), (lnum, len(line)), line)
- else:
- yield ((NL, COMMENT)[line[pos] == '#'], line[pos:],
- (lnum, pos), (lnum, len(line)), line)
- continue
-
- if column > indents[-1]: # count indents or dedents
- indents.append(column)
- yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
- while column < indents[-1]:
- if column not in indents:
- raise IndentationError(
- "unindent does not match any outer indentation level",
- ("", lnum, pos, line))
- indents = indents[:-1]
- yield (DEDENT, '', (lnum, pos), (lnum, pos), line)
-
- else: # continued statement
- if not line:
- raise TokenError("EOF in multi-line statement", (lnum, 0))
- continued = 0
-
- while pos < max:
- pseudomatch = pseudoprog.match(line, pos)
- if pseudomatch: # scan for tokens
- start, end = pseudomatch.span(1)
- spos, epos, pos = (lnum, start), (lnum, end), end
- if start == end:
- continue
- token, initial = line[start:end], line[start]
-
- if initial in numchars or \
- (initial == '.' and token != '.'): # ordinary number
- yield (NUMBER, token, spos, epos, line)
- elif initial in '\r\n':
- if parenlev > 0:
- n = NL
- else:
- n = NEWLINE
- yield (n, token, spos, epos, line)
- elif initial == '#':
- assert not token.endswith("\n")
- yield (COMMENT, token, spos, epos, line)
- elif token in triple_quoted:
- endprog = endprogs[token]
- endmatch = endprog.match(line, pos)
- if endmatch: # all on one line
- pos = endmatch.end(0)
- token = line[start:pos]
- yield (STRING, token, spos, (lnum, pos), line)
- else:
- strstart = (lnum, start) # multiple lines
- contstr = line[start:]
- contline = line
- break
- elif initial in single_quoted or \
- token[:2] in single_quoted or \
- token[:3] in single_quoted:
- if token[-1] == '\n': # continued string
- strstart = (lnum, start)
- endprog = (endprogs[initial] or endprogs[token[1]] or
- endprogs[token[2]])
- contstr, needcont = line[start:], 1
- contline = line
- break
- else: # ordinary string
- yield (STRING, token, spos, epos, line)
- elif initial in namechars: # ordinary name
- yield (NAME, token, spos, epos, line)
- elif initial == '\\': # continued stmt
- continued = 1
- else:
- if initial in '([{':
- parenlev += 1
- elif initial in ')]}':
- parenlev -= 1
- yield (OP, token, spos, epos, line)
- else:
- yield (ERRORTOKEN, line[pos],
- (lnum, pos), (lnum, pos+1), line)
- pos += 1
-
- for indent in indents[1:]: # pop remaining indent levels
- yield (DEDENT, '', (lnum, 0), (lnum, 0), '')
- yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '')
-
-if __name__ == '__main__': # testing
- import sys
- if len(sys.argv) > 1:
- tokenize(open(sys.argv[1]).readline)
- else:
- tokenize(sys.stdin.readline)
diff --git a/mitogen-0.2.7/mitogen/core.py b/mitogen-0.2.7/mitogen/core.py
deleted file mode 100644
index ff77bba..000000000
--- a/mitogen-0.2.7/mitogen/core.py
+++ /dev/null
@@ -1,3409 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-"""
-This module implements most package functionality, but remains separate from
-non-essential code in order to reduce its size, since it is also serves as the
-bootstrap implementation sent to every new slave context.
-"""
-
-import binascii
-import collections
-import encodings.latin_1
-import errno
-import fcntl
-import itertools
-import linecache
-import logging
-import os
-import pickle as py_pickle
-import pstats
-import signal
-import socket
-import struct
-import sys
-import threading
-import time
-import traceback
-import warnings
-import weakref
-import zlib
-
-# Python >3.7 deprecated the imp module.
-warnings.filterwarnings('ignore', message='the imp module is deprecated')
-import imp
-
-# Absolute imports for <2.5.
-select = __import__('select')
-
-try:
- import cProfile
-except ImportError:
- cProfile = None
-
-try:
- import thread
-except ImportError:
- import threading as thread
-
-try:
- import cPickle as pickle
-except ImportError:
- import pickle
-
-try:
- from cStringIO import StringIO as BytesIO
-except ImportError:
- from io import BytesIO
-
-try:
- BaseException
-except NameError:
- BaseException = Exception
-
-try:
- ModuleNotFoundError
-except NameError:
- ModuleNotFoundError = ImportError
-
-# TODO: usage of 'import' after setting __name__, but before fixing up
-# sys.modules generates a warning. This happens when profiling = True.
-warnings.filterwarnings('ignore',
- "Parent module 'mitogen' not found while handling absolute import")
-
-LOG = logging.getLogger('mitogen')
-IOLOG = logging.getLogger('mitogen.io')
-IOLOG.setLevel(logging.INFO)
-
-LATIN1_CODEC = encodings.latin_1.Codec()
-# str.encode() may take import lock. Deadlock possible if broker calls
-# .encode() on behalf of thread currently waiting for module.
-UTF8_CODEC = encodings.latin_1.Codec()
-
-_v = False
-_vv = False
-
-GET_MODULE = 100
-CALL_FUNCTION = 101
-FORWARD_LOG = 102
-ADD_ROUTE = 103
-DEL_ROUTE = 104
-ALLOCATE_ID = 105
-SHUTDOWN = 106
-LOAD_MODULE = 107
-FORWARD_MODULE = 108
-DETACHING = 109
-CALL_SERVICE = 110
-
-#: Special value used to signal disconnection or the inability to route a
-#: message, when it appears in the `reply_to` field. Usually causes
-#: :class:`mitogen.core.ChannelError` to be raised when it is received.
-#:
-#: It indicates the sender did not know how to process the message, or wishes
-#: no further messages to be delivered to it. It is used when:
-#:
-#: * a remote receiver is disconnected or explicitly closed.
-#: * a related message could not be delivered due to no route existing for it.
-#: * a router is being torn down, as a sentinel value to notify
-#: :meth:`mitogen.core.Router.add_handler` callbacks to clean up.
-IS_DEAD = 999
-
-try:
- BaseException
-except NameError:
- BaseException = Exception
-
-PY24 = sys.version_info < (2, 5)
-PY3 = sys.version_info > (3,)
-if PY3:
- b = str.encode
- BytesType = bytes
- UnicodeType = str
- FsPathTypes = (str,)
- BufferType = lambda buf, start: memoryview(buf)[start:]
- long = int
-else:
- b = str
- BytesType = str
- FsPathTypes = (str, unicode)
- BufferType = buffer
- UnicodeType = unicode
-
-AnyTextType = (BytesType, UnicodeType)
-
-try:
- next
-except NameError:
- next = lambda it: it.next()
-
-# #550: prehistoric WSL did not advertise itself in uname output.
-try:
- fp = open('/proc/sys/kernel/osrelease')
- IS_WSL = 'Microsoft' in fp.read()
- fp.close()
-except IOError:
- IS_WSL = False
-
-
-#: Default size for calls to :meth:`Side.read` or :meth:`Side.write`, and the
-#: size of buffers configured by :func:`mitogen.parent.create_socketpair`. This
-#: value has many performance implications, 128KiB seems to be a sweet spot.
-#:
-#: * When set low, large messages cause many :class:`Broker` IO loop
-#: iterations, burning CPU and reducing throughput.
-#: * When set high, excessive RAM is reserved by the OS for socket buffers (2x
-#: per child), and an identically sized temporary userspace buffer is
-#: allocated on each read that requires zeroing, and over a particular size
-#: may require two system calls to allocate/deallocate.
-#:
-#: Care must be taken to ensure the underlying kernel object and receiving
-#: program support the desired size. For example,
-#:
-#: * Most UNIXes have TTYs with fixed 2KiB-4KiB buffers, making them unsuitable
-#: for efficient IO.
-#: * Different UNIXes have varying presets for pipes, which may not be
-#: configurable. On recent Linux the default pipe buffer size is 64KiB, but
-#: under memory pressure may be as low as 4KiB for unprivileged processes.
-#: * When communication is via an intermediary process, its internal buffers
-#: effect the speed OS buffers will drain. For example OpenSSH uses 64KiB
-#: reads.
-#:
-#: An ideal :class:`Message` has a size that is a multiple of
-#: :data:`CHUNK_SIZE` inclusive of headers, to avoid wasting IO loop iterations
-#: writing small trailer chunks.
-CHUNK_SIZE = 131072
-
-_tls = threading.local()
-
-
-if __name__ == 'mitogen.core':
- # When loaded using import mechanism, ExternalContext.main() will not have
- # a chance to set the synthetic mitogen global, so just import it here.
- import mitogen
-else:
- # When loaded as __main__, ensure classes and functions gain a __module__
- # attribute consistent with the host process, so that pickling succeeds.
- __name__ = 'mitogen.core'
-
-
-class Error(Exception):
- """Base for all exceptions raised by Mitogen.
-
- :param str fmt:
- Exception text, or format string if `args` is non-empty.
- :param tuple args:
- Format string arguments.
- """
- def __init__(self, fmt=None, *args):
- if args:
- fmt %= args
- if fmt and not isinstance(fmt, UnicodeType):
- fmt = fmt.decode('utf-8')
- Exception.__init__(self, fmt)
-
-
-class LatchError(Error):
- """Raised when an attempt is made to use a :class:`mitogen.core.Latch`
- that has been marked closed."""
- pass
-
-
-class Blob(BytesType):
- """A serializable bytes subclass whose content is summarized in repr()
- output, making it suitable for logging binary data."""
- def __repr__(self):
- return '[blob: %d bytes]' % len(self)
-
- def __reduce__(self):
- return (Blob, (BytesType(self),))
-
-
-class Secret(UnicodeType):
- """A serializable unicode subclass whose content is masked in repr()
- output, making it suitable for logging passwords."""
- def __repr__(self):
- return '[secret]'
-
- if not PY3:
- # TODO: what is this needed for in 2.x?
- def __str__(self):
- return UnicodeType(self)
-
- def __reduce__(self):
- return (Secret, (UnicodeType(self),))
-
-
-class Kwargs(dict):
- """
- A serializable dict subclass that indicates its keys should be coerced to
- Unicode on Python 3 and bytes on Python<2.6.
-
- Python 2 produces keyword argument dicts whose keys are bytes, requiring a
- helper to ensure compatibility with Python 3 where Unicode is required,
- whereas Python 3 produces keyword argument dicts whose keys are Unicode,
- requiring a helper for Python 2.4/2.5, where bytes are required.
- """
- if PY3:
- def __init__(self, dct):
- for k, v in dct.items():
- if type(k) is bytes:
- self[k.decode()] = v
- else:
- self[k] = v
- elif sys.version_info < (2, 6, 5):
- def __init__(self, dct):
- for k, v in dct.iteritems():
- if type(k) is unicode:
- k, _ = UTF8_CODEC.encode(k)
- self[k] = v
-
- def __repr__(self):
- return 'Kwargs(%s)' % (dict.__repr__(self),)
-
- def __reduce__(self):
- return (Kwargs, (dict(self),))
-
-
-class CallError(Error):
- """
- Serializable :class:`Error` subclass raised when :meth:`Context.call()
- ` fails. A copy of the traceback from the
- external context is appended to the exception message.
- """
- def __init__(self, fmt=None, *args):
- if not isinstance(fmt, BaseException):
- Error.__init__(self, fmt, *args)
- else:
- e = fmt
- cls = e.__class__
- fmt = '%s.%s: %s' % (cls.__module__, cls.__name__, e)
- tb = sys.exc_info()[2]
- if tb:
- fmt += '\n'
- fmt += ''.join(traceback.format_tb(tb))
- Error.__init__(self, fmt)
-
- def __reduce__(self):
- return (_unpickle_call_error, (self.args[0],))
-
-
-def _unpickle_call_error(s):
- if not (type(s) is UnicodeType and len(s) < 10000):
- raise TypeError('cannot unpickle CallError: bad input')
- return CallError(s)
-
-
-class ChannelError(Error):
- """Raised when a channel dies or has been closed."""
- remote_msg = 'Channel closed by remote end.'
- local_msg = 'Channel closed by local end.'
-
-
-class StreamError(Error):
- """Raised when a stream cannot be established."""
- pass
-
-
-class TimeoutError(Error):
- """Raised when a timeout occurs on a stream."""
- pass
-
-
-def to_text(o):
- """Coerce `o` to Unicode by decoding it from UTF-8 if it is an instance of
- :class:`bytes`, otherwise pass it to the :class:`str` constructor. The
- returned object is always a plain :class:`str`, any subclass is removed."""
- if isinstance(o, BytesType):
- return o.decode('utf-8')
- return UnicodeType(o)
-
-
-# Python 2.4
-try:
- any
-except NameError:
- def any(it):
- for elem in it:
- if elem:
- return True
-
-
-def _partition(s, sep, find):
- """
- (str|unicode).(partition|rpartition) for Python 2.4/2.5.
- """
- idx = find(sep)
- if idx != -1:
- left = s[0:idx]
- return left, sep, s[len(left)+len(sep):]
-
-
-if hasattr(UnicodeType, 'rpartition'):
- str_partition = UnicodeType.partition
- str_rpartition = UnicodeType.rpartition
- bytes_partition = BytesType.partition
-else:
- def str_partition(s, sep):
- return _partition(s, sep, s.find) or (s, u'', u'')
- def str_rpartition(s, sep):
- return _partition(s, sep, s.rfind) or (u'', u'', s)
- def bytes_partition(s, sep):
- return _partition(s, sep, s.find) or (s, '', '')
-
-
-def has_parent_authority(msg, _stream=None):
- """Policy function for use with :class:`Receiver` and
- :meth:`Router.add_handler` that requires incoming messages to originate
- from a parent context, or on a :class:`Stream` whose :attr:`auth_id
- ` has been set to that of a parent context or the current
- context."""
- return (msg.auth_id == mitogen.context_id or
- msg.auth_id in mitogen.parent_ids)
-
-
-def listen(obj, name, func):
- """
- Arrange for `func(*args, **kwargs)` to be invoked when the named signal is
- fired by `obj`.
- """
- signals = vars(obj).setdefault('_signals', {})
- signals.setdefault(name, []).append(func)
-
-
-def fire(obj, name, *args, **kwargs):
- """
- Arrange for `func(*args, **kwargs)` to be invoked for every function
- registered for the named signal on `obj`.
- """
- signals = vars(obj).get('_signals', {})
- for func in signals.get(name, ()):
- func(*args, **kwargs)
-
-
-def takes_econtext(func):
- func.mitogen_takes_econtext = True
- return func
-
-
-def takes_router(func):
- func.mitogen_takes_router = True
- return func
-
-
-def is_blacklisted_import(importer, fullname):
- """
- Return :data:`True` if `fullname` is part of a blacklisted package, or if
- any packages have been whitelisted and `fullname` is not part of one.
-
- NB:
- - If a package is on both lists, then it is treated as blacklisted.
- - If any package is whitelisted, then all non-whitelisted packages are
- treated as blacklisted.
- """
- return ((not any(fullname.startswith(s) for s in importer.whitelist)) or
- (any(fullname.startswith(s) for s in importer.blacklist)))
-
-
-def set_cloexec(fd):
- """Set the file descriptor `fd` to automatically close on
- :func:`os.execve`. This has no effect on file descriptors inherited across
- :func:`os.fork`, they must be explicitly closed through some other means,
- such as :func:`mitogen.fork.on_fork`."""
- flags = fcntl.fcntl(fd, fcntl.F_GETFD)
- assert fd > 2
- fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)
-
-
-def set_nonblock(fd):
- """Set the file descriptor `fd` to non-blocking mode. For most underlying
- file types, this causes :func:`os.read` or :func:`os.write` to raise
- :class:`OSError` with :data:`errno.EAGAIN` rather than block the thread
- when the underlying kernel buffer is exhausted."""
- flags = fcntl.fcntl(fd, fcntl.F_GETFL)
- fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK)
-
-
-def set_block(fd):
- """Inverse of :func:`set_nonblock`, i.e. cause `fd` to block the thread
- when the underlying kernel buffer is exhausted."""
- flags = fcntl.fcntl(fd, fcntl.F_GETFL)
- fcntl.fcntl(fd, fcntl.F_SETFL, flags & ~os.O_NONBLOCK)
-
-
-def io_op(func, *args):
- """Wrap `func(*args)` that may raise :class:`select.error`,
- :class:`IOError`, or :class:`OSError`, trapping UNIX error codes relating
- to disconnection and retry events in various subsystems:
-
- * When a signal is delivered to the process on Python 2, system call retry
- is signalled through :data:`errno.EINTR`. The invocation is automatically
- restarted.
- * When performing IO against a TTY, disconnection of the remote end is
- signalled by :data:`errno.EIO`.
- * When performing IO against a socket, disconnection of the remote end is
- signalled by :data:`errno.ECONNRESET`.
- * When performing IO against a pipe, disconnection of the remote end is
- signalled by :data:`errno.EPIPE`.
-
- :returns:
- Tuple of `(return_value, disconnect_reason)`, where `return_value` is
- the return value of `func(*args)`, and `disconnected` is an exception
- instance when disconnection was detected, otherwise :data:`None`.
- """
- while True:
- try:
- return func(*args), None
- except (select.error, OSError, IOError):
- e = sys.exc_info()[1]
- _vv and IOLOG.debug('io_op(%r) -> OSError: %s', func, e)
- if e.args[0] == errno.EINTR:
- continue
- if e.args[0] in (errno.EIO, errno.ECONNRESET, errno.EPIPE):
- return None, e
- raise
-
-
-class PidfulStreamHandler(logging.StreamHandler):
- """A :class:`logging.StreamHandler` subclass used when
- :meth:`Router.enable_debug() ` has been
- called, or the `debug` parameter was specified during context construction.
- Verifies the process ID has not changed on each call to :meth:`emit`,
- reopening the associated log file when a change is detected.
-
- This ensures logging to the per-process output files happens correctly even
- when uncooperative third party components call :func:`os.fork`.
- """
- #: PID that last opened the log file.
- open_pid = None
-
- #: Output path template.
- template = '/tmp/mitogen.%s.%s.log'
-
- def _reopen(self):
- self.acquire()
- try:
- if self.open_pid == os.getpid():
- return
- ts = time.strftime('%Y%m%d_%H%M%S')
- path = self.template % (os.getpid(), ts)
- self.stream = open(path, 'w', 1)
- set_cloexec(self.stream.fileno())
- self.stream.write('Parent PID: %s\n' % (os.getppid(),))
- self.stream.write('Created by:\n\n%s\n' % (
- ''.join(traceback.format_stack()),
- ))
- self.open_pid = os.getpid()
- finally:
- self.release()
-
- def emit(self, record):
- if self.open_pid != os.getpid():
- self._reopen()
- logging.StreamHandler.emit(self, record)
-
-
-def enable_debug_logging():
- global _v, _vv
- _v = True
- _vv = True
- root = logging.getLogger()
- root.setLevel(logging.DEBUG)
- IOLOG.setLevel(logging.DEBUG)
- handler = PidfulStreamHandler()
- handler.formatter = logging.Formatter(
- '%(asctime)s %(levelname).1s %(name)s: %(message)s',
- '%H:%M:%S'
- )
- root.handlers.insert(0, handler)
-
-
-_profile_hook = lambda name, func, *args: func(*args)
-_profile_fmt = os.environ.get(
- 'MITOGEN_PROFILE_FMT',
- '/tmp/mitogen.stats.%(pid)s.%(identity)s.%(now)s.%(ext)s',
-)
-
-
-def _profile_hook(name, func, *args):
- """
- Call `func(*args)` and return its result. This function is replaced by
- :func:`_real_profile_hook` when :func:`enable_profiling` is called. This
- interface is obsolete and will be replaced by a signals-based integration
- later on.
- """
- return func(*args)
-
-
-def _real_profile_hook(name, func, *args):
- profiler = cProfile.Profile()
- profiler.enable()
- try:
- return func(*args)
- finally:
- path = _profile_fmt % {
- 'now': int(1e6 * time.time()),
- 'identity': name,
- 'pid': os.getpid(),
- 'ext': '%s'
- }
- profiler.dump_stats(path % ('pstats',))
- profiler.create_stats()
- fp = open(path % ('log',), 'w')
- try:
- stats = pstats.Stats(profiler, stream=fp)
- stats.sort_stats('cumulative')
- stats.print_stats()
- finally:
- fp.close()
-
-
-def enable_profiling(econtext=None):
- global _profile_hook
- _profile_hook = _real_profile_hook
-
-
-def import_module(modname):
- """
- Import `module` and return the attribute named `attr`.
- """
- return __import__(modname, None, None, [''])
-
-
-class Py24Pickler(py_pickle.Pickler):
- """
- Exceptions were classic classes until Python 2.5. Sadly for 2.4, cPickle
- offers little control over how a classic instance is pickled. Therefore 2.4
- uses a pure-Python pickler, so CallError can be made to look as it does on
- newer Pythons.
-
- This mess will go away once proper serialization exists.
- """
- @classmethod
- def dumps(cls, obj, protocol):
- bio = BytesIO()
- self = cls(bio, protocol=protocol)
- self.dump(obj)
- return bio.getvalue()
-
- def save_exc_inst(self, obj):
- if isinstance(obj, CallError):
- func, args = obj.__reduce__()
- self.save(func)
- self.save(args)
- self.write(py_pickle.REDUCE)
- else:
- py_pickle.Pickler.save_inst(self, obj)
-
- if PY24:
- dispatch = py_pickle.Pickler.dispatch.copy()
- dispatch[py_pickle.InstanceType] = save_exc_inst
-
-
-if PY3:
- # In 3.x Unpickler is a class exposing find_class as an overridable, but it
- # cannot be overridden without subclassing.
- class _Unpickler(pickle.Unpickler):
- def find_class(self, module, func):
- return self.find_global(module, func)
- pickle__dumps = pickle.dumps
-elif PY24:
- # On Python 2.4, we must use a pure-Python pickler.
- pickle__dumps = Py24Pickler.dumps
- _Unpickler = pickle.Unpickler
-else:
- pickle__dumps = pickle.dumps
- # In 2.x Unpickler is a function exposing a writeable find_global
- # attribute.
- _Unpickler = pickle.Unpickler
-
-
-class Message(object):
- """
- Messages are the fundamental unit of communication, comprising fields from
- the :ref:`stream-protocol` header, an optional reference to the receiving
- :class:`mitogen.core.Router` for ingress messages, and helper methods for
- deserialization and generating replies.
- """
- #: Integer target context ID. :class:`Router` delivers messages locally
- #: when their :attr:`dst_id` matches :data:`mitogen.context_id`, otherwise
- #: they are routed up or downstream.
- dst_id = None
-
- #: Integer source context ID. Used as the target of replies if any are
- #: generated.
- src_id = None
-
- #: Context ID under whose authority the message is acting. See
- #: :ref:`source-verification`.
- auth_id = None
-
- #: Integer target handle in the destination context. This is one of the
- #: :ref:`standard-handles`, or a dynamically generated handle used to
- #: receive a one-time reply, such as the return value of a function call.
- handle = None
-
- #: Integer target handle to direct any reply to this message. Used to
- #: receive a one-time reply, such as the return value of a function call.
- #: :data:`IS_DEAD` has a special meaning when it appears in this field.
- reply_to = None
-
- #: Raw message data bytes.
- data = b('')
-
- _unpickled = object()
-
- #: The :class:`Router` responsible for routing the message. This is
- #: :data:`None` for locally originated messages.
- router = None
-
- #: The :class:`Receiver` over which the message was last received. Part of
- #: the :class:`mitogen.select.Select` interface. Defaults to :data:`None`.
- receiver = None
-
- def __init__(self, **kwargs):
- """
- Construct a message from from the supplied `kwargs`. :attr:`src_id` and
- :attr:`auth_id` are always set to :data:`mitogen.context_id`.
- """
- self.src_id = mitogen.context_id
- self.auth_id = mitogen.context_id
- vars(self).update(kwargs)
- assert isinstance(self.data, BytesType)
-
- def _unpickle_context(self, context_id, name):
- return _unpickle_context(context_id, name, router=self.router)
-
- def _unpickle_sender(self, context_id, dst_handle):
- return _unpickle_sender(self.router, context_id, dst_handle)
-
- def _unpickle_bytes(self, s, encoding):
- s, n = LATIN1_CODEC.encode(s)
- return s
-
- def _find_global(self, module, func):
- """Return the class implementing `module_name.class_name` or raise
- `StreamError` if the module is not whitelisted."""
- if module == __name__:
- if func == '_unpickle_call_error' or func == 'CallError':
- return _unpickle_call_error
- elif func == '_unpickle_sender':
- return self._unpickle_sender
- elif func == '_unpickle_context':
- return self._unpickle_context
- elif func == 'Blob':
- return Blob
- elif func == 'Secret':
- return Secret
- elif func == 'Kwargs':
- return Kwargs
- elif module == '_codecs' and func == 'encode':
- return self._unpickle_bytes
- elif module == '__builtin__' and func == 'bytes':
- return BytesType
- raise StreamError('cannot unpickle %r/%r', module, func)
-
- @property
- def is_dead(self):
- """
- :data:`True` if :attr:`reply_to` is set to the magic value
- :data:`IS_DEAD`, indicating the sender considers the channel dead. Dead
- messages can be raised in a variety of circumstances, see
- :data:`IS_DEAD` for more information.
- """
- return self.reply_to == IS_DEAD
-
- @classmethod
- def dead(cls, reason=None, **kwargs):
- """
- Syntax helper to construct a dead message.
- """
- kwargs['data'], _ = UTF8_CODEC.encode(reason or u'')
- return cls(reply_to=IS_DEAD, **kwargs)
-
- @classmethod
- def pickled(cls, obj, **kwargs):
- """
- Construct a pickled message, setting :attr:`data` to the serialization
- of `obj`, and setting remaining fields using `kwargs`.
-
- :returns:
- The new message.
- """
- self = cls(**kwargs)
- try:
- self.data = pickle__dumps(obj, protocol=2)
- except pickle.PicklingError:
- e = sys.exc_info()[1]
- self.data = pickle__dumps(CallError(e), protocol=2)
- return self
-
- def reply(self, msg, router=None, **kwargs):
- """
- Compose a reply to this message and send it using :attr:`router`, or
- `router` is :attr:`router` is :data:`None`.
-
- :param obj:
- Either a :class:`Message`, or an object to be serialized in order
- to construct a new message.
- :param router:
- Optional router to use if :attr:`router` is :data:`None`.
- :param kwargs:
- Optional keyword parameters overriding message fields in the reply.
- """
- if not isinstance(msg, Message):
- msg = Message.pickled(msg)
- msg.dst_id = self.src_id
- msg.handle = self.reply_to
- vars(msg).update(kwargs)
- if msg.handle:
- (self.router or router).route(msg)
- else:
- LOG.debug('Message.reply(): discarding due to zero handle: %r', msg)
-
- if PY3:
- UNPICKLER_KWARGS = {'encoding': 'bytes'}
- else:
- UNPICKLER_KWARGS = {}
-
- def _throw_dead(self):
- if len(self.data):
- raise ChannelError(self.data.decode('utf-8', 'replace'))
- elif self.src_id == mitogen.context_id:
- raise ChannelError(ChannelError.local_msg)
- else:
- raise ChannelError(ChannelError.remote_msg)
-
- def unpickle(self, throw=True, throw_dead=True):
- """
- Unpickle :attr:`data`, optionally raising any exceptions present.
-
- :param bool throw_dead:
- If :data:`True`, raise exceptions, otherwise it is the caller's
- responsibility.
-
- :raises CallError:
- The serialized data contained CallError exception.
- :raises ChannelError:
- The `is_dead` field was set.
- """
- _vv and IOLOG.debug('%r.unpickle()', self)
- if throw_dead and self.is_dead:
- self._throw_dead()
-
- obj = self._unpickled
- if obj is Message._unpickled:
- fp = BytesIO(self.data)
- unpickler = _Unpickler(fp, **self.UNPICKLER_KWARGS)
- unpickler.find_global = self._find_global
- try:
- # Must occur off the broker thread.
- obj = unpickler.load()
- self._unpickled = obj
- except (TypeError, ValueError):
- e = sys.exc_info()[1]
- raise StreamError('invalid message: %s', e)
-
- if throw:
- if isinstance(obj, CallError):
- raise obj
-
- return obj
-
- def __repr__(self):
- return 'Message(%r, %r, %r, %r, %r, %r..%d)' % (
- self.dst_id, self.src_id, self.auth_id, self.handle,
- self.reply_to, (self.data or '')[:50], len(self.data)
- )
-
-
-class Sender(object):
- """
- Senders are used to send pickled messages to a handle in another context,
- it is the inverse of :class:`mitogen.core.Receiver`.
-
- Senders may be serialized, making them convenient to wire up data flows.
- See :meth:`mitogen.core.Receiver.to_sender` for more information.
-
- :param Context context:
- Context to send messages to.
- :param int dst_handle:
- Destination handle to send messages to.
- """
- def __init__(self, context, dst_handle):
- self.context = context
- self.dst_handle = dst_handle
-
- def send(self, data):
- """
- Send `data` to the remote end.
- """
- _vv and IOLOG.debug('%r.send(%r..)', self, repr(data)[:100])
- self.context.send(Message.pickled(data, handle=self.dst_handle))
-
- explicit_close_msg = 'Sender was explicitly closed'
-
- def close(self):
- """
- Send a dead message to the remote, causing :meth:`ChannelError` to be
- raised in any waiting thread.
- """
- _vv and IOLOG.debug('%r.close()', self)
- self.context.send(
- Message.dead(
- reason=self.explicit_close_msg,
- handle=self.dst_handle
- )
- )
-
- def __repr__(self):
- return 'Sender(%r, %r)' % (self.context, self.dst_handle)
-
- def __reduce__(self):
- return _unpickle_sender, (self.context.context_id, self.dst_handle)
-
-
-def _unpickle_sender(router, context_id, dst_handle):
- if not (isinstance(router, Router) and
- isinstance(context_id, (int, long)) and context_id >= 0 and
- isinstance(dst_handle, (int, long)) and dst_handle > 0):
- raise TypeError('cannot unpickle Sender: bad input')
- return Sender(Context(router, context_id), dst_handle)
-
-
-class Receiver(object):
- """
- Receivers maintain a thread-safe queue of messages sent to a handle of this
- context from another context.
-
- :param mitogen.core.Router router:
- Router to register the handler on.
-
- :param int handle:
- If not :data:`None`, an explicit handle to register, otherwise an
- unused handle is chosen.
-
- :param bool persist:
- If :data:`False`, unregister the handler after one message is received.
- Single-message receivers are intended for RPC-like transactions, such
- as in the case of :meth:`mitogen.parent.Context.call_async`.
-
- :param mitogen.core.Context respondent:
- Context this receiver is receiving from. If not :data:`None`, arranges
- for the receiver to receive a dead message if messages can no longer be
- routed to the context due to disconnection, and ignores messages that
- did not originate from the respondent context.
- """
- #: If not :data:`None`, a reference to a function invoked as
- #: `notify(receiver)` when a new message is delivered to this receiver. The
- #: function is invoked on the broker thread, therefore it must not block.
- #: Used by :class:`mitogen.select.Select` to implement waiting on multiple
- #: receivers.
- notify = None
-
- raise_channelerror = True
-
- def __init__(self, router, handle=None, persist=True,
- respondent=None, policy=None, overwrite=False):
- self.router = router
- #: The handle.
- self.handle = handle # Avoid __repr__ crash in add_handler()
- self._latch = Latch() # Must exist prior to .add_handler()
- self.handle = router.add_handler(
- fn=self._on_receive,
- handle=handle,
- policy=policy,
- persist=persist,
- respondent=respondent,
- overwrite=overwrite,
- )
-
- def __repr__(self):
- return 'Receiver(%r, %r)' % (self.router, self.handle)
-
- def __enter__(self):
- return self
-
- def __exit__(self, _1, _2, _3):
- self.close()
-
- def to_sender(self):
- """
- Return a :class:`Sender` configured to deliver messages to this
- receiver. As senders are serializable, this makes it convenient to pass
- `(context_id, handle)` pairs around::
-
- def deliver_monthly_report(sender):
- for line in open('monthly_report.txt'):
- sender.send(line)
- sender.close()
-
- @mitogen.main()
- def main(router):
- remote = router.ssh(hostname='mainframe')
- recv = mitogen.core.Receiver(router)
- remote.call(deliver_monthly_report, recv.to_sender())
- for msg in recv:
- print(msg)
- """
- return Sender(self.router.myself(), self.handle)
-
- def _on_receive(self, msg):
- """
- Callback registered for the handle with :class:`Router`; appends data
- to the internal queue.
- """
- _vv and IOLOG.debug('%r._on_receive(%r)', self, msg)
- self._latch.put(msg)
- if self.notify:
- self.notify(self)
-
- closed_msg = 'the Receiver has been closed'
-
- def close(self):
- """
- Unregister the receiver's handle from its associated router, and cause
- :class:`ChannelError` to be raised in any thread waiting in :meth:`get`
- on this receiver.
- """
- if self.handle:
- self.router.del_handler(self.handle)
- self.handle = None
- self._latch.close()
-
- def empty(self):
- """
- Return :data:`True` if calling :meth:`get` would block.
-
- As with :class:`Queue.Queue`, :data:`True` may be returned even though
- a subsequent call to :meth:`get` will succeed, since a message may be
- posted at any moment between :meth:`empty` and :meth:`get`.
- """
- return self._latch.empty()
-
- def get(self, timeout=None, block=True, throw_dead=True):
- """
- Sleep waiting for a message to arrive on this receiver.
-
- :param float timeout:
- If not :data:`None`, specifies a timeout in seconds.
-
- :raises mitogen.core.ChannelError:
- The remote end indicated the channel should be closed,
- communication with it was lost, or :meth:`close` was called in the
- local process.
-
- :raises mitogen.core.TimeoutError:
- Timeout was reached.
-
- :returns:
- :class:`Message` that was received.
- """
- _vv and IOLOG.debug('%r.get(timeout=%r, block=%r)', self, timeout, block)
- try:
- msg = self._latch.get(timeout=timeout, block=block)
- except LatchError:
- raise ChannelError(self.closed_msg)
- if msg.is_dead and throw_dead:
- msg._throw_dead()
- return msg
-
- def __iter__(self):
- """
- Yield consecutive :class:`Message` instances delivered to this receiver
- until :class:`ChannelError` is raised.
- """
- while True:
- try:
- msg = self.get()
- except ChannelError:
- return
- yield msg
-
-
-class Channel(Sender, Receiver):
- """
- A channel inherits from :class:`mitogen.core.Sender` and
- `mitogen.core.Receiver` to provide bidirectional functionality.
-
- This class is incomplete and obsolete, it will be removed in Mitogen 0.3.
- Channels were an early attempt at syntax sugar. It is always easier to pass
- around unidirectional pairs of senders/receivers, even though the syntax is
- baroque:
-
- .. literalinclude:: ../examples/ping_pong.py
-
- Since all handles aren't known until after both ends are constructed, for
- both ends to communicate through a channel, it is necessary for one end to
- retrieve the handle allocated to the other and reconfigure its own channel
- to match. Currently this is a manual task.
- """
- def __init__(self, router, context, dst_handle, handle=None):
- Sender.__init__(self, context, dst_handle)
- Receiver.__init__(self, router, handle)
-
- def close(self):
- Receiver.close(self)
- Sender.close(self)
-
- def __repr__(self):
- return 'Channel(%s, %s)' % (
- Sender.__repr__(self),
- Receiver.__repr__(self)
- )
-
-
-class Importer(object):
- """
- Import protocol implementation that fetches modules from the parent
- process.
-
- :param context: Context to communicate via.
- """
- # The Mitogen package is handled specially, since the child context must
- # construct it manually during startup.
- MITOGEN_PKG_CONTENT = [
- 'compat',
- 'debug',
- 'doas',
- 'docker',
- 'kubectl',
- 'fakessh',
- 'fork',
- 'jail',
- 'lxc',
- 'lxd',
- 'master',
- 'minify',
- 'os_fork',
- 'parent',
- 'select',
- 'service',
- 'setns',
- 'ssh',
- 'su',
- 'sudo',
- 'utils',
- ]
-
- ALWAYS_BLACKLIST = [
- # 2.x generates needless imports for 'builtins', while 3.x does the
- # same for '__builtin__'. The correct one is built-in, the other always
- # a negative round-trip.
- 'builtins',
- '__builtin__',
- 'thread',
-
- # org.python.core imported by copy, pickle, xml.sax; breaks Jython, but
- # very unlikely to trigger a bug report.
- 'org',
- ]
-
- if PY3:
- ALWAYS_BLACKLIST += ['cStringIO']
-
- def __init__(self, router, context, core_src, whitelist=(), blacklist=()):
- self._context = context
- self._present = {'mitogen': self.MITOGEN_PKG_CONTENT}
- self._lock = threading.Lock()
- self.whitelist = list(whitelist) or ['']
- self.blacklist = list(blacklist) + self.ALWAYS_BLACKLIST
-
- # Preserve copies of the original server-supplied whitelist/blacklist
- # for later use by children.
- self.master_whitelist = self.whitelist[:]
- self.master_blacklist = self.blacklist[:]
-
- # Presence of an entry in this map indicates in-flight GET_MODULE.
- self._callbacks = {}
- self._cache = {}
- if core_src:
- self._update_linecache('x/mitogen/core.py', core_src)
- self._cache['mitogen.core'] = (
- 'mitogen.core',
- None,
- 'x/mitogen/core.py',
- zlib.compress(core_src, 9),
- [],
- )
- self._install_handler(router)
-
- def _update_linecache(self, path, data):
- """
- The Python 2.4 linecache module, used to fetch source code for
- tracebacks and :func:`inspect.getsource`, does not support PEP-302,
- meaning it needs extra help to for Mitogen-loaded modules. Directly
- populate its cache if a loaded module belongs to the Mitogen package.
- """
- if PY24 and 'mitogen' in path:
- linecache.cache[path] = (
- len(data),
- 0.0,
- [line+'\n' for line in data.splitlines()],
- path,
- )
-
- def _install_handler(self, router):
- router.add_handler(
- fn=self._on_load_module,
- handle=LOAD_MODULE,
- policy=has_parent_authority,
- )
-
- def __repr__(self):
- return 'Importer()'
-
- def builtin_find_module(self, fullname):
- # imp.find_module() will always succeed for __main__, because it is a
- # built-in module. That means it exists on a special linked list deep
- # within the bowels of the interpreter. We must special case it.
- if fullname == '__main__':
- raise ModuleNotFoundError()
-
- parent, _, modname = str_rpartition(fullname, '.')
- if parent:
- path = sys.modules[parent].__path__
- else:
- path = None
-
- fp, pathname, description = imp.find_module(modname, path)
- if fp:
- fp.close()
-
- def find_module(self, fullname, path=None):
- if hasattr(_tls, 'running'):
- return None
-
- _tls.running = True
- try:
- _v and LOG.debug('%r.find_module(%r)', self, fullname)
- fullname = to_text(fullname)
- pkgname, dot, _ = str_rpartition(fullname, '.')
- pkg = sys.modules.get(pkgname)
- if pkgname and getattr(pkg, '__loader__', None) is not self:
- LOG.debug('%r: %r is submodule of a package we did not load',
- self, fullname)
- return None
-
- suffix = fullname[len(pkgname+dot):]
- if pkgname and suffix not in self._present.get(pkgname, ()):
- LOG.debug('%r: master doesn\'t know %r', self, fullname)
- return None
-
- # #114: explicitly whitelisted prefixes override any
- # system-installed package.
- if self.whitelist != ['']:
- if any(fullname.startswith(s) for s in self.whitelist):
- return self
-
- try:
- self.builtin_find_module(fullname)
- _vv and IOLOG.debug('%r: %r is available locally',
- self, fullname)
- except ImportError:
- _vv and IOLOG.debug('find_module(%r) returning self', fullname)
- return self
- finally:
- del _tls.running
-
- blacklisted_msg = (
- '%r is present in the Mitogen importer blacklist, therefore this '
- 'context will not attempt to request it from the master, as the '
- 'request will always be refused.'
- )
- pkg_resources_msg = (
- 'pkg_resources is prohibited from importing __main__, as it causes '
- 'problems in applications whose main module is not designed to be '
- 're-imported by children.'
- )
- absent_msg = (
- 'The Mitogen master process was unable to serve %r. It may be a '
- 'native Python extension, or it may be missing entirely. Check the '
- 'importer debug logs on the master for more information.'
- )
-
- def _refuse_imports(self, fullname):
- if is_blacklisted_import(self, fullname):
- raise ModuleNotFoundError(self.blacklisted_msg % (fullname,))
-
- f = sys._getframe(2)
- requestee = f.f_globals['__name__']
-
- if fullname == '__main__' and requestee == 'pkg_resources':
- # Anything that imports pkg_resources will eventually cause
- # pkg_resources to try and scan __main__ for its __requires__
- # attribute (pkg_resources/__init__.py::_build_master()). This
- # breaks any app that is not expecting its __main__ to suddenly be
- # sucked over a network and injected into a remote process, like
- # py.test.
- raise ModuleNotFoundError(self.pkg_resources_msg)
-
- if fullname == 'pbr':
- # It claims to use pkg_resources to read version information, which
- # would result in PEP-302 being used, but it actually does direct
- # filesystem access. So instead smodge the environment to override
- # any version that was defined. This will probably break something
- # later.
- os.environ['PBR_VERSION'] = '0.0.0'
-
- def _on_load_module(self, msg):
- if msg.is_dead:
- return
-
- tup = msg.unpickle()
- fullname = tup[0]
- _v and LOG.debug('Importer._on_load_module(%r)', fullname)
-
- self._lock.acquire()
- try:
- self._cache[fullname] = tup
- if tup[2] is not None and PY24:
- self._update_linecache(
- path='master:' + tup[2],
- data=zlib.decompress(tup[3])
- )
- callbacks = self._callbacks.pop(fullname, [])
- finally:
- self._lock.release()
-
- for callback in callbacks:
- callback()
-
- def _request_module(self, fullname, callback):
- self._lock.acquire()
- try:
- present = fullname in self._cache
- if not present:
- funcs = self._callbacks.get(fullname)
- if funcs is not None:
- _v and LOG.debug('_request_module(%r): in flight', fullname)
- funcs.append(callback)
- else:
- _v and LOG.debug('_request_module(%r): new request', fullname)
- self._callbacks[fullname] = [callback]
- self._context.send(
- Message(data=b(fullname), handle=GET_MODULE)
- )
- finally:
- self._lock.release()
-
- if present:
- callback()
-
- def load_module(self, fullname):
- fullname = to_text(fullname)
- _v and LOG.debug('Importer.load_module(%r)', fullname)
- self._refuse_imports(fullname)
-
- event = threading.Event()
- self._request_module(fullname, event.set)
- event.wait()
-
- ret = self._cache[fullname]
- if ret[2] is None:
- raise ModuleNotFoundError(self.absent_msg % (fullname,))
-
- pkg_present = ret[1]
- mod = sys.modules.setdefault(fullname, imp.new_module(fullname))
- mod.__file__ = self.get_filename(fullname)
- mod.__loader__ = self
- if pkg_present is not None: # it's a package.
- mod.__path__ = []
- mod.__package__ = fullname
- self._present[fullname] = pkg_present
- else:
- mod.__package__ = str_rpartition(fullname, '.')[0] or None
-
- if mod.__package__ and not PY3:
- # 2.x requires __package__ to be exactly a string.
- mod.__package__, _ = UTF8_CODEC.encode(mod.__package__)
-
- source = self.get_source(fullname)
- try:
- code = compile(source, mod.__file__, 'exec', 0, 1)
- except SyntaxError:
- LOG.exception('while importing %r', fullname)
- raise
-
- if PY3:
- exec(code, vars(mod))
- else:
- exec('exec code in vars(mod)')
- return mod
-
- def get_filename(self, fullname):
- if fullname in self._cache:
- path = self._cache[fullname][2]
- if path is None:
- # If find_loader() returns self but a subsequent master RPC
- # reveals the module can't be loaded, and so load_module()
- # throws ImportError, on Python 3.x it is still possible for
- # the loader to be called to fetch metadata.
- raise ModuleNotFoundError(self.absent_msg % (fullname,))
- return u'master:' + self._cache[fullname][2]
-
- def get_source(self, fullname):
- if fullname in self._cache:
- compressed = self._cache[fullname][3]
- if compressed is None:
- raise ModuleNotFoundError(self.absent_msg % (fullname,))
-
- source = zlib.decompress(self._cache[fullname][3])
- if PY3:
- return to_text(source)
- return source
-
-
-class LogHandler(logging.Handler):
- def __init__(self, context):
- logging.Handler.__init__(self)
- self.context = context
- self.local = threading.local()
- self._buffer = []
-
- def uncork(self):
- """
- #305: during startup :class:`LogHandler` may be installed before it is
- possible to route messages, therefore messages are buffered until
- :meth:`uncork` is called by :class:`ExternalContext`.
- """
- self._send = self.context.send
- for msg in self._buffer:
- self._send(msg)
- self._buffer = None
-
- def _send(self, msg):
- self._buffer.append(msg)
-
- def emit(self, rec):
- if rec.name == 'mitogen.io' or \
- getattr(self.local, 'in_emit', False):
- return
-
- self.local.in_emit = True
- try:
- msg = self.format(rec)
- encoded = '%s\x00%s\x00%s' % (rec.name, rec.levelno, msg)
- if isinstance(encoded, UnicodeType):
- # Logging package emits both :(
- encoded = encoded.encode('utf-8')
- self._send(Message(data=encoded, handle=FORWARD_LOG))
- finally:
- self.local.in_emit = False
-
-
-class Side(object):
- """
- Represent a single side of a :class:`BasicStream`. This exists to allow
- streams implemented using unidirectional (e.g. UNIX pipe) and bidirectional
- (e.g. UNIX socket) file descriptors to operate identically.
-
- :param mitogen.core.Stream stream:
- The stream this side is associated with.
-
- :param int fd:
- Underlying file descriptor.
-
- :param bool keep_alive:
- Value for :attr:`keep_alive`
-
- During construction, the file descriptor has its :data:`os.O_NONBLOCK` flag
- enabled using :func:`fcntl.fcntl`.
- """
- _fork_refs = weakref.WeakValueDictionary()
-
- def __init__(self, stream, fd, cloexec=True, keep_alive=True, blocking=False):
- #: The :class:`Stream` for which this is a read or write side.
- self.stream = stream
- #: Integer file descriptor to perform IO on, or :data:`None` if
- #: :meth:`close` has been called.
- self.fd = fd
- self.closed = False
- #: If :data:`True`, causes presence of this side in
- #: :class:`Broker`'s active reader set to defer shutdown until the
- #: side is disconnected.
- self.keep_alive = keep_alive
- self._fork_refs[id(self)] = self
- if cloexec:
- set_cloexec(fd)
- if not blocking:
- set_nonblock(fd)
-
- def __repr__(self):
- return '' % (self.stream, self.fd)
-
- @classmethod
- def _on_fork(cls):
- while cls._fork_refs:
- _, side = cls._fork_refs.popitem()
- _vv and IOLOG.debug('Side._on_fork() closing %r', side)
- side.close()
-
- def close(self):
- """
- Call :func:`os.close` on :attr:`fd` if it is not :data:`None`,
- then set it to :data:`None`.
- """
- if not self.closed:
- _vv and IOLOG.debug('%r.close()', self)
- self.closed = True
- os.close(self.fd)
-
- def read(self, n=CHUNK_SIZE):
- """
- Read up to `n` bytes from the file descriptor, wrapping the underlying
- :func:`os.read` call with :func:`io_op` to trap common disconnection
- conditions.
-
- :meth:`read` always behaves as if it is reading from a regular UNIX
- file; socket, pipe, and TTY disconnection errors are masked and result
- in a 0-sized read like a regular file.
-
- :returns:
- Bytes read, or the empty to string to indicate disconnection was
- detected.
- """
- if self.closed:
- # Refuse to touch the handle after closed, it may have been reused
- # by another thread. TODO: synchronize read()/write()/close().
- return b('')
- s, disconnected = io_op(os.read, self.fd, n)
- if disconnected:
- LOG.debug('%r.read(): disconnected: %s', self, disconnected)
- return b('')
- return s
-
- def write(self, s):
- """
- Write as much of the bytes from `s` as possible to the file descriptor,
- wrapping the underlying :func:`os.write` call with :func:`io_op` to
- trap common disconnection conditions.
-
- :returns:
- Number of bytes written, or :data:`None` if disconnection was
- detected.
- """
- if self.closed or self.fd is None:
- # Refuse to touch the handle after closed, it may have been reused
- # by another thread.
- return None
-
- written, disconnected = io_op(os.write, self.fd, s)
- if disconnected:
- LOG.debug('%r.write(): disconnected: %s', self, disconnected)
- return None
- return written
-
-
-class BasicStream(object):
- #: A :class:`Side` representing the stream's receive file descriptor.
- receive_side = None
-
- #: A :class:`Side` representing the stream's transmit file descriptor.
- transmit_side = None
-
- def on_receive(self, broker):
- """
- Called by :class:`Broker` when the stream's :attr:`receive_side` has
- been marked readable using :meth:`Broker.start_receive` and the broker
- has detected the associated file descriptor is ready for reading.
-
- Subclasses must implement this if :meth:`Broker.start_receive` is ever
- called on them, and the method must call :meth:`on_disconect` if
- reading produces an empty string.
- """
- pass
-
- def on_transmit(self, broker):
- """
- Called by :class:`Broker` when the stream's :attr:`transmit_side`
- has been marked writeable using :meth:`Broker._start_transmit` and
- the broker has detected the associated file descriptor is ready for
- writing.
-
- Subclasses must implement this if :meth:`Broker._start_transmit` is
- ever called on them.
- """
- pass
-
- def on_shutdown(self, broker):
- """
- Called by :meth:`Broker.shutdown` to allow the stream time to
- gracefully shutdown. The base implementation simply called
- :meth:`on_disconnect`.
- """
- _v and LOG.debug('%r.on_shutdown()', self)
- fire(self, 'shutdown')
- self.on_disconnect(broker)
-
- def on_disconnect(self, broker):
- """
- Called by :class:`Broker` to force disconnect the stream. The base
- implementation simply closes :attr:`receive_side` and
- :attr:`transmit_side` and unregisters the stream from the broker.
- """
- LOG.debug('%r.on_disconnect()', self)
- if self.receive_side:
- broker.stop_receive(self)
- self.receive_side.close()
- if self.transmit_side:
- broker._stop_transmit(self)
- self.transmit_side.close()
- fire(self, 'disconnect')
-
-
-class Stream(BasicStream):
- """
- :class:`BasicStream` subclass implementing mitogen's :ref:`stream
- protocol `.
- """
- #: If not :data:`None`, :class:`Router` stamps this into
- #: :attr:`Message.auth_id` of every message received on this stream.
- auth_id = None
-
- #: If not :data:`False`, indicates the stream has :attr:`auth_id` set and
- #: its value is the same as :data:`mitogen.context_id` or appears in
- #: :data:`mitogen.parent_ids`.
- is_privileged = False
-
- def __init__(self, router, remote_id, **kwargs):
- self._router = router
- self.remote_id = remote_id
- self.name = u'default'
- self.sent_modules = set(['mitogen', 'mitogen.core'])
- self.construct(**kwargs)
- self._input_buf = collections.deque()
- self._output_buf = collections.deque()
- self._input_buf_len = 0
- self._output_buf_len = 0
- #: Routing records the dst_id of every message arriving from this
- #: stream. Any arriving DEL_ROUTE is rebroadcast for any such ID.
- self.egress_ids = set()
-
- def construct(self):
- pass
-
- def _internal_receive(self, broker, buf):
- if self._input_buf and self._input_buf_len < 128:
- self._input_buf[0] += buf
- else:
- self._input_buf.append(buf)
-
- self._input_buf_len += len(buf)
- while self._receive_one(broker):
- pass
-
- def on_receive(self, broker):
- """Handle the next complete message on the stream. Raise
- :class:`StreamError` on failure."""
- _vv and IOLOG.debug('%r.on_receive()', self)
-
- buf = self.receive_side.read()
- if not buf:
- return self.on_disconnect(broker)
-
- self._internal_receive(broker, buf)
-
- HEADER_FMT = '>hLLLLLL'
- HEADER_LEN = struct.calcsize(HEADER_FMT)
- HEADER_MAGIC = 0x4d49 # 'MI'
-
- corrupt_msg = (
- 'Corruption detected: frame signature incorrect. This likely means '
- 'some external process is interfering with the connection. Received:'
- '\n\n'
- '%r'
- )
-
- def _receive_one(self, broker):
- if self._input_buf_len < self.HEADER_LEN:
- return False
-
- msg = Message()
- msg.router = self._router
- (magic, msg.dst_id, msg.src_id, msg.auth_id,
- msg.handle, msg.reply_to, msg_len) = struct.unpack(
- self.HEADER_FMT,
- self._input_buf[0][:self.HEADER_LEN],
- )
-
- if magic != self.HEADER_MAGIC:
- LOG.error(self.corrupt_msg, self._input_buf[0][:2048])
- self.on_disconnect(broker)
- return False
-
- if msg_len > self._router.max_message_size:
- LOG.error('Maximum message size exceeded (got %d, max %d)',
- msg_len, self._router.max_message_size)
- self.on_disconnect(broker)
- return False
-
- total_len = msg_len + self.HEADER_LEN
- if self._input_buf_len < total_len:
- _vv and IOLOG.debug(
- '%r: Input too short (want %d, got %d)',
- self, msg_len, self._input_buf_len - self.HEADER_LEN
- )
- return False
-
- start = self.HEADER_LEN
- prev_start = start
- remain = total_len
- bits = []
- while remain:
- buf = self._input_buf.popleft()
- bit = buf[start:remain]
- bits.append(bit)
- remain -= len(bit) + start
- prev_start = start
- start = 0
-
- msg.data = b('').join(bits)
- self._input_buf.appendleft(buf[prev_start+len(bit):])
- self._input_buf_len -= total_len
- self._router._async_route(msg, self)
- return True
-
- def pending_bytes(self):
- """
- Return the number of bytes queued for transmission on this stream. This
- can be used to limit the amount of data buffered in RAM by an otherwise
- unlimited consumer.
-
- For an accurate result, this method should be called from the Broker
- thread, for example by using :meth:`Broker.defer_sync`.
- """
- return self._output_buf_len
-
- def on_transmit(self, broker):
- """Transmit buffered messages."""
- _vv and IOLOG.debug('%r.on_transmit()', self)
-
- if self._output_buf:
- buf = self._output_buf.popleft()
- written = self.transmit_side.write(buf)
- if not written:
- _v and LOG.debug('%r.on_transmit(): disconnection detected', self)
- self.on_disconnect(broker)
- return
- elif written != len(buf):
- self._output_buf.appendleft(BufferType(buf, written))
-
- _vv and IOLOG.debug('%r.on_transmit() -> len %d', self, written)
- self._output_buf_len -= written
-
- if not self._output_buf:
- broker._stop_transmit(self)
-
- def _send(self, msg):
- _vv and IOLOG.debug('%r._send(%r)', self, msg)
- pkt = struct.pack(self.HEADER_FMT, self.HEADER_MAGIC, msg.dst_id,
- msg.src_id, msg.auth_id, msg.handle,
- msg.reply_to or 0, len(msg.data)) + msg.data
-
- if not self._output_buf_len:
- # Modifying epoll/Kqueue state is expensive, as are needless broker
- # loops. Rather than wait for writeability, just write immediately,
- # and fall back to the broker loop on error or full buffer.
- try:
- n = self.transmit_side.write(pkt)
- if n:
- if n == len(pkt):
- return
- pkt = pkt[n:]
- except OSError:
- pass
-
- self._router.broker._start_transmit(self)
- self._output_buf.append(pkt)
- self._output_buf_len += len(pkt)
-
- def send(self, msg):
- """Send `data` to `handle`, and tell the broker we have output. May
- be called from any thread."""
- self._router.broker.defer(self._send, msg)
-
- def on_shutdown(self, broker):
- """Override BasicStream behaviour of immediately disconnecting."""
- _v and LOG.debug('%r.on_shutdown(%r)', self, broker)
-
- def accept(self, rfd, wfd):
- # TODO: what is this os.dup for?
- self.receive_side = Side(self, os.dup(rfd))
- self.transmit_side = Side(self, os.dup(wfd))
-
- def __repr__(self):
- cls = type(self)
- return "%s.%s('%s')" % (cls.__module__, cls.__name__, self.name)
-
-
-class Context(object):
- """
- Represent a remote context regardless of the underlying connection method.
- Context objects are simple facades that emit messages through an
- associated router, and have :ref:`signals` raised against them in response
- to various events relating to the context.
-
- **Note:** This is the somewhat limited core version, used by child
- contexts. The master subclass is documented below this one.
-
- Contexts maintain no internal state and are thread-safe.
-
- Prefer :meth:`Router.context_by_id` over constructing context objects
- explicitly, as that method is deduplicating, and returns the only context
- instance :ref:`signals` will be raised on.
-
- :param Router router:
- Router to emit messages through.
- :param int context_id:
- Context ID.
- :param str name:
- Context name.
- """
- remote_name = None
-
- def __init__(self, router, context_id, name=None):
- self.router = router
- self.context_id = context_id
- self.name = name
-
- def __reduce__(self):
- name = self.name
- if name and not isinstance(name, UnicodeType):
- name = UnicodeType(name, 'utf-8')
- return _unpickle_context, (self.context_id, name)
-
- def on_disconnect(self):
- _v and LOG.debug('%r.on_disconnect()', self)
- fire(self, 'disconnect')
-
- def send_async(self, msg, persist=False):
- """
- Arrange for `msg` to be delivered to this context, with replies
- directed to a newly constructed receiver. :attr:`dst_id
- ` is set to the target context ID, and :attr:`reply_to
- ` is set to the newly constructed receiver's handle.
-
- :param bool persist:
- If :data:`False`, the handler will be unregistered after a single
- message has been received.
-
- :param mitogen.core.Message msg:
- The message.
-
- :returns:
- :class:`Receiver` configured to receive any replies sent to the
- message's `reply_to` handle.
- """
- if self.router.broker._thread == threading.currentThread(): # TODO
- raise SystemError('Cannot making blocking call on broker thread')
-
- receiver = Receiver(self.router, persist=persist, respondent=self)
- msg.dst_id = self.context_id
- msg.reply_to = receiver.handle
-
- _v and LOG.debug('%r.send_async(%r)', self, msg)
- self.send(msg)
- return receiver
-
- def call_service_async(self, service_name, method_name, **kwargs):
- _v and LOG.debug('%r.call_service_async(%r, %r, %r)',
- self, service_name, method_name, kwargs)
- if isinstance(service_name, BytesType):
- service_name = service_name.encode('utf-8')
- elif not isinstance(service_name, UnicodeType):
- service_name = service_name.name() # Service.name()
- tup = (service_name, to_text(method_name), Kwargs(kwargs))
- msg = Message.pickled(tup, handle=CALL_SERVICE)
- return self.send_async(msg)
-
- def send(self, msg):
- """
- Arrange for `msg` to be delivered to this context. :attr:`dst_id
- ` is set to the target context ID.
-
- :param Message msg:
- Message.
- """
- msg.dst_id = self.context_id
- self.router.route(msg)
-
- def call_service(self, service_name, method_name, **kwargs):
- recv = self.call_service_async(service_name, method_name, **kwargs)
- return recv.get().unpickle()
-
- def send_await(self, msg, deadline=None):
- """
- Like :meth:`send_async`, but expect a single reply (`persist=False`)
- delivered within `deadline` seconds.
-
- :param mitogen.core.Message msg:
- The message.
- :param float deadline:
- If not :data:`None`, seconds before timing out waiting for a reply.
- :returns:
- Deserialized reply.
- :raises TimeoutError:
- No message was received and `deadline` passed.
- """
- receiver = self.send_async(msg)
- response = receiver.get(deadline)
- data = response.unpickle()
- _vv and IOLOG.debug('%r._send_await() -> %r', self, data)
- return data
-
- def __repr__(self):
- return 'Context(%s, %r)' % (self.context_id, self.name)
-
-
-def _unpickle_context(context_id, name, router=None):
- if not (isinstance(context_id, (int, long)) and context_id >= 0 and (
- (name is None) or
- (isinstance(name, UnicodeType) and len(name) < 100))
- ):
- raise TypeError('cannot unpickle Context: bad input')
-
- if isinstance(router, Router):
- return router.context_by_id(context_id, name=name)
- return Context(None, context_id, name) # For plain Jane pickle.
-
-
-class Poller(object):
- """
- A poller manages OS file descriptors the user is waiting to become
- available for IO. The :meth:`poll` method blocks the calling thread
- until one or more become ready. The default implementation is based on
- :func:`select.poll`.
-
- Each descriptor has an associated `data` element, which is unique for each
- readiness type, and defaults to being the same as the file descriptor. The
- :meth:`poll` method yields the data associated with a descriptor, rather
- than the descriptor itself, allowing concise loops like::
-
- p = Poller()
- p.start_receive(conn.fd, data=conn.on_read)
- p.start_transmit(conn.fd, data=conn.on_write)
-
- for callback in p.poll():
- callback() # invoke appropriate bound instance method
-
- Pollers may be modified while :meth:`poll` is yielding results. Removals
- are processed immediately, causing pending events for the descriptor to be
- discarded.
-
- The :meth:`close` method must be called when a poller is discarded to avoid
- a resource leak.
-
- Pollers may only be used by one thread at a time.
- """
- SUPPORTED = True
-
- # This changed from select() to poll() in Mitogen 0.2.4. Since poll() has
- # no upper FD limit, it is suitable for use with Latch, which must handle
- # FDs larger than select's limit during many-host runs. We want this
- # because poll() requires no setup and teardown: just a single system call,
- # which is important because Latch.get() creates a Poller on each
- # invocation. In a microbenchmark, poll() vs. epoll_ctl() is 30% faster in
- # this scenario. If select() must return in future, it is important
- # Latch.poller_class is set from parent.py to point to the industrial
- # strength poller for the OS, otherwise Latch will fail randomly.
-
- #: Increments on every poll(). Used to version _rfds and _wfds.
- _generation = 1
-
- def __init__(self):
- self._rfds = {}
- self._wfds = {}
-
- def __repr__(self):
- return '%s(%#x)' % (type(self).__name__, id(self))
-
- def _update(self, fd):
- """
- Required by PollPoller subclass.
- """
- pass
-
- @property
- def readers(self):
- """
- Return a list of `(fd, data)` tuples for every FD registered for
- receive readiness.
- """
- return list((fd, data) for fd, (data, gen) in self._rfds.items())
-
- @property
- def writers(self):
- """
- Return a list of `(fd, data)` tuples for every FD registered for
- transmit readiness.
- """
- return list((fd, data) for fd, (data, gen) in self._wfds.items())
-
- def close(self):
- """
- Close any underlying OS resource used by the poller.
- """
- pass
-
- def start_receive(self, fd, data=None):
- """
- Cause :meth:`poll` to yield `data` when `fd` is readable.
- """
- self._rfds[fd] = (data or fd, self._generation)
- self._update(fd)
-
- def stop_receive(self, fd):
- """
- Stop yielding readability events for `fd`.
-
- Redundant calls to :meth:`stop_receive` are silently ignored, this may
- change in future.
- """
- self._rfds.pop(fd, None)
- self._update(fd)
-
- def start_transmit(self, fd, data=None):
- """
- Cause :meth:`poll` to yield `data` when `fd` is writeable.
- """
- self._wfds[fd] = (data or fd, self._generation)
- self._update(fd)
-
- def stop_transmit(self, fd):
- """
- Stop yielding writeability events for `fd`.
-
- Redundant calls to :meth:`stop_transmit` are silently ignored, this may
- change in future.
- """
- self._wfds.pop(fd, None)
- self._update(fd)
-
- def _poll(self, timeout):
- (rfds, wfds, _), _ = io_op(select.select,
- self._rfds,
- self._wfds,
- (), timeout
- )
-
- for fd in rfds:
- _vv and IOLOG.debug('%r: POLLIN for %r', self, fd)
- data, gen = self._rfds.get(fd, (None, None))
- if gen and gen < self._generation:
- yield data
-
- for fd in wfds:
- _vv and IOLOG.debug('%r: POLLOUT for %r', self, fd)
- data, gen = self._wfds.get(fd, (None, None))
- if gen and gen < self._generation:
- yield data
-
- if timeout:
- timeout *= 1000
-
- def poll(self, timeout=None):
- """
- Block the calling thread until one or more FDs are ready for IO.
-
- :param float timeout:
- If not :data:`None`, seconds to wait without an event before
- returning an empty iterable.
- :returns:
- Iterable of `data` elements associated with ready FDs.
- """
- _vv and IOLOG.debug('%r.poll(%r)', self, timeout)
- self._generation += 1
- return self._poll(timeout)
-
-
-class Latch(object):
- """
- A latch is a :class:`Queue.Queue`-like object that supports mutation and
- waiting from multiple threads, however unlike :class:`Queue.Queue`,
- waiting threads always remain interruptible, so CTRL+C always succeeds, and
- waits where a timeout is set experience no wake up latency. These
- properties are not possible in combination using the built-in threading
- primitives available in Python 2.x.
-
- Latches implement queues using the UNIX self-pipe trick, and a per-thread
- :func:`socket.socketpair` that is lazily created the first time any
- latch attempts to sleep on a thread, and dynamically associated with the
- waiting Latch only for duration of the wait.
-
- See :ref:`waking-sleeping-threads` for further discussion.
- """
- poller_class = Poller
-
- notify = None
-
- # The _cls_ prefixes here are to make it crystal clear in the code which
- # state mutation isn't covered by :attr:`_lock`.
-
- #: List of reusable :func:`socket.socketpair` tuples. The list is mutated
- #: from multiple threads, the only safe operations are `append()` and
- #: `pop()`.
- _cls_idle_socketpairs = []
-
- #: List of every socket object that must be closed by :meth:`_on_fork`.
- #: Inherited descriptors cannot be reused, as the duplicated handles
- #: reference the same underlying kernel object in use by the parent.
- _cls_all_sockets = []
-
- def __init__(self):
- self.closed = False
- self._lock = threading.Lock()
- #: List of unconsumed enqueued items.
- self._queue = []
- #: List of `(wsock, cookie)` awaiting an element, where `wsock` is the
- #: socketpair's write side, and `cookie` is the string to write.
- self._sleeping = []
- #: Number of elements of :attr:`_sleeping` that have already been
- #: woken, and have a corresponding element index from :attr:`_queue`
- #: assigned to them.
- self._waking = 0
-
- @classmethod
- def _on_fork(cls):
- """
- Clean up any files belonging to the parent process after a fork.
- """
- cls._cls_idle_socketpairs = []
- while cls._cls_all_sockets:
- cls._cls_all_sockets.pop().close()
-
- def close(self):
- """
- Mark the latch as closed, and cause every sleeping thread to be woken,
- with :class:`mitogen.core.LatchError` raised in each thread.
- """
- self._lock.acquire()
- try:
- self.closed = True
- while self._waking < len(self._sleeping):
- wsock, cookie = self._sleeping[self._waking]
- self._wake(wsock, cookie)
- self._waking += 1
- finally:
- self._lock.release()
-
- def empty(self):
- """
- Return :data:`True` if calling :meth:`get` would block.
-
- As with :class:`Queue.Queue`, :data:`True` may be returned even
- though a subsequent call to :meth:`get` will succeed, since a
- message may be posted at any moment between :meth:`empty` and
- :meth:`get`.
-
- As with :class:`Queue.Queue`, :data:`False` may be returned even
- though a subsequent call to :meth:`get` will block, since another
- waiting thread may be woken at any moment between :meth:`empty` and
- :meth:`get`.
-
- :raises LatchError:
- The latch has already been marked closed.
- """
- self._lock.acquire()
- try:
- if self.closed:
- raise LatchError()
- return len(self._queue) == 0
- finally:
- self._lock.release()
-
- def _get_socketpair(self):
- """
- Return an unused socketpair, creating one if none exist.
- """
- try:
- return self._cls_idle_socketpairs.pop() # pop() must be atomic
- except IndexError:
- rsock, wsock = socket.socketpair()
- set_cloexec(rsock.fileno())
- set_cloexec(wsock.fileno())
- self._cls_all_sockets.extend((rsock, wsock))
- return rsock, wsock
-
- COOKIE_MAGIC, = struct.unpack('L', b('LTCH') * (struct.calcsize('L')//4))
- COOKIE_FMT = '>Qqqq' # #545: id() and get_ident() may exceed long on armhfp.
- COOKIE_SIZE = struct.calcsize(COOKIE_FMT)
-
- def _make_cookie(self):
- """
- Return a string encoding the ID of the process, instance and thread.
- This disambiguates legitimate wake-ups, accidental writes to the FD,
- and buggy internal FD sharing.
- """
- return struct.pack(self.COOKIE_FMT, self.COOKIE_MAGIC,
- os.getpid(), id(self), thread.get_ident())
-
- def get(self, timeout=None, block=True):
- """
- Return the next enqueued object, or sleep waiting for one.
-
- :param float timeout:
- If not :data:`None`, specifies a timeout in seconds.
-
- :param bool block:
- If :data:`False`, immediately raise
- :class:`mitogen.core.TimeoutError` if the latch is empty.
-
- :raises mitogen.core.LatchError:
- :meth:`close` has been called, and the object is no longer valid.
-
- :raises mitogen.core.TimeoutError:
- Timeout was reached.
-
- :returns:
- The de-queued object.
- """
- _vv and IOLOG.debug('%r.get(timeout=%r, block=%r)',
- self, timeout, block)
- self._lock.acquire()
- try:
- if self.closed:
- raise LatchError()
- i = len(self._sleeping)
- if len(self._queue) > i:
- _vv and IOLOG.debug('%r.get() -> %r', self, self._queue[i])
- return self._queue.pop(i)
- if not block:
- raise TimeoutError()
- rsock, wsock = self._get_socketpair()
- cookie = self._make_cookie()
- self._sleeping.append((wsock, cookie))
- finally:
- self._lock.release()
-
- poller = self.poller_class()
- poller.start_receive(rsock.fileno())
- try:
- return self._get_sleep(poller, timeout, block, rsock, wsock, cookie)
- finally:
- poller.close()
-
- def _get_sleep(self, poller, timeout, block, rsock, wsock, cookie):
- """
- When a result is not immediately available, sleep waiting for
- :meth:`put` to write a byte to our socket pair.
- """
- _vv and IOLOG.debug(
- '%r._get_sleep(timeout=%r, block=%r, rfd=%d, wfd=%d)',
- self, timeout, block, rsock.fileno(), wsock.fileno()
- )
-
- e = None
- woken = None
- try:
- woken = list(poller.poll(timeout))
- except Exception:
- e = sys.exc_info()[1]
-
- self._lock.acquire()
- try:
- i = self._sleeping.index((wsock, cookie))
- del self._sleeping[i]
- if not woken:
- raise e or TimeoutError()
-
- got_cookie = rsock.recv(self.COOKIE_SIZE)
- self._cls_idle_socketpairs.append((rsock, wsock))
-
- assert cookie == got_cookie, (
- "Cookie incorrect; got %r, expected %r" \
- % (binascii.hexlify(got_cookie),
- binascii.hexlify(cookie))
- )
- assert i < self._waking, (
- "Cookie correct, but no queue element assigned."
- )
- self._waking -= 1
- if self.closed:
- raise LatchError()
- _vv and IOLOG.debug('%r.get() wake -> %r', self, self._queue[i])
- return self._queue.pop(i)
- finally:
- self._lock.release()
-
- def put(self, obj=None):
- """
- Enqueue an object, waking the first thread waiting for a result, if one
- exists.
-
- :param obj:
- Object to enqueue. Defaults to :data:`None` as a convenience when
- using :class:`Latch` only for synchronization.
- :raises mitogen.core.LatchError:
- :meth:`close` has been called, and the object is no longer valid.
- """
- _vv and IOLOG.debug('%r.put(%r)', self, obj)
- self._lock.acquire()
- try:
- if self.closed:
- raise LatchError()
- self._queue.append(obj)
-
- if self._waking < len(self._sleeping):
- wsock, cookie = self._sleeping[self._waking]
- self._waking += 1
- _vv and IOLOG.debug('%r.put() -> waking wfd=%r',
- self, wsock.fileno())
- self._wake(wsock, cookie)
- elif self.notify:
- self.notify(self)
- finally:
- self._lock.release()
-
- def _wake(self, wsock, cookie):
- written, disconnected = io_op(os.write, wsock.fileno(), cookie)
- assert written == len(cookie) and not disconnected
-
- def __repr__(self):
- return 'Latch(%#x, size=%d, t=%r)' % (
- id(self),
- len(self._queue),
- threading.currentThread().getName(),
- )
-
-
-class Waker(BasicStream):
- """
- :class:`BasicStream` subclass implementing the `UNIX self-pipe trick`_.
- Used to wake the multiplexer when another thread needs to modify its state
- (via a cross-thread function call).
-
- .. _UNIX self-pipe trick: https://cr.yp.to/docs/selfpipe.html
- """
- broker_ident = None
-
- def __init__(self, broker):
- self._broker = broker
- self._lock = threading.Lock()
- self._deferred = []
-
- rfd, wfd = os.pipe()
- self.receive_side = Side(self, rfd)
- self.transmit_side = Side(self, wfd)
-
- def __repr__(self):
- return 'Waker(%r rfd=%r, wfd=%r)' % (
- self._broker,
- self.receive_side and self.receive_side.fd,
- self.transmit_side and self.transmit_side.fd,
- )
-
- @property
- def keep_alive(self):
- """
- Prevent immediate Broker shutdown while deferred functions remain.
- """
- self._lock.acquire()
- try:
- return len(self._deferred)
- finally:
- self._lock.release()
-
- def on_receive(self, broker):
- """
- Drain the pipe and fire callbacks. Since :attr:`_deferred` is
- synchronized, :meth:`defer` and :meth:`on_receive` can conspire to
- ensure only one byte needs to be pending regardless of queue length.
- """
- _vv and IOLOG.debug('%r.on_receive()', self)
- self._lock.acquire()
- try:
- self.receive_side.read(1)
- deferred = self._deferred
- self._deferred = []
- finally:
- self._lock.release()
-
- for func, args, kwargs in deferred:
- try:
- func(*args, **kwargs)
- except Exception:
- LOG.exception('defer() crashed: %r(*%r, **%r)',
- func, args, kwargs)
- self._broker.shutdown()
-
- def _wake(self):
- """
- Wake the multiplexer by writing a byte. If Broker is midway through
- teardown, the FD may already be closed, so ignore EBADF.
- """
- try:
- self.transmit_side.write(b(' '))
- except OSError:
- e = sys.exc_info()[1]
- if e.args[0] != errno.EBADF:
- raise
-
- broker_shutdown_msg = (
- "An attempt was made to enqueue a message with a Broker that has "
- "already exitted. It is likely your program called Broker.shutdown() "
- "too early."
- )
-
- def defer(self, func, *args, **kwargs):
- """
- Arrange for `func()` to execute on the broker thread. This function
- returns immediately without waiting the result of `func()`. Use
- :meth:`defer_sync` to block until a result is available.
-
- :raises mitogen.core.Error:
- :meth:`defer` was called after :class:`Broker` has begun shutdown.
- """
- if thread.get_ident() == self.broker_ident:
- _vv and IOLOG.debug('%r.defer() [immediate]', self)
- return func(*args, **kwargs)
- if self._broker._exitted:
- raise Error(self.broker_shutdown_msg)
-
- _vv and IOLOG.debug('%r.defer() [fd=%r]', self, self.transmit_side.fd)
- self._lock.acquire()
- try:
- if not self._deferred:
- self._wake()
- self._deferred.append((func, args, kwargs))
- finally:
- self._lock.release()
-
-
-class IoLogger(BasicStream):
- """
- :class:`BasicStream` subclass that sets up redirection of a standard
- UNIX file descriptor back into the Python :mod:`logging` package.
- """
- _buf = ''
-
- def __init__(self, broker, name, dest_fd):
- self._broker = broker
- self._name = name
- self._rsock, self._wsock = socket.socketpair()
- os.dup2(self._wsock.fileno(), dest_fd)
- set_cloexec(self._wsock.fileno())
-
- self._log = logging.getLogger(name)
- # #453: prevent accidental log initialization in a child creating a
- # feedback loop.
- self._log.propagate = False
- self._log.handlers = logging.getLogger().handlers[:]
-
- self.receive_side = Side(self, self._rsock.fileno())
- self.transmit_side = Side(self, dest_fd, cloexec=False, blocking=True)
- self._broker.start_receive(self)
-
- def __repr__(self):
- return '' % (self._name,)
-
- def _log_lines(self):
- while self._buf.find('\n') != -1:
- line, _, self._buf = str_partition(self._buf, '\n')
- self._log.info('%s', line.rstrip('\n'))
-
- def on_shutdown(self, broker):
- """Shut down the write end of the logging socket."""
- _v and LOG.debug('%r.on_shutdown()', self)
- if not IS_WSL:
- # #333: WSL generates invalid readiness indication on shutdown()
- self._wsock.shutdown(socket.SHUT_WR)
- self._wsock.close()
- self.transmit_side.close()
-
- def on_receive(self, broker):
- _vv and IOLOG.debug('%r.on_receive()', self)
- buf = self.receive_side.read()
- if not buf:
- return self.on_disconnect(broker)
-
- self._buf += buf.decode('latin1')
- self._log_lines()
-
-
-class Router(object):
- """
- Route messages between contexts, and invoke local handlers for messages
- addressed to this context. :meth:`Router.route() ` straddles the
- :class:`Broker` thread and user threads, it is safe to call anywhere.
-
- **Note:** This is the somewhat limited core version of the Router class
- used by child contexts. The master subclass is documented below this one.
- """
- context_class = Context
- max_message_size = 128 * 1048576
-
- #: When :data:`True`, permit children to only communicate with the current
- #: context or a parent of the current context. Routing between siblings or
- #: children of parents is prohibited, ensuring no communication is possible
- #: between intentionally partitioned networks, such as when a program
- #: simultaneously manipulates hosts spread across a corporate and a
- #: production network, or production networks that are otherwise
- #: air-gapped.
- #:
- #: Sending a prohibited message causes an error to be logged and a dead
- #: message to be sent in reply to the errant message, if that message has
- #: ``reply_to`` set.
- #:
- #: The value of :data:`unidirectional` becomes the default for the
- #: :meth:`local() ` `unidirectional`
- #: parameter.
- unidirectional = False
-
- def __init__(self, broker):
- self.broker = broker
- listen(broker, 'exit', self._on_broker_exit)
- self._setup_logging()
-
- self._write_lock = threading.Lock()
- #: context ID -> Stream; must hold _write_lock to edit or iterate
- self._stream_by_id = {}
- #: List of contexts to notify of shutdown; must hold _write_lock
- self._context_by_id = {}
- self._last_handle = itertools.count(1000)
- #: handle -> (persistent?, func(msg))
- self._handle_map = {}
- #: Context -> set { handle, .. }
- self._handles_by_respondent = {}
- self.add_handler(self._on_del_route, DEL_ROUTE)
-
- def __repr__(self):
- return 'Router(%r)' % (self.broker,)
-
- def _setup_logging(self):
- """
- This is done in the :class:`Router` constructor for historical reasons.
- It must be called before ExternalContext logs its first messages, but
- after logging has been setup. It must also be called when any router is
- constructed for a consumer app.
- """
- # Here seems as good a place as any.
- global _v, _vv
- _v = logging.getLogger().level <= logging.DEBUG
- _vv = IOLOG.level <= logging.DEBUG
-
- def _on_del_route(self, msg):
- """
- Stub :data:`DEL_ROUTE` handler; fires 'disconnect' events on the
- corresponding :attr:`_context_by_id` member. This is replaced by
- :class:`mitogen.parent.RouteMonitor` in an upgraded context.
- """
- LOG.error('%r._on_del_route() %r', self, msg)
- if msg.is_dead:
- return
-
- target_id_s, _, name = bytes_partition(msg.data, b(':'))
- target_id = int(target_id_s, 10)
- context = self._context_by_id.get(target_id)
- if context:
- fire(context, 'disconnect')
- else:
- LOG.debug('DEL_ROUTE for unknown ID %r: %r', target_id, msg)
-
- def _on_stream_disconnect(self, stream):
- notify = []
- self._write_lock.acquire()
- try:
- for context in list(self._context_by_id.values()):
- stream_ = self._stream_by_id.get(context.context_id)
- if stream_ is stream:
- del self._stream_by_id[context.context_id]
- notify.append(context)
- finally:
- self._write_lock.release()
-
- # Happens outside lock as e.g. RouteMonitor wants the same lock.
- for context in notify:
- context.on_disconnect()
-
- broker_exit_msg = 'Broker has exitted'
-
- def _on_broker_exit(self):
- while self._handle_map:
- _, (_, func, _, _) = self._handle_map.popitem()
- func(Message.dead(self.broker_exit_msg))
-
- def myself(self):
- """
- Return a :class:`Context` referring to the current process.
- """
- return self.context_class(
- router=self,
- context_id=mitogen.context_id,
- name='self',
- )
-
- def context_by_id(self, context_id, via_id=None, create=True, name=None):
- """
- Messy factory/lookup function to find a context by its ID, or construct
- it. This will eventually be replaced by a more sensible interface.
- """
- context = self._context_by_id.get(context_id)
- if context:
- return context
-
- if create and via_id is not None:
- via = self.context_by_id(via_id)
- else:
- via = None
-
- self._write_lock.acquire()
- try:
- context = self._context_by_id.get(context_id)
- if create and not context:
- context = self.context_class(self, context_id, name=name)
- context.via = via
- self._context_by_id[context_id] = context
- finally:
- self._write_lock.release()
-
- return context
-
- def register(self, context, stream):
- """
- Register a newly constructed context and its associated stream, and add
- the stream's receive side to the I/O multiplexer. This method remains
- public while the design has not yet settled.
- """
- _v and LOG.debug('register(%r, %r)', context, stream)
- self._write_lock.acquire()
- try:
- self._stream_by_id[context.context_id] = stream
- self._context_by_id[context.context_id] = context
- finally:
- self._write_lock.release()
-
- self.broker.start_receive(stream)
- listen(stream, 'disconnect', lambda: self._on_stream_disconnect(stream))
-
- def stream_by_id(self, dst_id):
- """
- Return the :class:`Stream` that should be used to communicate with
- `dst_id`. If a specific route for `dst_id` is not known, a reference to
- the parent context's stream is returned.
- """
- return (
- self._stream_by_id.get(dst_id) or
- self._stream_by_id.get(mitogen.parent_id)
- )
-
- def del_handler(self, handle):
- """
- Remove the handle registered for `handle`
-
- :raises KeyError:
- The handle wasn't registered.
- """
- _, _, _, respondent = self._handle_map.pop(handle)
- if respondent:
- self._handles_by_respondent[respondent].discard(handle)
-
- def add_handler(self, fn, handle=None, persist=True,
- policy=None, respondent=None,
- overwrite=False):
- """
- Invoke `fn(msg)` on the :class:`Broker` thread for each Message sent to
- `handle` from this context. Unregister after one invocation if
- `persist` is :data:`False`. If `handle` is :data:`None`, a new handle
- is allocated and returned.
-
- :param int handle:
- If not :data:`None`, an explicit handle to register, usually one of
- the ``mitogen.core.*`` constants. If unspecified, a new unused
- handle will be allocated.
-
- :param bool persist:
- If :data:`False`, the handler will be unregistered after a single
- message has been received.
-
- :param Context respondent:
- Context that messages to this handle are expected to be sent from.
- If specified, arranges for a dead message to be delivered to `fn`
- when disconnection of the context is detected.
-
- In future `respondent` will likely also be used to prevent other
- contexts from sending messages to the handle.
-
- :param function policy:
- Function invoked as `policy(msg, stream)` where `msg` is a
- :class:`mitogen.core.Message` about to be delivered, and `stream`
- is the :class:`mitogen.core.Stream` on which it was received. The
- function must return :data:`True`, otherwise an error is logged and
- delivery is refused.
-
- Two built-in policy functions exist:
-
- * :func:`has_parent_authority`: requires the message arrived from a
- parent context, or a context acting with a parent context's
- authority (``auth_id``).
-
- * :func:`mitogen.parent.is_immediate_child`: requires the
- message arrived from an immediately connected child, for use in
- messaging patterns where either something becomes buggy or
- insecure by permitting indirect upstream communication.
-
- In case of refusal, and the message's ``reply_to`` field is
- nonzero, a :class:`mitogen.core.CallError` is delivered to the
- sender indicating refusal occurred.
-
- :param bool overwrite:
- If :data:`True`, allow existing handles to be silently overwritten.
-
- :return:
- `handle`, or if `handle` was :data:`None`, the newly allocated
- handle.
- :raises Error:
- Attemp to register handle that was already registered.
- """
- handle = handle or next(self._last_handle)
- _vv and IOLOG.debug('%r.add_handler(%r, %r, %r)', self, fn, handle, persist)
- if handle in self._handle_map and not overwrite:
- raise Error(self.duplicate_handle_msg)
-
- self._handle_map[handle] = persist, fn, policy, respondent
- if respondent:
- if respondent not in self._handles_by_respondent:
- self._handles_by_respondent[respondent] = set()
- listen(respondent, 'disconnect',
- lambda: self._on_respondent_disconnect(respondent))
- self._handles_by_respondent[respondent].add(handle)
-
- return handle
-
- duplicate_handle_msg = 'cannot register a handle that is already exists'
- refused_msg = 'refused by policy'
- invalid_handle_msg = 'invalid handle'
- too_large_msg = 'message too large (max %d bytes)'
- respondent_disconnect_msg = 'the respondent Context has disconnected'
- broker_shutdown_msg = 'Broker is shutting down'
- no_route_msg = 'no route to %r, my ID is %r'
- unidirectional_msg = (
- 'routing mode prevents forward of message from context %d via '
- 'context %d'
- )
-
- def _on_respondent_disconnect(self, context):
- for handle in self._handles_by_respondent.pop(context, ()):
- _, fn, _, _ = self._handle_map[handle]
- fn(Message.dead(self.respondent_disconnect_msg))
- del self._handle_map[handle]
-
- def on_shutdown(self, broker):
- """Called during :meth:`Broker.shutdown`, informs callbacks registered
- with :meth:`add_handle_cb` the connection is dead."""
- _v and LOG.debug('%r.on_shutdown(%r)', self, broker)
- fire(self, 'shutdown')
- for handle, (persist, fn) in self._handle_map.iteritems():
- _v and LOG.debug('%r.on_shutdown(): killing %r: %r', self, handle, fn)
- fn(Message.dead(self.broker_shutdown_msg))
-
- def _maybe_send_dead(self, msg, reason, *args):
- if args:
- reason %= args
- LOG.debug('%r: %r is dead: %r', self, msg, reason)
- if msg.reply_to and not msg.is_dead:
- msg.reply(Message.dead(reason=reason), router=self)
-
- def _invoke(self, msg, stream):
- # IOLOG.debug('%r._invoke(%r)', self, msg)
- try:
- persist, fn, policy, respondent = self._handle_map[msg.handle]
- except KeyError:
- self._maybe_send_dead(msg, reason=self.invalid_handle_msg)
- return
-
- if respondent and not (msg.is_dead or
- msg.src_id == respondent.context_id):
- self._maybe_send_dead(msg, 'reply from unexpected context')
- return
-
- if policy and not policy(msg, stream):
- self._maybe_send_dead(msg, self.refused_msg)
- return
-
- if not persist:
- self.del_handler(msg.handle)
-
- try:
- fn(msg)
- except Exception:
- LOG.exception('%r._invoke(%r): %r crashed', self, msg, fn)
-
- def _async_route(self, msg, in_stream=None):
- """
- Arrange for `msg` to be forwarded towards its destination. If its
- destination is the local context, then arrange for it to be dispatched
- using the local handlers.
-
- This is a lower overhead version of :meth:`route` that may only be
- called from the :class:`Broker` thread.
-
- :param Stream in_stream:
- If not :data:`None`, the stream the message arrived on. Used for
- performing source route verification, to ensure sensitive messages
- such as ``CALL_FUNCTION`` arrive only from trusted contexts.
- """
- _vv and IOLOG.debug('%r._async_route(%r, %r)', self, msg, in_stream)
-
- if len(msg.data) > self.max_message_size:
- self._maybe_send_dead(msg, self.too_large_msg % (
- self.max_message_size,
- ))
- return
-
- # Perform source verification.
- if in_stream:
- parent = self._stream_by_id.get(mitogen.parent_id)
- expect = self._stream_by_id.get(msg.auth_id, parent)
- if in_stream != expect:
- LOG.error('%r: bad auth_id: got %r via %r, not %r: %r',
- self, msg.auth_id, in_stream, expect, msg)
- return
-
- if msg.src_id != msg.auth_id:
- expect = self._stream_by_id.get(msg.src_id, parent)
- if in_stream != expect:
- LOG.error('%r: bad src_id: got %r via %r, not %r: %r',
- self, msg.src_id, in_stream, expect, msg)
- return
-
- if in_stream.auth_id is not None:
- msg.auth_id = in_stream.auth_id
-
- # Maintain a set of IDs the source ever communicated with.
- in_stream.egress_ids.add(msg.dst_id)
-
- if msg.dst_id == mitogen.context_id:
- return self._invoke(msg, in_stream)
-
- out_stream = self._stream_by_id.get(msg.dst_id)
- if out_stream is None:
- out_stream = self._stream_by_id.get(mitogen.parent_id)
-
- if out_stream is None:
- self._maybe_send_dead(msg, self.no_route_msg,
- msg.dst_id, mitogen.context_id)
- return
-
- if in_stream and self.unidirectional and not \
- (in_stream.is_privileged or out_stream.is_privileged):
- self._maybe_send_dead(msg, self.unidirectional_msg,
- in_stream.remote_id, out_stream.remote_id)
- return
-
- out_stream._send(msg)
-
- def route(self, msg):
- """
- Arrange for the :class:`Message` `msg` to be delivered to its
- destination using any relevant downstream context, or if none is found,
- by forwarding the message upstream towards the master context. If `msg`
- is destined for the local context, it is dispatched using the handles
- registered with :meth:`add_handler`.
-
- This may be called from any thread.
- """
- self.broker.defer(self._async_route, msg)
-
-
-class Broker(object):
- """
- Responsible for handling I/O multiplexing in a private thread.
-
- **Note:** This is the somewhat limited core version of the Broker class
- used by child contexts. The master subclass is documented below.
- """
- poller_class = Poller
- _waker = None
- _thread = None
-
- #: Seconds grace to allow :class:`streams ` to shutdown gracefully
- #: before force-disconnecting them during :meth:`shutdown`.
- shutdown_timeout = 3.0
-
- def __init__(self, poller_class=None, activate_compat=True):
- self._alive = True
- self._exitted = False
- self._waker = Waker(self)
- #: Arrange for `func(\*args, \**kwargs)` to be executed on the broker
- #: thread, or immediately if the current thread is the broker thread.
- #: Safe to call from any thread.
- self.defer = self._waker.defer
- self.poller = self.poller_class()
- self.poller.start_receive(
- self._waker.receive_side.fd,
- (self._waker.receive_side, self._waker.on_receive)
- )
- self._thread = threading.Thread(
- target=self._broker_main,
- name='mitogen.broker'
- )
- self._thread.start()
- if activate_compat:
- self._py24_25_compat()
-
- def _py24_25_compat(self):
- """
- Python 2.4/2.5 have grave difficulties with threads/fork. We
- mandatorily quiesce all running threads during fork using a
- monkey-patch there.
- """
- if sys.version_info < (2, 6):
- # import_module() is used to avoid dep scanner.
- os_fork = import_module('mitogen.os_fork')
- mitogen.os_fork._notice_broker_or_pool(self)
-
- def start_receive(self, stream):
- """
- Mark the :attr:`receive_side ` on `stream` as
- ready for reading. Safe to call from any thread. When the associated
- file descriptor becomes ready for reading,
- :meth:`BasicStream.on_receive` will be called.
- """
- _vv and IOLOG.debug('%r.start_receive(%r)', self, stream)
- side = stream.receive_side
- assert side and side.fd is not None
- self.defer(self.poller.start_receive,
- side.fd, (side, stream.on_receive))
-
- def stop_receive(self, stream):
- """
- Mark the :attr:`receive_side ` on `stream` as not
- ready for reading. Safe to call from any thread.
- """
- _vv and IOLOG.debug('%r.stop_receive(%r)', self, stream)
- self.defer(self.poller.stop_receive, stream.receive_side.fd)
-
- def _start_transmit(self, stream):
- """
- Mark the :attr:`transmit_side ` on `stream` as
- ready for writing. Must only be called from the Broker thread. When the
- associated file descriptor becomes ready for writing,
- :meth:`BasicStream.on_transmit` will be called.
- """
- _vv and IOLOG.debug('%r._start_transmit(%r)', self, stream)
- side = stream.transmit_side
- assert side and side.fd is not None
- self.poller.start_transmit(side.fd, (side, stream.on_transmit))
-
- def _stop_transmit(self, stream):
- """
- Mark the :attr:`transmit_side ` on `stream` as not
- ready for writing.
- """
- _vv and IOLOG.debug('%r._stop_transmit(%r)', self, stream)
- self.poller.stop_transmit(stream.transmit_side.fd)
-
- def keep_alive(self):
- """
- Return :data:`True` if any reader's :attr:`Side.keep_alive` attribute
- is :data:`True`, or any :class:`Context` is still registered that is
- not the master. Used to delay shutdown while some important work is in
- progress (e.g. log draining).
- """
- it = (side.keep_alive for (_, (side, _)) in self.poller.readers)
- return sum(it, 0)
-
- def defer_sync(self, func):
- """
- Arrange for `func()` to execute on :class:`Broker` thread, blocking the
- current thread until a result or exception is available.
-
- :returns:
- Return value of `func()`.
- """
- latch = Latch()
- def wrapper():
- try:
- latch.put(func())
- except Exception:
- latch.put(sys.exc_info()[1])
- self.defer(wrapper)
- res = latch.get()
- if isinstance(res, Exception):
- raise res
- return res
-
- def _call(self, stream, func):
- """
- Call `func(self)`, catching any exception that might occur, logging it,
- and force-disconnecting the related `stream`.
- """
- try:
- func(self)
- except Exception:
- LOG.exception('%r crashed', stream)
- stream.on_disconnect(self)
-
- def _loop_once(self, timeout=None):
- """
- Execute a single :class:`Poller` wait, dispatching any IO events that
- caused the wait to complete.
-
- :param float timeout:
- If not :data:`None`, maximum time in seconds to wait for events.
- """
- _vv and IOLOG.debug('%r._loop_once(%r, %r)',
- self, timeout, self.poller)
- #IOLOG.debug('readers =\n%s', pformat(self.poller.readers))
- #IOLOG.debug('writers =\n%s', pformat(self.poller.writers))
- for side, func in self.poller.poll(timeout):
- self._call(side.stream, func)
-
- def _broker_exit(self):
- """
- Forcefully call :meth:`Stream.on_disconnect` on any streams that failed
- to shut down gracefully, then discard the :class:`Poller`.
- """
- for _, (side, _) in self.poller.readers + self.poller.writers:
- LOG.debug('_broker_main() force disconnecting %r', side)
- side.stream.on_disconnect(self)
-
- self.poller.close()
-
- def _broker_shutdown(self):
- """
- Invoke :meth:`Stream.on_shutdown` for every active stream, then allow
- up to :attr:`shutdown_timeout` seconds for the streams to unregister
- themselves, logging an error if any did not unregister during the grace
- period.
- """
- for _, (side, _) in self.poller.readers + self.poller.writers:
- self._call(side.stream, side.stream.on_shutdown)
-
- deadline = time.time() + self.shutdown_timeout
- while self.keep_alive() and time.time() < deadline:
- self._loop_once(max(0, deadline - time.time()))
-
- if self.keep_alive():
- LOG.error('%r: some streams did not close gracefully. '
- 'The most likely cause for this is one or '
- 'more child processes still connected to '
- 'our stdout/stderr pipes.', self)
-
- def _do_broker_main(self):
- """
- Broker thread main function. Dispatches IO events until
- :meth:`shutdown` is called.
- """
- # For Python 2.4, no way to retrieve ident except on thread.
- self._waker.broker_ident = thread.get_ident()
- try:
- while self._alive:
- self._loop_once()
-
- fire(self, 'shutdown')
- self._broker_shutdown()
- except Exception:
- LOG.exception('_broker_main() crashed')
-
- self._alive = False # Ensure _alive is consistent on crash.
- self._exitted = True
- self._broker_exit()
-
- def _broker_main(self):
- _profile_hook('mitogen.broker', self._do_broker_main)
- fire(self, 'exit')
-
- def shutdown(self):
- """
- Request broker gracefully disconnect streams and stop. Safe to call
- from any thread.
- """
- _v and LOG.debug('%r.shutdown()', self)
- def _shutdown():
- self._alive = False
- if self._alive and not self._exitted:
- self.defer(_shutdown)
-
- def join(self):
- """
- Wait for the broker to stop, expected to be called after
- :meth:`shutdown`.
- """
- self._thread.join()
-
- def __repr__(self):
- return 'Broker(%#x)' % (id(self),)
-
-
-class Dispatcher(object):
- """
- Implementation of the :data:`CALL_FUNCTION` handle for a child context.
- Listens on the child's main thread for messages sent by
- :class:`mitogen.parent.CallChain` and dispatches the function calls they
- describe.
-
- If a :class:`mitogen.parent.CallChain` sending a message is in pipelined
- mode, any exception that occurs is recorded, and causes all subsequent
- calls with the same `chain_id` to fail with the same exception.
- """
- def __init__(self, econtext):
- self.econtext = econtext
- #: Chain ID -> CallError if prior call failed.
- self._error_by_chain_id = {}
- self.recv = Receiver(router=econtext.router,
- handle=CALL_FUNCTION,
- policy=has_parent_authority)
- listen(econtext.broker, 'shutdown', self.recv.close)
-
- @classmethod
- @takes_econtext
- def forget_chain(cls, chain_id, econtext):
- econtext.dispatcher._error_by_chain_id.pop(chain_id, None)
-
- def _parse_request(self, msg):
- data = msg.unpickle(throw=False)
- _v and LOG.debug('_dispatch_one(%r)', data)
-
- chain_id, modname, klass, func, args, kwargs = data
- obj = import_module(modname)
- if klass:
- obj = getattr(obj, klass)
- fn = getattr(obj, func)
- if getattr(fn, 'mitogen_takes_econtext', None):
- kwargs.setdefault('econtext', self.econtext)
- if getattr(fn, 'mitogen_takes_router', None):
- kwargs.setdefault('router', self.econtext.router)
-
- return chain_id, fn, args, kwargs
-
- def _dispatch_one(self, msg):
- try:
- chain_id, fn, args, kwargs = self._parse_request(msg)
- except Exception:
- return None, CallError(sys.exc_info()[1])
-
- if chain_id in self._error_by_chain_id:
- return chain_id, self._error_by_chain_id[chain_id]
-
- try:
- return chain_id, fn(*args, **kwargs)
- except Exception:
- e = CallError(sys.exc_info()[1])
- if chain_id is not None:
- self._error_by_chain_id[chain_id] = e
- return chain_id, e
-
- def _dispatch_calls(self):
- for msg in self.recv:
- chain_id, ret = self._dispatch_one(msg)
- _v and LOG.debug('_dispatch_calls: %r -> %r', msg, ret)
- if msg.reply_to:
- msg.reply(ret)
- elif isinstance(ret, CallError) and chain_id is None:
- LOG.error('No-reply function call failed: %s', ret)
-
- def run(self):
- if self.econtext.config.get('on_start'):
- self.econtext.config['on_start'](self.econtext)
-
- _profile_hook('mitogen.child_main', self._dispatch_calls)
-
-
-class ExternalContext(object):
- """
- External context implementation.
-
- .. attribute:: broker
- The :class:`mitogen.core.Broker` instance.
-
- .. attribute:: context
- The :class:`mitogen.core.Context` instance.
-
- .. attribute:: channel
- The :class:`mitogen.core.Channel` over which :data:`CALL_FUNCTION`
- requests are received.
-
- .. attribute:: stdout_log
- The :class:`mitogen.core.IoLogger` connected to ``stdout``.
-
- .. attribute:: importer
- The :class:`mitogen.core.Importer` instance.
-
- .. attribute:: stdout_log
- The :class:`IoLogger` connected to ``stdout``.
-
- .. attribute:: stderr_log
- The :class:`IoLogger` connected to ``stderr``.
-
- .. method:: _dispatch_calls
- Implementation for the main thread in every child context.
- """
- detached = False
-
- def __init__(self, config):
- self.config = config
-
- def _on_broker_exit(self):
- if not self.config['profiling']:
- os.kill(os.getpid(), signal.SIGTERM)
-
- #: On Python >3.4, the global importer lock has been sharded into a
- #: per-module lock, meaning there is no guarantee the import statement in
- #: service_stub_main will be truly complete before a second thread
- #: attempting the same import will see a partially initialized module.
- #: Sigh. Therefore serialize execution of the stub itself.
- service_stub_lock = threading.Lock()
-
- def _service_stub_main(self, msg):
- self.service_stub_lock.acquire()
- try:
- import mitogen.service
- pool = mitogen.service.get_or_create_pool(router=self.router)
- pool._receiver._on_receive(msg)
- finally:
- self.service_stub_lock.release()
-
- def _on_call_service_msg(self, msg):
- """
- Stub service handler. Start a thread to import the mitogen.service
- implementation from, and deliver the message to the newly constructed
- pool. This must be done as CALL_SERVICE for e.g. PushFileService may
- race with a CALL_FUNCTION blocking the main thread waiting for a result
- from that service.
- """
- if not msg.is_dead:
- th = threading.Thread(target=self._service_stub_main, args=(msg,))
- th.start()
-
- def _on_shutdown_msg(self, msg):
- _v and LOG.debug('_on_shutdown_msg(%r)', msg)
- if not msg.is_dead:
- self.broker.shutdown()
-
- def _on_parent_disconnect(self):
- if self.detached:
- mitogen.parent_ids = []
- mitogen.parent_id = None
- LOG.info('Detachment complete')
- else:
- _v and LOG.debug('%r: parent stream is gone, dying.', self)
- self.broker.shutdown()
-
- def detach(self):
- self.detached = True
- stream = self.router.stream_by_id(mitogen.parent_id)
- if stream: # not double-detach()'d
- os.setsid()
- self.parent.send_await(Message(handle=DETACHING))
- LOG.info('Detaching from %r; parent is %s', stream, self.parent)
- for x in range(20):
- pending = self.broker.defer_sync(lambda: stream.pending_bytes())
- if not pending:
- break
- time.sleep(0.05)
- if pending:
- LOG.error('Stream had %d bytes after 2000ms', pending)
- self.broker.defer(stream.on_disconnect, self.broker)
-
- def _setup_master(self):
- Router.max_message_size = self.config['max_message_size']
- if self.config['profiling']:
- enable_profiling()
- self.broker = Broker(activate_compat=False)
- self.router = Router(self.broker)
- self.router.debug = self.config.get('debug', False)
- self.router.undirectional = self.config['unidirectional']
- self.router.add_handler(
- fn=self._on_shutdown_msg,
- handle=SHUTDOWN,
- policy=has_parent_authority,
- )
- self.router.add_handler(
- fn=self._on_call_service_msg,
- handle=CALL_SERVICE,
- policy=has_parent_authority,
- )
- self.master = Context(self.router, 0, 'master')
- parent_id = self.config['parent_ids'][0]
- if parent_id == 0:
- self.parent = self.master
- else:
- self.parent = Context(self.router, parent_id, 'parent')
-
- in_fd = self.config.get('in_fd', 100)
- out_fd = self.config.get('out_fd', 1)
- self.stream = Stream(self.router, parent_id)
- self.stream.name = 'parent'
- self.stream.accept(in_fd, out_fd)
- self.stream.receive_side.keep_alive = False
-
- listen(self.stream, 'disconnect', self._on_parent_disconnect)
- listen(self.broker, 'exit', self._on_broker_exit)
-
- os.close(in_fd)
-
- def _reap_first_stage(self):
- try:
- os.wait() # Reap first stage.
- except OSError:
- pass # No first stage exists (e.g. fakessh)
-
- def _setup_logging(self):
- self.log_handler = LogHandler(self.master)
- root = logging.getLogger()
- root.setLevel(self.config['log_level'])
- root.handlers = [self.log_handler]
- if self.config['debug']:
- enable_debug_logging()
-
- def _setup_importer(self):
- importer = self.config.get('importer')
- if importer:
- importer._install_handler(self.router)
- importer._context = self.parent
- else:
- core_src_fd = self.config.get('core_src_fd', 101)
- if core_src_fd:
- fp = os.fdopen(core_src_fd, 'rb', 1)
- try:
- core_src = fp.read()
- # Strip "ExternalContext.main()" call from last line.
- core_src = b('\n').join(core_src.splitlines()[:-1])
- finally:
- fp.close()
- else:
- core_src = None
-
- importer = Importer(
- self.router,
- self.parent,
- core_src,
- self.config.get('whitelist', ()),
- self.config.get('blacklist', ()),
- )
-
- self.importer = importer
- self.router.importer = importer
- sys.meta_path.insert(0, self.importer)
-
- def _setup_package(self):
- global mitogen
- mitogen = imp.new_module('mitogen')
- mitogen.__package__ = 'mitogen'
- mitogen.__path__ = []
- mitogen.__loader__ = self.importer
- mitogen.main = lambda *args, **kwargs: (lambda func: None)
- mitogen.core = sys.modules['__main__']
- mitogen.core.__file__ = 'x/mitogen/core.py' # For inspect.getsource()
- mitogen.core.__loader__ = self.importer
- sys.modules['mitogen'] = mitogen
- sys.modules['mitogen.core'] = mitogen.core
- del sys.modules['__main__']
-
- def _setup_globals(self):
- mitogen.is_master = False
- mitogen.__version__ = self.config['version']
- mitogen.context_id = self.config['context_id']
- mitogen.parent_ids = self.config['parent_ids'][:]
- mitogen.parent_id = mitogen.parent_ids[0]
-
- def _nullify_stdio(self):
- """
- Open /dev/null to replace stdin, and stdout/stderr temporarily. In case
- of odd startup, assume we may be allocated a standard handle.
- """
- fd = os.open('/dev/null', os.O_RDWR)
- try:
- for stdfd in (0, 1, 2):
- if fd != stdfd:
- os.dup2(fd, stdfd)
- finally:
- if fd not in (0, 1, 2):
- os.close(fd)
-
- def _setup_stdio(self):
- # #481: when stderr is a TTY due to being started via
- # tty_create_child()/hybrid_tty_create_child(), and some privilege
- # escalation tool like prehistoric versions of sudo exec this process
- # over the top of itself, there is nothing left to keep the slave PTY
- # open after we replace our stdio. Therefore if stderr is a TTY, keep
- # around a permanent dup() to avoid receiving SIGHUP.
- try:
- if os.isatty(2):
- self.reserve_tty_fd = os.dup(2)
- set_cloexec(self.reserve_tty_fd)
- except OSError:
- pass
- # When sys.stdout was opened by the runtime, overwriting it will not
- # close FD 1. However when forking from a child that previously used
- # fdopen(), overwriting it /will/ close FD 1. So we must swallow the
- # close before IoLogger overwrites FD 1, otherwise its new FD 1 will be
- # clobbered. Additionally, stdout must be replaced with /dev/null prior
- # to stdout.close(), since if block buffering was active in the parent,
- # any pre-fork buffered data will be flushed on close(), corrupting the
- # connection to the parent.
- self._nullify_stdio()
- sys.stdout.close()
- self._nullify_stdio()
-
- self.stdout_log = IoLogger(self.broker, 'stdout', 1)
- self.stderr_log = IoLogger(self.broker, 'stderr', 2)
- # Reopen with line buffering.
- sys.stdout = os.fdopen(1, 'w', 1)
-
- def main(self):
- self._setup_master()
- try:
- try:
- self._setup_logging()
- self._setup_importer()
- self._reap_first_stage()
- if self.config.get('setup_package', True):
- self._setup_package()
- self._setup_globals()
- if self.config.get('setup_stdio', True):
- self._setup_stdio()
-
- self.dispatcher = Dispatcher(self)
- self.router.register(self.parent, self.stream)
- self.router._setup_logging()
- self.log_handler.uncork()
-
- sys.executable = os.environ.pop('ARGV0', sys.executable)
- _v and LOG.debug('Connected to context %s; my ID is %r',
- self.parent, mitogen.context_id)
- _v and LOG.debug('pid:%r ppid:%r uid:%r/%r, gid:%r/%r host:%r',
- os.getpid(), os.getppid(), os.geteuid(),
- os.getuid(), os.getegid(), os.getgid(),
- socket.gethostname())
- _v and LOG.debug('Recovered sys.executable: %r', sys.executable)
-
- self.broker._py24_25_compat()
- self.dispatcher.run()
- _v and LOG.debug('ExternalContext.main() normal exit')
- except KeyboardInterrupt:
- LOG.debug('KeyboardInterrupt received, exiting gracefully.')
- except BaseException:
- LOG.exception('ExternalContext.main() crashed')
- raise
- finally:
- self.broker.shutdown()
- self.broker.join()
diff --git a/mitogen-0.2.7/mitogen/debug.py b/mitogen-0.2.7/mitogen/debug.py
deleted file mode 100644
index 3d13347..000000000
--- a/mitogen-0.2.7/mitogen/debug.py
+++ /dev/null
@@ -1,236 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-"""
-Basic signal handler for dumping thread stacks.
-"""
-
-import difflib
-import logging
-import os
-import gc
-import signal
-import sys
-import threading
-import time
-import traceback
-
-import mitogen.core
-import mitogen.parent
-
-
-LOG = logging.getLogger(__name__)
-_last = None
-
-
-def enable_evil_interrupts():
- signal.signal(signal.SIGALRM, (lambda a, b: None))
- signal.setitimer(signal.ITIMER_REAL, 0.01, 0.01)
-
-
-def disable_evil_interrupts():
- signal.setitimer(signal.ITIMER_REAL, 0, 0)
-
-
-def _hex(n):
- return '%08x' % n
-
-
-def get_subclasses(klass):
- """
- Rather than statically import every interesting subclass, forcing it all to
- be transferred and potentially disrupting the debugged environment,
- enumerate only those loaded in memory. Also returns the original class.
- """
- stack = [klass]
- seen = set()
- while stack:
- klass = stack.pop()
- seen.add(klass)
- stack.extend(klass.__subclasses__())
- return seen
-
-
-def get_routers():
- return dict(
- (_hex(id(router)), router)
- for klass in get_subclasses(mitogen.core.Router)
- for router in gc.get_referrers(klass)
- if isinstance(router, mitogen.core.Router)
- )
-
-
-def get_router_info():
- return {
- 'routers': dict(
- (id_, {
- 'id': id_,
- 'streams': len(set(router._stream_by_id.values())),
- 'contexts': len(set(router._context_by_id.values())),
- 'handles': len(router._handle_map),
- })
- for id_, router in get_routers().items()
- )
- }
-
-
-def get_stream_info(router_id):
- router = get_routers().get(router_id)
- return {
- 'streams': dict(
- (_hex(id(stream)), ({
- 'name': stream.name,
- 'remote_id': stream.remote_id,
- 'sent_module_count': len(getattr(stream, 'sent_modules', [])),
- 'routes': sorted(getattr(stream, 'routes', [])),
- 'type': type(stream).__module__,
- }))
- for via_id, stream in router._stream_by_id.items()
- )
- }
-
-
-def format_stacks():
- name_by_id = dict(
- (t.ident, t.name)
- for t in threading.enumerate()
- )
-
- l = ['', '']
- for threadId, stack in sys._current_frames().items():
- l += ["# PID %d ThreadID: (%s) %s; %r" % (
- os.getpid(),
- name_by_id.get(threadId, ''),
- threadId,
- stack,
- )]
- #stack = stack.f_back.f_back
-
- for filename, lineno, name, line in traceback.extract_stack(stack):
- l += [
- 'File: "%s", line %d, in %s' % (
- filename,
- lineno,
- name
- )
- ]
- if line:
- l += [' ' + line.strip()]
- l += ['']
-
- l += ['', '']
- return '\n'.join(l)
-
-
-def get_snapshot():
- global _last
-
- s = format_stacks()
- snap = s
- if _last:
- snap += '\n'
- diff = list(difflib.unified_diff(
- a=_last.splitlines(),
- b=s.splitlines(),
- fromfile='then',
- tofile='now'
- ))
-
- if diff:
- snap += '\n'.join(diff) + '\n'
- else:
- snap += '(no change since last time)\n'
- _last = s
- return snap
-
-
-def _handler(*_):
- fp = open('/dev/tty', 'w', 1)
- fp.write(get_snapshot())
- fp.close()
-
-
-def install_handler():
- signal.signal(signal.SIGUSR2, _handler)
-
-
-def _logging_main(secs):
- while True:
- time.sleep(secs)
- LOG.info('PERIODIC THREAD DUMP\n\n%s', get_snapshot())
-
-
-def dump_to_logger(secs=5):
- th = threading.Thread(
- target=_logging_main,
- kwargs={'secs': secs},
- name='mitogen.debug.dump_to_logger',
- )
- th.setDaemon(True)
- th.start()
-
-
-class ContextDebugger(object):
- @classmethod
- @mitogen.core.takes_econtext
- def _configure_context(cls, econtext):
- mitogen.parent.upgrade_router(econtext)
- econtext.debugger = cls(econtext.router)
-
- def __init__(self, router):
- self.router = router
- self.router.add_handler(
- func=self._on_debug_msg,
- handle=mitogen.core.DEBUG,
- persist=True,
- policy=mitogen.core.has_parent_authority,
- )
- mitogen.core.listen(router, 'register', self._on_stream_register)
- LOG.debug('Context debugging configured.')
-
- def _on_stream_register(self, context, stream):
- LOG.debug('_on_stream_register: sending configure() to %r', stream)
- context.call_async(ContextDebugger._configure_context)
-
- def _on_debug_msg(self, msg):
- if msg != mitogen.core._DEAD:
- threading.Thread(
- target=self._handle_debug_msg,
- name='ContextDebuggerHandler',
- args=(msg,)
- ).start()
-
- def _handle_debug_msg(self, msg):
- try:
- method, args, kwargs = msg.unpickle()
- msg.reply(getattr(cls, method)(*args, **kwargs))
- except Exception:
- e = sys.exc_info()[1]
- msg.reply(mitogen.core.CallError(e))
diff --git a/mitogen-0.2.7/mitogen/doas.py b/mitogen-0.2.7/mitogen/doas.py
deleted file mode 100644
index 1b687fb..000000000
--- a/mitogen-0.2.7/mitogen/doas.py
+++ /dev/null
@@ -1,113 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-import logging
-
-import mitogen.core
-import mitogen.parent
-from mitogen.core import b
-
-
-LOG = logging.getLogger(__name__)
-
-
-class PasswordError(mitogen.core.StreamError):
- pass
-
-
-class Stream(mitogen.parent.Stream):
- create_child = staticmethod(mitogen.parent.hybrid_tty_create_child)
- child_is_immediate_subprocess = False
-
- username = 'root'
- password = None
- doas_path = 'doas'
- password_prompt = b('Password:')
- incorrect_prompts = (
- b('doas: authentication failed'),
- )
-
- def construct(self, username=None, password=None, doas_path=None,
- password_prompt=None, incorrect_prompts=None, **kwargs):
- super(Stream, self).construct(**kwargs)
- if username is not None:
- self.username = username
- if password is not None:
- self.password = password
- if doas_path is not None:
- self.doas_path = doas_path
- if password_prompt is not None:
- self.password_prompt = password_prompt.lower()
- if incorrect_prompts is not None:
- self.incorrect_prompts = map(str.lower, incorrect_prompts)
-
- def _get_name(self):
- return u'doas.' + mitogen.core.to_text(self.username)
-
- def get_boot_command(self):
- bits = [self.doas_path, '-u', self.username, '--']
- bits = bits + super(Stream, self).get_boot_command()
- LOG.debug('doas command line: %r', bits)
- return bits
-
- password_incorrect_msg = 'doas password is incorrect'
- password_required_msg = 'doas password is required'
-
- def _connect_input_loop(self, it):
- password_sent = False
- for buf in it:
- LOG.debug('%r: received %r', self, buf)
- if buf.endswith(self.EC0_MARKER):
- self._ec0_received()
- return
- if any(s in buf.lower() for s in self.incorrect_prompts):
- if password_sent:
- raise PasswordError(self.password_incorrect_msg)
- elif self.password_prompt in buf.lower():
- if self.password is None:
- raise PasswordError(self.password_required_msg)
- if password_sent:
- raise PasswordError(self.password_incorrect_msg)
- LOG.debug('sending password')
- self.diag_stream.transmit_side.write(
- mitogen.core.to_text(self.password + '\n').encode('utf-8')
- )
- password_sent = True
- raise mitogen.core.StreamError('bootstrap failed')
-
- def _connect_bootstrap(self):
- it = mitogen.parent.iter_read(
- fds=[self.receive_side.fd, self.diag_stream.receive_side.fd],
- deadline=self.connect_deadline,
- )
- try:
- self._connect_input_loop(it)
- finally:
- it.close()
diff --git a/mitogen-0.2.7/mitogen/docker.py b/mitogen-0.2.7/mitogen/docker.py
deleted file mode 100644
index 0c0d40e..000000000
--- a/mitogen-0.2.7/mitogen/docker.py
+++ /dev/null
@@ -1,81 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-import logging
-
-import mitogen.core
-import mitogen.parent
-
-
-LOG = logging.getLogger(__name__)
-
-
-class Stream(mitogen.parent.Stream):
- child_is_immediate_subprocess = False
-
- container = None
- image = None
- username = None
- docker_path = 'docker'
-
- # TODO: better way of capturing errors such as "No such container."
- create_child_args = {
- 'merge_stdio': True
- }
-
- def construct(self, container=None, image=None,
- docker_path=None, username=None,
- **kwargs):
- assert container or image
- super(Stream, self).construct(**kwargs)
- if container:
- self.container = container
- if image:
- self.image = image
- if docker_path:
- self.docker_path = docker_path
- if username:
- self.username = username
-
- def _get_name(self):
- return u'docker.' + (self.container or self.image)
-
- def get_boot_command(self):
- args = ['--interactive']
- if self.username:
- args += ['--user=' + self.username]
-
- bits = [self.docker_path]
- if self.container:
- bits += ['exec'] + args + [self.container]
- elif self.image:
- bits += ['run'] + args + ['--rm', self.image]
-
- return bits + super(Stream, self).get_boot_command()
diff --git a/mitogen-0.2.7/mitogen/fakessh.py b/mitogen-0.2.7/mitogen/fakessh.py
deleted file mode 100644
index d39a710..000000000
--- a/mitogen-0.2.7/mitogen/fakessh.py
+++ /dev/null
@@ -1,461 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-"""
-:mod:`mitogen.fakessh` is a stream implementation that starts a subprocess with
-its environment modified such that ``PATH`` searches for `ssh` return a Mitogen
-implementation of SSH. When invoked, this implementation arranges for the
-command line supplied by the caller to be executed in a remote context, reusing
-the parent context's (possibly proxied) connection to that remote context.
-
-This allows tools like `rsync` and `scp` to transparently reuse the connections
-and tunnels already established by the host program to connect to a target
-machine, without wasteful redundant SSH connection setup, 3-way handshakes, or
-firewall hopping configurations, and enables these tools to be used in
-impossible scenarios, such as over `sudo` with ``requiretty`` enabled.
-
-The fake `ssh` command source is written to a temporary file on disk, and
-consists of a copy of the :py:mod:`mitogen.core` source code (just like any
-other child context), with a line appended to cause it to connect back to the
-host process over an FD it inherits. As there is no reliance on an existing
-filesystem file, it is possible for child contexts to use fakessh.
-
-As a consequence of connecting back through an inherited FD, only one SSH
-invocation is possible, which is fine for tools like `rsync`, however in future
-this restriction will be lifted.
-
-Sequence:
-
- 1. ``fakessh`` Context and Stream created by parent context. The stream's
- buffer has a :py:func:`_fakessh_main` :py:data:`CALL_FUNCTION
- ` enqueued.
- 2. Target program (`rsync/scp/sftp`) invoked, which internally executes
- `ssh` from ``PATH``.
- 3. :py:mod:`mitogen.core` bootstrap begins, recovers the stream FD
- inherited via the target program, established itself as the fakessh
- context.
- 4. :py:func:`_fakessh_main` :py:data:`CALL_FUNCTION
- ` is read by fakessh context,
-
- a. sets up :py:class:`IoPump` for stdio, registers
- stdin_handle for local context.
- b. Enqueues :py:data:`CALL_FUNCTION ` for
- :py:func:`_start_slave` invoked in target context,
-
- i. the program from the `ssh` command line is started
- ii. sets up :py:class:`IoPump` for `ssh` command line process's
- stdio pipes
- iii. returns `(control_handle, stdin_handle)` to
- :py:func:`_fakessh_main`
-
- 5. :py:func:`_fakessh_main` receives control/stdin handles from from
- :py:func:`_start_slave`,
-
- a. registers remote's stdin_handle with local :py:class:`IoPump`.
- b. sends `("start", local_stdin_handle)` to remote's control_handle
- c. registers local :py:class:`IoPump` with
- :py:class:`mitogen.core.Broker`.
- d. loops waiting for `local stdout closed && remote stdout closed`
-
- 6. :py:func:`_start_slave` control channel receives `("start", stdin_handle)`,
-
- a. registers remote's stdin_handle with local :py:class:`IoPump`
- b. registers local :py:class:`IoPump` with
- :py:class:`mitogen.core.Broker`.
- c. loops waiting for `local stdout closed && remote stdout closed`
-"""
-
-import getopt
-import inspect
-import os
-import shutil
-import socket
-import subprocess
-import sys
-import tempfile
-import threading
-
-import mitogen.core
-import mitogen.master
-import mitogen.parent
-
-from mitogen.core import LOG, IOLOG
-
-
-SSH_GETOPTS = (
- "1246ab:c:e:fgi:kl:m:no:p:qstvx"
- "ACD:E:F:I:KL:MNO:PQ:R:S:TVw:W:XYy"
-)
-
-_mitogen = None
-
-
-class IoPump(mitogen.core.BasicStream):
- _output_buf = ''
- _closed = False
-
- def __init__(self, broker, stdin_fd, stdout_fd):
- self._broker = broker
- self.receive_side = mitogen.core.Side(self, stdout_fd)
- self.transmit_side = mitogen.core.Side(self, stdin_fd)
-
- def write(self, s):
- self._output_buf += s
- self._broker._start_transmit(self)
-
- def close(self):
- self._closed = True
- # If local process hasn't exitted yet, ensure its write buffer is
- # drained before lazily triggering disconnect in on_transmit.
- if self.transmit_side.fd is not None:
- self._broker._start_transmit(self)
-
- def on_shutdown(self, broker):
- self.close()
-
- def on_transmit(self, broker):
- written = self.transmit_side.write(self._output_buf)
- IOLOG.debug('%r.on_transmit() -> len %r', self, written)
- if written is None:
- self.on_disconnect(broker)
- else:
- self._output_buf = self._output_buf[written:]
-
- if not self._output_buf:
- broker._stop_transmit(self)
- if self._closed:
- self.on_disconnect(broker)
-
- def on_receive(self, broker):
- s = self.receive_side.read()
- IOLOG.debug('%r.on_receive() -> len %r', self, len(s))
- if s:
- mitogen.core.fire(self, 'receive', s)
- else:
- self.on_disconnect(broker)
-
- def __repr__(self):
- return 'IoPump(%r, %r)' % (
- self.receive_side.fd,
- self.transmit_side.fd,
- )
-
-
-class Process(object):
- """
- Manages the lifetime and pipe connections of the SSH command running in the
- slave.
- """
- def __init__(self, router, stdin_fd, stdout_fd, proc=None):
- self.router = router
- self.stdin_fd = stdin_fd
- self.stdout_fd = stdout_fd
- self.proc = proc
- self.control_handle = router.add_handler(self._on_control)
- self.stdin_handle = router.add_handler(self._on_stdin)
- self.pump = IoPump(router.broker, stdin_fd, stdout_fd)
- self.stdin = None
- self.control = None
- self.wake_event = threading.Event()
-
- mitogen.core.listen(self.pump, 'disconnect', self._on_pump_disconnect)
- mitogen.core.listen(self.pump, 'receive', self._on_pump_receive)
-
- if proc:
- pmon = mitogen.parent.ProcessMonitor.instance()
- pmon.add(proc.pid, self._on_proc_exit)
-
- def __repr__(self):
- return 'Process(%r, %r)' % (self.stdin_fd, self.stdout_fd)
-
- def _on_proc_exit(self, status):
- LOG.debug('%r._on_proc_exit(%r)', self, status)
- self.control.put(('exit', status))
-
- def _on_stdin(self, msg):
- if msg.is_dead:
- IOLOG.debug('%r._on_stdin() -> %r', self, data)
- self.pump.close()
- return
-
- data = msg.unpickle()
- IOLOG.debug('%r._on_stdin() -> len %d', self, len(data))
- self.pump.write(data)
-
- def _on_control(self, msg):
- if not msg.is_dead:
- command, arg = msg.unpickle(throw=False)
- LOG.debug('%r._on_control(%r, %s)', self, command, arg)
-
- func = getattr(self, '_on_%s' % (command,), None)
- if func:
- return func(msg, arg)
-
- LOG.warning('%r: unknown command %r', self, command)
-
- def _on_start(self, msg, arg):
- dest = mitogen.core.Context(self.router, msg.src_id)
- self.control = mitogen.core.Sender(dest, arg[0])
- self.stdin = mitogen.core.Sender(dest, arg[1])
- self.router.broker.start_receive(self.pump)
-
- def _on_exit(self, msg, arg):
- LOG.debug('on_exit: proc = %r', self.proc)
- if self.proc:
- self.proc.terminate()
- else:
- self.router.broker.shutdown()
-
- def _on_pump_receive(self, s):
- IOLOG.info('%r._on_pump_receive(len %d)', self, len(s))
- self.stdin.put(s)
-
- def _on_pump_disconnect(self):
- LOG.debug('%r._on_pump_disconnect()', self)
- mitogen.core.fire(self, 'disconnect')
- self.stdin.close()
- self.wake_event.set()
-
- def start_master(self, stdin, control):
- self.stdin = stdin
- self.control = control
- control.put(('start', (self.control_handle, self.stdin_handle)))
- self.router.broker.start_receive(self.pump)
-
- def wait(self):
- while not self.wake_event.isSet():
- # Timeout is used so that sleep is interruptible, as blocking
- # variants of libc thread operations cannot be interrupted e.g. via
- # KeyboardInterrupt. isSet() test and wait() are separate since in
- # <2.7 wait() always returns None.
- self.wake_event.wait(0.1)
-
-
-@mitogen.core.takes_router
-def _start_slave(src_id, cmdline, router):
- """
- This runs in the target context, it is invoked by _fakessh_main running in
- the fakessh context immediately after startup. It starts the slave process
- (the the point where it has a stdin_handle to target but not stdout_chan to
- write to), and waits for main to.
- """
- LOG.debug('_start_slave(%r, %r)', router, cmdline)
-
- proc = subprocess.Popen(
- cmdline,
- # SSH server always uses user's shell.
- shell=True,
- # SSH server always executes new commands in the user's HOME.
- cwd=os.path.expanduser('~'),
-
- stdin=subprocess.PIPE,
- stdout=subprocess.PIPE,
- )
-
- process = Process(
- router,
- proc.stdin.fileno(),
- proc.stdout.fileno(),
- proc,
- )
-
- return process.control_handle, process.stdin_handle
-
-
-#
-# SSH client interface.
-#
-
-
-def exit():
- _mitogen.broker.shutdown()
-
-
-def die(msg, *args):
- if args:
- msg %= args
- sys.stderr.write('%s\n' % (msg,))
- exit()
-
-
-def parse_args():
- hostname = None
- remain = sys.argv[1:]
- allopts = []
- restarted = 0
-
- while remain and restarted < 2:
- opts, args = getopt.getopt(remain, SSH_GETOPTS)
- remain = remain[:] # getopt bug!
- allopts += opts
- if not args:
- break
-
- if not hostname:
- hostname = args.pop(0)
- remain = remain[remain.index(hostname) + 1:]
-
- restarted += 1
-
- return hostname, allopts, args
-
-
-@mitogen.core.takes_econtext
-def _fakessh_main(dest_context_id, econtext):
- hostname, opts, args = parse_args()
- if not hostname:
- die('Missing hostname')
-
- subsystem = False
- for opt, optarg in opts:
- if opt == '-s':
- subsystem = True
- else:
- LOG.debug('Warning option %s %s is ignored.', opt, optarg)
-
- LOG.debug('hostname: %r', hostname)
- LOG.debug('opts: %r', opts)
- LOG.debug('args: %r', args)
-
- if subsystem:
- die('-s is not yet supported')
-
- if not args:
- die('fakessh: login mode not supported and no command specified')
-
- dest = mitogen.parent.Context(econtext.router, dest_context_id)
-
- # Even though SSH receives an argument vector, it still cats the vector
- # together before sending to the server, the server just uses /bin/sh -c to
- # run the command. We must remain puke-for-puke compatible.
- control_handle, stdin_handle = dest.call(_start_slave,
- mitogen.context_id, ' '.join(args))
-
- LOG.debug('_fakessh_main: received control_handle=%r, stdin_handle=%r',
- control_handle, stdin_handle)
-
- process = Process(econtext.router, 1, 0)
- process.start_master(
- stdin=mitogen.core.Sender(dest, stdin_handle),
- control=mitogen.core.Sender(dest, control_handle),
- )
- process.wait()
- process.control.put(('exit', None))
-
-
-def _get_econtext_config(context, sock2):
- parent_ids = mitogen.parent_ids[:]
- parent_ids.insert(0, mitogen.context_id)
- return {
- 'context_id': context.context_id,
- 'core_src_fd': None,
- 'debug': getattr(context.router, 'debug', False),
- 'in_fd': sock2.fileno(),
- 'log_level': mitogen.parent.get_log_level(),
- 'max_message_size': context.router.max_message_size,
- 'out_fd': sock2.fileno(),
- 'parent_ids': parent_ids,
- 'profiling': getattr(context.router, 'profiling', False),
- 'unidirectional': getattr(context.router, 'unidirectional', False),
- 'setup_stdio': False,
- 'version': mitogen.__version__,
- }
-
-
-#
-# Public API.
-#
-
-@mitogen.core.takes_econtext
-@mitogen.core.takes_router
-def run(dest, router, args, deadline=None, econtext=None):
- """
- Run the command specified by `args` such that ``PATH`` searches for SSH by
- the command will cause its attempt to use SSH to execute a remote program
- to be redirected to use mitogen to execute that program using the context
- `dest` instead.
-
- :param list args:
- Argument vector.
- :param mitogen.core.Context dest:
- The destination context to execute the SSH command line in.
-
- :param mitogen.core.Router router:
-
- :param list[str] args:
- Command line arguments for local program, e.g.
- ``['rsync', '/tmp', 'remote:/tmp']``
-
- :returns:
- Exit status of the child process.
- """
- if econtext is not None:
- mitogen.parent.upgrade_router(econtext)
-
- context_id = router.allocate_id()
- fakessh = mitogen.parent.Context(router, context_id)
- fakessh.name = u'fakessh.%d' % (context_id,)
-
- sock1, sock2 = socket.socketpair()
-
- stream = mitogen.core.Stream(router, context_id)
- stream.name = u'fakessh'
- stream.accept(sock1.fileno(), sock1.fileno())
- router.register(fakessh, stream)
-
- # Held in socket buffer until process is booted.
- fakessh.call_async(_fakessh_main, dest.context_id)
-
- tmp_path = tempfile.mkdtemp(prefix='mitogen_fakessh')
- try:
- ssh_path = os.path.join(tmp_path, 'ssh')
- fp = open(ssh_path, 'w')
- try:
- fp.write('#!%s\n' % (mitogen.parent.get_sys_executable(),))
- fp.write(inspect.getsource(mitogen.core))
- fp.write('\n')
- fp.write('ExternalContext(%r).main()\n' % (
- _get_econtext_config(context, sock2),
- ))
- finally:
- fp.close()
-
- os.chmod(ssh_path, int('0755', 8))
- env = os.environ.copy()
- env.update({
- 'PATH': '%s:%s' % (tmp_path, env.get('PATH', '')),
- 'ARGV0': mitogen.parent.get_sys_executable(),
- 'SSH_PATH': ssh_path,
- })
-
- proc = subprocess.Popen(args, env=env)
- return proc.wait()
- finally:
- shutil.rmtree(tmp_path)
diff --git a/mitogen-0.2.7/mitogen/fork.py b/mitogen-0.2.7/mitogen/fork.py
deleted file mode 100644
index d6685d7..000000000
--- a/mitogen-0.2.7/mitogen/fork.py
+++ /dev/null
@@ -1,223 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-import logging
-import os
-import random
-import sys
-import threading
-import traceback
-
-import mitogen.core
-import mitogen.parent
-
-
-LOG = logging.getLogger('mitogen')
-
-# Python 2.4/2.5 cannot support fork+threads whatsoever, it doesn't even fix up
-# interpreter state. So 2.4/2.5 interpreters start .local() contexts for
-# isolation instead. Since we don't have any crazy memory sharing problems to
-# avoid, there is no virginal fork parent either. The child is started directly
-# from the login/become process. In future this will be default everywhere,
-# fork is brainwrong from the stone age.
-FORK_SUPPORTED = sys.version_info >= (2, 6)
-
-
-class Error(mitogen.core.StreamError):
- pass
-
-
-def fixup_prngs():
- """
- Add 256 bits of /dev/urandom to OpenSSL's PRNG in the child, and re-seed
- the random package with the same data.
- """
- s = os.urandom(256 // 8)
- random.seed(s)
- if 'ssl' in sys.modules:
- sys.modules['ssl'].RAND_add(s, 75.0)
-
-
-def reset_logging_framework():
- """
- After fork, ensure any logging.Handler locks are recreated, as a variety of
- threads in the parent may have been using the logging package at the moment
- of fork.
-
- It is not possible to solve this problem in general; see
- https://github.com/dw/mitogen/issues/150 for a full discussion.
- """
- logging._lock = threading.RLock()
-
- # The root logger does not appear in the loggerDict.
- for name in [None] + list(logging.Logger.manager.loggerDict):
- for handler in logging.getLogger(name).handlers:
- handler.createLock()
-
- root = logging.getLogger()
- root.handlers = [
- handler
- for handler in root.handlers
- if not isinstance(handler, mitogen.core.LogHandler)
- ]
-
-
-def on_fork():
- """
- Should be called by any program integrating Mitogen each time the process
- is forked, in the context of the new child.
- """
- reset_logging_framework() # Must be first!
- fixup_prngs()
- mitogen.core.Latch._on_fork()
- mitogen.core.Side._on_fork()
- mitogen.core.ExternalContext.service_stub_lock = threading.Lock()
-
- mitogen__service = sys.modules.get('mitogen.service')
- if mitogen__service:
- mitogen__service._pool_lock = threading.Lock()
-
-
-def handle_child_crash():
- """
- Respond to _child_main() crashing by ensuring the relevant exception is
- logged to /dev/tty.
- """
- tty = open('/dev/tty', 'wb')
- tty.write('\n\nFORKED CHILD PID %d CRASHED\n%s\n\n' % (
- os.getpid(),
- traceback.format_exc(),
- ))
- tty.close()
- os._exit(1)
-
-
-class Stream(mitogen.parent.Stream):
- child_is_immediate_subprocess = True
-
- #: Reference to the importer, if any, recovered from the parent.
- importer = None
-
- #: User-supplied function for cleaning up child process state.
- on_fork = None
-
- python_version_msg = (
- "The mitogen.fork method is not supported on Python versions "
- "prior to 2.6, since those versions made no attempt to repair "
- "critical interpreter state following a fork. Please use the "
- "local() method instead."
- )
-
- def construct(self, old_router, max_message_size, on_fork=None,
- debug=False, profiling=False, unidirectional=False,
- on_start=None):
- if not FORK_SUPPORTED:
- raise Error(self.python_version_msg)
-
- # fork method only supports a tiny subset of options.
- super(Stream, self).construct(max_message_size=max_message_size,
- debug=debug, profiling=profiling,
- unidirectional=False)
- self.on_fork = on_fork
- self.on_start = on_start
-
- responder = getattr(old_router, 'responder', None)
- if isinstance(responder, mitogen.parent.ModuleForwarder):
- self.importer = responder.importer
-
- name_prefix = u'fork'
-
- def start_child(self):
- parentfp, childfp = mitogen.parent.create_socketpair()
- self.pid = os.fork()
- if self.pid:
- childfp.close()
- # Decouple the socket from the lifetime of the Python socket object.
- fd = os.dup(parentfp.fileno())
- parentfp.close()
- return self.pid, fd, None
- else:
- parentfp.close()
- self._wrap_child_main(childfp)
-
- def _wrap_child_main(self, childfp):
- try:
- self._child_main(childfp)
- except BaseException:
- handle_child_crash()
-
- def _child_main(self, childfp):
- on_fork()
- if self.on_fork:
- self.on_fork()
- mitogen.core.set_block(childfp.fileno())
-
- # Expected by the ExternalContext.main().
- os.dup2(childfp.fileno(), 1)
- os.dup2(childfp.fileno(), 100)
-
- # Overwritten by ExternalContext.main(); we must replace the
- # parent-inherited descriptors that were closed by Side._on_fork() to
- # avoid ExternalContext.main() accidentally allocating new files over
- # the standard handles.
- os.dup2(childfp.fileno(), 0)
-
- # Avoid corrupting the stream on fork crash by dupping /dev/null over
- # stderr. Instead, handle_child_crash() uses /dev/tty to log errors.
- devnull = os.open('/dev/null', os.O_WRONLY)
- if devnull != 2:
- os.dup2(devnull, 2)
- os.close(devnull)
-
- # If we're unlucky, childfp.fileno() may coincidentally be one of our
- # desired FDs. In that case closing it breaks ExternalContext.main().
- if childfp.fileno() not in (0, 1, 100):
- childfp.close()
-
- config = self.get_econtext_config()
- config['core_src_fd'] = None
- config['importer'] = self.importer
- config['setup_package'] = False
- if self.on_start:
- config['on_start'] = self.on_start
-
- try:
- try:
- mitogen.core.ExternalContext(config).main()
- except Exception:
- # TODO: report exception somehow.
- os._exit(72)
- finally:
- # Don't trigger atexit handlers, they were copied from the parent.
- os._exit(0)
-
- def _connect_bootstrap(self):
- # None required.
- pass
diff --git a/mitogen-0.2.7/mitogen/jail.py b/mitogen-0.2.7/mitogen/jail.py
deleted file mode 100644
index 6e0ac68..000000000
--- a/mitogen-0.2.7/mitogen/jail.py
+++ /dev/null
@@ -1,65 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-import logging
-
-import mitogen.core
-import mitogen.parent
-
-
-LOG = logging.getLogger(__name__)
-
-
-class Stream(mitogen.parent.Stream):
- child_is_immediate_subprocess = False
- create_child_args = {
- 'merge_stdio': True
- }
-
- container = None
- username = None
- jexec_path = '/usr/sbin/jexec'
-
- def construct(self, container, jexec_path=None, username=None, **kwargs):
- super(Stream, self).construct(**kwargs)
- self.container = container
- self.username = username
- if jexec_path:
- self.jexec_path = jexec_path
-
- def _get_name(self):
- return u'jail.' + self.container
-
- def get_boot_command(self):
- bits = [self.jexec_path]
- if self.username:
- bits += ['-U', self.username]
- bits += [self.container]
- return bits + super(Stream, self).get_boot_command()
diff --git a/mitogen-0.2.7/mitogen/kubectl.py b/mitogen-0.2.7/mitogen/kubectl.py
deleted file mode 100644
index ef626e1..000000000
--- a/mitogen-0.2.7/mitogen/kubectl.py
+++ /dev/null
@@ -1,65 +0,0 @@
-# Copyright 2018, Yannig Perre
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-import logging
-
-import mitogen.core
-import mitogen.parent
-
-
-LOG = logging.getLogger(__name__)
-
-
-class Stream(mitogen.parent.Stream):
- child_is_immediate_subprocess = True
-
- pod = None
- kubectl_path = 'kubectl'
- kubectl_args = None
-
- # TODO: better way of capturing errors such as "No such container."
- create_child_args = {
- 'merge_stdio': True
- }
-
- def construct(self, pod, kubectl_path=None, kubectl_args=None, **kwargs):
- super(Stream, self).construct(**kwargs)
- assert pod
- self.pod = pod
- if kubectl_path:
- self.kubectl_path = kubectl_path
- self.kubectl_args = kubectl_args or []
-
- def _get_name(self):
- return u'kubectl.%s%s' % (self.pod, self.kubectl_args)
-
- def get_boot_command(self):
- bits = [self.kubectl_path] + self.kubectl_args + ['exec', '-it', self.pod]
- return bits + ["--"] + super(Stream, self).get_boot_command()
diff --git a/mitogen-0.2.7/mitogen/lxc.py b/mitogen-0.2.7/mitogen/lxc.py
deleted file mode 100644
index 879d19a..000000000
--- a/mitogen-0.2.7/mitogen/lxc.py
+++ /dev/null
@@ -1,75 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-import logging
-
-import mitogen.core
-import mitogen.parent
-
-
-LOG = logging.getLogger(__name__)
-
-
-class Stream(mitogen.parent.Stream):
- child_is_immediate_subprocess = False
- create_child_args = {
- # If lxc-attach finds any of stdin, stdout, stderr connected to a TTY,
- # to prevent input injection it creates a proxy pty, forcing all IO to
- # be buffered in <4KiB chunks. So ensure stderr is also routed to the
- # socketpair.
- 'merge_stdio': True
- }
-
- container = None
- lxc_attach_path = 'lxc-attach'
-
- eof_error_hint = (
- 'Note: many versions of LXC do not report program execution failure '
- 'meaningfully. Please check the host logs (/var/log) for more '
- 'information.'
- )
-
- def construct(self, container, lxc_attach_path=None, **kwargs):
- super(Stream, self).construct(**kwargs)
- self.container = container
- if lxc_attach_path:
- self.lxc_attach_path = lxc_attach_path
-
- def _get_name(self):
- return u'lxc.' + self.container
-
- def get_boot_command(self):
- bits = [
- self.lxc_attach_path,
- '--clear-env',
- '--name', self.container,
- '--',
- ]
- return bits + super(Stream, self).get_boot_command()
diff --git a/mitogen-0.2.7/mitogen/lxd.py b/mitogen-0.2.7/mitogen/lxd.py
deleted file mode 100644
index faea256..000000000
--- a/mitogen-0.2.7/mitogen/lxd.py
+++ /dev/null
@@ -1,77 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-import logging
-
-import mitogen.core
-import mitogen.parent
-
-
-LOG = logging.getLogger(__name__)
-
-
-class Stream(mitogen.parent.Stream):
- child_is_immediate_subprocess = False
- create_child_args = {
- # If lxc finds any of stdin, stdout, stderr connected to a TTY, to
- # prevent input injection it creates a proxy pty, forcing all IO to be
- # buffered in <4KiB chunks. So ensure stderr is also routed to the
- # socketpair.
- 'merge_stdio': True
- }
-
- container = None
- lxc_path = 'lxc'
- python_path = 'python'
-
- eof_error_hint = (
- 'Note: many versions of LXC do not report program execution failure '
- 'meaningfully. Please check the host logs (/var/log) for more '
- 'information.'
- )
-
- def construct(self, container, lxc_path=None, **kwargs):
- super(Stream, self).construct(**kwargs)
- self.container = container
- if lxc_path:
- self.lxc_path = lxc_path
-
- def _get_name(self):
- return u'lxd.' + self.container
-
- def get_boot_command(self):
- bits = [
- self.lxc_path,
- 'exec',
- '--mode=noninteractive',
- self.container,
- '--',
- ]
- return bits + super(Stream, self).get_boot_command()
diff --git a/mitogen-0.2.7/mitogen/master.py b/mitogen-0.2.7/mitogen/master.py
deleted file mode 100644
index 1396f4e..000000000
--- a/mitogen-0.2.7/mitogen/master.py
+++ /dev/null
@@ -1,1173 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-"""
-This module implements functionality required by master processes, such as
-starting new contexts via SSH. Its size is also restricted, since it must
-be sent to any context that will be used to establish additional child
-contexts.
-"""
-
-import dis
-import imp
-import inspect
-import itertools
-import logging
-import os
-import pkgutil
-import re
-import string
-import sys
-import time
-import threading
-import types
-import zlib
-
-if not hasattr(pkgutil, 'find_loader'):
- # find_loader() was new in >=2.5, but the modern pkgutil.py syntax has
- # been kept intentionally 2.3 compatible so we can reuse it.
- from mitogen.compat import pkgutil
-
-import mitogen
-import mitogen.core
-import mitogen.minify
-import mitogen.parent
-
-from mitogen.core import b
-from mitogen.core import IOLOG
-from mitogen.core import LOG
-from mitogen.core import str_partition
-from mitogen.core import str_rpartition
-from mitogen.core import to_text
-
-imap = getattr(itertools, 'imap', map)
-izip = getattr(itertools, 'izip', zip)
-
-try:
- any
-except NameError:
- from mitogen.core import any
-
-try:
- next
-except NameError:
- from mitogen.core import next
-
-
-RLOG = logging.getLogger('mitogen.ctx')
-
-
-def _stdlib_paths():
- """Return a set of paths from which Python imports the standard library.
- """
- attr_candidates = [
- 'prefix',
- 'real_prefix', # virtualenv: only set inside a virtual environment.
- 'base_prefix', # venv: always set, equal to prefix if outside.
- ]
- prefixes = (getattr(sys, a) for a in attr_candidates if hasattr(sys, a))
- version = 'python%s.%s' % sys.version_info[0:2]
- return set(os.path.abspath(os.path.join(p, 'lib', version))
- for p in prefixes)
-
-
-def is_stdlib_name(modname):
- """Return :data:`True` if `modname` appears to come from the standard
- library.
- """
- if imp.is_builtin(modname) != 0:
- return True
-
- module = sys.modules.get(modname)
- if module is None:
- return False
-
- # six installs crap with no __file__
- modpath = os.path.abspath(getattr(module, '__file__', ''))
- return is_stdlib_path(modpath)
-
-
-_STDLIB_PATHS = _stdlib_paths()
-
-
-def is_stdlib_path(path):
- return any(
- os.path.commonprefix((libpath, path)) == libpath
- and 'site-packages' not in path
- and 'dist-packages' not in path
- for libpath in _STDLIB_PATHS
- )
-
-
-def get_child_modules(path):
- """Return the suffixes of submodules directly neated beneath of the package
- directory at `path`.
-
- :param str path:
- Path to the module's source code on disk, or some PEP-302-recognized
- equivalent. Usually this is the module's ``__file__`` attribute, but
- is specified explicitly to avoid loading the module.
-
- :return:
- List of submodule name suffixes.
- """
- it = pkgutil.iter_modules([os.path.dirname(path)])
- return [to_text(name) for _, name, _ in it]
-
-
-def _get_core_source():
- """
- Master version of parent.get_core_source().
- """
- source = inspect.getsource(mitogen.core)
- return mitogen.minify.minimize_source(source)
-
-
-if mitogen.is_master:
- # TODO: find a less surprising way of installing this.
- mitogen.parent._get_core_source = _get_core_source
-
-
-LOAD_CONST = dis.opname.index('LOAD_CONST')
-IMPORT_NAME = dis.opname.index('IMPORT_NAME')
-
-
-def _getarg(nextb, c):
- if c >= dis.HAVE_ARGUMENT:
- return nextb() | (nextb() << 8)
-
-
-if sys.version_info < (3, 0):
- def iter_opcodes(co):
- # Yield `(op, oparg)` tuples from the code object `co`.
- ordit = imap(ord, co.co_code)
- nextb = ordit.next
- return ((c, _getarg(nextb, c)) for c in ordit)
-elif sys.version_info < (3, 6):
- def iter_opcodes(co):
- # Yield `(op, oparg)` tuples from the code object `co`.
- ordit = iter(co.co_code)
- nextb = ordit.__next__
- return ((c, _getarg(nextb, c)) for c in ordit)
-else:
- def iter_opcodes(co):
- # Yield `(op, oparg)` tuples from the code object `co`.
- ordit = iter(co.co_code)
- nextb = ordit.__next__
- # https://github.com/abarnert/cpython/blob/c095a32f/Python/wordcode.md
- return ((c, nextb()) for c in ordit)
-
-
-def scan_code_imports(co):
- """
- Given a code object `co`, scan its bytecode yielding any ``IMPORT_NAME``
- and associated prior ``LOAD_CONST`` instructions representing an `Import`
- statement or `ImportFrom` statement.
-
- :return:
- Generator producing `(level, modname, namelist)` tuples, where:
-
- * `level`: -1 for normal import, 0, for absolute import, and >0 for
- relative import.
- * `modname`: Name of module to import, or from where `namelist` names
- are imported.
- * `namelist`: for `ImportFrom`, the list of names to be imported from
- `modname`.
- """
- opit = iter_opcodes(co)
- opit, opit2, opit3 = itertools.tee(opit, 3)
-
- try:
- next(opit2)
- next(opit3)
- next(opit3)
- except StopIteration:
- return
-
- if sys.version_info >= (2, 5):
- for oparg1, oparg2, (op3, arg3) in izip(opit, opit2, opit3):
- if op3 == IMPORT_NAME:
- op2, arg2 = oparg2
- op1, arg1 = oparg1
- if op1 == op2 == LOAD_CONST:
- yield (co.co_consts[arg1],
- co.co_names[arg3],
- co.co_consts[arg2] or ())
- else:
- # Python 2.4 did not yet have 'level', so stack format differs.
- for oparg1, (op2, arg2) in izip(opit, opit2):
- if op2 == IMPORT_NAME:
- op1, arg1 = oparg1
- if op1 == LOAD_CONST:
- yield (-1, co.co_names[arg2], co.co_consts[arg1] or ())
-
-
-class ThreadWatcher(object):
- """
- Manage threads that wait for another thread to shut down, before invoking
- `on_join()` for each associated ThreadWatcher.
-
- In CPython it seems possible to use this method to ensure a non-main thread
- is signalled when the main thread has exited, using a third thread as a
- proxy.
- """
- #: Protects remaining _cls_* members.
- _cls_lock = threading.Lock()
-
- #: PID of the process that last modified the class data. If the PID
- #: changes, it means the thread watch dict refers to threads that no longer
- #: exist in the current process (since it forked), and so must be reset.
- _cls_pid = None
-
- #: Map watched Thread -> list of ThreadWatcher instances.
- _cls_instances_by_target = {}
-
- #: Map watched Thread -> watcher Thread for each watched thread.
- _cls_thread_by_target = {}
-
- @classmethod
- def _reset(cls):
- """If we have forked since the watch dictionaries were initialized, all
- that has is garbage, so clear it."""
- if os.getpid() != cls._cls_pid:
- cls._cls_pid = os.getpid()
- cls._cls_instances_by_target.clear()
- cls._cls_thread_by_target.clear()
-
- def __init__(self, target, on_join):
- self.target = target
- self.on_join = on_join
-
- @classmethod
- def _watch(cls, target):
- target.join()
- for watcher in cls._cls_instances_by_target[target]:
- watcher.on_join()
-
- def install(self):
- self._cls_lock.acquire()
- try:
- self._reset()
- lst = self._cls_instances_by_target.setdefault(self.target, [])
- lst.append(self)
- if self.target not in self._cls_thread_by_target:
- self._cls_thread_by_target[self.target] = threading.Thread(
- name='mitogen.master.join_thread_async',
- target=self._watch,
- args=(self.target,)
- )
- self._cls_thread_by_target[self.target].start()
- finally:
- self._cls_lock.release()
-
- def remove(self):
- self._cls_lock.acquire()
- try:
- self._reset()
- lst = self._cls_instances_by_target.get(self.target, [])
- if self in lst:
- lst.remove(self)
- finally:
- self._cls_lock.release()
-
- @classmethod
- def watch(cls, target, on_join):
- watcher = cls(target, on_join)
- watcher.install()
- return watcher
-
-
-class LogForwarder(object):
- """
- Install a :data:`mitogen.core.FORWARD_LOG` handler that delivers forwarded
- log events into the local logging framework. This is used by the master's
- :class:`Router`.
-
- The forwarded :class:`logging.LogRecord` objects are delivered to loggers
- under ``mitogen.ctx.*`` corresponding to their
- :attr:`mitogen.core.Context.name`, with the message prefixed with the
- logger name used in the child. The records include some extra attributes:
-
- * ``mitogen_message``: Unicode original message without the logger name
- prepended.
- * ``mitogen_context``: :class:`mitogen.parent.Context` reference to the
- source context.
- * ``mitogen_name``: Original logger name.
-
- :param mitogen.master.Router router:
- Router to install the handler on.
- """
- def __init__(self, router):
- self._router = router
- self._cache = {}
- router.add_handler(
- fn=self._on_forward_log,
- handle=mitogen.core.FORWARD_LOG,
- )
-
- def _on_forward_log(self, msg):
- if msg.is_dead:
- return
-
- logger = self._cache.get(msg.src_id)
- if logger is None:
- context = self._router.context_by_id(msg.src_id)
- if context is None:
- LOG.error('%s: dropping log from unknown context ID %d',
- self, msg.src_id)
- return
-
- name = '%s.%s' % (RLOG.name, context.name)
- self._cache[msg.src_id] = logger = logging.getLogger(name)
-
- name, level_s, s = msg.data.decode('latin1').split('\x00', 2)
-
- # See logging.Handler.makeRecord()
- record = logging.LogRecord(
- name=logger.name,
- level=int(level_s),
- pathname='(unknown file)',
- lineno=0,
- msg=('%s: %s' % (name, s)),
- args=(),
- exc_info=None,
- )
- record.mitogen_message = s
- record.mitogen_context = self._router.context_by_id(msg.src_id)
- record.mitogen_name = name
- logger.handle(record)
-
- def __repr__(self):
- return 'LogForwarder(%r)' % (self._router,)
-
-
-class ModuleFinder(object):
- """
- Given the name of a loaded module, make a best-effort attempt at finding
- related modules likely needed by a child context requesting the original
- module.
- """
- def __init__(self):
- #: Import machinery is expensive, keep :py:meth`:get_module_source`
- #: results around.
- self._found_cache = {}
-
- #: Avoid repeated dependency scanning, which is expensive.
- self._related_cache = {}
-
- def __repr__(self):
- return 'ModuleFinder()'
-
- def _looks_like_script(self, path):
- """
- Return :data:`True` if the (possibly extensionless) file at `path`
- resembles a Python script. For now we simply verify the file contains
- ASCII text.
- """
- fp = open(path, 'rb')
- try:
- sample = fp.read(512).decode('latin-1')
- return not set(sample).difference(string.printable)
- finally:
- fp.close()
-
- def _py_filename(self, path):
- if not path:
- return None
-
- if path[-4:] in ('.pyc', '.pyo'):
- path = path.rstrip('co')
-
- if path.endswith('.py'):
- return path
-
- if os.path.exists(path) and self._looks_like_script(path):
- return path
-
- def _get_main_module_defective_python_3x(self, fullname):
- """
- Recent versions of Python 3.x introduced an incomplete notion of
- importer specs, and in doing so created permanent asymmetry in the
- :mod:`pkgutil` interface handling for the `__main__` module. Therefore
- we must handle `__main__` specially.
- """
- if fullname != '__main__':
- return None
-
- mod = sys.modules.get(fullname)
- if not mod:
- return None
-
- path = getattr(mod, '__file__', None)
- if not (os.path.exists(path) and self._looks_like_script(path)):
- return None
-
- fp = open(path, 'rb')
- try:
- source = fp.read()
- finally:
- fp.close()
-
- return path, source, False
-
- def _get_module_via_pkgutil(self, fullname):
- """
- Attempt to fetch source code via pkgutil. In an ideal world, this would
- be the only required implementation of get_module().
- """
- try:
- # Pre-'import spec' this returned None, in Python3.6 it raises
- # ImportError.
- loader = pkgutil.find_loader(fullname)
- except ImportError:
- e = sys.exc_info()[1]
- LOG.debug('%r._get_module_via_pkgutil(%r): %s',
- self, fullname, e)
- return None
-
- IOLOG.debug('%r._get_module_via_pkgutil(%r) -> %r',
- self, fullname, loader)
- if not loader:
- return
-
- try:
- path = self._py_filename(loader.get_filename(fullname))
- source = loader.get_source(fullname)
- is_pkg = loader.is_package(fullname)
- except (AttributeError, ImportError):
- # - Per PEP-302, get_source() and is_package() are optional,
- # calling them may throw AttributeError.
- # - get_filename() may throw ImportError if pkgutil.find_loader()
- # picks a "parent" package's loader for some crap that's been
- # stuffed in sys.modules, for example in the case of urllib3:
- # "loader for urllib3.contrib.pyopenssl cannot handle
- # requests.packages.urllib3.contrib.pyopenssl"
- e = sys.exc_info()[1]
- LOG.debug('%r: loading %r using %r failed: %s',
- self, fullname, loader, e)
- return
-
- if path is None or source is None:
- return
-
- if isinstance(source, mitogen.core.UnicodeType):
- # get_source() returns "string" according to PEP-302, which was
- # reinterpreted for Python 3 to mean a Unicode string.
- source = source.encode('utf-8')
-
- return path, source, is_pkg
-
- def _get_module_via_sys_modules(self, fullname):
- """
- Attempt to fetch source code via sys.modules. This is specifically to
- support __main__, but it may catch a few more cases.
- """
- module = sys.modules.get(fullname)
- LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module)
- if not isinstance(module, types.ModuleType):
- LOG.debug('sys.modules[%r] absent or not a regular module',
- fullname)
- return
-
- path = self._py_filename(getattr(module, '__file__', ''))
- if not path:
- return
-
- is_pkg = hasattr(module, '__path__')
- try:
- source = inspect.getsource(module)
- except IOError:
- # Work around inspect.getsourcelines() bug for 0-byte __init__.py
- # files.
- if not is_pkg:
- raise
- source = '\n'
-
- if isinstance(source, mitogen.core.UnicodeType):
- # get_source() returns "string" according to PEP-302, which was
- # reinterpreted for Python 3 to mean a Unicode string.
- source = source.encode('utf-8')
-
- return path, source, is_pkg
-
- def _get_module_via_parent_enumeration(self, fullname):
- """
- Attempt to fetch source code by examining the module's (hopefully less
- insane) parent package. Required for older versions of
- ansible.compat.six and plumbum.colors.
- """
- if fullname not in sys.modules:
- # Don't attempt this unless a module really exists in sys.modules,
- # else we could return junk.
- return
-
- pkgname, _, modname = str_rpartition(to_text(fullname), u'.')
- pkg = sys.modules.get(pkgname)
- if pkg is None or not hasattr(pkg, '__file__'):
- return
-
- pkg_path = os.path.dirname(pkg.__file__)
- try:
- fp, path, ext = imp.find_module(modname, [pkg_path])
- try:
- path = self._py_filename(path)
- if not path:
- fp.close()
- return
-
- source = fp.read()
- finally:
- if fp:
- fp.close()
-
- if isinstance(source, mitogen.core.UnicodeType):
- # get_source() returns "string" according to PEP-302, which was
- # reinterpreted for Python 3 to mean a Unicode string.
- source = source.encode('utf-8')
- return path, source, False
- except ImportError:
- e = sys.exc_info()[1]
- LOG.debug('imp.find_module(%r, %r) -> %s', modname, [pkg_path], e)
-
- def add_source_override(self, fullname, path, source, is_pkg):
- """
- Explicitly install a source cache entry, preventing usual lookup
- methods from being used.
-
- Beware the value of `path` is critical when `is_pkg` is specified,
- since it directs where submodules are searched for.
-
- :param str fullname:
- Name of the module to override.
- :param str path:
- Module's path as it will appear in the cache.
- :param bytes source:
- Module source code as a bytestring.
- :param bool is_pkg:
- :data:`True` if the module is a package.
- """
- self._found_cache[fullname] = (path, source, is_pkg)
-
- get_module_methods = [
- _get_main_module_defective_python_3x,
- _get_module_via_pkgutil,
- _get_module_via_sys_modules,
- _get_module_via_parent_enumeration,
- ]
-
- def get_module_source(self, fullname):
- """Given the name of a loaded module `fullname`, attempt to find its
- source code.
-
- :returns:
- Tuple of `(module path, source text, is package?)`, or :data:`None`
- if the source cannot be found.
- """
- tup = self._found_cache.get(fullname)
- if tup:
- return tup
-
- for method in self.get_module_methods:
- tup = method(self, fullname)
- if tup:
- #LOG.debug('%r returned %r', method, tup)
- break
- else:
- tup = None, None, None
- LOG.debug('get_module_source(%r): cannot find source', fullname)
-
- self._found_cache[fullname] = tup
- return tup
-
- def resolve_relpath(self, fullname, level):
- """Given an ImportFrom AST node, guess the prefix that should be tacked
- on to an alias name to produce a canonical name. `fullname` is the name
- of the module in which the ImportFrom appears.
- """
- mod = sys.modules.get(fullname, None)
- if hasattr(mod, '__path__'):
- fullname += '.__init__'
-
- if level == 0 or not fullname:
- return ''
-
- bits = fullname.split('.')
- if len(bits) <= level:
- # This would be an ImportError in real code.
- return ''
-
- return '.'.join(bits[:-level]) + '.'
-
- def generate_parent_names(self, fullname):
- while '.' in fullname:
- fullname, _, _ = str_rpartition(to_text(fullname), u'.')
- yield fullname
-
- def find_related_imports(self, fullname):
- """
- Return a list of non-stdlib modules that are directly imported by
- `fullname`, plus their parents.
-
- The list is determined by retrieving the source code of
- `fullname`, compiling it, and examining all IMPORT_NAME ops.
-
- :param fullname: Fully qualified name of an _already imported_ module
- for which source code can be retrieved
- :type fullname: str
- """
- related = self._related_cache.get(fullname)
- if related is not None:
- return related
-
- modpath, src, _ = self.get_module_source(fullname)
- if src is None:
- return []
-
- maybe_names = list(self.generate_parent_names(fullname))
-
- co = compile(src, modpath, 'exec')
- for level, modname, namelist in scan_code_imports(co):
- if level == -1:
- modnames = [modname, '%s.%s' % (fullname, modname)]
- else:
- modnames = [
- '%s%s' % (self.resolve_relpath(fullname, level), modname)
- ]
-
- maybe_names.extend(modnames)
- maybe_names.extend(
- '%s.%s' % (mname, name)
- for mname in modnames
- for name in namelist
- )
-
- return self._related_cache.setdefault(fullname, sorted(
- set(
- mitogen.core.to_text(name)
- for name in maybe_names
- if sys.modules.get(name) is not None
- and not is_stdlib_name(name)
- and u'six.moves' not in name # TODO: crap
- )
- ))
-
- def find_related(self, fullname):
- """
- Return a list of non-stdlib modules that are imported directly or
- indirectly by `fullname`, plus their parents.
-
- This method is like :py:meth:`find_related_imports`, but also
- recursively searches any modules which are imported by `fullname`.
-
- :param fullname: Fully qualified name of an _already imported_ module
- for which source code can be retrieved
- :type fullname: str
- """
- stack = [fullname]
- found = set()
-
- while stack:
- name = stack.pop(0)
- names = self.find_related_imports(name)
- stack.extend(set(names).difference(set(found).union(stack)))
- found.update(names)
-
- found.discard(fullname)
- return sorted(found)
-
-
-class ModuleResponder(object):
- def __init__(self, router):
- self._router = router
- self._finder = ModuleFinder()
- self._cache = {} # fullname -> pickled
- self.blacklist = []
- self.whitelist = ['']
-
- #: Context -> set([fullname, ..])
- self._forwarded_by_context = {}
-
- #: Number of GET_MODULE messages received.
- self.get_module_count = 0
- #: Total time spent in uncached GET_MODULE.
- self.get_module_secs = 0.0
- #: Total time spent minifying modules.
- self.minify_secs = 0.0
- #: Number of successful LOAD_MODULE messages sent.
- self.good_load_module_count = 0
- #: Total bytes in successful LOAD_MODULE payloads.
- self.good_load_module_size = 0
- #: Number of negative LOAD_MODULE messages sent.
- self.bad_load_module_count = 0
-
- router.add_handler(
- fn=self._on_get_module,
- handle=mitogen.core.GET_MODULE,
- )
-
- def __repr__(self):
- return 'ModuleResponder(%r)' % (self._router,)
-
- def add_source_override(self, fullname, path, source, is_pkg):
- """
- See :meth:`ModuleFinder.add_source_override.
- """
- self._finder.add_source_override(fullname, path, source, is_pkg)
-
- MAIN_RE = re.compile(b(r'^if\s+__name__\s*==\s*.__main__.\s*:'), re.M)
- main_guard_msg = (
- "A child context attempted to import __main__, however the main "
- "module present in the master process lacks an execution guard. "
- "Update %r to prevent unintended execution, using a guard like:\n"
- "\n"
- " if __name__ == '__main__':\n"
- " # your code here.\n"
- )
-
- def whitelist_prefix(self, fullname):
- if self.whitelist == ['']:
- self.whitelist = ['mitogen']
- self.whitelist.append(fullname)
-
- def blacklist_prefix(self, fullname):
- self.blacklist.append(fullname)
-
- def neutralize_main(self, path, src):
- """Given the source for the __main__ module, try to find where it
- begins conditional execution based on a "if __name__ == '__main__'"
- guard, and remove any code after that point."""
- match = self.MAIN_RE.search(src)
- if match:
- return src[:match.start()]
-
- if b('mitogen.main(') in src:
- return src
-
- LOG.error(self.main_guard_msg, path)
- raise ImportError('refused')
-
- def _make_negative_response(self, fullname):
- return (fullname, None, None, None, ())
-
- minify_safe_re = re.compile(b(r'\s+#\s*!mitogen:\s*minify_safe'))
-
- def _build_tuple(self, fullname):
- if fullname in self._cache:
- return self._cache[fullname]
-
- if mitogen.core.is_blacklisted_import(self, fullname):
- raise ImportError('blacklisted')
-
- path, source, is_pkg = self._finder.get_module_source(fullname)
- if path and is_stdlib_path(path):
- # Prevent loading of 2.x<->3.x stdlib modules! This costs one
- # RTT per hit, so a client-side solution is also required.
- LOG.debug('%r: refusing to serve stdlib module %r',
- self, fullname)
- tup = self._make_negative_response(fullname)
- self._cache[fullname] = tup
- return tup
-
- if source is None:
- # TODO: make this .warning() or similar again once importer has its
- # own logging category.
- LOG.debug('_build_tuple(%r): could not locate source', fullname)
- tup = self._make_negative_response(fullname)
- self._cache[fullname] = tup
- return tup
-
- if self.minify_safe_re.search(source):
- # If the module contains a magic marker, it's safe to minify.
- t0 = time.time()
- source = mitogen.minify.minimize_source(source).encode('utf-8')
- self.minify_secs += time.time() - t0
-
- if is_pkg:
- pkg_present = get_child_modules(path)
- LOG.debug('_build_tuple(%r, %r) -> %r',
- path, fullname, pkg_present)
- else:
- pkg_present = None
-
- if fullname == '__main__':
- source = self.neutralize_main(path, source)
- compressed = mitogen.core.Blob(zlib.compress(source, 9))
- related = [
- to_text(name)
- for name in self._finder.find_related(fullname)
- if not mitogen.core.is_blacklisted_import(self, name)
- ]
- # 0:fullname 1:pkg_present 2:path 3:compressed 4:related
- tup = (
- to_text(fullname),
- pkg_present,
- to_text(path),
- compressed,
- related
- )
- self._cache[fullname] = tup
- return tup
-
- def _send_load_module(self, stream, fullname):
- if fullname not in stream.sent_modules:
- tup = self._build_tuple(fullname)
- msg = mitogen.core.Message.pickled(
- tup,
- dst_id=stream.remote_id,
- handle=mitogen.core.LOAD_MODULE,
- )
- LOG.debug('%s: sending module %s (%.2f KiB)',
- stream.name, fullname, len(msg.data) / 1024.0)
- self._router._async_route(msg)
- stream.sent_modules.add(fullname)
- if tup[2] is not None:
- self.good_load_module_count += 1
- self.good_load_module_size += len(msg.data)
- else:
- self.bad_load_module_count += 1
-
- def _send_module_load_failed(self, stream, fullname):
- self.bad_load_module_count += 1
- stream.send(
- mitogen.core.Message.pickled(
- self._make_negative_response(fullname),
- dst_id=stream.remote_id,
- handle=mitogen.core.LOAD_MODULE,
- )
- )
-
- def _send_module_and_related(self, stream, fullname):
- if fullname in stream.sent_modules:
- return
-
- try:
- tup = self._build_tuple(fullname)
- for name in tup[4]: # related
- parent, _, _ = str_partition(name, '.')
- if parent != fullname and parent not in stream.sent_modules:
- # Parent hasn't been sent, so don't load submodule yet.
- continue
-
- self._send_load_module(stream, name)
- self._send_load_module(stream, fullname)
- except Exception:
- LOG.debug('While importing %r', fullname, exc_info=True)
- self._send_module_load_failed(stream, fullname)
-
- def _on_get_module(self, msg):
- if msg.is_dead:
- return
-
- stream = self._router.stream_by_id(msg.src_id)
- if stream is None:
- return
-
- fullname = msg.data.decode()
- LOG.debug('%s requested module %s', stream.name, fullname)
- self.get_module_count += 1
- if fullname in stream.sent_modules:
- LOG.warning('_on_get_module(): dup request for %r from %r',
- fullname, stream)
-
- t0 = time.time()
- try:
- self._send_module_and_related(stream, fullname)
- finally:
- self.get_module_secs += time.time() - t0
-
- def _send_forward_module(self, stream, context, fullname):
- if stream.remote_id != context.context_id:
- stream.send(
- mitogen.core.Message(
- data=b('%s\x00%s' % (context.context_id, fullname)),
- handle=mitogen.core.FORWARD_MODULE,
- dst_id=stream.remote_id,
- )
- )
-
- def _forward_one_module(self, context, fullname):
- forwarded = self._forwarded_by_context.get(context)
- if forwarded is None:
- forwarded = set()
- self._forwarded_by_context[context] = forwarded
-
- if fullname in forwarded:
- return
-
- path = []
- while fullname:
- path.append(fullname)
- fullname, _, _ = str_rpartition(fullname, u'.')
-
- stream = self._router.stream_by_id(context.context_id)
- if stream is None:
- LOG.debug('%r: dropping forward of %s to no longer existent '
- '%r', self, path[0], context)
- return
-
- for fullname in reversed(path):
- self._send_module_and_related(stream, fullname)
- self._send_forward_module(stream, context, fullname)
-
- def _forward_modules(self, context, fullnames):
- IOLOG.debug('%r._forward_modules(%r, %r)', self, context, fullnames)
- for fullname in fullnames:
- self._forward_one_module(context, mitogen.core.to_text(fullname))
-
- def forward_modules(self, context, fullnames):
- self._router.broker.defer(self._forward_modules, context, fullnames)
-
-
-class Broker(mitogen.core.Broker):
- """
- .. note::
-
- You may construct as many brokers as desired, and use the same broker
- for multiple routers, however usually only one broker need exist.
- Multiple brokers may be useful when dealing with sets of children with
- differing lifetimes. For example, a subscription service where
- non-payment results in termination for one customer.
-
- :param bool install_watcher:
- If :data:`True`, an additional thread is started to monitor the
- lifetime of the main thread, triggering :meth:`shutdown`
- automatically in case the user forgets to call it, or their code
- crashed.
-
- You should not rely on this functionality in your program, it is only
- intended as a fail-safe and to simplify the API for new users. In
- particular, alternative Python implementations may not be able to
- support watching the main thread.
- """
- shutdown_timeout = 5.0
- _watcher = None
- poller_class = mitogen.parent.PREFERRED_POLLER
-
- def __init__(self, install_watcher=True):
- if install_watcher:
- self._watcher = ThreadWatcher.watch(
- target=threading.currentThread(),
- on_join=self.shutdown,
- )
- super(Broker, self).__init__()
-
- def shutdown(self):
- super(Broker, self).shutdown()
- if self._watcher:
- self._watcher.remove()
-
-
-class Router(mitogen.parent.Router):
- """
- Extend :class:`mitogen.core.Router` with functionality useful to masters,
- and child contexts who later become masters. Currently when this class is
- required, the target context's router is upgraded at runtime.
-
- .. note::
-
- You may construct as many routers as desired, and use the same broker
- for multiple routers, however usually only one broker and router need
- exist. Multiple routers may be useful when dealing with separate trust
- domains, for example, manipulating infrastructure belonging to separate
- customers or projects.
-
- :param mitogen.master.Broker broker:
- Broker to use. If not specified, a private :class:`Broker` is created.
-
- :param int max_message_size:
- Override the maximum message size this router is willing to receive or
- transmit. Any value set here is automatically inherited by any children
- created by the router.
-
- This has a liberal default of 128 MiB, but may be set much lower.
- Beware that setting it below 64KiB may encourage unexpected failures as
- parents and children can no longer route large Python modules that may
- be required by your application.
- """
-
- broker_class = Broker
-
- #: When :data:`True`, cause the broker thread and any subsequent broker and
- #: main threads existing in any child to write
- #: ``/tmp/mitogen.stats...log`` containing a
- #: :mod:`cProfile` dump on graceful exit. Must be set prior to construction
- #: of any :class:`Broker`, e.g. via::
- #:
- #: mitogen.master.Router.profiling = True
- profiling = os.environ.get('MITOGEN_PROFILING') is not None
-
- def __init__(self, broker=None, max_message_size=None):
- if broker is None:
- broker = self.broker_class()
- if max_message_size:
- self.max_message_size = max_message_size
- super(Router, self).__init__(broker)
- self.upgrade()
-
- def upgrade(self):
- self.id_allocator = IdAllocator(self)
- self.responder = ModuleResponder(self)
- self.log_forwarder = LogForwarder(self)
- self.route_monitor = mitogen.parent.RouteMonitor(router=self)
- self.add_handler( # TODO: cutpaste.
- fn=self._on_detaching,
- handle=mitogen.core.DETACHING,
- persist=True,
- )
-
- def _on_broker_exit(self):
- super(Router, self)._on_broker_exit()
- dct = self.get_stats()
- dct['self'] = self
- dct['minify_ms'] = 1000 * dct['minify_secs']
- dct['get_module_ms'] = 1000 * dct['get_module_secs']
- dct['good_load_module_size_kb'] = dct['good_load_module_size'] / 1024.0
- dct['good_load_module_size_avg'] = (
- (
- dct['good_load_module_size'] /
- (float(dct['good_load_module_count']) or 1.0)
- ) / 1024.0
- )
-
- LOG.debug(
- '%(self)r: stats: '
- '%(get_module_count)d module requests in '
- '%(get_module_ms)d ms, '
- '%(good_load_module_count)d sent '
- '(%(minify_ms)d ms minify time), '
- '%(bad_load_module_count)d negative responses. '
- 'Sent %(good_load_module_size_kb).01f kb total, '
- '%(good_load_module_size_avg).01f kb avg.'
- % dct
- )
-
- def get_stats(self):
- """
- Return performance data for the module responder.
-
- :returns:
-
- Dict containing keys:
-
- * `get_module_count`: Integer count of
- :data:`mitogen.core.GET_MODULE` messages received.
- * `get_module_secs`: Floating point total seconds spent servicing
- :data:`mitogen.core.GET_MODULE` requests.
- * `good_load_module_count`: Integer count of successful
- :data:`mitogen.core.LOAD_MODULE` messages sent.
- * `good_load_module_size`: Integer total bytes sent in
- :data:`mitogen.core.LOAD_MODULE` message payloads.
- * `bad_load_module_count`: Integer count of negative
- :data:`mitogen.core.LOAD_MODULE` messages sent.
- * `minify_secs`: CPU seconds spent minifying modules marked
- minify-safe.
- """
- return {
- 'get_module_count': self.responder.get_module_count,
- 'get_module_secs': self.responder.get_module_secs,
- 'good_load_module_count': self.responder.good_load_module_count,
- 'good_load_module_size': self.responder.good_load_module_size,
- 'bad_load_module_count': self.responder.bad_load_module_count,
- 'minify_secs': self.responder.minify_secs,
- }
-
- def enable_debug(self):
- """
- Cause this context and any descendant child contexts to write debug
- logs to ``/tmp/mitogen..log``.
- """
- mitogen.core.enable_debug_logging()
- self.debug = True
-
- def __enter__(self):
- return self
-
- def __exit__(self, e_type, e_val, tb):
- self.broker.shutdown()
- self.broker.join()
-
- def disconnect_stream(self, stream):
- self.broker.defer(stream.on_disconnect, self.broker)
-
- def disconnect_all(self):
- for stream in self._stream_by_id.values():
- self.disconnect_stream(stream)
-
-
-class IdAllocator(object):
- def __init__(self, router):
- self.router = router
- self.next_id = 1
- self.lock = threading.Lock()
- router.add_handler(
- fn=self.on_allocate_id,
- handle=mitogen.core.ALLOCATE_ID,
- )
-
- def __repr__(self):
- return 'IdAllocator(%r)' % (self.router,)
-
- BLOCK_SIZE = 1000
-
- def allocate(self):
- """
- Arrange for a unique context ID to be allocated and associated with a
- route leading to the active context. In masters, the ID is generated
- directly, in children it is forwarded to the master via a
- :data:`mitogen.core.ALLOCATE_ID` message.
- """
- self.lock.acquire()
- try:
- id_ = self.next_id
- self.next_id += 1
- return id_
- finally:
- self.lock.release()
-
- def allocate_block(self):
- self.lock.acquire()
- try:
- id_ = self.next_id
- self.next_id += self.BLOCK_SIZE
- end_id = id_ + self.BLOCK_SIZE
- LOG.debug('%r: allocating [%d..%d)', self, id_, end_id)
- return id_, end_id
- finally:
- self.lock.release()
-
- def on_allocate_id(self, msg):
- if msg.is_dead:
- return
-
- id_, last_id = self.allocate_block()
- requestee = self.router.context_by_id(msg.src_id)
- LOG.debug('%r: allocating [%r..%r) to %r',
- self, id_, last_id, requestee)
- msg.reply((id_, last_id))
diff --git a/mitogen-0.2.7/mitogen/minify.py b/mitogen-0.2.7/mitogen/minify.py
deleted file mode 100644
index dc9f517..000000000
--- a/mitogen-0.2.7/mitogen/minify.py
+++ /dev/null
@@ -1,139 +0,0 @@
-# Copyright 2017, Alex Willmer
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-import sys
-
-try:
- from io import StringIO
-except ImportError:
- from StringIO import StringIO
-
-import mitogen.core
-
-if sys.version_info < (2, 7, 11):
- from mitogen.compat import tokenize
-else:
- import tokenize
-
-
-def minimize_source(source):
- """Remove comments and docstrings from Python `source`, preserving line
- numbers and syntax of empty blocks.
-
- :param str source:
- The source to minimize.
-
- :returns str:
- The minimized source.
- """
- source = mitogen.core.to_text(source)
- tokens = tokenize.generate_tokens(StringIO(source).readline)
- tokens = strip_comments(tokens)
- tokens = strip_docstrings(tokens)
- tokens = reindent(tokens)
- return tokenize.untokenize(tokens)
-
-
-def strip_comments(tokens):
- """Drop comment tokens from a `tokenize` stream.
-
- Comments on lines 1-2 are kept, to preserve hashbang and encoding.
- Trailing whitespace is remove from all lines.
- """
- prev_typ = None
- prev_end_col = 0
- for typ, tok, (start_row, start_col), (end_row, end_col), line in tokens:
- if typ in (tokenize.NL, tokenize.NEWLINE):
- if prev_typ in (tokenize.NL, tokenize.NEWLINE):
- start_col = 0
- else:
- start_col = prev_end_col
- end_col = start_col + 1
- elif typ == tokenize.COMMENT and start_row > 2:
- continue
- prev_typ = typ
- prev_end_col = end_col
- yield typ, tok, (start_row, start_col), (end_row, end_col), line
-
-
-def strip_docstrings(tokens):
- """Replace docstring tokens with NL tokens in a `tokenize` stream.
-
- Any STRING token not part of an expression is deemed a docstring.
- Indented docstrings are not yet recognised.
- """
- stack = []
- state = 'wait_string'
- for t in tokens:
- typ = t[0]
- if state == 'wait_string':
- if typ in (tokenize.NL, tokenize.COMMENT):
- yield t
- elif typ in (tokenize.DEDENT, tokenize.INDENT, tokenize.STRING):
- stack.append(t)
- elif typ == tokenize.NEWLINE:
- stack.append(t)
- start_line, end_line = stack[0][2][0], stack[-1][3][0]+1
- for i in range(start_line, end_line):
- yield tokenize.NL, '\n', (i, 0), (i,1), '\n'
- for t in stack:
- if t[0] in (tokenize.DEDENT, tokenize.INDENT):
- yield t[0], t[1], (i+1, t[2][1]), (i+1, t[3][1]), t[4]
- del stack[:]
- else:
- stack.append(t)
- for t in stack: yield t
- del stack[:]
- state = 'wait_newline'
- elif state == 'wait_newline':
- if typ == tokenize.NEWLINE:
- state = 'wait_string'
- yield t
-
-
-def reindent(tokens, indent=' '):
- """Replace existing indentation in a token steam, with `indent`.
- """
- old_levels = []
- old_level = 0
- new_level = 0
- for typ, tok, (start_row, start_col), (end_row, end_col), line in tokens:
- if typ == tokenize.INDENT:
- old_levels.append(old_level)
- old_level = len(tok)
- new_level += 1
- tok = indent * new_level
- elif typ == tokenize.DEDENT:
- old_level = old_levels.pop()
- new_level -= 1
- start_col = max(0, start_col - old_level + new_level)
- if start_row == end_row:
- end_col = start_col + len(tok)
- yield typ, tok, (start_row, start_col), (end_row, end_col), line
diff --git a/mitogen-0.2.7/mitogen/os_fork.py b/mitogen-0.2.7/mitogen/os_fork.py
deleted file mode 100644
index b27cfd5..000000000
--- a/mitogen-0.2.7/mitogen/os_fork.py
+++ /dev/null
@@ -1,183 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-"""
-Support for operating in a mixed threading/forking environment.
-"""
-
-import os
-import socket
-import sys
-import weakref
-
-import mitogen.core
-
-
-# List of weakrefs. On Python 2.4, mitogen.core registers its Broker on this
-# list and mitogen.service registers its Pool too.
-_brokers = weakref.WeakKeyDictionary()
-_pools = weakref.WeakKeyDictionary()
-
-
-def _notice_broker_or_pool(obj):
- """
- Used by :mod:`mitogen.core` and :mod:`mitogen.service` to automatically
- register every broker and pool on Python 2.4/2.5.
- """
- if isinstance(obj, mitogen.core.Broker):
- _brokers[obj] = True
- else:
- _pools[obj] = True
-
-
-def wrap_os__fork():
- corker = Corker(
- brokers=list(_brokers),
- pools=list(_pools),
- )
- try:
- corker.cork()
- return os__fork()
- finally:
- corker.uncork()
-
-
-# If Python 2.4/2.5 where threading state is not fixed up, subprocess.Popen()
-# may still deadlock due to the broker thread. In this case, pause os.fork() so
-# that all active threads are paused during fork.
-if sys.version_info < (2, 6):
- os__fork = os.fork
- os.fork = wrap_os__fork
-
-
-class Corker(object):
- """
- Arrange for :class:`mitogen.core.Broker` and optionally
- :class:`mitogen.service.Pool` to be temporarily "corked" while fork
- operations may occur.
-
- In a mixed threading/forking environment, it is critical no threads are
- active at the moment of fork, as they could hold mutexes whose state is
- unrecoverably snapshotted in the locked state in the fork child, causing
- deadlocks at random future moments.
-
- To ensure a target thread has all locks dropped, it is made to write a
- large string to a socket with a small buffer that has :data:`os.O_NONBLOCK`
- disabled. CPython will drop the GIL and enter the ``write()`` system call,
- where it will block until the socket buffer is drained, or the write side
- is closed.
-
- :class:`mitogen.core.Poller` is used to ensure the thread really has
- blocked outside any Python locks, by checking if the socket buffer has
- started to fill.
-
- Since this necessarily involves posting a message to every existent thread
- and verifying acknowledgement, it will never be a fast operation.
-
- This does not yet handle the case of corking being initiated from within a
- thread that is also a cork target.
-
- :param brokers:
- Sequence of :class:`mitogen.core.Broker` instances to cork.
- :param pools:
- Sequence of :class:`mitogen.core.Pool` instances to cork.
- """
- def __init__(self, brokers=(), pools=()):
- self.brokers = brokers
- self.pools = pools
-
- def _do_cork(self, s, wsock):
- try:
- try:
- while True:
- # at least EINTR is possible. Do our best to keep handling
- # outside the GIL in this case using sendall().
- wsock.sendall(s)
- except socket.error:
- pass
- finally:
- wsock.close()
-
- def _cork_one(self, s, obj):
- """
- Construct a socketpair, saving one side of it, and passing the other to
- `obj` to be written to by one of its threads.
- """
- rsock, wsock = mitogen.parent.create_socketpair(size=4096)
- mitogen.core.set_cloexec(rsock.fileno())
- mitogen.core.set_cloexec(wsock.fileno())
- mitogen.core.set_block(wsock) # gevent
- self._rsocks.append(rsock)
- obj.defer(self._do_cork, s, wsock)
-
- def _verify_one(self, rsock):
- """
- Pause until the socket `rsock` indicates readability, due to
- :meth:`_do_cork` triggering a blocking write on another thread.
- """
- poller = mitogen.core.Poller()
- poller.start_receive(rsock.fileno())
- try:
- while True:
- for fd in poller.poll():
- return
- finally:
- poller.close()
-
- def cork(self):
- """
- Arrange for any associated brokers and pools to be paused with no locks
- held. This will not return until each thread acknowledges it has ceased
- execution.
- """
- s = mitogen.core.b('CORK') * ((128 // 4) * 1024)
- self._rsocks = []
-
- # Pools must be paused first, as existing work may require the
- # participation of a broker in order to complete.
- for pool in self.pools:
- if not pool.closed:
- for x in range(pool.size):
- self._cork_one(s, pool)
-
- for broker in self.brokers:
- if broker._alive:
- self._cork_one(s, broker)
-
- # Pause until we can detect every thread has entered write().
- for rsock in self._rsocks:
- self._verify_one(rsock)
-
- def uncork(self):
- """
- Arrange for paused threads to resume operation.
- """
- for rsock in self._rsocks:
- rsock.close()
diff --git a/mitogen-0.2.7/mitogen/parent.py b/mitogen-0.2.7/mitogen/parent.py
deleted file mode 100644
index 3d02bc4..000000000
--- a/mitogen-0.2.7/mitogen/parent.py
+++ /dev/null
@@ -1,2330 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-"""
-This module defines functionality common to master and parent processes. It is
-sent to any child context that is due to become a parent, due to recursive
-connection.
-"""
-
-import codecs
-import errno
-import fcntl
-import getpass
-import inspect
-import logging
-import os
-import signal
-import socket
-import struct
-import subprocess
-import sys
-import termios
-import textwrap
-import threading
-import time
-import zlib
-
-# Absolute imports for <2.5.
-select = __import__('select')
-
-try:
- import thread
-except ImportError:
- import threading as thread
-
-import mitogen.core
-from mitogen.core import b
-from mitogen.core import bytes_partition
-from mitogen.core import LOG
-from mitogen.core import IOLOG
-
-try:
- next
-except NameError:
- # Python 2.4/2.5
- from mitogen.core import next
-
-
-itervalues = getattr(dict, 'itervalues', dict.values)
-
-if mitogen.core.PY3:
- xrange = range
- closure_attr = '__closure__'
- IM_SELF_ATTR = '__self__'
-else:
- closure_attr = 'func_closure'
- IM_SELF_ATTR = 'im_self'
-
-
-try:
- SC_OPEN_MAX = os.sysconf('SC_OPEN_MAX')
-except ValueError:
- SC_OPEN_MAX = 1024
-
-OPENPTY_MSG = (
- "Failed to create a PTY: %s. It is likely the maximum number of PTYs has "
- "been reached. Consider increasing the 'kern.tty.ptmx_max' sysctl on OS "
- "X, the 'kernel.pty.max' sysctl on Linux, or modifying your configuration "
- "to avoid PTY use."
-)
-
-SYS_EXECUTABLE_MSG = (
- "The Python sys.executable variable is unset, indicating Python was "
- "unable to determine its original program name. Unless explicitly "
- "configured otherwise, child contexts will be started using "
- "'/usr/bin/python'"
-)
-_sys_executable_warning_logged = False
-
-
-def _ioctl_cast(n):
- """
- Linux ioctl() request parameter is unsigned, whereas on BSD/Darwin it is
- signed. Until 2.5 Python exclusively implemented the BSD behaviour,
- preventing use of large unsigned int requests like the TTY layer uses
- below. So on 2.4, we cast our unsigned to look like signed for Python.
- """
- if sys.version_info < (2, 5):
- n, = struct.unpack('i', struct.pack('I', n))
- return n
-
-
-# If not :data:`None`, called prior to exec() of any new child process. Used by
-# :func:`mitogen.utils.reset_affinity` to allow the child to be freely
-# scheduled.
-_preexec_hook = None
-
-# Get PTY number; asm-generic/ioctls.h
-LINUX_TIOCGPTN = _ioctl_cast(2147767344)
-
-# Lock/unlock PTY; asm-generic/ioctls.h
-LINUX_TIOCSPTLCK = _ioctl_cast(1074025521)
-
-IS_LINUX = os.uname()[0] == 'Linux'
-
-SIGNAL_BY_NUM = dict(
- (getattr(signal, name), name)
- for name in sorted(vars(signal), reverse=True)
- if name.startswith('SIG') and not name.startswith('SIG_')
-)
-
-
-def get_log_level():
- return (LOG.level or logging.getLogger().level or logging.INFO)
-
-
-def get_sys_executable():
- """
- Return :data:`sys.executable` if it is set, otherwise return
- ``"/usr/bin/python"`` and log a warning.
- """
- if sys.executable:
- return sys.executable
-
- global _sys_executable_warning_logged
- if not _sys_executable_warning_logged:
- LOG.warn(SYS_EXECUTABLE_MSG)
- _sys_executable_warning_logged = True
-
- return '/usr/bin/python'
-
-
-_core_source_lock = threading.Lock()
-_core_source_partial = None
-
-
-def _get_core_source():
- """
- In non-masters, simply fetch the cached mitogen.core source code via the
- import mechanism. In masters, this function is replaced with a version that
- performs minification directly.
- """
- return inspect.getsource(mitogen.core)
-
-
-def get_core_source_partial():
- """
- _get_core_source() is expensive, even with @lru_cache in minify.py, threads
- can enter it simultaneously causing severe slowdowns.
- """
- global _core_source_partial
-
- if _core_source_partial is None:
- _core_source_lock.acquire()
- try:
- if _core_source_partial is None:
- _core_source_partial = PartialZlib(
- _get_core_source().encode('utf-8')
- )
- finally:
- _core_source_lock.release()
-
- return _core_source_partial
-
-
-def get_default_remote_name():
- """
- Return the default name appearing in argv[0] of remote machines.
- """
- s = u'%s@%s:%d'
- s %= (getpass.getuser(), socket.gethostname(), os.getpid())
- # In mixed UNIX/Windows environments, the username may contain slashes.
- return s.translate({
- ord(u'\\'): ord(u'_'),
- ord(u'/'): ord(u'_')
- })
-
-
-def is_immediate_child(msg, stream):
- """
- Handler policy that requires messages to arrive only from immediately
- connected children.
- """
- return msg.src_id == stream.remote_id
-
-
-def flags(names):
- """Return the result of ORing a set of (space separated) :py:mod:`termios`
- module constants together."""
- return sum(getattr(termios, name, 0)
- for name in names.split())
-
-
-def cfmakeraw(tflags):
- """Given a list returned by :py:func:`termios.tcgetattr`, return a list
- modified in a manner similar to the `cfmakeraw()` C library function, but
- additionally disabling local echo."""
- # BSD: https://github.com/freebsd/freebsd/blob/master/lib/libc/gen/termios.c#L162
- # Linux: https://github.com/lattera/glibc/blob/master/termios/cfmakeraw.c#L20
- iflag, oflag, cflag, lflag, ispeed, ospeed, cc = tflags
- iflag &= ~flags('IMAXBEL IXOFF INPCK BRKINT PARMRK ISTRIP INLCR ICRNL IXON IGNPAR')
- iflag &= ~flags('IGNBRK BRKINT PARMRK')
- oflag &= ~flags('OPOST')
- lflag &= ~flags('ECHO ECHOE ECHOK ECHONL ICANON ISIG IEXTEN NOFLSH TOSTOP PENDIN')
- cflag &= ~flags('CSIZE PARENB')
- cflag |= flags('CS8 CREAD')
- return [iflag, oflag, cflag, lflag, ispeed, ospeed, cc]
-
-
-def disable_echo(fd):
- old = termios.tcgetattr(fd)
- new = cfmakeraw(old)
- flags = getattr(termios, 'TCSASOFT', 0)
- if not mitogen.core.IS_WSL:
- # issue #319: Windows Subsystem for Linux as of July 2018 throws EINVAL
- # if TCSAFLUSH is specified.
- flags |= termios.TCSAFLUSH
- termios.tcsetattr(fd, flags, new)
-
-
-def close_nonstandard_fds():
- for fd in xrange(3, SC_OPEN_MAX):
- try:
- os.close(fd)
- except OSError:
- pass
-
-
-def create_socketpair(size=None):
- """
- Create a :func:`socket.socketpair` to use for use as a child process's UNIX
- stdio channels. As socket pairs are bidirectional, they are economical on
- file descriptor usage as the same descriptor can be used for ``stdin`` and
- ``stdout``. As they are sockets their buffers are tunable, allowing large
- buffers to be configured in order to improve throughput for file transfers
- and reduce :class:`mitogen.core.Broker` IO loop iterations.
- """
- parentfp, childfp = socket.socketpair()
- parentfp.setsockopt(socket.SOL_SOCKET,
- socket.SO_SNDBUF,
- size or mitogen.core.CHUNK_SIZE)
- childfp.setsockopt(socket.SOL_SOCKET,
- socket.SO_RCVBUF,
- size or mitogen.core.CHUNK_SIZE)
- return parentfp, childfp
-
-
-def detach_popen(**kwargs):
- """
- Use :class:`subprocess.Popen` to construct a child process, then hack the
- Popen so that it forgets the child it created, allowing it to survive a
- call to Popen.__del__.
-
- If the child process is not detached, there is a race between it exitting
- and __del__ being called. If it exits before __del__ runs, then __del__'s
- call to :func:`os.waitpid` will capture the one and only exit event
- delivered to this process, causing later 'legitimate' calls to fail with
- ECHILD.
-
- :param list close_on_error:
- Array of integer file descriptors to close on exception.
- :returns:
- Process ID of the new child.
- """
- # This allows Popen() to be used for e.g. graceful post-fork error
- # handling, without tying the surrounding code into managing a Popen
- # object, which isn't possible for at least :mod:`mitogen.fork`. This
- # should be replaced by a swappable helper class in a future version.
- real_preexec_fn = kwargs.pop('preexec_fn', None)
- def preexec_fn():
- if _preexec_hook:
- _preexec_hook()
- if real_preexec_fn:
- real_preexec_fn()
- proc = subprocess.Popen(preexec_fn=preexec_fn, **kwargs)
- proc._child_created = False
- return proc.pid
-
-
-def create_child(args, merge_stdio=False, stderr_pipe=False, preexec_fn=None):
- """
- Create a child process whose stdin/stdout is connected to a socket.
-
- :param args:
- Argument vector for execv() call.
- :param bool merge_stdio:
- If :data:`True`, arrange for `stderr` to be connected to the `stdout`
- socketpair, rather than inherited from the parent process. This may be
- necessary to ensure that not TTY is connected to any stdio handle, for
- instance when using LXC.
- :param bool stderr_pipe:
- If :data:`True` and `merge_stdio` is :data:`False`, arrange for
- `stderr` to be connected to a separate pipe, to allow any ongoing debug
- logs generated by e.g. SSH to be outpu as the session progresses,
- without interfering with `stdout`.
- :returns:
- `(pid, socket_obj, :data:`None` or pipe_fd)`
- """
- parentfp, childfp = create_socketpair()
- # When running under a monkey patches-enabled gevent, the socket module
- # yields file descriptors who already have O_NONBLOCK, which is
- # persisted across fork, totally breaking Python. Therefore, drop
- # O_NONBLOCK from Python's future stdin fd.
- mitogen.core.set_block(childfp.fileno())
-
- stderr_r = None
- extra = {}
- if merge_stdio:
- extra = {'stderr': childfp}
- elif stderr_pipe:
- stderr_r, stderr_w = os.pipe()
- mitogen.core.set_cloexec(stderr_r)
- mitogen.core.set_cloexec(stderr_w)
- extra = {'stderr': stderr_w}
-
- try:
- pid = detach_popen(
- args=args,
- stdin=childfp,
- stdout=childfp,
- close_fds=True,
- preexec_fn=preexec_fn,
- **extra
- )
- except Exception:
- childfp.close()
- parentfp.close()
- if stderr_pipe:
- os.close(stderr_r)
- os.close(stderr_w)
- raise
-
- if stderr_pipe:
- os.close(stderr_w)
- childfp.close()
- # Decouple the socket from the lifetime of the Python socket object.
- fd = os.dup(parentfp.fileno())
- parentfp.close()
-
- LOG.debug('create_child() child %d fd %d, parent %d, cmd: %s',
- pid, fd, os.getpid(), Argv(args))
- return pid, fd, stderr_r
-
-
-def _acquire_controlling_tty():
- os.setsid()
- if sys.platform in ('linux', 'linux2'):
- # On Linux, the controlling tty becomes the first tty opened by a
- # process lacking any prior tty.
- os.close(os.open(os.ttyname(2), os.O_RDWR))
- if hasattr(termios, 'TIOCSCTTY') and not mitogen.core.IS_WSL:
- # #550: prehistoric WSL does not like TIOCSCTTY.
- # On BSD an explicit ioctl is required. For some inexplicable reason,
- # Python 2.6 on Travis also requires it.
- fcntl.ioctl(2, termios.TIOCSCTTY)
-
-
-def _linux_broken_devpts_openpty():
- """
- #462: On broken Linux hosts with mismatched configuration (e.g. old
- /etc/fstab template installed), /dev/pts may be mounted without the gid=
- mount option, causing new slave devices to be created with the group ID of
- the calling process. This upsets glibc, whose openpty() is required by
- specification to produce a slave owned by a special group ID (which is
- always the 'tty' group).
-
- Glibc attempts to use "pt_chown" to fix ownership. If that fails, it
- chown()s the PTY directly, which fails due to non-root, causing openpty()
- to fail with EPERM ("Operation not permitted"). Since we don't need the
- magical TTY group to run sudo and su, open the PTY ourselves in this case.
- """
- master_fd = None
- try:
- # Opening /dev/ptmx causes a PTY pair to be allocated, and the
- # corresponding slave /dev/pts/* device to be created, owned by UID/GID
- # matching this process.
- master_fd = os.open('/dev/ptmx', os.O_RDWR)
- # Clear the lock bit from the PTY. This a prehistoric feature from a
- # time when slave device files were persistent.
- fcntl.ioctl(master_fd, LINUX_TIOCSPTLCK, struct.pack('i', 0))
- # Since v4.13 TIOCGPTPEER exists to open the slave in one step, but we
- # must support older kernels. Ask for the PTY number.
- pty_num_s = fcntl.ioctl(master_fd, LINUX_TIOCGPTN,
- struct.pack('i', 0))
- pty_num, = struct.unpack('i', pty_num_s)
- pty_name = '/dev/pts/%d' % (pty_num,)
- # Now open it with O_NOCTTY to ensure it doesn't change our controlling
- # TTY. Otherwise when we close the FD we get killed by the kernel, and
- # the child we spawn that should really attach to it will get EPERM
- # during _acquire_controlling_tty().
- slave_fd = os.open(pty_name, os.O_RDWR|os.O_NOCTTY)
- return master_fd, slave_fd
- except OSError:
- if master_fd is not None:
- os.close(master_fd)
- e = sys.exc_info()[1]
- raise mitogen.core.StreamError(OPENPTY_MSG, e)
-
-
-def openpty():
- """
- Call :func:`os.openpty`, raising a descriptive error if the call fails.
-
- :raises mitogen.core.StreamError:
- Creating a PTY failed.
- :returns:
- See :func`os.openpty`.
- """
- try:
- return os.openpty()
- except OSError:
- e = sys.exc_info()[1]
- if IS_LINUX and e.args[0] == errno.EPERM:
- return _linux_broken_devpts_openpty()
- raise mitogen.core.StreamError(OPENPTY_MSG, e)
-
-
-def tty_create_child(args):
- """
- Return a file descriptor connected to the master end of a pseudo-terminal,
- whose slave end is connected to stdin/stdout/stderr of a new child process.
- The child is created such that the pseudo-terminal becomes its controlling
- TTY, ensuring access to /dev/tty returns a new file descriptor open on the
- slave end.
-
- :param list args:
- :py:func:`os.execl` argument list.
-
- :returns:
- `(pid, tty_fd, None)`
- """
- master_fd, slave_fd = openpty()
- try:
- mitogen.core.set_block(slave_fd)
- disable_echo(master_fd)
- disable_echo(slave_fd)
-
- pid = detach_popen(
- args=args,
- stdin=slave_fd,
- stdout=slave_fd,
- stderr=slave_fd,
- preexec_fn=_acquire_controlling_tty,
- close_fds=True,
- )
- except Exception:
- os.close(master_fd)
- os.close(slave_fd)
- raise
-
- os.close(slave_fd)
- LOG.debug('tty_create_child() child %d fd %d, parent %d, cmd: %s',
- pid, master_fd, os.getpid(), Argv(args))
- return pid, master_fd, None
-
-
-def hybrid_tty_create_child(args):
- """
- Like :func:`tty_create_child`, except attach stdin/stdout to a socketpair
- like :func:`create_child`, but leave stderr and the controlling TTY
- attached to a TTY.
-
- :param list args:
- :py:func:`os.execl` argument list.
-
- :returns:
- `(pid, socketpair_fd, tty_fd)`
- """
- master_fd, slave_fd = openpty()
-
- try:
- disable_echo(master_fd)
- disable_echo(slave_fd)
- mitogen.core.set_block(slave_fd)
-
- parentfp, childfp = create_socketpair()
- try:
- mitogen.core.set_block(childfp)
- pid = detach_popen(
- args=args,
- stdin=childfp,
- stdout=childfp,
- stderr=slave_fd,
- preexec_fn=_acquire_controlling_tty,
- close_fds=True,
- )
- except Exception:
- parentfp.close()
- childfp.close()
- raise
- except Exception:
- os.close(master_fd)
- os.close(slave_fd)
- raise
-
- os.close(slave_fd)
- childfp.close()
- # Decouple the socket from the lifetime of the Python socket object.
- stdio_fd = os.dup(parentfp.fileno())
- parentfp.close()
-
- LOG.debug('hybrid_tty_create_child() pid=%d stdio=%d, tty=%d, cmd: %s',
- pid, stdio_fd, master_fd, Argv(args))
- return pid, stdio_fd, master_fd
-
-
-def write_all(fd, s, deadline=None):
- """Arrange for all of bytestring `s` to be written to the file descriptor
- `fd`.
-
- :param int fd:
- File descriptor to write to.
- :param bytes s:
- Bytestring to write to file descriptor.
- :param float deadline:
- If not :data:`None`, absolute UNIX timestamp after which timeout should
- occur.
-
- :raises mitogen.core.TimeoutError:
- Bytestring could not be written entirely before deadline was exceeded.
- :raises mitogen.parent.EofError:
- Stream indicated EOF, suggesting the child process has exitted.
- :raises mitogen.core.StreamError:
- File descriptor was disconnected before write could complete.
- """
- timeout = None
- written = 0
- poller = PREFERRED_POLLER()
- poller.start_transmit(fd)
-
- try:
- while written < len(s):
- if deadline is not None:
- timeout = max(0, deadline - time.time())
- if timeout == 0:
- raise mitogen.core.TimeoutError('write timed out')
-
- if mitogen.core.PY3:
- window = memoryview(s)[written:]
- else:
- window = buffer(s, written)
-
- for fd in poller.poll(timeout):
- n, disconnected = mitogen.core.io_op(os.write, fd, window)
- if disconnected:
- raise EofError('EOF on stream during write')
-
- written += n
- finally:
- poller.close()
-
-
-class PartialZlib(object):
- """
- Because the mitogen.core source has a line appended to it during bootstrap,
- it must be recompressed for each connection. This is not a problem for a
- small number of connections, but it amounts to 30 seconds CPU time by the
- time 500 targets are in use.
-
- For that reason, build a compressor containing mitogen.core and flush as
- much of it as possible into an initial buffer. Then to append the custom
- line, clone the compressor and compress just that line.
-
- A full compression costs ~6ms on a modern machine, this method costs ~35
- usec.
- """
- def __init__(self, s):
- self.s = s
- if sys.version_info > (2, 5):
- self._compressor = zlib.compressobj(9)
- self._out = self._compressor.compress(s)
- self._out += self._compressor.flush(zlib.Z_SYNC_FLUSH)
- else:
- self._compressor = None
-
- def append(self, s):
- """
- Append the bytestring `s` to the compressor state and return the
- final compressed output.
- """
- if self._compressor is None:
- return zlib.compress(self.s + s, 9)
- else:
- compressor = self._compressor.copy()
- out = self._out
- out += compressor.compress(s)
- return out + compressor.flush()
-
-
-class IteratingRead(object):
- def __init__(self, fds, deadline=None):
- self.deadline = deadline
- self.timeout = None
- self.poller = PREFERRED_POLLER()
- for fd in fds:
- self.poller.start_receive(fd)
-
- self.bits = []
- self.timeout = None
-
- def close(self):
- self.poller.close()
-
- def __iter__(self):
- return self
-
- def next(self):
- while self.poller.readers:
- if self.deadline is not None:
- self.timeout = max(0, self.deadline - time.time())
- if self.timeout == 0:
- break
-
- for fd in self.poller.poll(self.timeout):
- s, disconnected = mitogen.core.io_op(os.read, fd, 4096)
- if disconnected or not s:
- LOG.debug('iter_read(%r) -> disconnected: %s',
- fd, disconnected)
- self.poller.stop_receive(fd)
- else:
- IOLOG.debug('iter_read(%r) -> %r', fd, s)
- self.bits.append(s)
- return s
-
- if not self.poller.readers:
- raise EofError(u'EOF on stream; last 300 bytes received: %r' %
- (b('').join(self.bits)[-300:].decode('latin1'),))
-
- raise mitogen.core.TimeoutError('read timed out')
-
- __next__ = next
-
-
-def iter_read(fds, deadline=None):
- """Return a generator that arranges for up to 4096-byte chunks to be read
- at a time from the file descriptor `fd` until the generator is destroyed.
-
- :param int fd:
- File descriptor to read from.
- :param float deadline:
- If not :data:`None`, an absolute UNIX timestamp after which timeout
- should occur.
-
- :raises mitogen.core.TimeoutError:
- Attempt to read beyond deadline.
- :raises mitogen.parent.EofError:
- All streams indicated EOF, suggesting the child process has exitted.
- :raises mitogen.core.StreamError:
- Attempt to read past end of file.
- """
- return IteratingRead(fds=fds, deadline=deadline)
-
-
-def discard_until(fd, s, deadline):
- """Read chunks from `fd` until one is encountered that ends with `s`. This
- is used to skip output produced by ``/etc/profile``, ``/etc/motd`` and
- mandatory SSH banners while waiting for :attr:`Stream.EC0_MARKER` to
- appear, indicating the first stage is ready to receive the compressed
- :mod:`mitogen.core` source.
-
- :param int fd:
- File descriptor to read from.
- :param bytes s:
- Marker string to discard until encountered.
- :param float deadline:
- Absolute UNIX timestamp after which timeout should occur.
-
- :raises mitogen.core.TimeoutError:
- Attempt to read beyond deadline.
- :raises mitogen.parent.EofError:
- All streams indicated EOF, suggesting the child process has exitted.
- :raises mitogen.core.StreamError:
- Attempt to read past end of file.
- """
- it = iter_read([fd], deadline)
- try:
- for buf in it:
- if IOLOG.level == logging.DEBUG:
- for line in buf.splitlines():
- IOLOG.debug('discard_until: discarding %r', line)
- if buf.endswith(s):
- return
- finally:
- it.close() # ensure Poller.close() is called.
-
-
-def _upgrade_broker(broker):
- """
- Extract the poller state from Broker and replace it with the industrial
- strength poller for this OS. Must run on the Broker thread.
- """
- # This function is deadly! The act of calling start_receive() generates log
- # messages which must be silenced as the upgrade progresses, otherwise the
- # poller state will change as it is copied, resulting in write fds that are
- # lost. (Due to LogHandler->Router->Stream->Broker->Poller, where Stream
- # only calls start_transmit() when transitioning from empty to non-empty
- # buffer. If the start_transmit() is lost, writes from the child hang
- # permanently).
- root = logging.getLogger()
- old_level = root.level
- root.setLevel(logging.CRITICAL)
-
- old = broker.poller
- new = PREFERRED_POLLER()
- for fd, data in old.readers:
- new.start_receive(fd, data)
- for fd, data in old.writers:
- new.start_transmit(fd, data)
-
- old.close()
- broker.poller = new
- root.setLevel(old_level)
- LOG.debug('replaced %r with %r (new: %d readers, %d writers; '
- 'old: %d readers, %d writers)', old, new,
- len(new.readers), len(new.writers),
- len(old.readers), len(old.writers))
-
-
-@mitogen.core.takes_econtext
-def upgrade_router(econtext):
- if not isinstance(econtext.router, Router): # TODO
- econtext.broker.defer(_upgrade_broker, econtext.broker)
- econtext.router.__class__ = Router # TODO
- econtext.router.upgrade(
- importer=econtext.importer,
- parent=econtext.parent,
- )
-
-
-def stream_by_method_name(name):
- """
- Given the name of a Mitogen connection method, import its implementation
- module and return its Stream subclass.
- """
- if name == u'local':
- name = u'parent'
- module = mitogen.core.import_module(u'mitogen.' + name)
- return module.Stream
-
-
-@mitogen.core.takes_econtext
-def _proxy_connect(name, method_name, kwargs, econtext):
- """
- Implements the target portion of Router._proxy_connect() by upgrading the
- local context to a parent if it was not already, then calling back into
- Router._connect() using the arguments passed to the parent's
- Router.connect().
-
- :returns:
- Dict containing:
- * ``id``: :data:`None`, or integer new context ID.
- * ``name``: :data:`None`, or string name attribute of new Context.
- * ``msg``: :data:`None`, or StreamError exception text.
- """
- upgrade_router(econtext)
-
- try:
- context = econtext.router._connect(
- klass=stream_by_method_name(method_name),
- name=name,
- **kwargs
- )
- except mitogen.core.StreamError:
- return {
- u'id': None,
- u'name': None,
- u'msg': 'error occurred on host %s: %s' % (
- socket.gethostname(),
- sys.exc_info()[1],
- ),
- }
-
- return {
- u'id': context.context_id,
- u'name': context.name,
- u'msg': None,
- }
-
-
-def wstatus_to_str(status):
- """
- Parse and format a :func:`os.waitpid` exit status.
- """
- if os.WIFEXITED(status):
- return 'exited with return code %d' % (os.WEXITSTATUS(status),)
- if os.WIFSIGNALED(status):
- n = os.WTERMSIG(status)
- return 'exited due to signal %d (%s)' % (n, SIGNAL_BY_NUM.get(n))
- if os.WIFSTOPPED(status):
- n = os.WSTOPSIG(status)
- return 'stopped due to signal %d (%s)' % (n, SIGNAL_BY_NUM.get(n))
- return 'unknown wait status (%d)' % (status,)
-
-
-class EofError(mitogen.core.StreamError):
- """
- Raised by :func:`iter_read` and :func:`write_all` when EOF is detected by
- the child process.
- """
- # inherits from StreamError to maintain compatibility.
- pass
-
-
-class Argv(object):
- """
- Wrapper to defer argv formatting when debug logging is disabled.
- """
- def __init__(self, argv):
- self.argv = argv
-
- must_escape = frozenset('\\$"`!')
- must_escape_or_space = must_escape | frozenset(' ')
-
- def escape(self, x):
- if not self.must_escape_or_space.intersection(x):
- return x
-
- s = '"'
- for c in x:
- if c in self.must_escape:
- s += '\\'
- s += c
- s += '"'
- return s
-
- def __str__(self):
- return ' '.join(map(self.escape, self.argv))
-
-
-class CallSpec(object):
- """
- Wrapper to defer call argument formatting when debug logging is disabled.
- """
- def __init__(self, func, args, kwargs):
- self.func = func
- self.args = args
- self.kwargs = kwargs
-
- def _get_name(self):
- bits = [self.func.__module__]
- if inspect.ismethod(self.func):
- im_self = getattr(self.func, IM_SELF_ATTR)
- bits.append(getattr(im_self, '__name__', None) or
- getattr(type(im_self), '__name__', None))
- bits.append(self.func.__name__)
- return u'.'.join(bits)
-
- def _get_args(self):
- return u', '.join(repr(a) for a in self.args)
-
- def _get_kwargs(self):
- s = u''
- if self.kwargs:
- s = u', '.join('%s=%r' % (k, v) for k, v in self.kwargs.items())
- if self.args:
- s = u', ' + s
- return s
-
- def __repr__(self):
- return '%s(%s%s)' % (
- self._get_name(),
- self._get_args(),
- self._get_kwargs(),
- )
-
-
-class PollPoller(mitogen.core.Poller):
- """
- Poller based on the POSIX poll(2) interface. Not available on some versions
- of OS X, otherwise it is the preferred poller for small FD counts.
- """
- SUPPORTED = hasattr(select, 'poll')
- _repr = 'PollPoller()'
-
- def __init__(self):
- super(PollPoller, self).__init__()
- self._pollobj = select.poll()
-
- # TODO: no proof we dont need writemask too
- _readmask = (
- getattr(select, 'POLLIN', 0) |
- getattr(select, 'POLLHUP', 0)
- )
-
- def _update(self, fd):
- mask = (((fd in self._rfds) and self._readmask) |
- ((fd in self._wfds) and select.POLLOUT))
- if mask:
- self._pollobj.register(fd, mask)
- else:
- try:
- self._pollobj.unregister(fd)
- except KeyError:
- pass
-
- def _poll(self, timeout):
- if timeout:
- timeout *= 1000
-
- events, _ = mitogen.core.io_op(self._pollobj.poll, timeout)
- for fd, event in events:
- if event & self._readmask:
- IOLOG.debug('%r: POLLIN|POLLHUP for %r', self, fd)
- data, gen = self._rfds.get(fd, (None, None))
- if gen and gen < self._generation:
- yield data
- if event & select.POLLOUT:
- IOLOG.debug('%r: POLLOUT for %r', self, fd)
- data, gen = self._wfds.get(fd, (None, None))
- if gen and gen < self._generation:
- yield data
-
-
-class KqueuePoller(mitogen.core.Poller):
- """
- Poller based on the FreeBSD/Darwin kqueue(2) interface.
- """
- SUPPORTED = hasattr(select, 'kqueue')
- _repr = 'KqueuePoller()'
-
- def __init__(self):
- super(KqueuePoller, self).__init__()
- self._kqueue = select.kqueue()
- self._changelist = []
-
- def close(self):
- super(KqueuePoller, self).close()
- self._kqueue.close()
-
- def _control(self, fd, filters, flags):
- mitogen.core._vv and IOLOG.debug(
- '%r._control(%r, %r, %r)', self, fd, filters, flags)
- # TODO: at shutdown it is currently possible for KQ_EV_ADD/KQ_EV_DEL
- # pairs to be pending after the associated file descriptor has already
- # been closed. Fixing this requires maintaining extra state, or perhaps
- # making fd closure the poller's responsibility. In the meantime,
- # simply apply changes immediately.
- # self._changelist.append(select.kevent(fd, filters, flags))
- changelist = [select.kevent(fd, filters, flags)]
- events, _ = mitogen.core.io_op(self._kqueue.control, changelist, 0, 0)
- assert not events
-
- def start_receive(self, fd, data=None):
- mitogen.core._vv and IOLOG.debug('%r.start_receive(%r, %r)',
- self, fd, data)
- if fd not in self._rfds:
- self._control(fd, select.KQ_FILTER_READ, select.KQ_EV_ADD)
- self._rfds[fd] = (data or fd, self._generation)
-
- def stop_receive(self, fd):
- mitogen.core._vv and IOLOG.debug('%r.stop_receive(%r)', self, fd)
- if fd in self._rfds:
- self._control(fd, select.KQ_FILTER_READ, select.KQ_EV_DELETE)
- del self._rfds[fd]
-
- def start_transmit(self, fd, data=None):
- mitogen.core._vv and IOLOG.debug('%r.start_transmit(%r, %r)',
- self, fd, data)
- if fd not in self._wfds:
- self._control(fd, select.KQ_FILTER_WRITE, select.KQ_EV_ADD)
- self._wfds[fd] = (data or fd, self._generation)
-
- def stop_transmit(self, fd):
- mitogen.core._vv and IOLOG.debug('%r.stop_transmit(%r)', self, fd)
- if fd in self._wfds:
- self._control(fd, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE)
- del self._wfds[fd]
-
- def _poll(self, timeout):
- changelist = self._changelist
- self._changelist = []
- events, _ = mitogen.core.io_op(self._kqueue.control,
- changelist, 32, timeout)
- for event in events:
- fd = event.ident
- if event.flags & select.KQ_EV_ERROR:
- LOG.debug('ignoring stale event for fd %r: errno=%d: %s',
- fd, event.data, errno.errorcode.get(event.data))
- elif event.filter == select.KQ_FILTER_READ:
- data, gen = self._rfds.get(fd, (None, None))
- # Events can still be read for an already-discarded fd.
- if gen and gen < self._generation:
- mitogen.core._vv and IOLOG.debug('%r: POLLIN: %r', self, fd)
- yield data
- elif event.filter == select.KQ_FILTER_WRITE and fd in self._wfds:
- data, gen = self._wfds.get(fd, (None, None))
- if gen and gen < self._generation:
- mitogen.core._vv and IOLOG.debug('%r: POLLOUT: %r', self, fd)
- yield data
-
-
-class EpollPoller(mitogen.core.Poller):
- """
- Poller based on the Linux epoll(2) interface.
- """
- SUPPORTED = hasattr(select, 'epoll')
- _repr = 'EpollPoller()'
-
- def __init__(self):
- super(EpollPoller, self).__init__()
- self._epoll = select.epoll(32)
- self._registered_fds = set()
-
- def close(self):
- super(EpollPoller, self).close()
- self._epoll.close()
-
- def _control(self, fd):
- mitogen.core._vv and IOLOG.debug('%r._control(%r)', self, fd)
- mask = (((fd in self._rfds) and select.EPOLLIN) |
- ((fd in self._wfds) and select.EPOLLOUT))
- if mask:
- if fd in self._registered_fds:
- self._epoll.modify(fd, mask)
- else:
- self._epoll.register(fd, mask)
- self._registered_fds.add(fd)
- elif fd in self._registered_fds:
- self._epoll.unregister(fd)
- self._registered_fds.remove(fd)
-
- def start_receive(self, fd, data=None):
- mitogen.core._vv and IOLOG.debug('%r.start_receive(%r, %r)',
- self, fd, data)
- self._rfds[fd] = (data or fd, self._generation)
- self._control(fd)
-
- def stop_receive(self, fd):
- mitogen.core._vv and IOLOG.debug('%r.stop_receive(%r)', self, fd)
- self._rfds.pop(fd, None)
- self._control(fd)
-
- def start_transmit(self, fd, data=None):
- mitogen.core._vv and IOLOG.debug('%r.start_transmit(%r, %r)',
- self, fd, data)
- self._wfds[fd] = (data or fd, self._generation)
- self._control(fd)
-
- def stop_transmit(self, fd):
- mitogen.core._vv and IOLOG.debug('%r.stop_transmit(%r)', self, fd)
- self._wfds.pop(fd, None)
- self._control(fd)
-
- _inmask = (getattr(select, 'EPOLLIN', 0) |
- getattr(select, 'EPOLLHUP', 0))
-
- def _poll(self, timeout):
- the_timeout = -1
- if timeout is not None:
- the_timeout = timeout
-
- events, _ = mitogen.core.io_op(self._epoll.poll, the_timeout, 32)
- for fd, event in events:
- if event & self._inmask:
- data, gen = self._rfds.get(fd, (None, None))
- if gen and gen < self._generation:
- # Events can still be read for an already-discarded fd.
- mitogen.core._vv and IOLOG.debug('%r: POLLIN: %r', self, fd)
- yield data
- if event & select.EPOLLOUT:
- data, gen = self._wfds.get(fd, (None, None))
- if gen and gen < self._generation:
- mitogen.core._vv and IOLOG.debug('%r: POLLOUT: %r', self, fd)
- yield data
-
-
-# 2.4 and 2.5 only had select.select() and select.poll().
-for _klass in mitogen.core.Poller, PollPoller, KqueuePoller, EpollPoller:
- if _klass.SUPPORTED:
- PREFERRED_POLLER = _klass
-
-# For apps that start threads dynamically, it's possible Latch will also get
-# very high-numbered wait fds when there are many connections, and so select()
-# becomes useless there too. So swap in our favourite poller.
-if PollPoller.SUPPORTED:
- mitogen.core.Latch.poller_class = PollPoller
-else:
- mitogen.core.Latch.poller_class = PREFERRED_POLLER
-
-
-class DiagLogStream(mitogen.core.BasicStream):
- """
- For "hybrid TTY/socketpair" mode, after a connection has been setup, a
- spare TTY file descriptor will exist that cannot be closed, and to which
- SSH or sudo may continue writing log messages.
-
- The descriptor cannot be closed since the UNIX TTY layer will send a
- termination signal to any processes whose controlling TTY is the TTY that
- has been closed.
-
- DiagLogStream takes over this descriptor and creates corresponding log
- messages for anything written to it.
- """
-
- def __init__(self, fd, stream):
- self.receive_side = mitogen.core.Side(self, fd)
- self.transmit_side = self.receive_side
- self.stream = stream
- self.buf = ''
-
- def __repr__(self):
- return "mitogen.parent.DiagLogStream(fd=%r, '%s')" % (
- self.receive_side.fd,
- self.stream.name,
- )
-
- def on_receive(self, broker):
- """
- This handler is only called after the stream is registered with the IO
- loop, the descriptor is manually read/written by _connect_bootstrap()
- prior to that.
- """
- buf = self.receive_side.read()
- if not buf:
- return self.on_disconnect(broker)
-
- self.buf += buf.decode('utf-8', 'replace')
- while u'\n' in self.buf:
- lines = self.buf.split('\n')
- self.buf = lines[-1]
- for line in lines[:-1]:
- LOG.debug('%s: %s', self.stream.name, line.rstrip())
-
-
-class Stream(mitogen.core.Stream):
- """
- Base for streams capable of starting new slaves.
- """
- #: The path to the remote Python interpreter.
- python_path = get_sys_executable()
-
- #: Maximum time to wait for a connection attempt.
- connect_timeout = 30.0
-
- #: Derived from :py:attr:`connect_timeout`; absolute floating point
- #: UNIX timestamp after which the connection attempt should be abandoned.
- connect_deadline = None
-
- #: True to cause context to write verbose /tmp/mitogen..log.
- debug = False
-
- #: True to cause context to write /tmp/mitogen.stats...log.
- profiling = False
-
- #: Set to the child's PID by connect().
- pid = None
-
- #: Passed via Router wrapper methods, must eventually be passed to
- #: ExternalContext.main().
- max_message_size = None
-
- #: If :attr:`create_child` supplied a diag_fd, references the corresponding
- #: :class:`DiagLogStream`, allowing it to be disconnected when this stream
- #: is disconnected. Set to :data:`None` if no `diag_fd` was present.
- diag_stream = None
-
- #: Function with the semantics of :func:`create_child` used to create the
- #: child process.
- create_child = staticmethod(create_child)
-
- #: Dictionary of extra kwargs passed to :attr:`create_child`.
- create_child_args = {}
-
- #: :data:`True` if the remote has indicated that it intends to detach, and
- #: should not be killed on disconnect.
- detached = False
-
- #: If :data:`True`, indicates the child should not be killed during
- #: graceful detachment, as it the actual process implementing the child
- #: context. In all other cases, the subprocess is SSH, sudo, or a similar
- #: tool that should be reminded to quit during disconnection.
- child_is_immediate_subprocess = True
-
- #: Prefix given to default names generated by :meth:`connect`.
- name_prefix = u'local'
-
- _reaped = False
-
- def __init__(self, *args, **kwargs):
- super(Stream, self).__init__(*args, **kwargs)
- self.sent_modules = set(['mitogen', 'mitogen.core'])
-
- def construct(self, max_message_size, remote_name=None, python_path=None,
- debug=False, connect_timeout=None, profiling=False,
- unidirectional=False, old_router=None, **kwargs):
- """Get the named context running on the local machine, creating it if
- it does not exist."""
- super(Stream, self).construct(**kwargs)
- self.max_message_size = max_message_size
- if python_path:
- self.python_path = python_path
- if connect_timeout:
- self.connect_timeout = connect_timeout
- if remote_name is None:
- remote_name = get_default_remote_name()
- if '/' in remote_name or '\\' in remote_name:
- raise ValueError('remote_name= cannot contain slashes')
- self.remote_name = remote_name
- self.debug = debug
- self.profiling = profiling
- self.unidirectional = unidirectional
- self.max_message_size = max_message_size
- self.connect_deadline = time.time() + self.connect_timeout
-
- def on_shutdown(self, broker):
- """Request the slave gracefully shut itself down."""
- LOG.debug('%r closing CALL_FUNCTION channel', self)
- self._send(
- mitogen.core.Message(
- src_id=mitogen.context_id,
- dst_id=self.remote_id,
- handle=mitogen.core.SHUTDOWN,
- )
- )
-
- def _reap_child(self):
- """
- Reap the child process during disconnection.
- """
- if self.detached and self.child_is_immediate_subprocess:
- LOG.debug('%r: immediate child is detached, won\'t reap it', self)
- return
-
- if self.profiling:
- LOG.info('%r: wont kill child because profiling=True', self)
- return
-
- if self._reaped:
- # on_disconnect() may be invoked more than once, for example, if
- # there is still a pending message to be sent after the first
- # on_disconnect() call.
- return
-
- try:
- pid, status = os.waitpid(self.pid, os.WNOHANG)
- except OSError:
- e = sys.exc_info()[1]
- if e.args[0] == errno.ECHILD:
- LOG.warn('%r: waitpid(%r) produced ECHILD', self, self.pid)
- return
- raise
-
- self._reaped = True
- if pid:
- LOG.debug('%r: PID %d %s', self, pid, wstatus_to_str(status))
- return
-
- if not self._router.profiling:
- # For processes like sudo we cannot actually send sudo a signal,
- # because it is setuid, so this is best-effort only.
- LOG.debug('%r: child process still alive, sending SIGTERM', self)
- try:
- os.kill(self.pid, signal.SIGTERM)
- except OSError:
- e = sys.exc_info()[1]
- if e.args[0] != errno.EPERM:
- raise
-
- def on_disconnect(self, broker):
- super(Stream, self).on_disconnect(broker)
- if self.diag_stream is not None:
- self.diag_stream.on_disconnect(broker)
- self._reap_child()
-
- # Minimised, gzipped, base64'd and passed to 'python -c'. It forks, dups
- # file descriptor 0 as 100, creates a pipe, then execs a new interpreter
- # with a custom argv.
- # * Optimized for minimum byte count after minification & compression.
- # * 'CONTEXT_NAME' and 'PREAMBLE_COMPRESSED_LEN' are substituted with
- # their respective values.
- # * CONTEXT_NAME must be prefixed with the name of the Python binary in
- # order to allow virtualenvs to detect their install prefix.
- # * For Darwin, OS X installs a craptacular argv0-introspecting Python
- # version switcher as /usr/bin/python. Override attempts to call it
- # with an explicit call to python2.7
- #
- # Locals:
- # R: read side of interpreter stdin.
- # W: write side of interpreter stdin.
- # r: read side of core_src FD.
- # w: write side of core_src FD.
- # C: the decompressed core source.
-
- # Final os.close(2) to avoid --py-debug build from corrupting stream with
- # "[1234 refs]" during exit.
- @staticmethod
- def _first_stage():
- R,W=os.pipe()
- r,w=os.pipe()
- if os.fork():
- os.dup2(0,100)
- os.dup2(R,0)
- os.dup2(r,101)
- os.close(R)
- os.close(r)
- os.close(W)
- os.close(w)
- if sys.platform == 'darwin' and sys.executable == '/usr/bin/python':
- sys.executable += sys.version[:3]
- os.environ['ARGV0']=sys.executable
- os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)')
- os.write(1,'MITO000\n'.encode())
- C=_(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN),'zip')
- fp=os.fdopen(W,'wb',0)
- fp.write(C)
- fp.close()
- fp=os.fdopen(w,'wb',0)
- fp.write(C)
- fp.close()
- os.write(1,'MITO001\n'.encode())
- os.close(2)
-
- def get_python_argv(self):
- """
- Return the initial argument vector elements necessary to invoke Python,
- by returning a 1-element list containing :attr:`python_path` if it is a
- string, or simply returning it if it is already a list.
-
- This allows emulation of existing tools where the Python invocation may
- be set to e.g. `['/usr/bin/env', 'python']`.
- """
- if isinstance(self.python_path, list):
- return self.python_path
- return [self.python_path]
-
- def get_boot_command(self):
- source = inspect.getsource(self._first_stage)
- source = textwrap.dedent('\n'.join(source.strip().split('\n')[2:]))
- source = source.replace(' ', '\t')
- source = source.replace('CONTEXT_NAME', self.remote_name)
- preamble_compressed = self.get_preamble()
- source = source.replace('PREAMBLE_COMPRESSED_LEN',
- str(len(preamble_compressed)))
- compressed = zlib.compress(source.encode(), 9)
- encoded = codecs.encode(compressed, 'base64').replace(b('\n'), b(''))
- # We can't use bytes.decode() in 3.x since it was restricted to always
- # return unicode, so codecs.decode() is used instead. In 3.x
- # codecs.decode() requires a bytes object. Since we must be compatible
- # with 2.4 (no bytes literal), an extra .encode() either returns the
- # same str (2.x) or an equivalent bytes (3.x).
- return self.get_python_argv() + [
- '-c',
- 'import codecs,os,sys;_=codecs.decode;'
- 'exec(_(_("%s".encode(),"base64"),"zip"))' % (encoded.decode(),)
- ]
-
- def get_econtext_config(self):
- assert self.max_message_size is not None
- parent_ids = mitogen.parent_ids[:]
- parent_ids.insert(0, mitogen.context_id)
- return {
- 'parent_ids': parent_ids,
- 'context_id': self.remote_id,
- 'debug': self.debug,
- 'profiling': self.profiling,
- 'unidirectional': self.unidirectional,
- 'log_level': get_log_level(),
- 'whitelist': self._router.get_module_whitelist(),
- 'blacklist': self._router.get_module_blacklist(),
- 'max_message_size': self.max_message_size,
- 'version': mitogen.__version__,
- }
-
- def get_preamble(self):
- suffix = (
- '\nExternalContext(%r).main()\n' %\
- (self.get_econtext_config(),)
- )
- partial = get_core_source_partial()
- return partial.append(suffix.encode('utf-8'))
-
- def start_child(self):
- args = self.get_boot_command()
- try:
- return self.create_child(args, **self.create_child_args)
- except OSError:
- e = sys.exc_info()[1]
- msg = 'Child start failed: %s. Command was: %s' % (e, Argv(args))
- raise mitogen.core.StreamError(msg)
-
- eof_error_hint = None
-
- def _adorn_eof_error(self, e):
- """
- Used by subclasses to provide additional information in the case of a
- failed connection.
- """
- if self.eof_error_hint:
- e.args = ('%s\n\n%s' % (e.args[0], self.eof_error_hint),)
-
- def _get_name(self):
- """
- Called by :meth:`connect` after :attr:`pid` is known. Subclasses can
- override it to specify a default stream name, or set
- :attr:`name_prefix` to generate a default format.
- """
- return u'%s.%s' % (self.name_prefix, self.pid)
-
- def connect(self):
- LOG.debug('%r.connect()', self)
- self.pid, fd, diag_fd = self.start_child()
- self.name = self._get_name()
- self.receive_side = mitogen.core.Side(self, fd)
- self.transmit_side = mitogen.core.Side(self, os.dup(fd))
- if diag_fd is not None:
- self.diag_stream = DiagLogStream(diag_fd, self)
- else:
- self.diag_stream = None
-
- LOG.debug('%r.connect(): pid:%r stdin:%r, stdout:%r, diag:%r',
- self, self.pid, self.receive_side.fd, self.transmit_side.fd,
- self.diag_stream and self.diag_stream.receive_side.fd)
-
- try:
- self._connect_bootstrap()
- except EofError:
- self.on_disconnect(self._router.broker)
- e = sys.exc_info()[1]
- self._adorn_eof_error(e)
- raise
- except Exception:
- self.on_disconnect(self._router.broker)
- self._reap_child()
- raise
-
- #: Sentinel value emitted by the first stage to indicate it is ready to
- #: receive the compressed bootstrap. For :mod:`mitogen.ssh` this must have
- #: length of at least `max(len('password'), len('debug1:'))`
- EC0_MARKER = mitogen.core.b('MITO000\n')
- EC1_MARKER = mitogen.core.b('MITO001\n')
-
- def _ec0_received(self):
- LOG.debug('%r._ec0_received()', self)
- write_all(self.transmit_side.fd, self.get_preamble())
- discard_until(self.receive_side.fd, self.EC1_MARKER,
- self.connect_deadline)
- if self.diag_stream:
- self._router.broker.start_receive(self.diag_stream)
-
- def _connect_bootstrap(self):
- discard_until(self.receive_side.fd, self.EC0_MARKER,
- self.connect_deadline)
- self._ec0_received()
-
-
-class ChildIdAllocator(object):
- def __init__(self, router):
- self.router = router
- self.lock = threading.Lock()
- self.it = iter(xrange(0))
-
- def allocate(self):
- self.lock.acquire()
- try:
- for id_ in self.it:
- return id_
-
- master = mitogen.core.Context(self.router, 0)
- start, end = master.send_await(
- mitogen.core.Message(dst_id=0, handle=mitogen.core.ALLOCATE_ID)
- )
- self.it = iter(xrange(start, end))
- finally:
- self.lock.release()
-
- return self.allocate()
-
-
-class CallChain(object):
- """
- Deliver :data:`mitogen.core.CALL_FUNCTION` messages to a target context,
- optionally threading related calls so an exception in an earlier call
- cancels subsequent calls.
-
- :param mitogen.core.Context context:
- Target context.
- :param bool pipelined:
- Enable pipelining.
-
- :meth:`call`, :meth:`call_no_reply` and :meth:`call_async`
- normally issue calls and produce responses with no memory of prior
- exceptions. If a call made with :meth:`call_no_reply` fails, the exception
- is logged to the target context's logging framework.
-
- **Pipelining**
-
- When pipelining is enabled, if an exception occurs during a call,
- subsequent calls made by the same :class:`CallChain` fail with the same
- exception, including those already in-flight on the network, and no further
- calls execute until :meth:`reset` is invoked.
-
- No exception is logged for calls made with :meth:`call_no_reply`, instead
- the exception is saved and reported as the result of subsequent
- :meth:`call` or :meth:`call_async` calls.
-
- Sequences of asynchronous calls can be made without wasting network
- round-trips to discover if prior calls succeed, and chains originating from
- multiple unrelated source contexts may overlap concurrently at a target
- context without interference.
-
- In this example, 4 calls complete in one round-trip::
-
- chain = mitogen.parent.CallChain(context, pipelined=True)
- chain.call_no_reply(os.mkdir, '/tmp/foo')
-
- # If previous mkdir() failed, this never runs:
- chain.call_no_reply(os.mkdir, '/tmp/foo/bar')
-
- # If either mkdir() failed, this never runs, and the exception is
- # asynchronously delivered to the receiver.
- recv = chain.call_async(subprocess.check_output, '/tmp/foo')
-
- # If anything so far failed, this never runs, and raises the exception.
- chain.call(do_something)
-
- # If this code was executed, the exception would also be raised.
- if recv.get().unpickle() == 'baz':
- pass
-
- When pipelining is enabled, :meth:`reset` must be invoked to ensure any
- exception is discarded, otherwise unbounded memory usage is possible in
- long-running programs. The context manager protocol is supported to ensure
- :meth:`reset` is always invoked::
-
- with mitogen.parent.CallChain(context, pipelined=True) as chain:
- chain.call_no_reply(...)
- chain.call_no_reply(...)
- chain.call_no_reply(...)
- chain.call(...)
-
- # chain.reset() automatically invoked.
- """
- def __init__(self, context, pipelined=False):
- self.context = context
- if pipelined:
- self.chain_id = self.make_chain_id()
- else:
- self.chain_id = None
-
- @classmethod
- def make_chain_id(cls):
- return '%s-%s-%x-%x' % (
- socket.gethostname(),
- os.getpid(),
- thread.get_ident(),
- int(1e6 * time.time()),
- )
-
- def __repr__(self):
- return '%s(%s)' % (self.__class__.__name__, self.context)
-
- def __enter__(self):
- return self
-
- def __exit__(self, _1, _2, _3):
- self.reset()
-
- def reset(self):
- """
- Instruct the target to forget any related exception.
- """
- if not self.chain_id:
- return
-
- saved, self.chain_id = self.chain_id, None
- try:
- self.call_no_reply(mitogen.core.Dispatcher.forget_chain, saved)
- finally:
- self.chain_id = saved
-
- closures_msg = (
- 'Mitogen cannot invoke closures, as doing so would require '
- 'serializing arbitrary program state, and no universal '
- 'method exists to recover a reference to them.'
- )
-
- lambda_msg = (
- 'Mitogen cannot invoke anonymous functions, as no universal method '
- 'exists to recover a reference to an anonymous function.'
- )
-
- method_msg = (
- 'Mitogen cannot invoke instance methods, as doing so would require '
- 'serializing arbitrary program state.'
- )
-
- def make_msg(self, fn, *args, **kwargs):
- if getattr(fn, closure_attr, None) is not None:
- raise TypeError(self.closures_msg)
- if fn.__name__ == '':
- raise TypeError(self.lambda_msg)
-
- if inspect.ismethod(fn):
- im_self = getattr(fn, IM_SELF_ATTR)
- if not inspect.isclass(im_self):
- raise TypeError(self.method_msg)
- klass = mitogen.core.to_text(im_self.__name__)
- else:
- klass = None
-
- tup = (
- self.chain_id,
- mitogen.core.to_text(fn.__module__),
- klass,
- mitogen.core.to_text(fn.__name__),
- args,
- mitogen.core.Kwargs(kwargs)
- )
- return mitogen.core.Message.pickled(tup,
- handle=mitogen.core.CALL_FUNCTION)
-
- def call_no_reply(self, fn, *args, **kwargs):
- """
- Like :meth:`call_async`, but do not wait for a return value, and inform
- the target context no reply is expected. If the call fails and
- pipelining is disabled, the exception will be logged to the target
- context's logging framework.
- """
- LOG.debug('%r.call_no_reply(): %r', self, CallSpec(fn, args, kwargs))
- self.context.send(self.make_msg(fn, *args, **kwargs))
-
- def call_async(self, fn, *args, **kwargs):
- """
- Arrange for `fn(*args, **kwargs)` to be invoked on the context's main
- thread.
-
- :param fn:
- A free function in module scope or a class method of a class
- directly reachable from module scope:
-
- .. code-block:: python
-
- # mymodule.py
-
- def my_func():
- '''A free function reachable as mymodule.my_func'''
-
- class MyClass:
- @classmethod
- def my_classmethod(cls):
- '''Reachable as mymodule.MyClass.my_classmethod'''
-
- def my_instancemethod(self):
- '''Unreachable: requires a class instance!'''
-
- class MyEmbeddedClass:
- @classmethod
- def my_classmethod(cls):
- '''Not directly reachable from module scope!'''
-
- :param tuple args:
- Function arguments, if any. See :ref:`serialization-rules` for
- permitted types.
- :param dict kwargs:
- Function keyword arguments, if any. See :ref:`serialization-rules`
- for permitted types.
- :returns:
- :class:`mitogen.core.Receiver` configured to receive the result of
- the invocation:
-
- .. code-block:: python
-
- recv = context.call_async(os.check_output, 'ls /tmp/')
- try:
- # Prints output once it is received.
- msg = recv.get()
- print(msg.unpickle())
- except mitogen.core.CallError, e:
- print('Call failed:', str(e))
-
- Asynchronous calls may be dispatched in parallel to multiple
- contexts and consumed as they complete using
- :class:`mitogen.select.Select`.
- """
- LOG.debug('%r.call_async(): %r', self, CallSpec(fn, args, kwargs))
- return self.context.send_async(self.make_msg(fn, *args, **kwargs))
-
- def call(self, fn, *args, **kwargs):
- """
- Like :meth:`call_async`, but block until the return value is available.
- Equivalent to::
-
- call_async(fn, *args, **kwargs).get().unpickle()
-
- :returns:
- The function's return value.
- :raises mitogen.core.CallError:
- An exception was raised in the remote context during execution.
- """
- receiver = self.call_async(fn, *args, **kwargs)
- return receiver.get().unpickle(throw_dead=False)
-
-
-class Context(mitogen.core.Context):
- """
- Extend :class:`mitogen.core.Context` with functionality useful to masters,
- and child contexts who later become parents. Currently when this class is
- required, the target context's router is upgraded at runtime.
- """
- #: A :class:`CallChain` instance constructed by default, with pipelining
- #: disabled. :meth:`call`, :meth:`call_async` and :meth:`call_no_reply` use
- #: this instance.
- call_chain_class = CallChain
-
- via = None
-
- def __init__(self, *args, **kwargs):
- super(Context, self).__init__(*args, **kwargs)
- self.default_call_chain = self.call_chain_class(self)
-
- def __ne__(self, other):
- return not (self == other)
-
- def __eq__(self, other):
- return (isinstance(other, mitogen.core.Context) and
- (other.context_id == self.context_id) and
- (other.router == self.router))
-
- def __hash__(self):
- return hash((self.router, self.context_id))
-
- def call_async(self, fn, *args, **kwargs):
- """
- See :meth:`CallChain.call_async`.
- """
- return self.default_call_chain.call_async(fn, *args, **kwargs)
-
- def call(self, fn, *args, **kwargs):
- """
- See :meth:`CallChain.call`.
- """
- return self.default_call_chain.call(fn, *args, **kwargs)
-
- def call_no_reply(self, fn, *args, **kwargs):
- """
- See :meth:`CallChain.call_no_reply`.
- """
- self.default_call_chain.call_no_reply(fn, *args, **kwargs)
-
- def shutdown(self, wait=False):
- """
- Arrange for the context to receive a ``SHUTDOWN`` message, triggering
- graceful shutdown.
-
- Due to a lack of support for timers, no attempt is made yet to force
- terminate a hung context using this method. This will be fixed shortly.
-
- :param bool wait:
- If :data:`True`, block the calling thread until the context has
- completely terminated.
-
- :returns:
- If `wait` is :data:`False`, returns a :class:`mitogen.core.Latch`
- whose :meth:`get() ` method returns
- :data:`None` when shutdown completes. The `timeout` parameter may
- be used to implement graceful timeouts.
- """
- LOG.debug('%r.shutdown() sending SHUTDOWN', self)
- latch = mitogen.core.Latch()
- mitogen.core.listen(self, 'disconnect', lambda: latch.put(None))
- self.send(
- mitogen.core.Message(
- handle=mitogen.core.SHUTDOWN,
- )
- )
-
- if wait:
- latch.get()
- else:
- return latch
-
-
-class RouteMonitor(object):
- """
- Generate and respond to :data:`mitogen.core.ADD_ROUTE` and
- :data:`mitogen.core.DEL_ROUTE` messages sent to the local context by
- maintaining a table of available routes, and propagating messages towards
- parents and siblings as appropriate.
-
- :class:`RouteMonitor` is responsible for generating routing messages for
- directly attached children. It learns of new children via
- :meth:`notice_stream` called by :class:`Router`, and subscribes to their
- ``disconnect`` event to learn when they disappear.
-
- In children, constructing this class overwrites the stub
- :data:`mitogen.core.DEL_ROUTE` handler installed by
- :class:`mitogen.core.ExternalContext`, which is expected behaviour when a
- child is beging upgraded in preparation to become a parent of children of
- its own.
-
- By virtue of only being active while responding to messages from a handler,
- RouteMonitor lives entirely on the broker thread, so its data requires no
- locking.
-
- :param Router router:
- Router to install handlers on.
- :param Context parent:
- :data:`None` in the master process, or reference to the parent context
- we should propagate route updates towards.
- """
- def __init__(self, router, parent=None):
- self.router = router
- self.parent = parent
- #: Mapping of Stream instance to integer context IDs reachable via the
- #: stream; used to cleanup routes during disconnection.
- self._routes_by_stream = {}
- self.router.add_handler(
- fn=self._on_add_route,
- handle=mitogen.core.ADD_ROUTE,
- persist=True,
- policy=is_immediate_child,
- overwrite=True,
- )
- self.router.add_handler(
- fn=self._on_del_route,
- handle=mitogen.core.DEL_ROUTE,
- persist=True,
- policy=is_immediate_child,
- overwrite=True,
- )
-
- def __repr__(self):
- return 'RouteMonitor()'
-
- def _send_one(self, stream, handle, target_id, name):
- """
- Compose and send an update message on a stream.
-
- :param mitogen.core.Stream stream:
- Stream to send it on.
- :param int handle:
- :data:`mitogen.core.ADD_ROUTE` or :data:`mitogen.core.DEL_ROUTE`
- :param int target_id:
- ID of the connecting or disconnecting context.
- :param str name:
- Context name or :data:`None`.
- """
- if not stream:
- # We may not have a stream during shutdown.
- return
-
- data = str(target_id)
- if name:
- data = '%s:%s' % (target_id, name)
- stream.send(
- mitogen.core.Message(
- handle=handle,
- data=data.encode('utf-8'),
- dst_id=stream.remote_id,
- )
- )
-
- def _propagate_up(self, handle, target_id, name=None):
- """
- In a non-master context, propagate an update towards the master.
-
- :param int handle:
- :data:`mitogen.core.ADD_ROUTE` or :data:`mitogen.core.DEL_ROUTE`
- :param int target_id:
- ID of the connecting or disconnecting context.
- :param str name:
- For :data:`mitogen.core.ADD_ROUTE`, the name of the new context
- assigned by its parent. This is used by parents to assign the
- :attr:`mitogen.core.Context.name` attribute.
- """
- if self.parent:
- stream = self.router.stream_by_id(self.parent.context_id)
- self._send_one(stream, handle, target_id, name)
-
- def _propagate_down(self, handle, target_id):
- """
- For DEL_ROUTE, we additionally want to broadcast the message to any
- stream that has ever communicated with the disconnecting ID, so
- core.py's :meth:`mitogen.core.Router._on_del_route` can turn the
- message into a disconnect event.
-
- :param int handle:
- :data:`mitogen.core.ADD_ROUTE` or :data:`mitogen.core.DEL_ROUTE`
- :param int target_id:
- ID of the connecting or disconnecting context.
- """
- for stream in self.router.get_streams():
- if target_id in stream.egress_ids and (
- (self.parent is None) or
- (self.parent.context_id != stream.remote_id)
- ):
- self._send_one(stream, mitogen.core.DEL_ROUTE, target_id, None)
-
- def notice_stream(self, stream):
- """
- When this parent is responsible for a new directly connected child
- stream, we're also responsible for broadcasting DEL_ROUTE upstream
- if/when that child disconnects.
- """
- self._routes_by_stream[stream] = set([stream.remote_id])
- self._propagate_up(mitogen.core.ADD_ROUTE, stream.remote_id,
- stream.name)
- mitogen.core.listen(
- obj=stream,
- name='disconnect',
- func=lambda: self._on_stream_disconnect(stream),
- )
-
- def get_routes(self, stream):
- """
- Return the set of context IDs reachable on a stream.
-
- :param mitogen.core.Stream stream:
- :returns: set([int])
- """
- return self._routes_by_stream.get(stream) or set()
-
- def _on_stream_disconnect(self, stream):
- """
- Respond to disconnection of a local stream by propagating DEL_ROUTE for
- any contexts we know were attached to it.
- """
- # During a stream crash it is possible for disconnect signal to fire
- # twice, in which case ignore the second instance.
- routes = self._routes_by_stream.pop(stream, None)
- if routes is None:
- return
-
- LOG.debug('%r: %r is gone; propagating DEL_ROUTE for %r',
- self, stream, routes)
- for target_id in routes:
- self.router.del_route(target_id)
- self._propagate_up(mitogen.core.DEL_ROUTE, target_id)
- self._propagate_down(mitogen.core.DEL_ROUTE, target_id)
-
- context = self.router.context_by_id(target_id, create=False)
- if context:
- mitogen.core.fire(context, 'disconnect')
-
- def _on_add_route(self, msg):
- """
- Respond to :data:`mitogen.core.ADD_ROUTE` by validating the source of
- the message, updating the local table, and propagating the message
- upwards.
- """
- if msg.is_dead:
- return
-
- target_id_s, _, target_name = bytes_partition(msg.data, b(':'))
- target_name = target_name.decode()
- target_id = int(target_id_s)
- self.router.context_by_id(target_id).name = target_name
- stream = self.router.stream_by_id(msg.auth_id)
- current = self.router.stream_by_id(target_id)
- if current and current.remote_id != mitogen.parent_id:
- LOG.error('Cannot add duplicate route to %r via %r, '
- 'already have existing route via %r',
- target_id, stream, current)
- return
-
- LOG.debug('Adding route to %d via %r', target_id, stream)
- self._routes_by_stream[stream].add(target_id)
- self.router.add_route(target_id, stream)
- self._propagate_up(mitogen.core.ADD_ROUTE, target_id, target_name)
-
- def _on_del_route(self, msg):
- """
- Respond to :data:`mitogen.core.DEL_ROUTE` by validating the source of
- the message, updating the local table, propagating the message
- upwards, and downwards towards any stream that every had a message
- forwarded from it towards the disconnecting context.
- """
- if msg.is_dead:
- return
-
- target_id = int(msg.data)
- registered_stream = self.router.stream_by_id(target_id)
- if registered_stream is None:
- return
-
- stream = self.router.stream_by_id(msg.auth_id)
- if registered_stream != stream:
- LOG.error('%r: received DEL_ROUTE for %d from %r, expected %r',
- self, target_id, stream, registered_stream)
- return
-
- context = self.router.context_by_id(target_id, create=False)
- if context:
- LOG.debug('%r: firing local disconnect for %r', self, context)
- mitogen.core.fire(context, 'disconnect')
-
- LOG.debug('%r: deleting route to %d via %r', self, target_id, stream)
- routes = self._routes_by_stream.get(stream)
- if routes:
- routes.discard(target_id)
-
- self.router.del_route(target_id)
- if stream.remote_id != mitogen.parent_id:
- self._propagate_up(mitogen.core.DEL_ROUTE, target_id)
- self._propagate_down(mitogen.core.DEL_ROUTE, target_id)
-
-
-class Router(mitogen.core.Router):
- context_class = Context
- debug = False
- profiling = False
-
- id_allocator = None
- responder = None
- log_forwarder = None
- route_monitor = None
-
- def upgrade(self, importer, parent):
- LOG.debug('%r.upgrade()', self)
- self.id_allocator = ChildIdAllocator(router=self)
- self.responder = ModuleForwarder(
- router=self,
- parent_context=parent,
- importer=importer,
- )
- self.route_monitor = RouteMonitor(self, parent)
- self.add_handler(
- fn=self._on_detaching,
- handle=mitogen.core.DETACHING,
- persist=True,
- )
-
- def _on_detaching(self, msg):
- if msg.is_dead:
- return
- stream = self.stream_by_id(msg.src_id)
- if stream.remote_id != msg.src_id or stream.detached:
- LOG.warning('bad DETACHING received on %r: %r', stream, msg)
- return
- LOG.debug('%r: marking as detached', stream)
- stream.detached = True
- msg.reply(None)
-
- def get_streams(self):
- """
- Return a snapshot of all streams in existence at time of call.
- """
- self._write_lock.acquire()
- try:
- return itervalues(self._stream_by_id)
- finally:
- self._write_lock.release()
-
- def add_route(self, target_id, stream):
- """
- Arrange for messages whose `dst_id` is `target_id` to be forwarded on
- the directly connected stream for `via_id`. This method is called
- automatically in response to :data:`mitogen.core.ADD_ROUTE` messages,
- but remains public while the design has not yet settled, and situations
- may arise where routing is not fully automatic.
- """
- LOG.debug('%r.add_route(%r, %r)', self, target_id, stream)
- assert isinstance(target_id, int)
- assert isinstance(stream, Stream)
-
- self._write_lock.acquire()
- try:
- self._stream_by_id[target_id] = stream
- finally:
- self._write_lock.release()
-
- def del_route(self, target_id):
- LOG.debug('%r.del_route(%r)', self, target_id)
- # DEL_ROUTE may be sent by a parent if it knows this context sent
- # messages to a peer that has now disconnected, to let us raise
- # 'disconnect' event on the appropriate Context instance. In that case,
- # we won't a matching _stream_by_id entry for the disappearing route,
- # so don't raise an error for a missing key here.
- self._write_lock.acquire()
- try:
- self._stream_by_id.pop(target_id, None)
- finally:
- self._write_lock.release()
-
- def get_module_blacklist(self):
- if mitogen.context_id == 0:
- return self.responder.blacklist
- return self.importer.master_blacklist
-
- def get_module_whitelist(self):
- if mitogen.context_id == 0:
- return self.responder.whitelist
- return self.importer.master_whitelist
-
- def allocate_id(self):
- return self.id_allocator.allocate()
-
- connection_timeout_msg = u"Connection timed out."
-
- def _connect(self, klass, name=None, **kwargs):
- context_id = self.allocate_id()
- context = self.context_class(self, context_id)
- kwargs['old_router'] = self
- kwargs['max_message_size'] = self.max_message_size
- stream = klass(self, context_id, **kwargs)
- if name is not None:
- stream.name = name
- try:
- stream.connect()
- except mitogen.core.TimeoutError:
- raise mitogen.core.StreamError(self.connection_timeout_msg)
- context.name = stream.name
- self.route_monitor.notice_stream(stream)
- self.register(context, stream)
- return context
-
- def connect(self, method_name, name=None, **kwargs):
- klass = stream_by_method_name(method_name)
- kwargs.setdefault(u'debug', self.debug)
- kwargs.setdefault(u'profiling', self.profiling)
- kwargs.setdefault(u'unidirectional', self.unidirectional)
-
- via = kwargs.pop(u'via', None)
- if via is not None:
- return self.proxy_connect(via, method_name, name=name,
- **mitogen.core.Kwargs(kwargs))
- return self._connect(klass, name=name,
- **mitogen.core.Kwargs(kwargs))
-
- def proxy_connect(self, via_context, method_name, name=None, **kwargs):
- resp = via_context.call(_proxy_connect,
- name=name,
- method_name=method_name,
- kwargs=mitogen.core.Kwargs(kwargs),
- )
- if resp['msg'] is not None:
- raise mitogen.core.StreamError(resp['msg'])
-
- name = u'%s.%s' % (via_context.name, resp['name'])
- context = self.context_class(self, resp['id'], name=name)
- context.via = via_context
- self._write_lock.acquire()
- try:
- self._context_by_id[context.context_id] = context
- finally:
- self._write_lock.release()
- return context
-
- def doas(self, **kwargs):
- return self.connect(u'doas', **kwargs)
-
- def docker(self, **kwargs):
- return self.connect(u'docker', **kwargs)
-
- def kubectl(self, **kwargs):
- return self.connect(u'kubectl', **kwargs)
-
- def fork(self, **kwargs):
- return self.connect(u'fork', **kwargs)
-
- def jail(self, **kwargs):
- return self.connect(u'jail', **kwargs)
-
- def local(self, **kwargs):
- return self.connect(u'local', **kwargs)
-
- def lxc(self, **kwargs):
- return self.connect(u'lxc', **kwargs)
-
- def lxd(self, **kwargs):
- return self.connect(u'lxd', **kwargs)
-
- def setns(self, **kwargs):
- return self.connect(u'setns', **kwargs)
-
- def su(self, **kwargs):
- return self.connect(u'su', **kwargs)
-
- def sudo(self, **kwargs):
- return self.connect(u'sudo', **kwargs)
-
- def ssh(self, **kwargs):
- return self.connect(u'ssh', **kwargs)
-
-
-class ProcessMonitor(object):
- """
- Install a :data:`signal.SIGCHLD` handler that generates callbacks when a
- specific child process has exitted. This class is obsolete, do not use.
- """
- def __init__(self):
- # pid -> callback()
- self.callback_by_pid = {}
- signal.signal(signal.SIGCHLD, self._on_sigchld)
-
- def _on_sigchld(self, _signum, _frame):
- for pid, callback in self.callback_by_pid.items():
- pid, status = os.waitpid(pid, os.WNOHANG)
- if pid:
- callback(status)
- del self.callback_by_pid[pid]
-
- def add(self, pid, callback):
- """
- Add a callback function to be notified of the exit status of a process.
-
- :param int pid:
- Process ID to be notified of.
-
- :param callback:
- Function invoked as `callback(status)`, where `status` is the raw
- exit status of the child process.
- """
- self.callback_by_pid[pid] = callback
-
- _instance = None
-
- @classmethod
- def instance(cls):
- if cls._instance is None:
- cls._instance = cls()
- return cls._instance
-
-
-class ModuleForwarder(object):
- """
- Respond to GET_MODULE requests in a slave by forwarding the request to our
- parent context, or satisfying the request from our local Importer cache.
- """
- def __init__(self, router, parent_context, importer):
- self.router = router
- self.parent_context = parent_context
- self.importer = importer
- router.add_handler(
- fn=self._on_forward_module,
- handle=mitogen.core.FORWARD_MODULE,
- persist=True,
- policy=mitogen.core.has_parent_authority,
- )
- router.add_handler(
- fn=self._on_get_module,
- handle=mitogen.core.GET_MODULE,
- persist=True,
- policy=is_immediate_child,
- )
-
- def __repr__(self):
- return 'ModuleForwarder(%r)' % (self.router,)
-
- def _on_forward_module(self, msg):
- if msg.is_dead:
- return
-
- context_id_s, _, fullname = bytes_partition(msg.data, b('\x00'))
- fullname = mitogen.core.to_text(fullname)
- context_id = int(context_id_s)
- stream = self.router.stream_by_id(context_id)
- if stream.remote_id == mitogen.parent_id:
- LOG.error('%r: dropping FORWARD_MODULE(%d, %r): no route to child',
- self, context_id, fullname)
- return
-
- if fullname in stream.sent_modules:
- return
-
- LOG.debug('%r._on_forward_module() sending %r to %r via %r',
- self, fullname, context_id, stream.remote_id)
- self._send_module_and_related(stream, fullname)
- if stream.remote_id != context_id:
- stream._send(
- mitogen.core.Message(
- data=msg.data,
- handle=mitogen.core.FORWARD_MODULE,
- dst_id=stream.remote_id,
- )
- )
-
- def _on_get_module(self, msg):
- LOG.debug('%r._on_get_module(%r)', self, msg)
- if msg.is_dead:
- return
-
- fullname = msg.data.decode('utf-8')
- callback = lambda: self._on_cache_callback(msg, fullname)
- self.importer._request_module(fullname, callback)
-
- def _on_cache_callback(self, msg, fullname):
- LOG.debug('%r._on_get_module(): sending %r', self, fullname)
- stream = self.router.stream_by_id(msg.src_id)
- self._send_module_and_related(stream, fullname)
-
- def _send_module_and_related(self, stream, fullname):
- tup = self.importer._cache[fullname]
- for related in tup[4]:
- rtup = self.importer._cache.get(related)
- if rtup:
- self._send_one_module(stream, rtup)
- else:
- LOG.debug('%r._send_module_and_related(%r): absent: %r',
- self, fullname, related)
-
- self._send_one_module(stream, tup)
-
- def _send_one_module(self, stream, tup):
- if tup[0] not in stream.sent_modules:
- stream.sent_modules.add(tup[0])
- self.router._async_route(
- mitogen.core.Message.pickled(
- tup,
- dst_id=stream.remote_id,
- handle=mitogen.core.LOAD_MODULE,
- )
- )
diff --git a/mitogen-0.2.7/mitogen/profiler.py b/mitogen-0.2.7/mitogen/profiler.py
deleted file mode 100644
index 74bbdb2..000000000
--- a/mitogen-0.2.7/mitogen/profiler.py
+++ /dev/null
@@ -1,166 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-"""mitogen.profiler
- Record and report cProfile statistics from a run. Creates one aggregated
- output file, one aggregate containing only workers, and one for the
- top-level process.
-
-Usage:
- mitogen.profiler record [args ..]
- mitogen.profiler report [sort_mode]
- mitogen.profiler stat [args ..]
-
-Mode:
- record: Record a trace.
- report: Report on a previously recorded trace.
- stat: Record and report in a single step.
-
-Where:
- dest_path: Filesystem prefix to write .pstats files to.
- sort_mode: Sorting mode; defaults to "cumulative". See:
- https://docs.python.org/2/library/profile.html#pstats.Stats.sort_stats
-
-Example:
- mitogen.profiler record /tmp/mypatch ansible-playbook foo.yml
- mitogen.profiler dump /tmp/mypatch-worker.pstats
-"""
-
-from __future__ import print_function
-import os
-import pstats
-import cProfile
-import shutil
-import subprocess
-import sys
-import tempfile
-import time
-
-import mitogen.core
-
-
-def try_merge(stats, path):
- try:
- stats.add(path)
- return True
- except Exception as e:
- print('Failed. Race? Will retry. %s' % (e,))
- return False
-
-
-def merge_stats(outpath, inpaths):
- first, rest = inpaths[0], inpaths[1:]
- for x in range(5):
- try:
- stats = pstats.Stats(first)
- except EOFError:
- time.sleep(0.2)
- continue
-
- print("Writing %r..." % (outpath,))
- for path in rest:
- #print("Merging %r into %r.." % (os.path.basename(path), outpath))
- for x in range(5):
- if try_merge(stats, path):
- break
- time.sleep(0.2)
-
- stats.dump_stats(outpath)
-
-
-def generate_stats(outpath, tmpdir):
- print('Generating stats..')
- all_paths = []
- paths_by_ident = {}
-
- for name in os.listdir(tmpdir):
- if name.endswith('-dump.pstats'):
- ident, _, pid = name.partition('-')
- path = os.path.join(tmpdir, name)
- all_paths.append(path)
- paths_by_ident.setdefault(ident, []).append(path)
-
- merge_stats('%s-all.pstat' % (outpath,), all_paths)
- for ident, paths in paths_by_ident.items():
- merge_stats('%s-%s.pstat' % (outpath, ident), paths)
-
-
-def do_record(tmpdir, path, *args):
- env = os.environ.copy()
- fmt = '%(identity)s-%(pid)s.%(now)s-dump.%(ext)s'
- env['MITOGEN_PROFILING'] = '1'
- env['MITOGEN_PROFILE_FMT'] = os.path.join(tmpdir, fmt)
- rc = subprocess.call(args, env=env)
- generate_stats(path, tmpdir)
- return rc
-
-
-def do_report(tmpdir, path, sort='cumulative'):
- stats = pstats.Stats(path).sort_stats(sort)
- stats.print_stats(100)
-
-
-def do_stat(tmpdir, sort, *args):
- valid_sorts = pstats.Stats.sort_arg_dict_default
- if sort not in valid_sorts:
- sys.stderr.write('Invalid sort %r, must be one of %s\n' %
- (sort, ', '.join(sorted(valid_sorts))))
- sys.exit(1)
-
- outfile = os.path.join(tmpdir, 'combined')
- do_record(tmpdir, outfile, *args)
- aggs = ('app.main', 'mitogen.broker', 'mitogen.child_main',
- 'mitogen.service.pool', 'Strategy', 'WorkerProcess',
- 'all')
- for agg in aggs:
- path = '%s-%s.pstat' % (outfile, agg)
- if os.path.exists(path):
- print()
- print()
- print('------ Aggregation %r ------' % (agg,))
- print()
- do_report(tmpdir, path, sort)
- print()
-
-
-def main():
- if len(sys.argv) < 2 or sys.argv[1] not in ('record', 'report', 'stat'):
- sys.stderr.write(__doc__)
- sys.exit(1)
-
- func = globals()['do_' + sys.argv[1]]
- tmpdir = tempfile.mkdtemp(prefix='mitogen.profiler')
- try:
- sys.exit(func(tmpdir, *sys.argv[2:]) or 0)
- finally:
- shutil.rmtree(tmpdir)
-
-if __name__ == '__main__':
- main()
diff --git a/mitogen-0.2.7/mitogen/select.py b/mitogen-0.2.7/mitogen/select.py
deleted file mode 100644
index 51aebc2..000000000
--- a/mitogen-0.2.7/mitogen/select.py
+++ /dev/null
@@ -1,333 +0,0 @@
-# Copyright 2019, David Wilson
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright notice,
-# this list of conditions and the following disclaimer in the documentation
-# and/or other materials provided with the distribution.
-#
-# 3. Neither the name of the copyright holder nor the names of its contributors
-# may be used to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-# POSSIBILITY OF SUCH DAMAGE.
-
-# !mitogen: minify_safe
-
-import mitogen.core
-
-
-class Error(mitogen.core.Error):
- pass
-
-
-class Event(object):
- """
- Represents one selected event.
- """
- #: The first Receiver or Latch the event traversed.
- source = None
-
- #: The :class:`mitogen.core.Message` delivered to a receiver, or the object
- #: posted to a latch.
- data = None
-
-
-class Select(object):
- """
- Support scatter/gather asynchronous calls and waiting on multiple
- :class:`receivers `,
- :class:`channels `,
- :class:`latches `, and
- :class:`sub-selects