diff --git a/ansible.cfg b/ansible.cfg index e57103f..6749b25 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -1,6 +1,8 @@ [defaults] host_key_checking = False retry_files_enabled = False +strategy_plugins = ./mitogen-0.2.7/ansible_mitogen/plugins/strategy +strategy = mitogen_linear [ssh_connection] pipelining = True diff --git a/mitogen-0.2.7/LICENSE b/mitogen-0.2.7/LICENSE new file mode 100644 index 000000000..70e43a9 --- /dev/null +++ b/mitogen-0.2.7/LICENSE @@ -0,0 +1,26 @@ +Copyright 2019, David Wilson + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/mitogen-0.2.7/MANIFEST.in b/mitogen-0.2.7/MANIFEST.in new file mode 100644 index 000000000..1aba38f --- /dev/null +++ b/mitogen-0.2.7/MANIFEST.in @@ -0,0 +1 @@ +include LICENSE diff --git a/mitogen-0.2.7/PKG-INFO b/mitogen-0.2.7/PKG-INFO new file mode 100644 index 000000000..f63ba91 --- /dev/null +++ b/mitogen-0.2.7/PKG-INFO @@ -0,0 +1,23 @@ +Metadata-Version: 1.1 +Name: mitogen +Version: 0.2.7 +Summary: Library for writing distributed self-replicating programs. +Home-page: https://github.com/dw/mitogen/ +Author: David Wilson +Author-email: UNKNOWN +License: New BSD +Description: UNKNOWN +Platform: UNKNOWN +Classifier: Environment :: Console +Classifier: Intended Audience :: System Administrators +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: POSIX +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.4 +Classifier: Programming Language :: Python :: 2.5 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Topic :: System :: Distributed Computing +Classifier: Topic :: System :: Systems Administration diff --git a/mitogen-0.2.7/README.md b/mitogen-0.2.7/README.md new file mode 100644 index 000000000..5ef2447 --- /dev/null +++ b/mitogen-0.2.7/README.md @@ -0,0 +1,13 @@ + +# Mitogen + + +Please see the documentation. + +![](https://i.imgur.com/eBM6LhJ.gif) + +[![Total alerts](https://img.shields.io/lgtm/alerts/g/dw/mitogen.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/dw/mitogen/alerts/) + +[![Build Status](https://travis-ci.org/dw/mitogen.svg?branch=master)](https://travis-ci.org/dw/mitogen) + +[![Pipelines Status](https://dev.azure.com/dw-mitogen/Mitogen/_apis/build/status/dw.mitogen?branchName=master)](https://dev.azure.com/dw-mitogen/Mitogen/_build/latest?definitionId=1?branchName=master) diff --git a/mitogen-0.2.7/ansible_mitogen/__init__.py b/mitogen-0.2.7/ansible_mitogen/__init__.py new file mode 100644 index 000000000..e69de29 diff --git a/mitogen-0.2.7/ansible_mitogen/__init__.pyc b/mitogen-0.2.7/ansible_mitogen/__init__.pyc new file mode 100644 index 000000000..004bfb0 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/__init__.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/__pycache__/__init__.cpython-37.pyc b/mitogen-0.2.7/ansible_mitogen/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 000000000..d15a0c5 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/__pycache__/__init__.cpython-37.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/__pycache__/affinity.cpython-37.pyc b/mitogen-0.2.7/ansible_mitogen/__pycache__/affinity.cpython-37.pyc new file mode 100644 index 000000000..b021719 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/__pycache__/affinity.cpython-37.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/__pycache__/connection.cpython-37.pyc b/mitogen-0.2.7/ansible_mitogen/__pycache__/connection.cpython-37.pyc new file mode 100644 index 000000000..74d1aa3 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/__pycache__/connection.cpython-37.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/__pycache__/loaders.cpython-37.pyc b/mitogen-0.2.7/ansible_mitogen/__pycache__/loaders.cpython-37.pyc new file mode 100644 index 000000000..4d75377 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/__pycache__/loaders.cpython-37.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/__pycache__/logging.cpython-37.pyc b/mitogen-0.2.7/ansible_mitogen/__pycache__/logging.cpython-37.pyc new file mode 100644 index 000000000..083cc40 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/__pycache__/logging.cpython-37.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/__pycache__/mixins.cpython-37.pyc b/mitogen-0.2.7/ansible_mitogen/__pycache__/mixins.cpython-37.pyc new file mode 100644 index 000000000..fb59de3 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/__pycache__/mixins.cpython-37.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/__pycache__/module_finder.cpython-37.pyc b/mitogen-0.2.7/ansible_mitogen/__pycache__/module_finder.cpython-37.pyc new file mode 100644 index 000000000..3c6ea03 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/__pycache__/module_finder.cpython-37.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/__pycache__/parsing.cpython-37.pyc b/mitogen-0.2.7/ansible_mitogen/__pycache__/parsing.cpython-37.pyc new file mode 100644 index 000000000..cc0a1c0 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/__pycache__/parsing.cpython-37.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/__pycache__/planner.cpython-37.pyc b/mitogen-0.2.7/ansible_mitogen/__pycache__/planner.cpython-37.pyc new file mode 100644 index 000000000..054b7e1 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/__pycache__/planner.cpython-37.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/__pycache__/process.cpython-37.pyc b/mitogen-0.2.7/ansible_mitogen/__pycache__/process.cpython-37.pyc new file mode 100644 index 000000000..a95a431 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/__pycache__/process.cpython-37.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/__pycache__/runner.cpython-37.pyc b/mitogen-0.2.7/ansible_mitogen/__pycache__/runner.cpython-37.pyc new file mode 100644 index 000000000..aa45d24 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/__pycache__/runner.cpython-37.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/__pycache__/services.cpython-37.pyc b/mitogen-0.2.7/ansible_mitogen/__pycache__/services.cpython-37.pyc new file mode 100644 index 000000000..2cdf1be Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/__pycache__/services.cpython-37.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/__pycache__/strategy.cpython-37.pyc b/mitogen-0.2.7/ansible_mitogen/__pycache__/strategy.cpython-37.pyc new file mode 100644 index 000000000..4535988 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/__pycache__/strategy.cpython-37.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/__pycache__/target.cpython-37.pyc b/mitogen-0.2.7/ansible_mitogen/__pycache__/target.cpython-37.pyc new file mode 100644 index 000000000..1032825 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/__pycache__/target.cpython-37.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/__pycache__/transport_config.cpython-37.pyc b/mitogen-0.2.7/ansible_mitogen/__pycache__/transport_config.cpython-37.pyc new file mode 100644 index 000000000..2ccb6a4 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/__pycache__/transport_config.cpython-37.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/affinity.py b/mitogen-0.2.7/ansible_mitogen/affinity.py new file mode 100644 index 000000000..09a6ace --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/affinity.py @@ -0,0 +1,269 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +""" +As Mitogen separates asynchronous IO out to a broker thread, communication +necessarily involves context switching and waking that thread. When application +threads and the broker share a CPU, this can be almost invisibly fast - around +25 microseconds for a full A->B->A round-trip. + +However when threads are scheduled on different CPUs, round-trip delays +regularly vary wildly, and easily into milliseconds. Many contributing factors +exist, not least scenarios like: + +1. A is preempted immediately after waking B, but before releasing the GIL. +2. B wakes from IO wait only to immediately enter futex wait. +3. A may wait 10ms or more for another timeslice, as the scheduler on its CPU + runs threads unrelated to its transaction (i.e. not B), wake only to release + its GIL, before entering IO sleep waiting for a reply from B, which cannot + exist yet. +4. B wakes, acquires GIL, performs work, and sends reply to A, causing it to + wake. B is preempted before releasing GIL. +5. A wakes from IO wait only to immediately enter futex wait. +6. B may wait 10ms or more for another timeslice, wake only to release its GIL, + before sleeping again. +7. A wakes, acquires GIL, finally receives reply. + +Per above if we are unlucky, on an even moderately busy machine it is possible +to lose milliseconds just in scheduling delay, and the effect is compounded +when pairs of threads in process A are communicating with pairs of threads in +process B using the same scheme, such as when Ansible WorkerProcess is +communicating with ContextService in the connection multiplexer. In the worst +case it could involve 4 threads working in lockstep spread across 4 busy CPUs. + +Since multithreading in Python is essentially useless except for waiting on IO +due to the presence of the GIL, at least in Ansible there is no good reason for +threads in the same process to run on distinct CPUs - they always operate in +lockstep due to the GIL, and are thus vulnerable to issues like above. + +Linux lacks any natural API to describe what we want, it only permits +individual threads to be constrained to run on specific CPUs, and for that +constraint to be inherited by new threads and forks of the constrained thread. + +This module therefore implements a CPU pinning policy for Ansible processes, +providing methods that should be called early in any new process, either to +rebalance which CPU it is pinned to, or in the case of subprocesses, to remove +the pinning entirely. It is likely to require ongoing tweaking, since pinning +necessarily involves preventing the scheduler from making load balancing +decisions. +""" + +import ctypes +import mmap +import multiprocessing +import os +import struct + +import mitogen.core +import mitogen.parent + + +try: + _libc = ctypes.CDLL(None, use_errno=True) + _strerror = _libc.strerror + _strerror.restype = ctypes.c_char_p + _pthread_mutex_init = _libc.pthread_mutex_init + _pthread_mutex_lock = _libc.pthread_mutex_lock + _pthread_mutex_unlock = _libc.pthread_mutex_unlock + _sched_setaffinity = _libc.sched_setaffinity +except (OSError, AttributeError): + _libc = None + _strerror = None + _pthread_mutex_init = None + _pthread_mutex_lock = None + _pthread_mutex_unlock = None + _sched_setaffinity = None + + +class pthread_mutex_t(ctypes.Structure): + """ + Wrap pthread_mutex_t to allow storing a lock in shared memory. + """ + _fields_ = [ + ('data', ctypes.c_uint8 * 512), + ] + + def init(self): + if _pthread_mutex_init(self.data, 0): + raise Exception(_strerror(ctypes.get_errno())) + + def acquire(self): + if _pthread_mutex_lock(self.data): + raise Exception(_strerror(ctypes.get_errno())) + + def release(self): + if _pthread_mutex_unlock(self.data): + raise Exception(_strerror(ctypes.get_errno())) + + +class State(ctypes.Structure): + """ + Contents of shared memory segment. This allows :meth:`Manager.assign` to be + called from any child, since affinity assignment must happen from within + the context of the new child process. + """ + _fields_ = [ + ('lock', pthread_mutex_t), + ('counter', ctypes.c_uint8), + ] + + +class Policy(object): + """ + Process affinity policy. + """ + def assign_controller(self): + """ + Assign the Ansible top-level policy to this process. + """ + + def assign_muxprocess(self): + """ + Assign the MuxProcess policy to this process. + """ + + def assign_worker(self): + """ + Assign the WorkerProcess policy to this process. + """ + + def assign_subprocess(self): + """ + Assign the helper subprocess policy to this process. + """ + +class FixedPolicy(Policy): + """ + :class:`Policy` for machines where the only control method available is + fixed CPU placement. The scheme here was tested on an otherwise idle 16 + thread machine. + + - The connection multiplexer is pinned to CPU 0. + - The Ansible top-level (strategy) is pinned to CPU 1. + - WorkerProcesses are pinned sequentually to 2..N, wrapping around when no + more CPUs exist. + - Children such as SSH may be scheduled on any CPU except 0/1. + + If the machine has less than 4 cores available, the top-level and workers + are pinned between CPU 2..N, i.e. no CPU is reserved for the top-level + process. + + This could at least be improved by having workers pinned to independent + cores, before reusing the second hyperthread of an existing core. + + A hook is installed that causes :meth:`reset` to run in the child of any + process created with :func:`mitogen.parent.detach_popen`, ensuring + CPU-intensive children like SSH are not forced to share the same core as + the (otherwise potentially very busy) parent. + """ + def __init__(self, cpu_count=None): + #: For tests. + self.cpu_count = cpu_count or multiprocessing.cpu_count() + self.mem = mmap.mmap(-1, 4096) + self.state = State.from_buffer(self.mem) + self.state.lock.init() + + if self.cpu_count < 2: + # uniprocessor + self._reserve_mux = False + self._reserve_controller = False + self._reserve_mask = 0 + self._reserve_shift = 0 + elif self.cpu_count < 4: + # small SMP + self._reserve_mux = True + self._reserve_controller = False + self._reserve_mask = 1 + self._reserve_shift = 1 + else: + # big SMP + self._reserve_mux = True + self._reserve_controller = True + self._reserve_mask = 3 + self._reserve_shift = 2 + + def _set_affinity(self, mask): + mitogen.parent._preexec_hook = self._clear + self._set_cpu_mask(mask) + + def _balance(self): + self.state.lock.acquire() + try: + n = self.state.counter + self.state.counter += 1 + finally: + self.state.lock.release() + + self._set_cpu(self._reserve_shift + ( + (n % (self.cpu_count - self._reserve_shift)) + )) + + def _set_cpu(self, cpu): + self._set_affinity(1 << cpu) + + def _clear(self): + all_cpus = (1 << self.cpu_count) - 1 + self._set_affinity(all_cpus & ~self._reserve_mask) + + def assign_controller(self): + if self._reserve_controller: + self._set_cpu(1) + else: + self._balance() + + def assign_muxprocess(self): + self._set_cpu(0) + + def assign_worker(self): + self._balance() + + def assign_subprocess(self): + self._clear() + + +class LinuxPolicy(FixedPolicy): + def _mask_to_bytes(self, mask): + """ + Convert the (type long) mask to a cpu_set_t. + """ + chunks = [] + shiftmask = (2 ** 64) - 1 + for x in range(16): + chunks.append(struct.pack('>= 64 + return mitogen.core.b('').join(chunks) + + def _set_cpu_mask(self, mask): + s = self._mask_to_bytes(mask) + _sched_setaffinity(os.getpid(), len(s), s) + + +if _sched_setaffinity is not None: + policy = LinuxPolicy() +else: + policy = Policy() diff --git a/mitogen-0.2.7/ansible_mitogen/affinity.pyc b/mitogen-0.2.7/ansible_mitogen/affinity.pyc new file mode 100644 index 000000000..1aa3a74 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/affinity.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/compat/__init__.py b/mitogen-0.2.7/ansible_mitogen/compat/__init__.py new file mode 100644 index 000000000..e69de29 diff --git a/mitogen-0.2.7/ansible_mitogen/compat/simplejson/__init__.py b/mitogen-0.2.7/ansible_mitogen/compat/simplejson/__init__.py new file mode 100644 index 000000000..d5b4d39 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/compat/simplejson/__init__.py @@ -0,0 +1,318 @@ +r"""JSON (JavaScript Object Notation) is a subset of +JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data +interchange format. + +:mod:`simplejson` exposes an API familiar to users of the standard library +:mod:`marshal` and :mod:`pickle` modules. It is the externally maintained +version of the :mod:`json` library contained in Python 2.6, but maintains +compatibility with Python 2.4 and Python 2.5 and (currently) has +significant performance advantages, even without using the optional C +extension for speedups. + +Encoding basic Python object hierarchies:: + + >>> import simplejson as json + >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) + '["foo", {"bar": ["baz", null, 1.0, 2]}]' + >>> print json.dumps("\"foo\bar") + "\"foo\bar" + >>> print json.dumps(u'\u1234') + "\u1234" + >>> print json.dumps('\\') + "\\" + >>> print json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True) + {"a": 0, "b": 0, "c": 0} + >>> from StringIO import StringIO + >>> io = StringIO() + >>> json.dump(['streaming API'], io) + >>> io.getvalue() + '["streaming API"]' + +Compact encoding:: + + >>> import simplejson as json + >>> json.dumps([1,2,3,{'4': 5, '6': 7}], separators=(',',':')) + '[1,2,3,{"4":5,"6":7}]' + +Pretty printing:: + + >>> import simplejson as json + >>> s = json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4) + >>> print '\n'.join([l.rstrip() for l in s.splitlines()]) + { + "4": 5, + "6": 7 + } + +Decoding JSON:: + + >>> import simplejson as json + >>> obj = [u'foo', {u'bar': [u'baz', None, 1.0, 2]}] + >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') == obj + True + >>> json.loads('"\\"foo\\bar"') == u'"foo\x08ar' + True + >>> from StringIO import StringIO + >>> io = StringIO('["streaming API"]') + >>> json.load(io)[0] == 'streaming API' + True + +Specializing JSON object decoding:: + + >>> import simplejson as json + >>> def as_complex(dct): + ... if '__complex__' in dct: + ... return complex(dct['real'], dct['imag']) + ... return dct + ... + >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', + ... object_hook=as_complex) + (1+2j) + >>> import decimal + >>> json.loads('1.1', parse_float=decimal.Decimal) == decimal.Decimal('1.1') + True + +Specializing JSON object encoding:: + + >>> import simplejson as json + >>> def encode_complex(obj): + ... if isinstance(obj, complex): + ... return [obj.real, obj.imag] + ... raise TypeError(repr(o) + " is not JSON serializable") + ... + >>> json.dumps(2 + 1j, default=encode_complex) + '[2.0, 1.0]' + >>> json.JSONEncoder(default=encode_complex).encode(2 + 1j) + '[2.0, 1.0]' + >>> ''.join(json.JSONEncoder(default=encode_complex).iterencode(2 + 1j)) + '[2.0, 1.0]' + + +Using simplejson.tool from the shell to validate and pretty-print:: + + $ echo '{"json":"obj"}' | python -m simplejson.tool + { + "json": "obj" + } + $ echo '{ 1.2:3.4}' | python -m simplejson.tool + Expecting property name: line 1 column 2 (char 2) +""" +__version__ = '2.0.9' +__all__ = [ + 'dump', 'dumps', 'load', 'loads', + 'JSONDecoder', 'JSONEncoder', +] + +__author__ = 'Bob Ippolito ' + +from decoder import JSONDecoder +from encoder import JSONEncoder + +_default_encoder = JSONEncoder( + skipkeys=False, + ensure_ascii=True, + check_circular=True, + allow_nan=True, + indent=None, + separators=None, + encoding='utf-8', + default=None, +) + +def dump(obj, fp, skipkeys=False, ensure_ascii=True, check_circular=True, + allow_nan=True, cls=None, indent=None, separators=None, + encoding='utf-8', default=None, **kw): + """Serialize ``obj`` as a JSON formatted stream to ``fp`` (a + ``.write()``-supporting file-like object). + + If ``skipkeys`` is true then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + will be skipped instead of raising a ``TypeError``. + + If ``ensure_ascii`` is false, then the some chunks written to ``fp`` + may be ``unicode`` instances, subject to normal Python ``str`` to + ``unicode`` coercion rules. Unless ``fp.write()`` explicitly + understands ``unicode`` (as in ``codecs.getwriter()``) this is likely + to cause an error. + + If ``check_circular`` is false, then the circular reference check + for container types will be skipped and a circular reference will + result in an ``OverflowError`` (or worse). + + If ``allow_nan`` is false, then it will be a ``ValueError`` to + serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) + in strict compliance of the JSON specification, instead of using the + JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). + + If ``indent`` is a non-negative integer, then JSON array elements and object + members will be pretty-printed with that indent level. An indent level + of 0 will only insert newlines. ``None`` is the most compact representation. + + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. + + ``encoding`` is the character encoding for str instances, default is UTF-8. + + ``default(obj)`` is a function that should return a serializable version + of obj or raise TypeError. The default simply raises TypeError. + + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the + ``.default()`` method to serialize additional types), specify it with + the ``cls`` kwarg. + + """ + # cached encoder + if (not skipkeys and ensure_ascii and + check_circular and allow_nan and + cls is None and indent is None and separators is None and + encoding == 'utf-8' and default is None and not kw): + iterable = _default_encoder.iterencode(obj) + else: + if cls is None: + cls = JSONEncoder + iterable = cls(skipkeys=skipkeys, ensure_ascii=ensure_ascii, + check_circular=check_circular, allow_nan=allow_nan, indent=indent, + separators=separators, encoding=encoding, + default=default, **kw).iterencode(obj) + # could accelerate with writelines in some versions of Python, at + # a debuggability cost + for chunk in iterable: + fp.write(chunk) + + +def dumps(obj, skipkeys=False, ensure_ascii=True, check_circular=True, + allow_nan=True, cls=None, indent=None, separators=None, + encoding='utf-8', default=None, **kw): + """Serialize ``obj`` to a JSON formatted ``str``. + + If ``skipkeys`` is false then ``dict`` keys that are not basic types + (``str``, ``unicode``, ``int``, ``long``, ``float``, ``bool``, ``None``) + will be skipped instead of raising a ``TypeError``. + + If ``ensure_ascii`` is false, then the return value will be a + ``unicode`` instance subject to normal Python ``str`` to ``unicode`` + coercion rules instead of being escaped to an ASCII ``str``. + + If ``check_circular`` is false, then the circular reference check + for container types will be skipped and a circular reference will + result in an ``OverflowError`` (or worse). + + If ``allow_nan`` is false, then it will be a ``ValueError`` to + serialize out of range ``float`` values (``nan``, ``inf``, ``-inf``) in + strict compliance of the JSON specification, instead of using the + JavaScript equivalents (``NaN``, ``Infinity``, ``-Infinity``). + + If ``indent`` is a non-negative integer, then JSON array elements and + object members will be pretty-printed with that indent level. An indent + level of 0 will only insert newlines. ``None`` is the most compact + representation. + + If ``separators`` is an ``(item_separator, dict_separator)`` tuple + then it will be used instead of the default ``(', ', ': ')`` separators. + ``(',', ':')`` is the most compact JSON representation. + + ``encoding`` is the character encoding for str instances, default is UTF-8. + + ``default(obj)`` is a function that should return a serializable version + of obj or raise TypeError. The default simply raises TypeError. + + To use a custom ``JSONEncoder`` subclass (e.g. one that overrides the + ``.default()`` method to serialize additional types), specify it with + the ``cls`` kwarg. + + """ + # cached encoder + if (not skipkeys and ensure_ascii and + check_circular and allow_nan and + cls is None and indent is None and separators is None and + encoding == 'utf-8' and default is None and not kw): + return _default_encoder.encode(obj) + if cls is None: + cls = JSONEncoder + return cls( + skipkeys=skipkeys, ensure_ascii=ensure_ascii, + check_circular=check_circular, allow_nan=allow_nan, indent=indent, + separators=separators, encoding=encoding, default=default, + **kw).encode(obj) + + +_default_decoder = JSONDecoder(encoding=None, object_hook=None) + + +def load(fp, encoding=None, cls=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, **kw): + """Deserialize ``fp`` (a ``.read()``-supporting file-like object containing + a JSON document) to a Python object. + + If the contents of ``fp`` is encoded with an ASCII based encoding other + than utf-8 (e.g. latin-1), then an appropriate ``encoding`` name must + be specified. Encodings that are not ASCII based (such as UCS-2) are + not allowed, and should be wrapped with + ``codecs.getreader(fp)(encoding)``, or simply decoded to a ``unicode`` + object and passed to ``loads()`` + + ``object_hook`` is an optional function that will be called with the + result of any object literal decode (a ``dict``). The return value of + ``object_hook`` will be used instead of the ``dict``. This feature + can be used to implement custom decoders (e.g. JSON-RPC class hinting). + + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` + kwarg. + + """ + return loads(fp.read(), + encoding=encoding, cls=cls, object_hook=object_hook, + parse_float=parse_float, parse_int=parse_int, + parse_constant=parse_constant, **kw) + + +def loads(s, encoding=None, cls=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, **kw): + """Deserialize ``s`` (a ``str`` or ``unicode`` instance containing a JSON + document) to a Python object. + + If ``s`` is a ``str`` instance and is encoded with an ASCII based encoding + other than utf-8 (e.g. latin-1) then an appropriate ``encoding`` name + must be specified. Encodings that are not ASCII based (such as UCS-2) + are not allowed and should be decoded to ``unicode`` first. + + ``object_hook`` is an optional function that will be called with the + result of any object literal decode (a ``dict``). The return value of + ``object_hook`` will be used instead of the ``dict``. This feature + can be used to implement custom decoders (e.g. JSON-RPC class hinting). + + ``parse_float``, if specified, will be called with the string + of every JSON float to be decoded. By default this is equivalent to + float(num_str). This can be used to use another datatype or parser + for JSON floats (e.g. decimal.Decimal). + + ``parse_int``, if specified, will be called with the string + of every JSON int to be decoded. By default this is equivalent to + int(num_str). This can be used to use another datatype or parser + for JSON integers (e.g. float). + + ``parse_constant``, if specified, will be called with one of the + following strings: -Infinity, Infinity, NaN, null, true, false. + This can be used to raise an exception if invalid JSON numbers + are encountered. + + To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` + kwarg. + + """ + if (cls is None and encoding is None and object_hook is None and + parse_int is None and parse_float is None and + parse_constant is None and not kw): + return _default_decoder.decode(s) + if cls is None: + cls = JSONDecoder + if object_hook is not None: + kw['object_hook'] = object_hook + if parse_float is not None: + kw['parse_float'] = parse_float + if parse_int is not None: + kw['parse_int'] = parse_int + if parse_constant is not None: + kw['parse_constant'] = parse_constant + return cls(encoding=encoding, **kw).decode(s) diff --git a/mitogen-0.2.7/ansible_mitogen/compat/simplejson/decoder.py b/mitogen-0.2.7/ansible_mitogen/compat/simplejson/decoder.py new file mode 100644 index 000000000..b769ea4 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/compat/simplejson/decoder.py @@ -0,0 +1,354 @@ +"""Implementation of JSONDecoder +""" +import re +import sys +import struct + +from simplejson.scanner import make_scanner +try: + from simplejson._speedups import scanstring as c_scanstring +except ImportError: + c_scanstring = None + +__all__ = ['JSONDecoder'] + +FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL + +def _floatconstants(): + _BYTES = '7FF80000000000007FF0000000000000'.decode('hex') + if sys.byteorder != 'big': + _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1] + nan, inf = struct.unpack('dd', _BYTES) + return nan, inf, -inf + +NaN, PosInf, NegInf = _floatconstants() + + +def linecol(doc, pos): + lineno = doc.count('\n', 0, pos) + 1 + if lineno == 1: + colno = pos + else: + colno = pos - doc.rindex('\n', 0, pos) + return lineno, colno + + +def errmsg(msg, doc, pos, end=None): + # Note that this function is called from _speedups + lineno, colno = linecol(doc, pos) + if end is None: + #fmt = '{0}: line {1} column {2} (char {3})' + #return fmt.format(msg, lineno, colno, pos) + fmt = '%s: line %d column %d (char %d)' + return fmt % (msg, lineno, colno, pos) + endlineno, endcolno = linecol(doc, end) + #fmt = '{0}: line {1} column {2} - line {3} column {4} (char {5} - {6})' + #return fmt.format(msg, lineno, colno, endlineno, endcolno, pos, end) + fmt = '%s: line %d column %d - line %d column %d (char %d - %d)' + return fmt % (msg, lineno, colno, endlineno, endcolno, pos, end) + + +_CONSTANTS = { + '-Infinity': NegInf, + 'Infinity': PosInf, + 'NaN': NaN, +} + +STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) +BACKSLASH = { + '"': u'"', '\\': u'\\', '/': u'/', + 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t', +} + +DEFAULT_ENCODING = "utf-8" + +def py_scanstring(s, end, encoding=None, strict=True, _b=BACKSLASH, _m=STRINGCHUNK.match): + """Scan the string s for a JSON string. End is the index of the + character in s after the quote that started the JSON string. + Unescapes all valid JSON string escape sequences and raises ValueError + on attempt to decode an invalid string. If strict is False then literal + control characters are allowed in the string. + + Returns a tuple of the decoded string and the index of the character in s + after the end quote.""" + if encoding is None: + encoding = DEFAULT_ENCODING + chunks = [] + _append = chunks.append + begin = end - 1 + while 1: + chunk = _m(s, end) + if chunk is None: + raise ValueError( + errmsg("Unterminated string starting at", s, begin)) + end = chunk.end() + content, terminator = chunk.groups() + # Content is contains zero or more unescaped string characters + if content: + if not isinstance(content, unicode): + content = unicode(content, encoding) + _append(content) + # Terminator is the end of string, a literal control character, + # or a backslash denoting that an escape sequence follows + if terminator == '"': + break + elif terminator != '\\': + if strict: + msg = "Invalid control character %r at" % (terminator,) + #msg = "Invalid control character {0!r} at".format(terminator) + raise ValueError(errmsg(msg, s, end)) + else: + _append(terminator) + continue + try: + esc = s[end] + except IndexError: + raise ValueError( + errmsg("Unterminated string starting at", s, begin)) + # If not a unicode escape sequence, must be in the lookup table + if esc != 'u': + try: + char = _b[esc] + except KeyError: + msg = "Invalid \\escape: " + repr(esc) + raise ValueError(errmsg(msg, s, end)) + end += 1 + else: + # Unicode escape sequence + esc = s[end + 1:end + 5] + next_end = end + 5 + if len(esc) != 4: + msg = "Invalid \\uXXXX escape" + raise ValueError(errmsg(msg, s, end)) + uni = int(esc, 16) + # Check for surrogate pair on UCS-4 systems + if 0xd800 <= uni <= 0xdbff and sys.maxunicode > 65535: + msg = "Invalid \\uXXXX\\uXXXX surrogate pair" + if not s[end + 5:end + 7] == '\\u': + raise ValueError(errmsg(msg, s, end)) + esc2 = s[end + 7:end + 11] + if len(esc2) != 4: + raise ValueError(errmsg(msg, s, end)) + uni2 = int(esc2, 16) + uni = 0x10000 + (((uni - 0xd800) << 10) | (uni2 - 0xdc00)) + next_end += 6 + char = unichr(uni) + end = next_end + # Append the unescaped character + _append(char) + return u''.join(chunks), end + + +# Use speedup if available +scanstring = c_scanstring or py_scanstring + +WHITESPACE = re.compile(r'[ \t\n\r]*', FLAGS) +WHITESPACE_STR = ' \t\n\r' + +def JSONObject((s, end), encoding, strict, scan_once, object_hook, _w=WHITESPACE.match, _ws=WHITESPACE_STR): + pairs = {} + # Use a slice to prevent IndexError from being raised, the following + # check will raise a more specific ValueError if the string is empty + nextchar = s[end:end + 1] + # Normally we expect nextchar == '"' + if nextchar != '"': + if nextchar in _ws: + end = _w(s, end).end() + nextchar = s[end:end + 1] + # Trivial empty object + if nextchar == '}': + return pairs, end + 1 + elif nextchar != '"': + raise ValueError(errmsg("Expecting property name", s, end)) + end += 1 + while True: + key, end = scanstring(s, end, encoding, strict) + + # To skip some function call overhead we optimize the fast paths where + # the JSON key separator is ": " or just ":". + if s[end:end + 1] != ':': + end = _w(s, end).end() + if s[end:end + 1] != ':': + raise ValueError(errmsg("Expecting : delimiter", s, end)) + + end += 1 + + try: + if s[end] in _ws: + end += 1 + if s[end] in _ws: + end = _w(s, end + 1).end() + except IndexError: + pass + + try: + value, end = scan_once(s, end) + except StopIteration: + raise ValueError(errmsg("Expecting object", s, end)) + pairs[key] = value + + try: + nextchar = s[end] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end] + except IndexError: + nextchar = '' + end += 1 + + if nextchar == '}': + break + elif nextchar != ',': + raise ValueError(errmsg("Expecting , delimiter", s, end - 1)) + + try: + nextchar = s[end] + if nextchar in _ws: + end += 1 + nextchar = s[end] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end] + except IndexError: + nextchar = '' + + end += 1 + if nextchar != '"': + raise ValueError(errmsg("Expecting property name", s, end - 1)) + + if object_hook is not None: + pairs = object_hook(pairs) + return pairs, end + +def JSONArray((s, end), scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): + values = [] + nextchar = s[end:end + 1] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end:end + 1] + # Look-ahead for trivial empty array + if nextchar == ']': + return values, end + 1 + _append = values.append + while True: + try: + value, end = scan_once(s, end) + except StopIteration: + raise ValueError(errmsg("Expecting object", s, end)) + _append(value) + nextchar = s[end:end + 1] + if nextchar in _ws: + end = _w(s, end + 1).end() + nextchar = s[end:end + 1] + end += 1 + if nextchar == ']': + break + elif nextchar != ',': + raise ValueError(errmsg("Expecting , delimiter", s, end)) + + try: + if s[end] in _ws: + end += 1 + if s[end] in _ws: + end = _w(s, end + 1).end() + except IndexError: + pass + + return values, end + +class JSONDecoder(object): + """Simple JSON decoder + + Performs the following translations in decoding by default: + + +---------------+-------------------+ + | JSON | Python | + +===============+===================+ + | object | dict | + +---------------+-------------------+ + | array | list | + +---------------+-------------------+ + | string | unicode | + +---------------+-------------------+ + | number (int) | int, long | + +---------------+-------------------+ + | number (real) | float | + +---------------+-------------------+ + | true | True | + +---------------+-------------------+ + | false | False | + +---------------+-------------------+ + | null | None | + +---------------+-------------------+ + + It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as + their corresponding ``float`` values, which is outside the JSON spec. + + """ + + def __init__(self, encoding=None, object_hook=None, parse_float=None, + parse_int=None, parse_constant=None, strict=True): + """``encoding`` determines the encoding used to interpret any ``str`` + objects decoded by this instance (utf-8 by default). It has no + effect when decoding ``unicode`` objects. + + Note that currently only encodings that are a superset of ASCII work, + strings of other encodings should be passed in as ``unicode``. + + ``object_hook``, if specified, will be called with the result + of every JSON object decoded and its return value will be used in + place of the given ``dict``. This can be used to provide custom + deserializations (e.g. to support JSON-RPC class hinting). + + ``parse_float``, if specified, will be called with the string + of every JSON float to be decoded. By default this is equivalent to + float(num_str). This can be used to use another datatype or parser + for JSON floats (e.g. decimal.Decimal). + + ``parse_int``, if specified, will be called with the string + of every JSON int to be decoded. By default this is equivalent to + int(num_str). This can be used to use another datatype or parser + for JSON integers (e.g. float). + + ``parse_constant``, if specified, will be called with one of the + following strings: -Infinity, Infinity, NaN. + This can be used to raise an exception if invalid JSON numbers + are encountered. + + """ + self.encoding = encoding + self.object_hook = object_hook + self.parse_float = parse_float or float + self.parse_int = parse_int or int + self.parse_constant = parse_constant or _CONSTANTS.__getitem__ + self.strict = strict + self.parse_object = JSONObject + self.parse_array = JSONArray + self.parse_string = scanstring + self.scan_once = make_scanner(self) + + def decode(self, s, _w=WHITESPACE.match): + """Return the Python representation of ``s`` (a ``str`` or ``unicode`` + instance containing a JSON document) + + """ + obj, end = self.raw_decode(s, idx=_w(s, 0).end()) + end = _w(s, end).end() + if end != len(s): + raise ValueError(errmsg("Extra data", s, end, len(s))) + return obj + + def raw_decode(self, s, idx=0): + """Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning + with a JSON document) and return a 2-tuple of the Python + representation and the index in ``s`` where the document ended. + + This can be used to decode a JSON document from a string that may + have extraneous data at the end. + + """ + try: + obj, end = self.scan_once(s, idx) + except StopIteration: + raise ValueError("No JSON object could be decoded") + return obj, end diff --git a/mitogen-0.2.7/ansible_mitogen/compat/simplejson/encoder.py b/mitogen-0.2.7/ansible_mitogen/compat/simplejson/encoder.py new file mode 100644 index 000000000..cf58290 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/compat/simplejson/encoder.py @@ -0,0 +1,440 @@ +"""Implementation of JSONEncoder +""" +import re + +try: + from simplejson._speedups import encode_basestring_ascii as c_encode_basestring_ascii +except ImportError: + c_encode_basestring_ascii = None +try: + from simplejson._speedups import make_encoder as c_make_encoder +except ImportError: + c_make_encoder = None + +ESCAPE = re.compile(r'[\x00-\x1f\\"\b\f\n\r\t]') +ESCAPE_ASCII = re.compile(r'([\\"]|[^\ -~])') +HAS_UTF8 = re.compile(r'[\x80-\xff]') +ESCAPE_DCT = { + '\\': '\\\\', + '"': '\\"', + '\b': '\\b', + '\f': '\\f', + '\n': '\\n', + '\r': '\\r', + '\t': '\\t', +} +for i in range(0x20): + #ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) + ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) + +# Assume this produces an infinity on all machines (probably not guaranteed) +INFINITY = float('1e66666') +FLOAT_REPR = repr + +def encode_basestring(s): + """Return a JSON representation of a Python string + + """ + def replace(match): + return ESCAPE_DCT[match.group(0)] + return '"' + ESCAPE.sub(replace, s) + '"' + + +def py_encode_basestring_ascii(s): + """Return an ASCII-only JSON representation of a Python string + + """ + if isinstance(s, str) and HAS_UTF8.search(s) is not None: + s = s.decode('utf-8') + def replace(match): + s = match.group(0) + try: + return ESCAPE_DCT[s] + except KeyError: + n = ord(s) + if n < 0x10000: + #return '\\u{0:04x}'.format(n) + return '\\u%04x' % (n,) + else: + # surrogate pair + n -= 0x10000 + s1 = 0xd800 | ((n >> 10) & 0x3ff) + s2 = 0xdc00 | (n & 0x3ff) + #return '\\u{0:04x}\\u{1:04x}'.format(s1, s2) + return '\\u%04x\\u%04x' % (s1, s2) + return '"' + str(ESCAPE_ASCII.sub(replace, s)) + '"' + + +encode_basestring_ascii = c_encode_basestring_ascii or py_encode_basestring_ascii + +class JSONEncoder(object): + """Extensible JSON encoder for Python data structures. + + Supports the following objects and types by default: + + +-------------------+---------------+ + | Python | JSON | + +===================+===============+ + | dict | object | + +-------------------+---------------+ + | list, tuple | array | + +-------------------+---------------+ + | str, unicode | string | + +-------------------+---------------+ + | int, long, float | number | + +-------------------+---------------+ + | True | true | + +-------------------+---------------+ + | False | false | + +-------------------+---------------+ + | None | null | + +-------------------+---------------+ + + To extend this to recognize other objects, subclass and implement a + ``.default()`` method with another method that returns a serializable + object for ``o`` if possible, otherwise it should call the superclass + implementation (to raise ``TypeError``). + + """ + item_separator = ', ' + key_separator = ': ' + def __init__(self, skipkeys=False, ensure_ascii=True, + check_circular=True, allow_nan=True, sort_keys=False, + indent=None, separators=None, encoding='utf-8', default=None): + """Constructor for JSONEncoder, with sensible defaults. + + If skipkeys is false, then it is a TypeError to attempt + encoding of keys that are not str, int, long, float or None. If + skipkeys is True, such items are simply skipped. + + If ensure_ascii is true, the output is guaranteed to be str + objects with all incoming unicode characters escaped. If + ensure_ascii is false, the output will be unicode object. + + If check_circular is true, then lists, dicts, and custom encoded + objects will be checked for circular references during encoding to + prevent an infinite recursion (which would cause an OverflowError). + Otherwise, no such check takes place. + + If allow_nan is true, then NaN, Infinity, and -Infinity will be + encoded as such. This behavior is not JSON specification compliant, + but is consistent with most JavaScript based encoders and decoders. + Otherwise, it will be a ValueError to encode such floats. + + If sort_keys is true, then the output of dictionaries will be + sorted by key; this is useful for regression tests to ensure + that JSON serializations can be compared on a day-to-day basis. + + If indent is a non-negative integer, then JSON array + elements and object members will be pretty-printed with that + indent level. An indent level of 0 will only insert newlines. + None is the most compact representation. + + If specified, separators should be a (item_separator, key_separator) + tuple. The default is (', ', ': '). To get the most compact JSON + representation you should specify (',', ':') to eliminate whitespace. + + If specified, default is a function that gets called for objects + that can't otherwise be serialized. It should return a JSON encodable + version of the object or raise a ``TypeError``. + + If encoding is not None, then all input strings will be + transformed into unicode using that encoding prior to JSON-encoding. + The default is UTF-8. + + """ + + self.skipkeys = skipkeys + self.ensure_ascii = ensure_ascii + self.check_circular = check_circular + self.allow_nan = allow_nan + self.sort_keys = sort_keys + self.indent = indent + if separators is not None: + self.item_separator, self.key_separator = separators + if default is not None: + self.default = default + self.encoding = encoding + + def default(self, o): + """Implement this method in a subclass such that it returns + a serializable object for ``o``, or calls the base implementation + (to raise a ``TypeError``). + + For example, to support arbitrary iterators, you could + implement default like this:: + + def default(self, o): + try: + iterable = iter(o) + except TypeError: + pass + else: + return list(iterable) + return JSONEncoder.default(self, o) + + """ + raise TypeError(repr(o) + " is not JSON serializable") + + def encode(self, o): + """Return a JSON string representation of a Python data structure. + + >>> JSONEncoder().encode({"foo": ["bar", "baz"]}) + '{"foo": ["bar", "baz"]}' + + """ + # This is for extremely simple cases and benchmarks. + if isinstance(o, basestring): + if isinstance(o, str): + _encoding = self.encoding + if (_encoding is not None + and not (_encoding == 'utf-8')): + o = o.decode(_encoding) + if self.ensure_ascii: + return encode_basestring_ascii(o) + else: + return encode_basestring(o) + # This doesn't pass the iterator directly to ''.join() because the + # exceptions aren't as detailed. The list call should be roughly + # equivalent to the PySequence_Fast that ''.join() would do. + chunks = self.iterencode(o, _one_shot=True) + if not isinstance(chunks, (list, tuple)): + chunks = list(chunks) + return ''.join(chunks) + + def iterencode(self, o, _one_shot=False): + """Encode the given object and yield each string + representation as available. + + For example:: + + for chunk in JSONEncoder().iterencode(bigobject): + mysocket.write(chunk) + + """ + if self.check_circular: + markers = {} + else: + markers = None + if self.ensure_ascii: + _encoder = encode_basestring_ascii + else: + _encoder = encode_basestring + if self.encoding != 'utf-8': + def _encoder(o, _orig_encoder=_encoder, _encoding=self.encoding): + if isinstance(o, str): + o = o.decode(_encoding) + return _orig_encoder(o) + + def floatstr(o, allow_nan=self.allow_nan, _repr=FLOAT_REPR, _inf=INFINITY, _neginf=-INFINITY): + # Check for specials. Note that this type of test is processor- and/or + # platform-specific, so do tests which don't depend on the internals. + + if o != o: + text = 'NaN' + elif o == _inf: + text = 'Infinity' + elif o == _neginf: + text = '-Infinity' + else: + return _repr(o) + + if not allow_nan: + raise ValueError( + "Out of range float values are not JSON compliant: " + + repr(o)) + + return text + + + if _one_shot and c_make_encoder is not None and not self.indent and not self.sort_keys: + _iterencode = c_make_encoder( + markers, self.default, _encoder, self.indent, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, self.allow_nan) + else: + _iterencode = _make_iterencode( + markers, self.default, _encoder, self.indent, floatstr, + self.key_separator, self.item_separator, self.sort_keys, + self.skipkeys, _one_shot) + return _iterencode(o, 0) + +def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _key_separator, _item_separator, _sort_keys, _skipkeys, _one_shot, + ## HACK: hand-optimized bytecode; turn globals into locals + False=False, + True=True, + ValueError=ValueError, + basestring=basestring, + dict=dict, + float=float, + id=id, + int=int, + isinstance=isinstance, + list=list, + long=long, + str=str, + tuple=tuple, + ): + + def _iterencode_list(lst, _current_indent_level): + if not lst: + yield '[]' + return + if markers is not None: + markerid = id(lst) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = lst + buf = '[' + if _indent is not None: + _current_indent_level += 1 + newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) + separator = _item_separator + newline_indent + buf += newline_indent + else: + newline_indent = None + separator = _item_separator + first = True + for value in lst: + if first: + first = False + else: + buf = separator + if isinstance(value, basestring): + yield buf + _encoder(value) + elif value is None: + yield buf + 'null' + elif value is True: + yield buf + 'true' + elif value is False: + yield buf + 'false' + elif isinstance(value, (int, long)): + yield buf + str(value) + elif isinstance(value, float): + yield buf + _floatstr(value) + else: + yield buf + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + for chunk in chunks: + yield chunk + if newline_indent is not None: + _current_indent_level -= 1 + yield '\n' + (' ' * (_indent * _current_indent_level)) + yield ']' + if markers is not None: + del markers[markerid] + + def _iterencode_dict(dct, _current_indent_level): + if not dct: + yield '{}' + return + if markers is not None: + markerid = id(dct) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = dct + yield '{' + if _indent is not None: + _current_indent_level += 1 + newline_indent = '\n' + (' ' * (_indent * _current_indent_level)) + item_separator = _item_separator + newline_indent + yield newline_indent + else: + newline_indent = None + item_separator = _item_separator + first = True + if _sort_keys: + items = dct.items() + items.sort(key=lambda kv: kv[0]) + else: + items = dct.iteritems() + for key, value in items: + if isinstance(key, basestring): + pass + # JavaScript is weakly typed for these, so it makes sense to + # also allow them. Many encoders seem to do something like this. + elif isinstance(key, float): + key = _floatstr(key) + elif key is True: + key = 'true' + elif key is False: + key = 'false' + elif key is None: + key = 'null' + elif isinstance(key, (int, long)): + key = str(key) + elif _skipkeys: + continue + else: + raise TypeError("key " + repr(key) + " is not a string") + if first: + first = False + else: + yield item_separator + yield _encoder(key) + yield _key_separator + if isinstance(value, basestring): + yield _encoder(value) + elif value is None: + yield 'null' + elif value is True: + yield 'true' + elif value is False: + yield 'false' + elif isinstance(value, (int, long)): + yield str(value) + elif isinstance(value, float): + yield _floatstr(value) + else: + if isinstance(value, (list, tuple)): + chunks = _iterencode_list(value, _current_indent_level) + elif isinstance(value, dict): + chunks = _iterencode_dict(value, _current_indent_level) + else: + chunks = _iterencode(value, _current_indent_level) + for chunk in chunks: + yield chunk + if newline_indent is not None: + _current_indent_level -= 1 + yield '\n' + (' ' * (_indent * _current_indent_level)) + yield '}' + if markers is not None: + del markers[markerid] + + def _iterencode(o, _current_indent_level): + if isinstance(o, basestring): + yield _encoder(o) + elif o is None: + yield 'null' + elif o is True: + yield 'true' + elif o is False: + yield 'false' + elif isinstance(o, (int, long)): + yield str(o) + elif isinstance(o, float): + yield _floatstr(o) + elif isinstance(o, (list, tuple)): + for chunk in _iterencode_list(o, _current_indent_level): + yield chunk + elif isinstance(o, dict): + for chunk in _iterencode_dict(o, _current_indent_level): + yield chunk + else: + if markers is not None: + markerid = id(o) + if markerid in markers: + raise ValueError("Circular reference detected") + markers[markerid] = o + o = _default(o) + for chunk in _iterencode(o, _current_indent_level): + yield chunk + if markers is not None: + del markers[markerid] + + return _iterencode diff --git a/mitogen-0.2.7/ansible_mitogen/compat/simplejson/scanner.py b/mitogen-0.2.7/ansible_mitogen/compat/simplejson/scanner.py new file mode 100644 index 000000000..adbc6ec --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/compat/simplejson/scanner.py @@ -0,0 +1,65 @@ +"""JSON token scanner +""" +import re +try: + from simplejson._speedups import make_scanner as c_make_scanner +except ImportError: + c_make_scanner = None + +__all__ = ['make_scanner'] + +NUMBER_RE = re.compile( + r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', + (re.VERBOSE | re.MULTILINE | re.DOTALL)) + +def py_make_scanner(context): + parse_object = context.parse_object + parse_array = context.parse_array + parse_string = context.parse_string + match_number = NUMBER_RE.match + encoding = context.encoding + strict = context.strict + parse_float = context.parse_float + parse_int = context.parse_int + parse_constant = context.parse_constant + object_hook = context.object_hook + + def _scan_once(string, idx): + try: + nextchar = string[idx] + except IndexError: + raise StopIteration + + if nextchar == '"': + return parse_string(string, idx + 1, encoding, strict) + elif nextchar == '{': + return parse_object((string, idx + 1), encoding, strict, _scan_once, object_hook) + elif nextchar == '[': + return parse_array((string, idx + 1), _scan_once) + elif nextchar == 'n' and string[idx:idx + 4] == 'null': + return None, idx + 4 + elif nextchar == 't' and string[idx:idx + 4] == 'true': + return True, idx + 4 + elif nextchar == 'f' and string[idx:idx + 5] == 'false': + return False, idx + 5 + + m = match_number(string, idx) + if m is not None: + integer, frac, exp = m.groups() + if frac or exp: + res = parse_float(integer + (frac or '') + (exp or '')) + else: + res = parse_int(integer) + return res, m.end() + elif nextchar == 'N' and string[idx:idx + 3] == 'NaN': + return parse_constant('NaN'), idx + 3 + elif nextchar == 'I' and string[idx:idx + 8] == 'Infinity': + return parse_constant('Infinity'), idx + 8 + elif nextchar == '-' and string[idx:idx + 9] == '-Infinity': + return parse_constant('-Infinity'), idx + 9 + else: + raise StopIteration + + return _scan_once + +make_scanner = c_make_scanner or py_make_scanner diff --git a/mitogen-0.2.7/ansible_mitogen/connection.py b/mitogen-0.2.7/ansible_mitogen/connection.py new file mode 100644 index 000000000..b5f28d3 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/connection.py @@ -0,0 +1,1021 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import unicode_literals + +import errno +import logging +import os +import pprint +import stat +import sys +import time + +import jinja2.runtime +import ansible.constants as C +import ansible.errors +import ansible.plugins.connection +import ansible.utils.shlex + +import mitogen.core +import mitogen.fork +import mitogen.unix +import mitogen.utils + +import ansible_mitogen.parsing +import ansible_mitogen.process +import ansible_mitogen.services +import ansible_mitogen.target +import ansible_mitogen.transport_config + + +LOG = logging.getLogger(__name__) + + +def get_remote_name(spec): + """ + Return the value to use for the "remote_name" parameter. + """ + if spec.mitogen_mask_remote_name(): + return 'ansible' + return None + + +def optional_int(value): + """ + Convert `value` to an integer if it is not :data:`None`, otherwise return + :data:`None`. + """ + try: + return int(value) + except (TypeError, ValueError): + return None + + +def convert_bool(obj): + if isinstance(obj, bool): + return obj + if str(obj).lower() in ('no', 'false', '0'): + return False + if str(obj).lower() not in ('yes', 'true', '1'): + raise ansible.errors.AnsibleConnectionFailure( + 'expected yes/no/true/false/0/1, got %r' % (obj,) + ) + return True + + +def default(value, default): + """ + Return `default` is `value` is :data:`None`, otherwise return `value`. + """ + if value is None: + return default + return value + + +def _connect_local(spec): + """ + Return ContextService arguments for a local connection. + """ + return { + 'method': 'local', + 'kwargs': { + 'python_path': spec.python_path(), + } + } + + +def _connect_ssh(spec): + """ + Return ContextService arguments for an SSH connection. + """ + if C.HOST_KEY_CHECKING: + check_host_keys = 'enforce' + else: + check_host_keys = 'ignore' + + # #334: tilde-expand private_key_file to avoid implementation difference + # between Python and OpenSSH. + private_key_file = spec.private_key_file() + if private_key_file is not None: + private_key_file = os.path.expanduser(private_key_file) + + return { + 'method': 'ssh', + 'kwargs': { + 'check_host_keys': check_host_keys, + 'hostname': spec.remote_addr(), + 'username': spec.remote_user(), + 'compression': convert_bool( + default(spec.mitogen_ssh_compression(), True) + ), + 'password': spec.password(), + 'port': spec.port(), + 'python_path': spec.python_path(), + 'identity_file': private_key_file, + 'identities_only': False, + 'ssh_path': spec.ssh_executable(), + 'connect_timeout': spec.ansible_ssh_timeout(), + 'ssh_args': spec.ssh_args(), + 'ssh_debug_level': spec.mitogen_ssh_debug_level(), + 'remote_name': get_remote_name(spec), + } + } + + +def _connect_docker(spec): + """ + Return ContextService arguments for a Docker connection. + """ + return { + 'method': 'docker', + 'kwargs': { + 'username': spec.remote_user(), + 'container': spec.remote_addr(), + 'python_path': spec.python_path(), + 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'remote_name': get_remote_name(spec), + } + } + + +def _connect_kubectl(spec): + """ + Return ContextService arguments for a Kubernetes connection. + """ + return { + 'method': 'kubectl', + 'kwargs': { + 'pod': spec.remote_addr(), + 'python_path': spec.python_path(), + 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'kubectl_path': spec.mitogen_kubectl_path(), + 'kubectl_args': spec.extra_args(), + 'remote_name': get_remote_name(spec), + } + } + + +def _connect_jail(spec): + """ + Return ContextService arguments for a FreeBSD jail connection. + """ + return { + 'method': 'jail', + 'kwargs': { + 'username': spec.remote_user(), + 'container': spec.remote_addr(), + 'python_path': spec.python_path(), + 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'remote_name': get_remote_name(spec), + } + } + + +def _connect_lxc(spec): + """ + Return ContextService arguments for an LXC Classic container connection. + """ + return { + 'method': 'lxc', + 'kwargs': { + 'container': spec.remote_addr(), + 'python_path': spec.python_path(), + 'lxc_attach_path': spec.mitogen_lxc_attach_path(), + 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'remote_name': get_remote_name(spec), + } + } + + +def _connect_lxd(spec): + """ + Return ContextService arguments for an LXD container connection. + """ + return { + 'method': 'lxd', + 'kwargs': { + 'container': spec.remote_addr(), + 'python_path': spec.python_path(), + 'lxc_path': spec.mitogen_lxc_path(), + 'connect_timeout': spec.ansible_ssh_timeout() or spec.timeout(), + 'remote_name': get_remote_name(spec), + } + } + + +def _connect_machinectl(spec): + """ + Return ContextService arguments for a machinectl connection. + """ + return _connect_setns(spec, kind='machinectl') + + +def _connect_setns(spec, kind=None): + """ + Return ContextService arguments for a mitogen_setns connection. + """ + return { + 'method': 'setns', + 'kwargs': { + 'container': spec.remote_addr(), + 'username': spec.remote_user(), + 'python_path': spec.python_path(), + 'kind': kind or spec.mitogen_kind(), + 'docker_path': spec.mitogen_docker_path(), + 'lxc_path': spec.mitogen_lxc_path(), + 'lxc_info_path': spec.mitogen_lxc_info_path(), + 'machinectl_path': spec.mitogen_machinectl_path(), + } + } + + +def _connect_su(spec): + """ + Return ContextService arguments for su as a become method. + """ + return { + 'method': 'su', + 'enable_lru': True, + 'kwargs': { + 'username': spec.become_user(), + 'password': spec.become_pass(), + 'python_path': spec.python_path(), + 'su_path': spec.become_exe(), + 'connect_timeout': spec.timeout(), + 'remote_name': get_remote_name(spec), + } + } + + +def _connect_sudo(spec): + """ + Return ContextService arguments for sudo as a become method. + """ + return { + 'method': 'sudo', + 'enable_lru': True, + 'kwargs': { + 'username': spec.become_user(), + 'password': spec.become_pass(), + 'python_path': spec.python_path(), + 'sudo_path': spec.become_exe(), + 'connect_timeout': spec.timeout(), + 'sudo_args': spec.sudo_args(), + 'remote_name': get_remote_name(spec), + } + } + + +def _connect_doas(spec): + """ + Return ContextService arguments for doas as a become method. + """ + return { + 'method': 'doas', + 'enable_lru': True, + 'kwargs': { + 'username': spec.become_user(), + 'password': spec.become_pass(), + 'python_path': spec.python_path(), + 'doas_path': spec.become_exe(), + 'connect_timeout': spec.timeout(), + 'remote_name': get_remote_name(spec), + } + } + + +def _connect_mitogen_su(spec): + """ + Return ContextService arguments for su as a first class connection. + """ + return { + 'method': 'su', + 'kwargs': { + 'username': spec.remote_user(), + 'password': spec.password(), + 'python_path': spec.python_path(), + 'su_path': spec.become_exe(), + 'connect_timeout': spec.timeout(), + 'remote_name': get_remote_name(spec), + } + } + + +def _connect_mitogen_sudo(spec): + """ + Return ContextService arguments for sudo as a first class connection. + """ + return { + 'method': 'sudo', + 'kwargs': { + 'username': spec.remote_user(), + 'password': spec.password(), + 'python_path': spec.python_path(), + 'sudo_path': spec.become_exe(), + 'connect_timeout': spec.timeout(), + 'sudo_args': spec.sudo_args(), + 'remote_name': get_remote_name(spec), + } + } + + +def _connect_mitogen_doas(spec): + """ + Return ContextService arguments for doas as a first class connection. + """ + return { + 'method': 'doas', + 'kwargs': { + 'username': spec.remote_user(), + 'password': spec.password(), + 'python_path': spec.python_path(), + 'doas_path': spec.become_exe(), + 'connect_timeout': spec.timeout(), + 'remote_name': get_remote_name(spec), + } + } + + +#: Mapping of connection method names to functions invoked as `func(spec)` +#: generating ContextService keyword arguments matching a connection +#: specification. +CONNECTION_METHOD = { + 'docker': _connect_docker, + 'kubectl': _connect_kubectl, + 'jail': _connect_jail, + 'local': _connect_local, + 'lxc': _connect_lxc, + 'lxd': _connect_lxd, + 'machinectl': _connect_machinectl, + 'setns': _connect_setns, + 'ssh': _connect_ssh, + 'smart': _connect_ssh, # issue #548. + 'su': _connect_su, + 'sudo': _connect_sudo, + 'doas': _connect_doas, + 'mitogen_su': _connect_mitogen_su, + 'mitogen_sudo': _connect_mitogen_sudo, + 'mitogen_doas': _connect_mitogen_doas, +} + + +class Broker(mitogen.master.Broker): + """ + WorkerProcess maintains at most 2 file descriptors, therefore does not need + the exuberant syscall expense of EpollPoller, so override it and restore + the poll() poller. + """ + poller_class = mitogen.core.Poller + + +class CallChain(mitogen.parent.CallChain): + """ + Extend :class:`mitogen.parent.CallChain` to additionally cause the + associated :class:`Connection` to be reset if a ChannelError occurs. + + This only catches failures that occur while a call is pending, it is a + stop-gap until a more general method is available to notice connection in + every situation. + """ + call_aborted_msg = ( + 'Mitogen was disconnected from the remote environment while a call ' + 'was in-progress. If you feel this is in error, please file a bug. ' + 'Original error was: %s' + ) + + def __init__(self, connection, context, pipelined=False): + super(CallChain, self).__init__(context, pipelined) + #: The connection to reset on CallError. + self._connection = connection + + def _rethrow(self, recv): + try: + return recv.get().unpickle() + except mitogen.core.ChannelError as e: + self._connection.reset() + raise ansible.errors.AnsibleConnectionFailure( + self.call_aborted_msg % (e,) + ) + + def call(self, func, *args, **kwargs): + """ + Like :meth:`mitogen.parent.CallChain.call`, but log timings. + """ + t0 = time.time() + try: + recv = self.call_async(func, *args, **kwargs) + return self._rethrow(recv) + finally: + LOG.debug('Call took %d ms: %r', 1000 * (time.time() - t0), + mitogen.parent.CallSpec(func, args, kwargs)) + + +class Connection(ansible.plugins.connection.ConnectionBase): + #: mitogen.master.Broker for this worker. + broker = None + + #: mitogen.master.Router for this worker. + router = None + + #: mitogen.parent.Context representing the parent Context, which is + #: presently always the connection multiplexer process. + parent = None + + #: mitogen.parent.Context for the target account on the target, possibly + #: reached via become. + context = None + + #: Context for the login account on the target. This is always the login + #: account, even when become=True. + login_context = None + + #: Only sudo, su, and doas are supported for now. + become_methods = ['sudo', 'su', 'doas'] + + #: Dict containing init_child() return value as recorded at startup by + #: ContextService. Contains: + #: + #: fork_context: Context connected to the fork parent : process in the + #: target account. + #: home_dir: Target context's home directory. + #: good_temp_dir: A writeable directory where new temporary directories + #: can be created. + init_child_result = None + + #: A :class:`mitogen.parent.CallChain` for calls made to the target + #: account, to ensure subsequent calls fail with the original exception if + #: pipelined directory creation or file transfer fails. + chain = None + + # + # Note: any of the attributes below may be :data:`None` if the connection + # plugin was constructed directly by a non-cooperative action, such as in + # the case of the synchronize module. + # + + #: Set to the host name as it appears in inventory by on_action_run(). + inventory_hostname = None + + #: Set to task_vars by on_action_run(). + _task_vars = None + + #: Set to 'hostvars' by on_action_run() + host_vars = None + + #: Set by on_action_run() + delegate_to_hostname = None + + #: Set to '_loader.get_basedir()' by on_action_run(). Used by mitogen_local + #: to change the working directory to that of the current playbook, + #: matching vanilla Ansible behaviour. + loader_basedir = None + + def __init__(self, play_context, new_stdin, **kwargs): + assert ansible_mitogen.process.MuxProcess.unix_listener_path, ( + 'Mitogen connection types may only be instantiated ' + 'while the "mitogen" strategy is active.' + ) + super(Connection, self).__init__(play_context, new_stdin) + + def __del__(self): + """ + Ansible cannot be trusted to always call close() e.g. the synchronize + action constructs a local connection like this. So provide a destructor + in the hopes of catching these cases. + """ + # https://github.com/dw/mitogen/issues/140 + self.close() + + def on_action_run(self, task_vars, delegate_to_hostname, loader_basedir): + """ + Invoked by ActionModuleMixin to indicate a new task is about to start + executing. We use the opportunity to grab relevant bits from the + task-specific data. + + :param dict task_vars: + Task variable dictionary. + :param str delegate_to_hostname: + :data:`None`, or the template-expanded inventory hostname this task + is being delegated to. A similar variable exists on PlayContext + when ``delegate_to:`` is active, however it is unexpanded. + :param str loader_basedir: + Loader base directory; see :attr:`loader_basedir`. + """ + self.inventory_hostname = task_vars['inventory_hostname'] + self._task_vars = task_vars + self.host_vars = task_vars['hostvars'] + self.delegate_to_hostname = delegate_to_hostname + self.loader_basedir = loader_basedir + self._mitogen_reset(mode='put') + + def get_task_var(self, key, default=None): + """ + Fetch the value of a task variable related to connection configuration, + or, if delegate_to is active, fetch the same variable via HostVars for + the delegated-to machine. + + When running with delegate_to, Ansible tasks have variables associated + with the original machine, not the delegated-to machine, therefore it + does not make sense to extract connection-related configuration for the + delegated-to machine from them. + """ + if self._task_vars: + if self.delegate_to_hostname is None: + if key in self._task_vars: + return self._task_vars[key] + else: + delegated_vars = self._task_vars['ansible_delegated_vars'] + if self.delegate_to_hostname in delegated_vars: + task_vars = delegated_vars[self.delegate_to_hostname] + if key in task_vars: + return task_vars[key] + + return default + + @property + def homedir(self): + self._connect() + return self.init_child_result['home_dir'] + + @property + def connected(self): + return self.context is not None + + def _spec_from_via(self, proxied_inventory_name, via_spec): + """ + Produce a dict connection specifiction given a string `via_spec`, of + the form `[[become_method:]become_user@]inventory_hostname`. + """ + become_user, _, inventory_name = via_spec.rpartition('@') + become_method, _, become_user = become_user.rpartition(':') + + # must use __contains__ to avoid a TypeError for a missing host on + # Ansible 2.3. + if self.host_vars is None or inventory_name not in self.host_vars: + raise ansible.errors.AnsibleConnectionFailure( + self.unknown_via_msg % ( + via_spec, + proxied_inventory_name, + ) + ) + + via_vars = self.host_vars[inventory_name] + return ansible_mitogen.transport_config.MitogenViaSpec( + inventory_name=inventory_name, + play_context=self._play_context, + host_vars=dict(via_vars), # TODO: make it lazy + become_method=become_method or None, + become_user=become_user or None, + ) + + unknown_via_msg = 'mitogen_via=%s of %s specifies an unknown hostname' + via_cycle_msg = 'mitogen_via=%s of %s creates a cycle (%s)' + + def _stack_from_spec(self, spec, stack=(), seen_names=()): + """ + Return a tuple of ContextService parameter dictionaries corresponding + to the connection described by `spec`, and any connection referenced by + its `mitogen_via` or `become` fields. Each element is a dict of the + form:: + + { + # Optional. If present and `True`, this hop is elegible for + # interpreter recycling. + "enable_lru": True, + # mitogen.master.Router method name. + "method": "ssh", + # mitogen.master.Router method kwargs. + "kwargs": { + "hostname": "..." + } + } + + :param ansible_mitogen.transport_config.Spec spec: + Connection specification. + :param tuple stack: + Stack elements from parent call (used for recursion). + :param tuple seen_names: + Inventory hostnames from parent call (cycle detection). + :returns: + Tuple `(stack, seen_names)`. + """ + if spec.inventory_name() in seen_names: + raise ansible.errors.AnsibleConnectionFailure( + self.via_cycle_msg % ( + spec.mitogen_via(), + spec.inventory_name(), + ' -> '.join(reversed( + seen_names + (spec.inventory_name(),) + )), + ) + ) + + if spec.mitogen_via(): + stack = self._stack_from_spec( + self._spec_from_via(spec.inventory_name(), spec.mitogen_via()), + stack=stack, + seen_names=seen_names + (spec.inventory_name(),), + ) + + stack += (CONNECTION_METHOD[spec.transport()](spec),) + if spec.become() and ((spec.become_user() != spec.remote_user()) or + C.BECOME_ALLOW_SAME_USER): + stack += (CONNECTION_METHOD[spec.become_method()](spec),) + + return stack + + def _connect_broker(self): + """ + Establish a reference to the Broker, Router and parent context used for + connections. + """ + if not self.broker: + self.broker = mitogen.master.Broker() + self.router, self.parent = mitogen.unix.connect( + path=ansible_mitogen.process.MuxProcess.unix_listener_path, + broker=self.broker, + ) + + def _build_stack(self): + """ + Construct a list of dictionaries representing the connection + configuration between the controller and the target. This is + additionally used by the integration tests "mitogen_get_stack" action + to fetch the would-be connection configuration. + """ + return self._stack_from_spec( + ansible_mitogen.transport_config.PlayContextSpec( + connection=self, + play_context=self._play_context, + transport=self.transport, + inventory_name=self.inventory_hostname, + ) + ) + + def _connect_stack(self, stack): + """ + Pass `stack` to ContextService, requesting a copy of the context object + representing the last tuple element. If no connection exists yet, + ContextService will recursively establish it before returning it or + throwing an error. + + See :meth:`ansible_mitogen.services.ContextService.get` docstring for + description of the returned dictionary. + """ + try: + dct = self.parent.call_service( + service_name='ansible_mitogen.services.ContextService', + method_name='get', + stack=mitogen.utils.cast(list(stack)), + ) + except mitogen.core.CallError: + LOG.warning('Connection failed; stack configuration was:\n%s', + pprint.pformat(stack)) + raise + + if dct['msg']: + if dct['method_name'] in self.become_methods: + raise ansible.errors.AnsibleModuleError(dct['msg']) + raise ansible.errors.AnsibleConnectionFailure(dct['msg']) + + self.context = dct['context'] + self.chain = CallChain(self, self.context, pipelined=True) + if self._play_context.become: + self.login_context = dct['via'] + else: + self.login_context = self.context + + self.init_child_result = dct['init_child_result'] + + def get_good_temp_dir(self): + """ + Return the 'good temporary directory' as discovered by + :func:`ansible_mitogen.target.init_child` immediately after + ContextService constructed the target context. + """ + self._connect() + return self.init_child_result['good_temp_dir'] + + def _connect(self): + """ + Establish a connection to the master process's UNIX listener socket, + constructing a mitogen.master.Router to communicate with the master, + and a mitogen.parent.Context to represent it. + + Depending on the original transport we should emulate, trigger one of + the _connect_*() service calls defined above to cause the master + process to establish the real connection on our behalf, or return a + reference to the existing one. + """ + if self.connected: + return + + self._connect_broker() + stack = self._build_stack() + self._connect_stack(stack) + + def _mitogen_reset(self, mode): + """ + Forget everything we know about the connected context. This function + cannot be called _reset() since that name is used as a public API by + Ansible 2.4 wait_for_connection plug-in. + + :param str mode: + Name of ContextService method to use to discard the context, either + 'put' or 'reset'. + """ + if not self.context: + return + + self.chain.reset() + self.parent.call_service( + service_name='ansible_mitogen.services.ContextService', + method_name=mode, + context=self.context + ) + + self.context = None + self.login_context = None + self.init_child_result = None + self.chain = None + + def _shutdown_broker(self): + """ + Shutdown the broker thread during :meth:`close` or :meth:`reset`. + """ + if self.broker: + self.broker.shutdown() + self.broker.join() + self.broker = None + self.router = None + + # #420: Ansible executes "meta" actions in the top-level process, + # meaning "reset_connection" will cause :class:`mitogen.core.Latch` + # FDs to be cached and erroneously shared by children on subsequent + # WorkerProcess forks. To handle that, call on_fork() to ensure any + # shared state is discarded. + # #490: only attempt to clean up when it's known that some + # resources exist to cleanup, otherwise later __del__ double-call + # to close() due to GC at random moment may obliterate an unrelated + # Connection's resources. + mitogen.fork.on_fork() + + def close(self): + """ + Arrange for the mitogen.master.Router running in the worker to + gracefully shut down, and wait for shutdown to complete. Safe to call + multiple times. + """ + self._mitogen_reset(mode='put') + self._shutdown_broker() + + def _reset_find_task_vars(self): + """ + Monsterous hack: since "meta: reset_connection" does not run from an + action, we cannot capture task variables via :meth:`on_action_run`. + Instead walk the parent frames searching for the `all_vars` local from + StrategyBase._execute_meta(). If this fails, just leave task_vars + unset, likely causing a subtly wrong configuration to be selected. + """ + frame = sys._getframe() + while frame and not self._task_vars: + self._task_vars = frame.f_locals.get('all_vars') + frame = frame.f_back + + reset_compat_msg = ( + 'Mitogen only supports "reset_connection" on Ansible 2.5.6 or later' + ) + + def reset(self): + """ + Explicitly terminate the connection to the remote host. This discards + any local state we hold for the connection, returns the Connection to + the 'disconnected' state, and informs ContextService the connection is + bad somehow, and should be shut down and discarded. + """ + if self._task_vars is None: + self._reset_find_task_vars() + + if self._play_context.remote_addr is None: + # <2.5.6 incorrectly populate PlayContext for reset_connection + # https://github.com/ansible/ansible/issues/27520 + raise ansible.errors.AnsibleConnectionFailure( + self.reset_compat_msg + ) + + self._connect() + self._mitogen_reset(mode='reset') + self._shutdown_broker() + + # Compatibility with Ansible 2.4 wait_for_connection plug-in. + _reset = reset + + def get_chain(self, use_login=False, use_fork=False): + """ + Return the :class:`mitogen.parent.CallChain` to use for executing + function calls. + + :param bool use_login: + If :data:`True`, always return the chain for the login account + rather than any active become user. + :param bool use_fork: + If :data:`True`, return the chain for the fork parent. + :returns mitogen.parent.CallChain: + """ + self._connect() + if use_login: + return self.login_context.default_call_chain + # See FORK_SUPPORTED comments in target.py. + if use_fork and self.init_child_result['fork_context'] is not None: + return self.init_child_result['fork_context'].default_call_chain + return self.chain + + def spawn_isolated_child(self): + """ + Fork or launch a new child off the target context. + + :returns: + mitogen.core.Context of the new child. + """ + return self.get_chain(use_fork=True).call( + ansible_mitogen.target.spawn_isolated_child + ) + + def get_extra_args(self): + """ + Overridden by connections/mitogen_kubectl.py to a list of additional + arguments for the command. + """ + # TODO: maybe use this for SSH too. + return [] + + def get_default_cwd(self): + """ + Overridden by connections/mitogen_local.py to emulate behaviour of CWD + being fixed to that of ActionBase._loader.get_basedir(). + """ + return None + + def get_default_env(self): + """ + Overridden by connections/mitogen_local.py to emulate behaviour of + WorkProcess environment inherited from WorkerProcess. + """ + return None + + def exec_command(self, cmd, in_data='', sudoable=True, mitogen_chdir=None): + """ + Implement exec_command() by calling the corresponding + ansible_mitogen.target function in the target. + + :param str cmd: + Shell command to execute. + :param bytes in_data: + Data to supply on ``stdin`` of the process. + :returns: + (return code, stdout bytes, stderr bytes) + """ + emulate_tty = (not in_data and sudoable) + rc, stdout, stderr = self.get_chain().call( + ansible_mitogen.target.exec_command, + cmd=mitogen.utils.cast(cmd), + in_data=mitogen.utils.cast(in_data), + chdir=mitogen_chdir or self.get_default_cwd(), + emulate_tty=emulate_tty, + ) + + stderr += b'Shared connection to %s closed.%s' % ( + self._play_context.remote_addr.encode(), + (b'\r\n' if emulate_tty else b'\n'), + ) + return rc, stdout, stderr + + def fetch_file(self, in_path, out_path): + """ + Implement fetch_file() by calling the corresponding + ansible_mitogen.target function in the target. + + :param str in_path: + Remote filesystem path to read. + :param str out_path: + Local filesystem path to write. + """ + output = self.get_chain().call( + ansible_mitogen.target.read_path, + mitogen.utils.cast(in_path), + ) + ansible_mitogen.target.write_path(out_path, output) + + def put_data(self, out_path, data, mode=None, utimes=None): + """ + Implement put_file() by caling the corresponding ansible_mitogen.target + function in the target, transferring small files inline. This is + pipelined and will return immediately; failed transfers are reported as + exceptions in subsequent functon calls. + + :param str out_path: + Remote filesystem path to write. + :param byte data: + File contents to put. + """ + self.get_chain().call_no_reply( + ansible_mitogen.target.write_path, + mitogen.utils.cast(out_path), + mitogen.core.Blob(data), + mode=mode, + utimes=utimes, + ) + + #: Maximum size of a small file before switching to streaming + #: transfer. This should really be the same as + #: mitogen.services.FileService.IO_SIZE, however the message format has + #: slightly more overhead, so just randomly subtract 4KiB. + SMALL_FILE_LIMIT = mitogen.core.CHUNK_SIZE - 4096 + + def _throw_io_error(self, e, path): + if e.args[0] == errno.ENOENT: + s = 'file or module does not exist: ' + path + raise ansible.errors.AnsibleFileNotFound(s) + + def put_file(self, in_path, out_path): + """ + Implement put_file() by streamily transferring the file via + FileService. + + :param str in_path: + Local filesystem path to read. + :param str out_path: + Remote filesystem path to write. + """ + try: + st = os.stat(in_path) + except OSError as e: + self._throw_io_error(e, in_path) + raise + + if not stat.S_ISREG(st.st_mode): + raise IOError('%r is not a regular file.' % (in_path,)) + + # If the file is sufficiently small, just ship it in the argument list + # rather than introducing an extra RTT for the child to request it from + # FileService. + if st.st_size <= self.SMALL_FILE_LIMIT: + try: + fp = open(in_path, 'rb') + try: + s = fp.read(self.SMALL_FILE_LIMIT + 1) + finally: + fp.close() + except OSError: + self._throw_io_error(e, in_path) + raise + + # Ensure did not grow during read. + if len(s) == st.st_size: + return self.put_data(out_path, s, mode=st.st_mode, + utimes=(st.st_atime, st.st_mtime)) + + self._connect() + self.parent.call_service( + service_name='mitogen.service.FileService', + method_name='register', + path=mitogen.utils.cast(in_path) + ) + + # For now this must remain synchronous, as the action plug-in may have + # passed us a temporary file to transfer. A future FileService could + # maintain an LRU list of open file descriptors to keep the temporary + # file alive, but that requires more work. + self.get_chain().call( + ansible_mitogen.target.transfer_file, + context=self.parent, + in_path=in_path, + out_path=out_path + ) diff --git a/mitogen-0.2.7/ansible_mitogen/connection.pyc b/mitogen-0.2.7/ansible_mitogen/connection.pyc new file mode 100644 index 000000000..729ffe7 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/connection.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/loaders.py b/mitogen-0.2.7/ansible_mitogen/loaders.py new file mode 100644 index 000000000..ff06c0c --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/loaders.py @@ -0,0 +1,48 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +""" +Stable names for PluginLoader instances across Ansible versions. +""" + +from __future__ import absolute_import + +try: + from ansible.plugins.loader import action_loader + from ansible.plugins.loader import connection_loader + from ansible.plugins.loader import module_loader + from ansible.plugins.loader import module_utils_loader + from ansible.plugins.loader import shell_loader + from ansible.plugins.loader import strategy_loader +except ImportError: # Ansible <2.4 + from ansible.plugins import action_loader + from ansible.plugins import connection_loader + from ansible.plugins import module_loader + from ansible.plugins import module_utils_loader + from ansible.plugins import shell_loader + from ansible.plugins import strategy_loader diff --git a/mitogen-0.2.7/ansible_mitogen/loaders.pyc b/mitogen-0.2.7/ansible_mitogen/loaders.pyc new file mode 100644 index 000000000..cc2f9d0 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/loaders.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/logging.py b/mitogen-0.2.7/ansible_mitogen/logging.py new file mode 100644 index 000000000..ce6f165 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/logging.py @@ -0,0 +1,127 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import logging +import os + +import mitogen.core +import mitogen.utils + +try: + from __main__ import display +except ImportError: + from ansible.utils.display import Display + display = Display() + + +#: The process name set via :func:`set_process_name`. +_process_name = None + +#: The PID of the process that last called :func:`set_process_name`, so its +#: value can be ignored in unknown fork children. +_process_pid = None + + +def set_process_name(name): + """ + Set a name to adorn log messages with. + """ + global _process_name + _process_name = name + + global _process_pid + _process_pid = os.getpid() + + +class Handler(logging.Handler): + """ + Use Mitogen's log format, but send the result to a Display method. + """ + def __init__(self, normal_method): + logging.Handler.__init__(self) + self.formatter = mitogen.utils.log_get_formatter() + self.normal_method = normal_method + + #: Set of target loggers that produce warnings and errors that spam the + #: console needlessly. Their log level is forced to INFO. A better strategy + #: may simply be to bury all target logs in DEBUG output, but not by + #: overriding their log level as done here. + NOISY_LOGGERS = frozenset([ + 'dnf', # issue #272; warns when a package is already installed. + 'boto', # issue #541; normal boto retry logic can cause ERROR logs. + ]) + + def emit(self, record): + mitogen_name = getattr(record, 'mitogen_name', '') + if mitogen_name == 'stderr': + record.levelno = logging.ERROR + if mitogen_name in self.NOISY_LOGGERS and record.levelno >= logging.WARNING: + record.levelno = logging.DEBUG + + if _process_pid == os.getpid(): + process_name = _process_name + else: + process_name = '?' + + s = '[%-4s %d] %s' % (process_name, os.getpid(), self.format(record)) + if record.levelno >= logging.ERROR: + display.error(s, wrap_text=False) + elif record.levelno >= logging.WARNING: + display.warning(s, formatted=True) + else: + self.normal_method(s) + + +def setup(): + """ + Install handlers for Mitogen loggers to redirect them into the Ansible + display framework. Ansible installs its own logging framework handlers when + C.DEFAULT_LOG_PATH is set, therefore disable propagation for our handlers. + """ + l_mitogen = logging.getLogger('mitogen') + l_mitogen_io = logging.getLogger('mitogen.io') + l_ansible_mitogen = logging.getLogger('ansible_mitogen') + + for logger in l_mitogen, l_mitogen_io, l_ansible_mitogen: + logger.handlers = [Handler(display.vvv)] + logger.propagate = False + + if display.verbosity > 2: + l_ansible_mitogen.setLevel(logging.DEBUG) + l_mitogen.setLevel(logging.DEBUG) + else: + # Mitogen copies the active log level into new children, allowing them + # to filter tiny messages before they hit the network, and therefore + # before they wake the IO loop. Explicitly setting INFO saves ~4% + # running against just the local machine. + l_mitogen.setLevel(logging.ERROR) + l_ansible_mitogen.setLevel(logging.ERROR) + + if display.verbosity > 3: + l_mitogen_io.setLevel(logging.DEBUG) diff --git a/mitogen-0.2.7/ansible_mitogen/logging.pyc b/mitogen-0.2.7/ansible_mitogen/logging.pyc new file mode 100644 index 000000000..cf32ee6 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/logging.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/mixins.py b/mitogen-0.2.7/ansible_mitogen/mixins.py new file mode 100644 index 000000000..890467f --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/mixins.py @@ -0,0 +1,432 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import logging +import os +import pwd +import random +import traceback + +try: + from shlex import quote as shlex_quote +except ImportError: + from pipes import quote as shlex_quote + +from ansible.module_utils._text import to_bytes +from ansible.parsing.utils.jsonify import jsonify + +import ansible +import ansible.constants +import ansible.plugins +import ansible.plugins.action + +import mitogen.core +import mitogen.select +import mitogen.utils + +import ansible_mitogen.connection +import ansible_mitogen.planner +import ansible_mitogen.target +from ansible.module_utils._text import to_text + + +LOG = logging.getLogger(__name__) + + +class ActionModuleMixin(ansible.plugins.action.ActionBase): + """ + The Mitogen-patched PluginLoader dynamically mixes this into every action + class that Ansible attempts to load. It exists to override all the + assumptions built into the base action class that should really belong in + some middle layer, or at least in the connection layer. + + Functionality is defined here for: + + * Capturing the final set of task variables and giving Connection a chance + to update its idea of the correct execution environment, before any + attempt is made to call a Connection method. While it's not expected for + the interpreter to change on a per-task basis, Ansible permits this, and + so it must be supported. + + * Overriding lots of methods that try to call out to shell for mundane + reasons, such as copying files around, changing file permissions, + creating temporary directories and suchlike. + + * Short-circuiting any use of Ansiballz or related code for executing a + module remotely using shell commands and SSH. + + * Short-circuiting most of the logic in dealing with the fact that Ansible + always runs become: tasks across at least the SSH user account and the + destination user account, and handling the security permission issues + that crop up due to this. Mitogen always runs a task completely within + the target user account, so it's not a problem for us. + """ + def __init__(self, task, connection, *args, **kwargs): + """ + Verify the received connection is really a Mitogen connection. If not, + transmute this instance back into the original unadorned base class. + + This allows running the Mitogen strategy in mixed-target playbooks, + where some targets use SSH while others use WinRM or some fancier UNIX + connection plug-in. That's because when the Mitogen strategy is active, + ActionModuleMixin is unconditionally mixed into any action module that + is instantiated, and there is no direct way for the monkey-patch to + know what kind of connection will be used upfront. + """ + super(ActionModuleMixin, self).__init__(task, connection, *args, **kwargs) + if not isinstance(connection, ansible_mitogen.connection.Connection): + _, self.__class__ = type(self).__bases__ + + def run(self, tmp=None, task_vars=None): + """ + Override run() to notify Connection of task-specific data, so it has a + chance to know e.g. the Python interpreter in use. + """ + self._connection.on_action_run( + task_vars=task_vars, + delegate_to_hostname=self._task.delegate_to, + loader_basedir=self._loader.get_basedir(), + ) + return super(ActionModuleMixin, self).run(tmp, task_vars) + + COMMAND_RESULT = { + 'rc': 0, + 'stdout': '', + 'stdout_lines': [], + 'stderr': '' + } + + def fake_shell(self, func, stdout=False): + """ + Execute a function and decorate its return value in the style of + _low_level_execute_command(). This produces a return value that looks + like some shell command was run, when really func() was implemented + entirely in Python. + + If the function raises :py:class:`mitogen.core.CallError`, this will be + translated into a failed shell command with a non-zero exit status. + + :param func: + Function invoked as `func()`. + :returns: + See :py:attr:`COMMAND_RESULT`. + """ + dct = self.COMMAND_RESULT.copy() + try: + rc = func() + if stdout: + dct['stdout'] = repr(rc) + except mitogen.core.CallError: + LOG.exception('While emulating a shell command') + dct['rc'] = 1 + dct['stderr'] = traceback.format_exc() + + return dct + + def _remote_file_exists(self, path): + """ + Determine if `path` exists by directly invoking os.path.exists() in the + target user account. + """ + LOG.debug('_remote_file_exists(%r)', path) + return self._connection.get_chain().call( + ansible_mitogen.target.file_exists, + mitogen.utils.cast(path) + ) + + def _configure_module(self, module_name, module_args, task_vars=None): + """ + Mitogen does not use the Ansiballz framework. This call should never + happen when ActionMixin is active, so crash if it does. + """ + assert False, "_configure_module() should never be called." + + def _is_pipelining_enabled(self, module_style, wrap_async=False): + """ + Mitogen does not use SSH pipelining. This call should never happen when + ActionMixin is active, so crash if it does. + """ + assert False, "_is_pipelining_enabled() should never be called." + + def _generate_tmp_path(self): + return os.path.join( + self._connection.get_good_temp_dir(), + 'ansible_mitogen_action_%016x' % ( + random.getrandbits(8*8), + ) + ) + + def _generate_tmp_path(self): + return os.path.join( + self._connection.get_good_temp_dir(), + 'ansible_mitogen_action_%016x' % ( + random.getrandbits(8*8), + ) + ) + + def _make_tmp_path(self, remote_user=None): + """ + Create a temporary subdirectory as a child of the temporary directory + managed by the remote interpreter. + """ + LOG.debug('_make_tmp_path(remote_user=%r)', remote_user) + path = self._generate_tmp_path() + LOG.debug('Temporary directory: %r', path) + self._connection.get_chain().call_no_reply(os.mkdir, path) + self._connection._shell.tmpdir = path + return path + + def _remove_tmp_path(self, tmp_path): + """ + Replace the base implementation's invocation of rm -rf, replacing it + with a pipelined call to :func:`ansible_mitogen.target.prune_tree`. + """ + LOG.debug('_remove_tmp_path(%r)', tmp_path) + if tmp_path is None and ansible.__version__ > '2.6': + tmp_path = self._connection._shell.tmpdir # 06f73ad578d + if tmp_path is not None: + self._connection.get_chain().call_no_reply( + ansible_mitogen.target.prune_tree, + tmp_path, + ) + self._connection._shell.tmpdir = None + + def _transfer_data(self, remote_path, data): + """ + Used by the base _execute_module(), and in <2.4 also by the template + action module, and probably others. + """ + if isinstance(data, dict): + data = jsonify(data) + if not isinstance(data, bytes): + data = to_bytes(data, errors='surrogate_or_strict') + + LOG.debug('_transfer_data(%r, %s ..%d bytes)', + remote_path, type(data), len(data)) + self._connection.put_data(remote_path, data) + return remote_path + + #: Actions listed here cause :func:`_fixup_perms2` to avoid a needless + #: roundtrip, as they modify file modes separately afterwards. This is due + #: to the method prototype having a default of `execute=True`. + FIXUP_PERMS_RED_HERRING = set(['copy']) + + def _fixup_perms2(self, remote_paths, remote_user=None, execute=True): + """ + Mitogen always executes ActionBase helper methods in the context of the + target user account, so it is never necessary to modify permissions + except to ensure the execute bit is set if requested. + """ + LOG.debug('_fixup_perms2(%r, remote_user=%r, execute=%r)', + remote_paths, remote_user, execute) + if execute and self._task.action not in self.FIXUP_PERMS_RED_HERRING: + return self._remote_chmod(remote_paths, mode='u+x') + return self.COMMAND_RESULT.copy() + + def _remote_chmod(self, paths, mode, sudoable=False): + """ + Issue an asynchronous set_file_mode() call for every path in `paths`, + then format the resulting return value list with fake_shell(). + """ + LOG.debug('_remote_chmod(%r, mode=%r, sudoable=%r)', + paths, mode, sudoable) + return self.fake_shell(lambda: mitogen.select.Select.all( + self._connection.get_chain().call_async( + ansible_mitogen.target.set_file_mode, path, mode + ) + for path in paths + )) + + def _remote_chown(self, paths, user, sudoable=False): + """ + Issue an asynchronous os.chown() call for every path in `paths`, then + format the resulting return value list with fake_shell(). + """ + LOG.debug('_remote_chown(%r, user=%r, sudoable=%r)', + paths, user, sudoable) + ent = self._connection.get_chain().call(pwd.getpwnam, user) + return self.fake_shell(lambda: mitogen.select.Select.all( + self._connection.get_chain().call_async( + os.chown, path, ent.pw_uid, ent.pw_gid + ) + for path in paths + )) + + def _remote_expand_user(self, path, sudoable=True): + """ + Replace the base implementation's attempt to emulate + os.path.expanduser() with an actual call to os.path.expanduser(). + + :param bool sudoable: + If :data:`True`, indicate unqualified tilde ("~" with no username) + should be evaluated in the context of the login account, not any + become_user. + """ + LOG.debug('_remote_expand_user(%r, sudoable=%r)', path, sudoable) + if not path.startswith('~'): + # /home/foo -> /home/foo + return path + if sudoable or not self._play_context.become: + if path == '~': + # ~ -> /home/dmw + return self._connection.homedir + if path.startswith('~/'): + # ~/.ansible -> /home/dmw/.ansible + return os.path.join(self._connection.homedir, path[2:]) + # ~root/.ansible -> /root/.ansible + return self._connection.get_chain(use_login=(not sudoable)).call( + os.path.expanduser, + mitogen.utils.cast(path), + ) + + def get_task_timeout_secs(self): + """ + Return the task "async:" value, portable across 2.4-2.5. + """ + try: + return self._task.async_val + except AttributeError: + return getattr(self._task, 'async') + + def _temp_file_gibberish(self, module_args, wrap_async): + # Ansible>2.5 module_utils reuses the action's temporary directory if + # one exists. Older versions error if this key is present. + if ansible.__version__ > '2.5': + if wrap_async: + # Sharing is not possible with async tasks, as in that case, + # the directory must outlive the action plug-in. + module_args['_ansible_tmpdir'] = None + else: + module_args['_ansible_tmpdir'] = self._connection._shell.tmpdir + + # If _ansible_tmpdir is unset, Ansible>2.6 module_utils will use + # _ansible_remote_tmp as the location to create the module's temporary + # directory. Older versions error if this key is present. + if ansible.__version__ > '2.6': + module_args['_ansible_remote_tmp'] = ( + self._connection.get_good_temp_dir() + ) + + def _execute_module(self, module_name=None, module_args=None, tmp=None, + task_vars=None, persist_files=False, + delete_remote_tmp=True, wrap_async=False): + """ + Collect up a module's execution environment then use it to invoke + target.run_module() or helpers.run_module_async() in the target + context. + """ + if module_name is None: + module_name = self._task.action + if module_args is None: + module_args = self._task.args + if task_vars is None: + task_vars = {} + + self._update_module_args(module_name, module_args, task_vars) + env = {} + self._compute_environment_string(env) + self._temp_file_gibberish(module_args, wrap_async) + + self._connection._connect() + result = ansible_mitogen.planner.invoke( + ansible_mitogen.planner.Invocation( + action=self, + connection=self._connection, + module_name=mitogen.core.to_text(module_name), + module_args=mitogen.utils.cast(module_args), + task_vars=task_vars, + templar=self._templar, + env=mitogen.utils.cast(env), + wrap_async=wrap_async, + timeout_secs=self.get_task_timeout_secs(), + ) + ) + + if ansible.__version__ < '2.5' and delete_remote_tmp and \ + getattr(self._connection._shell, 'tmpdir', None) is not None: + # Built-in actions expected tmpdir to be cleaned up automatically + # on _execute_module(). + self._remove_tmp_path(self._connection._shell.tmpdir) + + return result + + def _postprocess_response(self, result): + """ + Apply fixups mimicking ActionBase._execute_module(); this is copied + verbatim from action/__init__.py, the guts of _parse_returned_data are + garbage and should be removed or reimplemented once tests exist. + + :param dict result: + Dictionary with format:: + + { + "rc": int, + "stdout": "stdout data", + "stderr": "stderr data" + } + """ + data = self._parse_returned_data(result) + + # Cutpasted from the base implementation. + if 'stdout' in data and 'stdout_lines' not in data: + data['stdout_lines'] = (data['stdout'] or u'').splitlines() + if 'stderr' in data and 'stderr_lines' not in data: + data['stderr_lines'] = (data['stderr'] or u'').splitlines() + + return data + + def _low_level_execute_command(self, cmd, sudoable=True, in_data=None, + executable=None, + encoding_errors='surrogate_then_replace', + chdir=None): + """ + Override the base implementation by simply calling + target.exec_command() in the target context. + """ + LOG.debug('_low_level_execute_command(%r, in_data=%r, exe=%r, dir=%r)', + cmd, type(in_data), executable, chdir) + if executable is None: # executable defaults to False + executable = self._play_context.executable + if executable: + cmd = executable + ' -c ' + shlex_quote(cmd) + + rc, stdout, stderr = self._connection.exec_command( + cmd=cmd, + in_data=in_data, + sudoable=sudoable, + mitogen_chdir=chdir, + ) + stdout_text = to_text(stdout, errors=encoding_errors) + + return { + 'rc': rc, + 'stdout': stdout_text, + 'stdout_lines': stdout_text.splitlines(), + 'stderr': stderr, + } diff --git a/mitogen-0.2.7/ansible_mitogen/mixins.pyc b/mitogen-0.2.7/ansible_mitogen/mixins.pyc new file mode 100644 index 000000000..9be208e Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/mixins.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/module_finder.py b/mitogen-0.2.7/ansible_mitogen/module_finder.py new file mode 100644 index 000000000..633e3ca --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/module_finder.py @@ -0,0 +1,157 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import unicode_literals + +import collections +import imp +import os + +import mitogen.master + + +PREFIX = 'ansible.module_utils.' + + +Module = collections.namedtuple('Module', 'name path kind parent') + + +def get_fullname(module): + """ + Reconstruct a Module's canonical path by recursing through its parents. + """ + bits = [str(module.name)] + while module.parent: + bits.append(str(module.parent.name)) + module = module.parent + return '.'.join(reversed(bits)) + + +def get_code(module): + """ + Compile and return a Module's code object. + """ + fp = open(module.path) + try: + return compile(fp.read(), str(module.name), 'exec') + finally: + fp.close() + + +def is_pkg(module): + """ + Return :data:`True` if a Module represents a package. + """ + return module.kind == imp.PKG_DIRECTORY + + +def find(name, path=(), parent=None): + """ + Return a Module instance describing the first matching module found on the + search path. + + :param str name: + Module name. + :param list path: + List of directory names to search for the module. + :param Module parent: + Optional module parent. + """ + assert isinstance(path, tuple) + head, _, tail = name.partition('.') + try: + tup = imp.find_module(head, list(path)) + except ImportError: + return parent + + fp, modpath, (suffix, mode, kind) = tup + if fp: + fp.close() + + if parent and modpath == parent.path: + # 'from timeout import timeout', where 'timeout' is a function but also + # the name of the module being imported. + return None + + if kind == imp.PKG_DIRECTORY: + modpath = os.path.join(modpath, '__init__.py') + + module = Module(head, modpath, kind, parent) + # TODO: this code is entirely wrong on Python 3.x, but works well enough + # for Ansible. We need a new find_child() that only looks in the package + # directory, never falling back to the parent search path. + if tail and kind == imp.PKG_DIRECTORY: + return find_relative(module, tail, path) + return module + + +def find_relative(parent, name, path=()): + if parent.kind == imp.PKG_DIRECTORY: + path = (os.path.dirname(parent.path),) + path + return find(name, path, parent=parent) + + +def scan_fromlist(code): + for level, modname_s, fromlist in mitogen.master.scan_code_imports(code): + for name in fromlist: + yield level, '%s.%s' % (modname_s, name) + if not fromlist: + yield level, modname_s + + +def scan(module_name, module_path, search_path): + module = Module(module_name, module_path, imp.PY_SOURCE, None) + stack = [module] + seen = set() + + while stack: + module = stack.pop(0) + for level, fromname in scan_fromlist(get_code(module)): + if not fromname.startswith(PREFIX): + continue + + imported = find(fromname[len(PREFIX):], search_path) + if imported is None or imported in seen: + continue + + seen.add(imported) + stack.append(imported) + parent = imported.parent + while parent: + fullname = get_fullname(parent) + module = Module(fullname, parent.path, parent.kind, None) + if module not in seen: + seen.add(module) + stack.append(module) + parent = parent.parent + + return sorted( + (PREFIX + get_fullname(module), module.path, is_pkg(module)) + for module in seen + ) diff --git a/mitogen-0.2.7/ansible_mitogen/module_finder.pyc b/mitogen-0.2.7/ansible_mitogen/module_finder.pyc new file mode 100644 index 000000000..dde4948 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/module_finder.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/parsing.py b/mitogen-0.2.7/ansible_mitogen/parsing.py new file mode 100644 index 000000000..525e60c --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/parsing.py @@ -0,0 +1,84 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +""" +Classes to detect each case from [0] and prepare arguments necessary for the +corresponding Runner class within the target, including preloading requisite +files/modules known missing. + +[0] "Ansible Module Architecture", developing_program_flow_modules.html +""" + +from __future__ import absolute_import +from __future__ import unicode_literals + +import mitogen.core + + +def parse_script_interpreter(source): + """ + Parse the script interpreter portion of a UNIX hashbang using the rules + Linux uses. + + :param str source: String like "/usr/bin/env python". + + :returns: + Tuple of `(interpreter, arg)`, where `intepreter` is the script + interpreter and `arg` is its sole argument if present, otherwise + :py:data:`None`. + """ + # Find terminating newline. Assume last byte of binprm_buf if absent. + nl = source.find(b'\n', 0, 128) + if nl == -1: + nl = min(128, len(source)) + + # Split once on the first run of whitespace. If no whitespace exists, + # bits just contains the interpreter filename. + bits = source[0:nl].strip().split(None, 1) + if len(bits) == 1: + return mitogen.core.to_text(bits[0]), None + return mitogen.core.to_text(bits[0]), mitogen.core.to_text(bits[1]) + + +def parse_hashbang(source): + """ + Parse a UNIX "hashbang line" using the syntax supported by Linux. + + :param str source: String like "#!/usr/bin/env python". + + :returns: + Tuple of `(interpreter, arg)`, where `intepreter` is the script + interpreter and `arg` is its sole argument if present, otherwise + :py:data:`None`. + """ + # Linux requires first 2 bytes with no whitespace, pretty sure it's the + # same everywhere. See binfmt_script.c. + if not source.startswith(b'#!'): + return None, None + + return parse_script_interpreter(source[2:]) diff --git a/mitogen-0.2.7/ansible_mitogen/parsing.pyc b/mitogen-0.2.7/ansible_mitogen/parsing.pyc new file mode 100644 index 000000000..9872fba Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/parsing.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/planner.py b/mitogen-0.2.7/ansible_mitogen/planner.py new file mode 100644 index 000000000..2eebd36 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/planner.py @@ -0,0 +1,499 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +""" +Classes to detect each case from [0] and prepare arguments necessary for the +corresponding Runner class within the target, including preloading requisite +files/modules known missing. + +[0] "Ansible Module Architecture", developing_program_flow_modules.html +""" + +from __future__ import absolute_import +from __future__ import unicode_literals + +import json +import logging +import os +import random + +from ansible.executor import module_common +import ansible.errors +import ansible.module_utils +import mitogen.core +import mitogen.select + +import ansible_mitogen.loaders +import ansible_mitogen.parsing +import ansible_mitogen.target + + +LOG = logging.getLogger(__name__) +NO_METHOD_MSG = 'Mitogen: no invocation method found for: ' +NO_INTERPRETER_MSG = 'module (%s) is missing interpreter line' +NO_MODULE_MSG = 'The module %s was not found in configured module paths.' + + +class Invocation(object): + """ + Collect up a module's execution environment then use it to invoke + target.run_module() or helpers.run_module_async() in the target context. + """ + def __init__(self, action, connection, module_name, module_args, + task_vars, templar, env, wrap_async, timeout_secs): + #: ActionBase instance invoking the module. Required to access some + #: output postprocessing methods that don't belong in ActionBase at + #: all. + self.action = action + #: Ansible connection to use to contact the target. Must be an + #: ansible_mitogen connection. + self.connection = connection + #: Name of the module ('command', 'shell', etc.) to execute. + self.module_name = module_name + #: Final module arguments. + self.module_args = module_args + #: Task variables, needed to extract ansible_*_interpreter. + self.task_vars = task_vars + #: Templar, needed to extract ansible_*_interpreter. + self.templar = templar + #: Final module environment. + self.env = env + #: Boolean, if :py:data:`True`, launch the module asynchronously. + self.wrap_async = wrap_async + #: Integer, if >0, limit the time an asynchronous job may run for. + self.timeout_secs = timeout_secs + #: Initially ``None``, but set by :func:`invoke`. The path on the + #: master to the module's implementation file. + self.module_path = None + #: Initially ``None``, but set by :func:`invoke`. The raw source or + #: binary contents of the module. + self.module_source = None + + def __repr__(self): + return 'Invocation(module_name=%s)' % (self.module_name,) + + +class Planner(object): + """ + A Planner receives a module name and the contents of its implementation + file, indicates whether or not it understands how to run the module, and + exports a method to run the module. + """ + def __init__(self, invocation): + self._inv = invocation + + def detect(self): + """ + Return true if the supplied `invocation` matches the module type + implemented by this planner. + """ + raise NotImplementedError() + + def should_fork(self): + """ + Asynchronous tasks must always be forked. + """ + return self._inv.wrap_async + + def get_push_files(self): + """ + Return a list of files that should be propagated to the target context + using PushFileService. The default implementation pushes nothing. + """ + return [] + + def get_module_deps(self): + """ + Return a list of the Python module names imported by the module. + """ + return [] + + def get_kwargs(self, **kwargs): + """ + If :meth:`detect` returned :data:`True`, plan for the module's + execution, including granting access to or delivering any files to it + that are known to be absent, and finally return a dict:: + + { + # Name of the class from runners.py that implements the + # target-side execution of this module type. + "runner_name": "...", + + # Remaining keys are passed to the constructor of the class + # named by `runner_name`. + } + """ + new = dict((mitogen.core.UnicodeType(k), kwargs[k]) + for k in kwargs) + new.setdefault('good_temp_dir', + self._inv.connection.get_good_temp_dir()) + new.setdefault('cwd', self._inv.connection.get_default_cwd()) + new.setdefault('extra_env', self._inv.connection.get_default_env()) + new.setdefault('emulate_tty', True) + new.setdefault('service_context', self._inv.connection.parent) + return new + + def __repr__(self): + return '%s()' % (type(self).__name__,) + + +class BinaryPlanner(Planner): + """ + Binary modules take their arguments and will return data to Ansible in the + same way as want JSON modules. + """ + runner_name = 'BinaryRunner' + + def detect(self): + return module_common._is_binary(self._inv.module_source) + + def get_push_files(self): + return [mitogen.core.to_text(self._inv.module_path)] + + def get_kwargs(self, **kwargs): + return super(BinaryPlanner, self).get_kwargs( + runner_name=self.runner_name, + module=self._inv.module_name, + path=self._inv.module_path, + json_args=json.dumps(self._inv.module_args), + env=self._inv.env, + **kwargs + ) + + +class ScriptPlanner(BinaryPlanner): + """ + Common functionality for script module planners -- handle interpreter + detection and rewrite. + """ + def _rewrite_interpreter(self, path): + """ + Given the original interpreter binary extracted from the script's + interpreter line, look up the associated `ansible_*_interpreter` + variable, render it and return it. + + :param str path: + Absolute UNIX path to original interpreter. + + :returns: + Shell fragment prefix used to execute the script via "/bin/sh -c". + While `ansible_*_interpreter` documentation suggests shell isn't + involved here, the vanilla implementation uses it and that use is + exploited in common playbooks. + """ + key = u'ansible_%s_interpreter' % os.path.basename(path).strip() + try: + template = self._inv.task_vars[key] + except KeyError: + return path + + return mitogen.utils.cast(self._inv.templar.template(template)) + + def _get_interpreter(self): + path, arg = ansible_mitogen.parsing.parse_hashbang( + self._inv.module_source + ) + if path is None: + raise ansible.errors.AnsibleError(NO_INTERPRETER_MSG % ( + self._inv.module_name, + )) + + fragment = self._rewrite_interpreter(path) + if arg: + fragment += ' ' + arg + + return fragment, path.startswith('python') + + def get_kwargs(self, **kwargs): + interpreter_fragment, is_python = self._get_interpreter() + return super(ScriptPlanner, self).get_kwargs( + interpreter_fragment=interpreter_fragment, + is_python=is_python, + **kwargs + ) + + +class JsonArgsPlanner(ScriptPlanner): + """ + Script that has its interpreter directive and the task arguments + substituted into its source as a JSON string. + """ + runner_name = 'JsonArgsRunner' + + def detect(self): + return module_common.REPLACER_JSONARGS in self._inv.module_source + + +class WantJsonPlanner(ScriptPlanner): + """ + If a module has the string WANT_JSON in it anywhere, Ansible treats it as a + non-native module that accepts a filename as its only command line + parameter. The filename is for a temporary file containing a JSON string + containing the module's parameters. The module needs to open the file, read + and parse the parameters, operate on the data, and print its return data as + a JSON encoded dictionary to stdout before exiting. + + These types of modules are self-contained entities. As of Ansible 2.1, + Ansible only modifies them to change a shebang line if present. + """ + runner_name = 'WantJsonRunner' + + def detect(self): + return b'WANT_JSON' in self._inv.module_source + + +class NewStylePlanner(ScriptPlanner): + """ + The Ansiballz framework differs from module replacer in that it uses real + Python imports of things in ansible/module_utils instead of merely + preprocessing the module. + """ + runner_name = 'NewStyleRunner' + marker = b'from ansible.module_utils.' + + def detect(self): + return self.marker in self._inv.module_source + + def _get_interpreter(self): + return None, None + + def get_push_files(self): + return super(NewStylePlanner, self).get_push_files() + [ + mitogen.core.to_text(path) + for fullname, path, is_pkg in self.get_module_map()['custom'] + ] + + def get_module_deps(self): + return self.get_module_map()['builtin'] + + #: Module names appearing in this set always require forking, usually due + #: to some terminal leakage that cannot be worked around in any sane + #: manner. + ALWAYS_FORK_MODULES = frozenset([ + 'dnf', # issue #280; py-dnf/hawkey need therapy + 'firewalld', # issue #570: ansible module_utils caches dbus conn + ]) + + def should_fork(self): + """ + In addition to asynchronous tasks, new-style modules should be forked + if: + + * the user specifies mitogen_task_isolation=fork, or + * the new-style module has a custom module search path, or + * the module is known to leak like a sieve. + """ + return ( + super(NewStylePlanner, self).should_fork() or + (self._inv.task_vars.get('mitogen_task_isolation') == 'fork') or + (self._inv.module_name in self.ALWAYS_FORK_MODULES) or + (len(self.get_module_map()['custom']) > 0) + ) + + def get_search_path(self): + return tuple( + path + for path in ansible_mitogen.loaders.module_utils_loader._get_paths( + subdirs=False + ) + if os.path.isdir(path) + ) + + _module_map = None + + def get_module_map(self): + if self._module_map is None: + self._module_map = self._inv.connection.parent.call_service( + service_name='ansible_mitogen.services.ModuleDepService', + method_name='scan', + + module_name='ansible_module_%s' % (self._inv.module_name,), + module_path=self._inv.module_path, + search_path=self.get_search_path(), + builtin_path=module_common._MODULE_UTILS_PATH, + context=self._inv.connection.context, + ) + return self._module_map + + def get_kwargs(self): + return super(NewStylePlanner, self).get_kwargs( + module_map=self.get_module_map(), + ) + + +class ReplacerPlanner(NewStylePlanner): + """ + The Module Replacer framework is the original framework implementing + new-style modules. It is essentially a preprocessor (like the C + Preprocessor for those familiar with that programming language). It does + straight substitutions of specific substring patterns in the module file. + There are two types of substitutions. + + * Replacements that only happen in the module file. These are public + replacement strings that modules can utilize to get helpful boilerplate + or access to arguments. + + "from ansible.module_utils.MOD_LIB_NAME import *" is replaced with the + contents of the ansible/module_utils/MOD_LIB_NAME.py. These should only + be used with new-style Python modules. + + "#<>" is equivalent to + "from ansible.module_utils.basic import *" and should also only apply to + new-style Python modules. + + "# POWERSHELL_COMMON" substitutes the contents of + "ansible/module_utils/powershell.ps1". It should only be used with + new-style Powershell modules. + """ + runner_name = 'ReplacerRunner' + + def detect(self): + return module_common.REPLACER in self._inv.module_source + + +class OldStylePlanner(ScriptPlanner): + runner_name = 'OldStyleRunner' + + def detect(self): + # Everything else. + return True + + +_planners = [ + BinaryPlanner, + # ReplacerPlanner, + NewStylePlanner, + JsonArgsPlanner, + WantJsonPlanner, + OldStylePlanner, +] + + +def get_module_data(name): + path = ansible_mitogen.loaders.module_loader.find_plugin(name, '') + if path is None: + raise ansible.errors.AnsibleError(NO_MODULE_MSG % (name,)) + + with open(path, 'rb') as fp: + source = fp.read() + return mitogen.core.to_text(path), source + + +def _propagate_deps(invocation, planner, context): + invocation.connection.parent.call_service( + service_name='mitogen.service.PushFileService', + method_name='propagate_paths_and_modules', + context=context, + paths=planner.get_push_files(), + modules=planner.get_module_deps(), + ) + + +def _invoke_async_task(invocation, planner): + job_id = '%016x' % random.randint(0, 2**64) + context = invocation.connection.spawn_isolated_child() + _propagate_deps(invocation, planner, context) + + with mitogen.core.Receiver(context.router) as started_recv: + call_recv = context.call_async( + ansible_mitogen.target.run_module_async, + job_id=job_id, + timeout_secs=invocation.timeout_secs, + started_sender=started_recv.to_sender(), + kwargs=planner.get_kwargs(), + ) + + # Wait for run_module_async() to crash, or for AsyncRunner to indicate + # the job file has been written. + for msg in mitogen.select.Select([started_recv, call_recv]): + if msg.receiver is call_recv: + # It can only be an exception. + raise msg.unpickle() + break + + return { + 'stdout': json.dumps({ + # modules/utilities/logic/async_wrapper.py::_run_module(). + 'changed': True, + 'started': 1, + 'finished': 0, + 'ansible_job_id': job_id, + }) + } + + +def _invoke_isolated_task(invocation, planner): + context = invocation.connection.spawn_isolated_child() + _propagate_deps(invocation, planner, context) + try: + return context.call( + ansible_mitogen.target.run_module, + kwargs=planner.get_kwargs(), + ) + finally: + context.shutdown() + + +def _get_planner(invocation): + for klass in _planners: + planner = klass(invocation) + if planner.detect(): + LOG.debug('%r accepted %r (filename %r)', planner, + invocation.module_name, invocation.module_path) + return planner + LOG.debug('%r rejected %r', planner, invocation.module_name) + raise ansible.errors.AnsibleError(NO_METHOD_MSG + repr(invocation)) + + +def invoke(invocation): + """ + Find a Planner subclass corresnding to `invocation` and use it to invoke + the module. + + :param Invocation invocation: + :returns: + Module return dict. + :raises ansible.errors.AnsibleError: + Unrecognized/unsupported module type. + """ + (invocation.module_path, + invocation.module_source) = get_module_data(invocation.module_name) + planner = _get_planner(invocation) + + if invocation.wrap_async: + response = _invoke_async_task(invocation, planner) + elif planner.should_fork(): + response = _invoke_isolated_task(invocation, planner) + else: + _propagate_deps(invocation, planner, invocation.connection.context) + response = invocation.connection.get_chain().call( + ansible_mitogen.target.run_module, + kwargs=planner.get_kwargs(), + ) + + return invocation.action._postprocess_response(response) diff --git a/mitogen-0.2.7/ansible_mitogen/planner.pyc b/mitogen-0.2.7/ansible_mitogen/planner.pyc new file mode 100644 index 000000000..11bb62e Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/planner.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/__init__.py b/mitogen-0.2.7/ansible_mitogen/plugins/__init__.py new file mode 100644 index 000000000..e69de29 diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/action/__init__.py b/mitogen-0.2.7/ansible_mitogen/plugins/action/__init__.py new file mode 100644 index 000000000..e69de29 diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/action/mitogen_get_stack.py b/mitogen-0.2.7/ansible_mitogen/plugins/action/mitogen_get_stack.py new file mode 100644 index 000000000..12afbfb --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/action/mitogen_get_stack.py @@ -0,0 +1,54 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import unicode_literals + +""" +Fetch the connection configuration stack that would be used to connect to a +target, without actually connecting to it. +""" + +import ansible_mitogen.connection + +from ansible.plugins.action import ActionBase + + +class ActionModule(ActionBase): + def run(self, tmp=None, task_vars=None): + if not isinstance(self._connection, + ansible_mitogen.connection.Connection): + return { + 'skipped': True, + } + + return { + 'changed': True, + 'result': self._connection._build_stack(), + '_ansible_verbose_always': True, + } diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/__init__.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/__init__.py new file mode 100644 index 000000000..e69de29 diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_doas.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_doas.py new file mode 100644 index 000000000..1113d7c --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_doas.py @@ -0,0 +1,44 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +try: + import ansible_mitogen.connection +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'mitogen_doas' diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_docker.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_docker.py new file mode 100644 index 000000000..b71ef5f --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_docker.py @@ -0,0 +1,51 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +try: + import ansible_mitogen +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'docker' + + @property + def docker_cmd(self): + """ + Ansible 2.3 synchronize module wants to know how we run Docker. + """ + return 'docker' diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_jail.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_jail.py new file mode 100644 index 000000000..c7475fb --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_jail.py @@ -0,0 +1,44 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +try: + import ansible_mitogen +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'jail' diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_kubectl.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_kubectl.py new file mode 100644 index 000000000..2dab131 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_kubectl.py @@ -0,0 +1,71 @@ +# coding: utf-8 +# Copyright 2018, Yannig Perré +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +try: + from ansible.plugins.connection import kubectl +except ImportError: + kubectl = None + +from ansible.errors import AnsibleConnectionFailure +from ansible.module_utils.six import iteritems + +try: + import ansible_mitogen +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'kubectl' + + not_supported_msg = ( + 'The "mitogen_kubectl" plug-in requires a version of Ansible ' + 'that ships with the "kubectl" connection plug-in.' + ) + + def __init__(self, *args, **kwargs): + if kubectl is None: + raise AnsibleConnectionFailure(self.not_supported_msg) + super(Connection, self).__init__(*args, **kwargs) + + def get_extra_args(self): + parameters = [] + for key, option in iteritems(kubectl.CONNECTION_OPTIONS): + if self.get_task_var('ansible_' + key) is not None: + parameters += [ option, self.get_task_var('ansible_' + key) ] + + return parameters diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_local.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_local.py new file mode 100644 index 000000000..24b84a0 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_local.py @@ -0,0 +1,86 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +try: + import ansible_mitogen.connection +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.connection +import ansible_mitogen.process + + +if sys.version_info > (3,): + viewkeys = dict.keys +elif sys.version_info > (2, 7): + viewkeys = dict.viewkeys +else: + viewkeys = lambda dct: set(dct) + + +def dict_diff(old, new): + """ + Return a dict representing the differences between the dicts `old` and + `new`. Deleted keys appear as a key with the value :data:`None`, added and + changed keys appear as a key with the new value. + """ + old_keys = viewkeys(old) + new_keys = viewkeys(dict(new)) + out = {} + for key in new_keys - old_keys: + out[key] = new[key] + for key in old_keys - new_keys: + out[key] = None + for key in old_keys & new_keys: + if old[key] != new[key]: + out[key] = new[key] + return out + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'local' + + def get_default_cwd(self): + # https://github.com/ansible/ansible/issues/14489 + return self.loader_basedir + + def get_default_env(self): + """ + Vanilla Ansible local commands execute with an environment inherited + from WorkerProcess, we must emulate that. + """ + return dict_diff( + old=ansible_mitogen.process.MuxProcess.original_env, + new=os.environ, + ) diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_lxc.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_lxc.py new file mode 100644 index 000000000..696c9ab --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_lxc.py @@ -0,0 +1,44 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +try: + import ansible_mitogen +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'lxc' diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_lxd.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_lxd.py new file mode 100644 index 000000000..95e692a --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_lxd.py @@ -0,0 +1,44 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +try: + import ansible_mitogen +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'lxd' diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_machinectl.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_machinectl.py new file mode 100644 index 000000000..0f5a0d2 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_machinectl.py @@ -0,0 +1,44 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +try: + import ansible_mitogen.connection +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'machinectl' diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_setns.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_setns.py new file mode 100644 index 000000000..20c6f13 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_setns.py @@ -0,0 +1,44 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +try: + import ansible_mitogen.connection +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'setns' diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_ssh.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_ssh.py new file mode 100644 index 000000000..df0e87c --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_ssh.py @@ -0,0 +1,65 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +DOCUMENTATION = """ + author: David Wilson + connection: mitogen_ssh + short_description: Connect over SSH via Mitogen + description: + - This connects using an OpenSSH client controlled by the Mitogen for + Ansible extension. It accepts every option the vanilla ssh plugin + accepts. + version_added: "2.5" + options: +""" + +import ansible.plugins.connection.ssh + +try: + import ansible_mitogen.connection +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'ssh' + vanilla_class = ansible.plugins.connection.ssh.Connection + + @staticmethod + def _create_control_path(*args, **kwargs): + """Forward _create_control_path() to the implementation in ssh.py.""" + # https://github.com/dw/mitogen/issues/342 + return Connection.vanilla_class._create_control_path(*args, **kwargs) diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_ssh.pyc b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_ssh.pyc new file mode 100644 index 000000000..2bbfc09 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_ssh.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_su.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_su.py new file mode 100644 index 000000000..4ab2711 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_su.py @@ -0,0 +1,44 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +try: + import ansible_mitogen.connection +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'mitogen_su' diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_sudo.py b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_sudo.py new file mode 100644 index 000000000..130f544 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/connection/mitogen_sudo.py @@ -0,0 +1,44 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +try: + import ansible_mitogen.connection +except ImportError: + base_dir = os.path.dirname(__file__) + sys.path.insert(0, os.path.abspath(os.path.join(base_dir, '../../..'))) + del base_dir + +import ansible_mitogen.connection + + +class Connection(ansible_mitogen.connection.Connection): + transport = 'mitogen_sudo' diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/strategy/__init__.py b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/__init__.py new file mode 100644 index 000000000..e69de29 diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/strategy/__pycache__/mitogen_linear.cpython-37.pyc b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/__pycache__/mitogen_linear.cpython-37.pyc new file mode 100644 index 000000000..e6fad12 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/__pycache__/mitogen_linear.cpython-37.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen.py b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen.py new file mode 100644 index 000000000..6687266 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen.py @@ -0,0 +1,61 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +# +# This is not the real Strategy implementation module, it simply exists as a +# proxy to the real module, which is loaded using Python's regular import +# mechanism, to prevent Ansible's PluginLoader from making up a fake name that +# results in ansible_mitogen plugin modules being loaded twice: once by +# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is +# stuffed into sys.modules even though attempting to import it will trigger an +# ImportError, and once under its canonical name, "ansible_mitogen.strategy". +# +# Therefore we have a proxy module that imports it under the real name, and +# sets up the duff PluginLoader-imported module to just contain objects from +# the real module, so duplicate types don't exist in memory, and things like +# debuggers and isinstance() work predictably. +# + +BASE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), '../../..') +) + +if BASE_DIR not in sys.path: + sys.path.insert(0, BASE_DIR) + +import ansible_mitogen.strategy +import ansible.plugins.strategy.linear + + +class StrategyModule(ansible_mitogen.strategy.StrategyMixin, + ansible.plugins.strategy.linear.StrategyModule): + pass diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_free.py b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_free.py new file mode 100644 index 000000000..ffe2fbd --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_free.py @@ -0,0 +1,62 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +# +# This is not the real Strategy implementation module, it simply exists as a +# proxy to the real module, which is loaded using Python's regular import +# mechanism, to prevent Ansible's PluginLoader from making up a fake name that +# results in ansible_mitogen plugin modules being loaded twice: once by +# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is +# stuffed into sys.modules even though attempting to import it will trigger an +# ImportError, and once under its canonical name, "ansible_mitogen.strategy". +# +# Therefore we have a proxy module that imports it under the real name, and +# sets up the duff PluginLoader-imported module to just contain objects from +# the real module, so duplicate types don't exist in memory, and things like +# debuggers and isinstance() work predictably. +# + +BASE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), '../../..') +) + +if BASE_DIR not in sys.path: + sys.path.insert(0, BASE_DIR) + +import ansible_mitogen.loaders +import ansible_mitogen.strategy + + +Base = ansible_mitogen.loaders.strategy_loader.get('free', class_only=True) + +class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base): + pass diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py new file mode 100644 index 000000000..23eccd3 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_host_pinned.py @@ -0,0 +1,67 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +# +# This is not the real Strategy implementation module, it simply exists as a +# proxy to the real module, which is loaded using Python's regular import +# mechanism, to prevent Ansible's PluginLoader from making up a fake name that +# results in ansible_mitogen plugin modules being loaded twice: once by +# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is +# stuffed into sys.modules even though attempting to import it will trigger an +# ImportError, and once under its canonical name, "ansible_mitogen.strategy". +# +# Therefore we have a proxy module that imports it under the real name, and +# sets up the duff PluginLoader-imported module to just contain objects from +# the real module, so duplicate types don't exist in memory, and things like +# debuggers and isinstance() work predictably. +# + +BASE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), '../../..') +) + +if BASE_DIR not in sys.path: + sys.path.insert(0, BASE_DIR) + +import ansible_mitogen.loaders +import ansible_mitogen.strategy + + +Base = ansible_mitogen.loaders.strategy_loader.get('host_pinned', class_only=True) + +if Base is None: + raise ImportError( + 'The host_pinned strategy is only available in Ansible 2.7 or newer.' + ) + +class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base): + pass diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_linear.py b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_linear.py new file mode 100644 index 000000000..1b198e6 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_linear.py @@ -0,0 +1,62 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os.path +import sys + +# +# This is not the real Strategy implementation module, it simply exists as a +# proxy to the real module, which is loaded using Python's regular import +# mechanism, to prevent Ansible's PluginLoader from making up a fake name that +# results in ansible_mitogen plugin modules being loaded twice: once by +# PluginLoader with a name like "ansible.plugins.strategy.mitogen", which is +# stuffed into sys.modules even though attempting to import it will trigger an +# ImportError, and once under its canonical name, "ansible_mitogen.strategy". +# +# Therefore we have a proxy module that imports it under the real name, and +# sets up the duff PluginLoader-imported module to just contain objects from +# the real module, so duplicate types don't exist in memory, and things like +# debuggers and isinstance() work predictably. +# + +BASE_DIR = os.path.abspath( + os.path.join(os.path.dirname(__file__), '../../..') +) + +if BASE_DIR not in sys.path: + sys.path.insert(0, BASE_DIR) + +import ansible_mitogen.loaders +import ansible_mitogen.strategy + + +Base = ansible_mitogen.loaders.strategy_loader.get('linear', class_only=True) + +class StrategyModule(ansible_mitogen.strategy.StrategyMixin, Base): + pass diff --git a/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_linear.pyc b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_linear.pyc new file mode 100644 index 000000000..786dc37 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/plugins/strategy/mitogen_linear.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/process.py b/mitogen-0.2.7/ansible_mitogen/process.py new file mode 100644 index 000000000..e4e61e8 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/process.py @@ -0,0 +1,358 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import atexit +import errno +import logging +import os +import signal +import socket +import sys +import time + +try: + import faulthandler +except ImportError: + faulthandler = None + +import mitogen +import mitogen.core +import mitogen.debug +import mitogen.master +import mitogen.parent +import mitogen.service +import mitogen.unix +import mitogen.utils + +import ansible +import ansible.constants as C +import ansible_mitogen.logging +import ansible_mitogen.services + +from mitogen.core import b +import ansible_mitogen.affinity + + +LOG = logging.getLogger(__name__) + +ANSIBLE_PKG_OVERRIDE = ( + u"__version__ = %r\n" + u"__author__ = %r\n" +) + + +def clean_shutdown(sock): + """ + Shut the write end of `sock`, causing `recv` in the worker process to wake + up with a 0-byte read and initiate mux process exit, then wait for a 0-byte + read from the read end, which will occur after the the child closes the + descriptor on exit. + + This is done using :mod:`atexit` since Ansible lacks any more sensible hook + to run code during exit, and unless some synchronization exists with + MuxProcess, debug logs may appear on the user's terminal *after* the prompt + has been printed. + """ + sock.shutdown(socket.SHUT_WR) + sock.recv(1) + + +def getenv_int(key, default=0): + """ + Get an integer-valued environment variable `key`, if it exists and parses + as an integer, otherwise return `default`. + """ + try: + return int(os.environ.get(key, str(default))) + except ValueError: + return default + + +def save_pid(name): + """ + When debugging and profiling, it is very annoying to poke through the + process list to discover the currently running Ansible and MuxProcess IDs, + especially when trying to catch an issue during early startup. So here, if + a magic environment variable set, stash them in hidden files in the CWD:: + + alias muxpid="cat .ansible-mux.pid" + alias anspid="cat .ansible-controller.pid" + + gdb -p $(muxpid) + perf top -p $(anspid) + """ + if os.environ.get('MITOGEN_SAVE_PIDS'): + with open('.ansible-%s.pid' % (name,), 'w') as fp: + fp.write(str(os.getpid())) + + +class MuxProcess(object): + """ + Implement a subprocess forked from the Ansible top-level, as a safe place + to contain the Mitogen IO multiplexer thread, keeping its use of the + logging package (and the logging package's heavy use of locks) far away + from the clutches of os.fork(), which is used continuously by the + multiprocessing package in the top-level process. + + The problem with running the multiplexer in that process is that should the + multiplexer thread be in the process of emitting a log entry (and holding + its lock) at the point of fork, in the child, the first attempt to log any + log entry using the same handler will deadlock the child, as in the memory + image the child received, the lock will always be marked held. + + See https://bugs.python.org/issue6721 for a thorough description of the + class of problems this worker is intended to avoid. + """ + + #: In the top-level process, this references one end of a socketpair(), + #: which the MuxProcess blocks reading from in order to determine when + #: the master process dies. Once the read returns, the MuxProcess will + #: begin shutting itself down. + worker_sock = None + + #: In the worker process, this references the other end of + #: :py:attr:`worker_sock`. + child_sock = None + + #: In the top-level process, this is the PID of the single MuxProcess + #: that was spawned. + worker_pid = None + + #: A copy of :data:`os.environ` at the time the multiplexer process was + #: started. It's used by mitogen_local.py to find changes made to the + #: top-level environment (e.g. vars plugins -- issue #297) that must be + #: applied to locally executed commands and modules. + original_env = None + + #: In both processes, this is the temporary UNIX socket used for + #: forked WorkerProcesses to contact the MuxProcess + unix_listener_path = None + + #: Singleton. + _instance = None + + @classmethod + def start(cls, _init_logging=True): + """ + Arrange for the subprocess to be started, if it is not already running. + + The parent process picks a UNIX socket path the child will use prior to + fork, creates a socketpair used essentially as a semaphore, then blocks + waiting for the child to indicate the UNIX socket is ready for use. + + :param bool _init_logging: + For testing, if :data:`False`, don't initialize logging. + """ + if cls.worker_sock is not None: + return + + if faulthandler is not None: + faulthandler.enable() + + mitogen.utils.setup_gil() + cls.unix_listener_path = mitogen.unix.make_socket_path() + cls.worker_sock, cls.child_sock = socket.socketpair() + atexit.register(lambda: clean_shutdown(cls.worker_sock)) + mitogen.core.set_cloexec(cls.worker_sock.fileno()) + mitogen.core.set_cloexec(cls.child_sock.fileno()) + + cls.profiling = os.environ.get('MITOGEN_PROFILING') is not None + if cls.profiling: + mitogen.core.enable_profiling() + if _init_logging: + ansible_mitogen.logging.setup() + + cls.original_env = dict(os.environ) + cls.child_pid = os.fork() + if cls.child_pid: + save_pid('controller') + ansible_mitogen.logging.set_process_name('top') + ansible_mitogen.affinity.policy.assign_controller() + cls.child_sock.close() + cls.child_sock = None + mitogen.core.io_op(cls.worker_sock.recv, 1) + else: + save_pid('mux') + ansible_mitogen.logging.set_process_name('mux') + ansible_mitogen.affinity.policy.assign_muxprocess() + cls.worker_sock.close() + cls.worker_sock = None + self = cls() + self.worker_main() + + def worker_main(self): + """ + The main function of for the mux process: setup the Mitogen broker + thread and ansible_mitogen services, then sleep waiting for the socket + connected to the parent to be closed (indicating the parent has died). + """ + self._setup_master() + self._setup_services() + + try: + # Let the parent know our listening socket is ready. + mitogen.core.io_op(self.child_sock.send, b('1')) + # Block until the socket is closed, which happens on parent exit. + mitogen.core.io_op(self.child_sock.recv, 1) + finally: + self.broker.shutdown() + self.broker.join() + + # Test frameworks living somewhere higher on the stack of the + # original parent process may try to catch sys.exit(), so do a C + # level exit instead. + os._exit(0) + + def _enable_router_debug(self): + if 'MITOGEN_ROUTER_DEBUG' in os.environ: + self.router.enable_debug() + + def _enable_stack_dumps(self): + secs = getenv_int('MITOGEN_DUMP_THREAD_STACKS', default=0) + if secs: + mitogen.debug.dump_to_logger(secs=secs) + + def _setup_simplejson(self, responder): + """ + We support serving simplejson for Python 2.4 targets on Ansible 2.3, at + least so the package's own CI Docker scripts can run without external + help, however newer versions of simplejson no longer support Python + 2.4. Therefore override any installed/loaded version with a + 2.4-compatible version we ship in the compat/ directory. + """ + responder.whitelist_prefix('simplejson') + + # issue #536: must be at end of sys.path, in case existing newer + # version is already loaded. + compat_path = os.path.join(os.path.dirname(__file__), 'compat') + sys.path.append(compat_path) + + for fullname, is_pkg, suffix in ( + (u'simplejson', True, '__init__.py'), + (u'simplejson.decoder', False, 'decoder.py'), + (u'simplejson.encoder', False, 'encoder.py'), + (u'simplejson.scanner', False, 'scanner.py'), + ): + path = os.path.join(compat_path, 'simplejson', suffix) + fp = open(path, 'rb') + try: + source = fp.read() + finally: + fp.close() + + responder.add_source_override( + fullname=fullname, + path=path, + source=source, + is_pkg=is_pkg, + ) + + def _setup_responder(self, responder): + """ + Configure :class:`mitogen.master.ModuleResponder` to only permit + certain packages, and to generate custom responses for certain modules. + """ + responder.whitelist_prefix('ansible') + responder.whitelist_prefix('ansible_mitogen') + self._setup_simplejson(responder) + + # Ansible 2.3 is compatible with Python 2.4 targets, however + # ansible/__init__.py is not. Instead, executor/module_common.py writes + # out a 2.4-compatible namespace package for unknown reasons. So we + # copy it here. + responder.add_source_override( + fullname='ansible', + path=ansible.__file__, + source=(ANSIBLE_PKG_OVERRIDE % ( + ansible.__version__, + ansible.__author__, + )).encode(), + is_pkg=True, + ) + + def _setup_master(self): + """ + Construct a Router, Broker, and mitogen.unix listener + """ + self.broker = mitogen.master.Broker(install_watcher=False) + self.router = mitogen.master.Router( + broker=self.broker, + max_message_size=4096 * 1048576, + ) + self._setup_responder(self.router.responder) + mitogen.core.listen(self.broker, 'shutdown', self.on_broker_shutdown) + mitogen.core.listen(self.broker, 'exit', self.on_broker_exit) + self.listener = mitogen.unix.Listener( + router=self.router, + path=self.unix_listener_path, + backlog=C.DEFAULT_FORKS, + ) + self._enable_router_debug() + self._enable_stack_dumps() + + def _setup_services(self): + """ + Construct a ContextService and a thread to service requests for it + arriving from worker processes. + """ + self.pool = mitogen.service.Pool( + router=self.router, + services=[ + mitogen.service.FileService(router=self.router), + mitogen.service.PushFileService(router=self.router), + ansible_mitogen.services.ContextService(self.router), + ansible_mitogen.services.ModuleDepService(self.router), + ], + size=getenv_int('MITOGEN_POOL_SIZE', default=32), + ) + LOG.debug('Service pool configured: size=%d', self.pool.size) + + def on_broker_shutdown(self): + """ + Respond to broker shutdown by beginning service pool shutdown. Do not + join on the pool yet, since that would block the broker thread which + then cannot clean up pending handlers, which is required for the + threads to exit gracefully. + """ + # In normal operation we presently kill the process because there is + # not yet any way to cancel connect(). + self.pool.stop(join=self.profiling) + + def on_broker_exit(self): + """ + Respond to the broker thread about to exit by sending SIGTERM to + ourself. In future this should gracefully join the pool, but TERM is + fine for now. + """ + if not self.profiling: + # In normal operation we presently kill the process because there is + # not yet any way to cancel connect(). When profiling, threads + # including the broker must shut down gracefully, otherwise pstats + # won't be written. + os.kill(os.getpid(), signal.SIGTERM) diff --git a/mitogen-0.2.7/ansible_mitogen/process.pyc b/mitogen-0.2.7/ansible_mitogen/process.pyc new file mode 100644 index 000000000..4fde91e Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/process.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/runner.py b/mitogen-0.2.7/ansible_mitogen/runner.py new file mode 100644 index 000000000..30c36be --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/runner.py @@ -0,0 +1,928 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +""" +These classes implement execution for each style of Ansible module. They are +instantiated in the target context by way of target.py::run_module(). + +Each class in here has a corresponding Planner class in planners.py that knows +how to build arguments for it, preseed related data, etc. +""" + +import atexit +import codecs +import imp +import os +import re +import shlex +import shutil +import sys +import tempfile +import traceback +import types + +import mitogen.core +import ansible_mitogen.target # TODO: circular import +from mitogen.core import b +from mitogen.core import bytes_partition +from mitogen.core import str_partition +from mitogen.core import str_rpartition +from mitogen.core import to_text + +try: + import ctypes +except ImportError: + # Python 2.4 + ctypes = None + +try: + import json +except ImportError: + # Python 2.4 + import simplejson as json + +try: + # Cannot use cStringIO as it does not support Unicode. + from StringIO import StringIO +except ImportError: + from io import StringIO + +try: + from shlex import quote as shlex_quote +except ImportError: + from pipes import quote as shlex_quote + +# Absolute imports for <2.5. +logging = __import__('logging') + + +# Prevent accidental import of an Ansible module from hanging on stdin read. +import ansible.module_utils.basic +ansible.module_utils.basic._ANSIBLE_ARGS = '{}' + +# For tasks that modify /etc/resolv.conf, non-Debian derivative glibcs cache +# resolv.conf at startup and never implicitly reload it. Cope with that via an +# explicit call to res_init() on each task invocation. BSD-alikes export it +# directly, Linux #defines it as "__res_init". +libc__res_init = None +if ctypes: + libc = ctypes.CDLL(None) + for symbol in 'res_init', '__res_init': + try: + libc__res_init = getattr(libc, symbol) + except AttributeError: + pass + +iteritems = getattr(dict, 'iteritems', dict.items) +LOG = logging.getLogger(__name__) + + +if mitogen.core.PY3: + shlex_split = shlex.split +else: + def shlex_split(s, comments=False): + return [mitogen.core.to_text(token) + for token in shlex.split(str(s), comments=comments)] + + +class EnvironmentFileWatcher(object): + """ + Usually Ansible edits to /etc/environment and ~/.pam_environment are + reflected in subsequent tasks if become:true or SSH multiplexing is + disabled, due to sudo and/or SSH reinvoking pam_env. Rather than emulate + existing semantics, do our best to ensure edits are always reflected. + + This can't perfectly replicate the existing behaviour, but it can safely + update and remove keys that appear to originate in `path`, and that do not + conflict with any existing environment key inherited from elsewhere. + + A more robust future approach may simply be to arrange for the persistent + interpreter to restart when a change is detected. + """ + def __init__(self, path): + self.path = os.path.expanduser(path) + #: Inode data at time of last check. + self._st = self._stat() + #: List of inherited keys appearing to originated from this file. + self._keys = [key for key, value in self._load() + if value == os.environ.get(key)] + LOG.debug('%r installed; existing keys: %r', self, self._keys) + + def __repr__(self): + return 'EnvironmentFileWatcher(%r)' % (self.path,) + + def _stat(self): + try: + return os.stat(self.path) + except OSError: + return None + + def _load(self): + try: + fp = codecs.open(self.path, 'r', encoding='utf-8') + try: + return list(self._parse(fp)) + finally: + fp.close() + except IOError: + return [] + + def _parse(self, fp): + """ + linux-pam-1.3.1/modules/pam_env/pam_env.c#L207 + """ + for line in fp: + # ' #export foo=some var ' -> ['#export', 'foo=some var '] + bits = shlex_split(line, comments=True) + if (not bits) or bits[0].startswith('#'): + continue + + if bits[0] == u'export': + bits.pop(0) + + key, sep, value = str_partition(u' '.join(bits), u'=') + if key and sep: + yield key, value + + def _on_file_changed(self): + LOG.debug('%r: file changed, reloading', self) + for key, value in self._load(): + if key in os.environ: + LOG.debug('%r: existing key %r=%r exists, not setting %r', + self, key, os.environ[key], value) + else: + LOG.debug('%r: setting key %r to %r', self, key, value) + self._keys.append(key) + os.environ[key] = value + + def _remove_existing(self): + """ + When a change is detected, remove keys that existed in the old file. + """ + for key in self._keys: + if key in os.environ: + LOG.debug('%r: removing old key %r', self, key) + del os.environ[key] + self._keys = [] + + def check(self): + """ + Compare the :func:`os.stat` for the pam_env style environmnt file + `path` with the previous result `old_st`, which may be :data:`None` if + the previous stat attempt failed. Reload its contents if the file has + changed or appeared since last attempt. + + :returns: + New :func:`os.stat` result. The new call to :func:`reload_env` should + pass it as the value of `old_st`. + """ + st = self._stat() + if self._st == st: + return + + self._st = st + self._remove_existing() + + if st is None: + LOG.debug('%r: file has disappeared', self) + else: + self._on_file_changed() + +_pam_env_watcher = EnvironmentFileWatcher('~/.pam_environment') +_etc_env_watcher = EnvironmentFileWatcher('/etc/environment') + + +def utf8(s): + """ + Coerce an object to bytes if it is Unicode. + """ + if isinstance(s, mitogen.core.UnicodeType): + s = s.encode('utf-8') + return s + + +def reopen_readonly(fp): + """ + Replace the file descriptor belonging to the file object `fp` with one + open on the same file (`fp.name`), but opened with :py:data:`os.O_RDONLY`. + This enables temporary files to be executed on Linux, which usually throws + ``ETXTBUSY`` if any writeable handle exists pointing to a file passed to + `execve()`. + """ + fd = os.open(fp.name, os.O_RDONLY) + os.dup2(fd, fp.fileno()) + os.close(fd) + + +class Runner(object): + """ + Ansible module runner. After instantiation (with kwargs supplied by the + corresponding Planner), `.run()` is invoked, upon which `setup()`, + `_run()`, and `revert()` are invoked, with the return value of `_run()` + returned by `run()`. + + Subclasses may override `_run`()` and extend `setup()` and `revert()`. + + :param str module: + Name of the module to execute, e.g. "shell" + :param mitogen.core.Context service_context: + Context to which we should direct FileService calls. For now, always + the connection multiplexer process on the controller. + :param str json_args: + Ansible module arguments. A mixture of user and internal keys created + by :meth:`ansible.plugins.action.ActionBase._execute_module`. + + This is passed as a string rather than a dict in order to mimic the + implicit bytes/str conversion behaviour of a 2.x controller running + against a 3.x target. + :param str good_temp_dir: + The writeable temporary directory for this user account reported by + :func:`ansible_mitogen.target.init_child` passed via the controller. + This is specified explicitly to remain compatible with Ansible<2.5, and + for forked tasks where init_child never runs. + :param dict env: + Additional environment variables to set during the run. Keys with + :data:`None` are unset if present. + :param str cwd: + If not :data:`None`, change to this directory before executing. + :param mitogen.core.ExternalContext econtext: + When `detach` is :data:`True`, a reference to the ExternalContext the + runner is executing in. + :param bool detach: + When :data:`True`, indicate the runner should detach the context from + its parent after setup has completed successfully. + """ + def __init__(self, module, service_context, json_args, good_temp_dir, + extra_env=None, cwd=None, env=None, econtext=None, + detach=False): + self.module = module + self.service_context = service_context + self.econtext = econtext + self.detach = detach + self.args = json.loads(mitogen.core.to_text(json_args)) + self.good_temp_dir = good_temp_dir + self.extra_env = extra_env + self.env = env + self.cwd = cwd + #: If not :data:`None`, :meth:`get_temp_dir` had to create a temporary + #: directory for this run, because we're in an asynchronous task, or + #: because the originating action did not create a directory. + self._temp_dir = None + + def get_temp_dir(self): + path = self.args.get('_ansible_tmpdir') + if path is not None: + return path + + if self._temp_dir is None: + self._temp_dir = tempfile.mkdtemp( + prefix='ansible_mitogen_runner_', + dir=self.good_temp_dir, + ) + + return self._temp_dir + + def revert_temp_dir(self): + if self._temp_dir is not None: + ansible_mitogen.target.prune_tree(self._temp_dir) + self._temp_dir = None + + def setup(self): + """ + Prepare for running a module, including fetching necessary dependencies + from the parent, as :meth:`run` may detach prior to beginning + execution. The base implementation simply prepares the environment. + """ + self._setup_cwd() + self._setup_environ() + + def _setup_cwd(self): + """ + For situations like sudo to a non-privileged account, CWD could be + $HOME of the old account, which could have mode go=, which means it is + impossible to restore the old directory, so don't even try. + """ + if self.cwd: + os.chdir(self.cwd) + + def _setup_environ(self): + """ + Apply changes from /etc/environment files before creating a + TemporaryEnvironment to snapshot environment state prior to module run. + """ + _pam_env_watcher.check() + _etc_env_watcher.check() + env = dict(self.extra_env or {}) + if self.env: + env.update(self.env) + self._env = TemporaryEnvironment(env) + + def revert(self): + """ + Revert any changes made to the process after running a module. The base + implementation simply restores the original environment. + """ + self._env.revert() + self.revert_temp_dir() + + def _run(self): + """ + The _run() method is expected to return a dictionary in the form of + ActionBase._low_level_execute_command() output, i.e. having:: + + { + "rc": int, + "stdout": "stdout data", + "stderr": "stderr data" + } + """ + raise NotImplementedError() + + def run(self): + """ + Set up the process environment in preparation for running an Ansible + module. This monkey-patches the Ansible libraries in various places to + prevent it from trying to kill the process on completion, and to + prevent it from reading sys.stdin. + + :returns: + Module result dictionary. + """ + self.setup() + if self.detach: + self.econtext.detach() + + try: + return self._run() + finally: + self.revert() + + +class AtExitWrapper(object): + """ + issue #397, #454: Newer Ansibles use :func:`atexit.register` to trigger + tmpdir cleanup when AnsibleModule.tmpdir is responsible for creating its + own temporary directory, however with Mitogen processes are preserved + across tasks, meaning cleanup must happen earlier. + + Patch :func:`atexit.register`, catching :func:`shutil.rmtree` calls so they + can be executed on task completion, rather than on process shutdown. + """ + # Wrapped in a dict to avoid instance method decoration. + original = { + 'register': atexit.register + } + + def __init__(self): + assert atexit.register == self.original['register'], \ + "AtExitWrapper installed twice." + atexit.register = self._atexit__register + self.deferred = [] + + def revert(self): + """ + Restore the original :func:`atexit.register`. + """ + assert atexit.register == self._atexit__register, \ + "AtExitWrapper not installed." + atexit.register = self.original['register'] + + def run_callbacks(self): + while self.deferred: + func, targs, kwargs = self.deferred.pop() + try: + func(*targs, **kwargs) + except Exception: + LOG.exception('While running atexit callbacks') + + def _atexit__register(self, func, *targs, **kwargs): + """ + Intercept :func:`atexit.register` calls, diverting any to + :func:`shutil.rmtree` into a private list. + """ + if func == shutil.rmtree: + self.deferred.append((func, targs, kwargs)) + return + + self.original['register'](func, *targs, **kwargs) + + +class ModuleUtilsImporter(object): + """ + :param list module_utils: + List of `(fullname, path, is_pkg)` tuples. + """ + def __init__(self, context, module_utils): + self._context = context + self._by_fullname = dict( + (fullname, (path, is_pkg)) + for fullname, path, is_pkg in module_utils + ) + self._loaded = set() + sys.meta_path.insert(0, self) + + def revert(self): + sys.meta_path.remove(self) + for fullname in self._loaded: + sys.modules.pop(fullname, None) + + def find_module(self, fullname, path=None): + if fullname in self._by_fullname: + return self + + def load_module(self, fullname): + path, is_pkg = self._by_fullname[fullname] + source = ansible_mitogen.target.get_small_file(self._context, path) + code = compile(source, path, 'exec', 0, 1) + mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) + mod.__file__ = "master:%s" % (path,) + mod.__loader__ = self + if is_pkg: + mod.__path__ = [] + mod.__package__ = str(fullname) + else: + mod.__package__ = str(str_rpartition(to_text(fullname), '.')[0]) + exec(code, mod.__dict__) + self._loaded.add(fullname) + return mod + + +class TemporaryEnvironment(object): + """ + Apply environment changes from `env` until :meth:`revert` is called. Values + in the dict may be :data:`None` to indicate the relevant key should be + deleted. + """ + def __init__(self, env=None): + self.original = dict(os.environ) + self.env = env or {} + for key, value in iteritems(self.env): + key = mitogen.core.to_text(key) + value = mitogen.core.to_text(value) + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = str(value) + + def revert(self): + """ + Revert changes made by the module to the process environment. This must + always run, as some modules (e.g. git.py) set variables like GIT_SSH + that must be cleared out between runs. + """ + os.environ.clear() + os.environ.update(self.original) + + +class TemporaryArgv(object): + def __init__(self, argv): + self.original = sys.argv[:] + sys.argv[:] = map(str, argv) + + def revert(self): + sys.argv[:] = self.original + + +class NewStyleStdio(object): + """ + Patch ansible.module_utils.basic argument globals. + """ + def __init__(self, args, temp_dir): + self.temp_dir = temp_dir + self.original_stdout = sys.stdout + self.original_stderr = sys.stderr + self.original_stdin = sys.stdin + sys.stdout = StringIO() + sys.stderr = StringIO() + encoded = json.dumps({'ANSIBLE_MODULE_ARGS': args}) + ansible.module_utils.basic._ANSIBLE_ARGS = utf8(encoded) + sys.stdin = StringIO(mitogen.core.to_text(encoded)) + + self.original_get_path = getattr(ansible.module_utils.basic, + 'get_module_path', None) + ansible.module_utils.basic.get_module_path = self._get_path + + def _get_path(self): + return self.temp_dir + + def revert(self): + ansible.module_utils.basic.get_module_path = self.original_get_path + sys.stdout = self.original_stdout + sys.stderr = self.original_stderr + sys.stdin = self.original_stdin + ansible.module_utils.basic._ANSIBLE_ARGS = '{}' + + +class ProgramRunner(Runner): + """ + Base class for runners that run external programs. + + :param str path: + Absolute path to the program file on the master, as it can be retrieved + via :class:`mitogen.service.FileService`. + :param bool emulate_tty: + If :data:`True`, execute the program with `stdout` and `stderr` merged + into a single pipe, emulating Ansible behaviour when an SSH TTY is in + use. + """ + def __init__(self, path, emulate_tty=None, **kwargs): + super(ProgramRunner, self).__init__(**kwargs) + self.emulate_tty = emulate_tty + self.path = path + + def setup(self): + super(ProgramRunner, self).setup() + self._setup_program() + + def _get_program_filename(self): + """ + Return the filename used for program on disk. Ansible uses the original + filename for non-Ansiballz runs, and "ansible_module_+filename for + Ansiballz runs. + """ + return os.path.basename(self.path) + + program_fp = None + + def _setup_program(self): + """ + Create a temporary file containing the program code. The code is + fetched via :meth:`_get_program`. + """ + filename = self._get_program_filename() + path = os.path.join(self.get_temp_dir(), filename) + self.program_fp = open(path, 'wb') + self.program_fp.write(self._get_program()) + self.program_fp.flush() + os.chmod(self.program_fp.name, int('0700', 8)) + reopen_readonly(self.program_fp) + + def _get_program(self): + """ + Fetch the module binary from the master if necessary. + """ + return ansible_mitogen.target.get_small_file( + context=self.service_context, + path=self.path, + ) + + def _get_program_args(self): + """ + Return any arguments to pass to the program. + """ + return [] + + def revert(self): + """ + Delete the temporary program file. + """ + if self.program_fp: + self.program_fp.close() + super(ProgramRunner, self).revert() + + def _get_argv(self): + """ + Return the final argument vector used to execute the program. + """ + return [ + self.args.get('_ansible_shell_executable', '/bin/sh'), + '-c', + self._get_shell_fragment(), + ] + + def _get_shell_fragment(self): + return "%s %s" % ( + shlex_quote(self.program_fp.name), + ' '.join(map(shlex_quote, self._get_program_args())), + ) + + def _run(self): + try: + rc, stdout, stderr = ansible_mitogen.target.exec_args( + args=self._get_argv(), + emulate_tty=self.emulate_tty, + ) + except Exception: + LOG.exception('While running %s', self._get_argv()) + e = sys.exc_info()[1] + return { + u'rc': 1, + u'stdout': u'', + u'stderr': u'%s: %s' % (type(e), e), + } + + return { + u'rc': rc, + u'stdout': mitogen.core.to_text(stdout), + u'stderr': mitogen.core.to_text(stderr), + } + + +class ArgsFileRunner(Runner): + def setup(self): + super(ArgsFileRunner, self).setup() + self._setup_args() + + def _setup_args(self): + """ + Create a temporary file containing the module's arguments. The + arguments are formatted via :meth:`_get_args`. + """ + self.args_fp = tempfile.NamedTemporaryFile( + prefix='ansible_mitogen', + suffix='-args', + dir=self.get_temp_dir(), + ) + self.args_fp.write(utf8(self._get_args_contents())) + self.args_fp.flush() + reopen_readonly(self.program_fp) + + def _get_args_contents(self): + """ + Return the module arguments formatted as JSON. + """ + return json.dumps(self.args) + + def _get_program_args(self): + return [self.args_fp.name] + + def revert(self): + """ + Delete the temporary argument file. + """ + self.args_fp.close() + super(ArgsFileRunner, self).revert() + + +class BinaryRunner(ArgsFileRunner, ProgramRunner): + pass + + +class ScriptRunner(ProgramRunner): + def __init__(self, interpreter_fragment, is_python, **kwargs): + super(ScriptRunner, self).__init__(**kwargs) + self.interpreter_fragment = interpreter_fragment + self.is_python = is_python + + b_ENCODING_STRING = b('# -*- coding: utf-8 -*-') + + def _get_program(self): + return self._rewrite_source( + super(ScriptRunner, self)._get_program() + ) + + def _get_argv(self): + return [ + self.args.get('_ansible_shell_executable', '/bin/sh'), + '-c', + self._get_shell_fragment(), + ] + + def _get_shell_fragment(self): + """ + Scripts are eligible for having their hashbang line rewritten, and to + be executed via /bin/sh using the ansible_*_interpreter value used as a + shell fragment prefixing to the invocation. + """ + return "%s %s %s" % ( + self.interpreter_fragment, + shlex_quote(self.program_fp.name), + ' '.join(map(shlex_quote, self._get_program_args())), + ) + + def _rewrite_source(self, s): + """ + Mutate the source according to the per-task parameters. + """ + # While Ansible rewrites the #! using ansible_*_interpreter, it is + # never actually used to execute the script, instead it is a shell + # fragment consumed by shell/__init__.py::build_module_command(). + new = [b('#!') + utf8(self.interpreter_fragment)] + if self.is_python: + new.append(self.b_ENCODING_STRING) + + _, _, rest = bytes_partition(s, b('\n')) + new.append(rest) + return b('\n').join(new) + + +class NewStyleRunner(ScriptRunner): + """ + Execute a new-style Ansible module, where Module Replacer-related tricks + aren't required. + """ + #: path => new-style module bytecode. + _code_by_path = {} + + def __init__(self, module_map, **kwargs): + super(NewStyleRunner, self).__init__(**kwargs) + self.module_map = module_map + + def _setup_imports(self): + """ + Ensure the local importer and PushFileService has everything for the + Ansible module before setup() completes, but before detach() is called + in an asynchronous task. + + The master automatically streams modules towards us concurrent to the + runner invocation, however there is no public API to synchronize on the + completion of those preloads. Instead simply reuse the importer's + synchronization mechanism by importing everything the module will need + prior to detaching. + """ + for fullname, _, _ in self.module_map['custom']: + mitogen.core.import_module(fullname) + for fullname in self.module_map['builtin']: + mitogen.core.import_module(fullname) + + def _setup_excepthook(self): + """ + Starting with Ansible 2.6, some modules (file.py) install a + sys.excepthook and never clean it up. So we must preserve the original + excepthook and restore it after the run completes. + """ + self.original_excepthook = sys.excepthook + + def setup(self): + super(NewStyleRunner, self).setup() + + self._stdio = NewStyleStdio(self.args, self.get_temp_dir()) + # It is possible that not supplying the script filename will break some + # module, but this has never been a bug report. Instead act like an + # interpreter that had its script piped on stdin. + self._argv = TemporaryArgv(['']) + self._importer = ModuleUtilsImporter( + context=self.service_context, + module_utils=self.module_map['custom'], + ) + self._setup_imports() + self._setup_excepthook() + self.atexit_wrapper = AtExitWrapper() + if libc__res_init: + libc__res_init() + + def _revert_excepthook(self): + sys.excepthook = self.original_excepthook + + def revert(self): + self.atexit_wrapper.revert() + self._argv.revert() + self._stdio.revert() + self._revert_excepthook() + super(NewStyleRunner, self).revert() + + def _get_program_filename(self): + """ + See ProgramRunner._get_program_filename(). + """ + return 'ansible_module_' + os.path.basename(self.path) + + def _setup_args(self): + pass + + # issue #555: in old times it was considered good form to reload sys and + # change the default encoding. This hack was removed from Ansible long ago, + # but not before permeating into many third party modules. + PREHISTORIC_HACK_RE = re.compile( + b(r'reload\s*\(\s*sys\s*\)\s*' + r'sys\s*\.\s*setdefaultencoding\([^)]+\)') + ) + + def _setup_program(self): + source = ansible_mitogen.target.get_small_file( + context=self.service_context, + path=self.path, + ) + self.source = self.PREHISTORIC_HACK_RE.sub(b(''), source) + + def _get_code(self): + try: + return self._code_by_path[self.path] + except KeyError: + return self._code_by_path.setdefault(self.path, compile( + # Py2.4 doesn't support kwargs. + self.source, # source + "master:" + self.path, # filename + 'exec', # mode + 0, # flags + True, # dont_inherit + )) + + if mitogen.core.PY3: + main_module_name = '__main__' + else: + main_module_name = b('__main__') + + def _handle_magic_exception(self, mod, exc): + """ + Beginning with Ansible >2.6, some modules (file.py) install a + sys.excepthook which is a closure over AnsibleModule, redirecting the + magical exception to AnsibleModule.fail_json(). + + For extra special needs bonus points, the class is not defined in + module_utils, but is defined in the module itself, meaning there is no + type for isinstance() that outlasts the invocation. + """ + klass = getattr(mod, 'AnsibleModuleError', None) + if klass and isinstance(exc, klass): + mod.module.fail_json(**exc.results) + + def _run_code(self, code, mod): + try: + if mitogen.core.PY3: + exec(code, vars(mod)) + else: + exec('exec code in vars(mod)') + except Exception: + self._handle_magic_exception(mod, sys.exc_info()[1]) + raise + + def _run(self): + mod = types.ModuleType(self.main_module_name) + mod.__package__ = None + # Some Ansible modules use __file__ to find the Ansiballz temporary + # directory. We must provide some temporary path in __file__, but we + # don't want to pointlessly write the module to disk when it never + # actually needs to exist. So just pass the filename as it would exist. + mod.__file__ = os.path.join( + self.get_temp_dir(), + 'ansible_module_' + os.path.basename(self.path), + ) + + code = self._get_code() + rc = 2 + try: + try: + self._run_code(code, mod) + except SystemExit: + exc = sys.exc_info()[1] + rc = exc.args[0] + except Exception: + # This writes to stderr by default. + traceback.print_exc() + rc = 1 + + finally: + self.atexit_wrapper.run_callbacks() + + return { + u'rc': rc, + u'stdout': mitogen.core.to_text(sys.stdout.getvalue()), + u'stderr': mitogen.core.to_text(sys.stderr.getvalue()), + } + + +class JsonArgsRunner(ScriptRunner): + JSON_ARGS = b('<>') + + def _get_args_contents(self): + return json.dumps(self.args).encode() + + def _rewrite_source(self, s): + return ( + super(JsonArgsRunner, self)._rewrite_source(s) + .replace(self.JSON_ARGS, self._get_args_contents()) + ) + + +class WantJsonRunner(ArgsFileRunner, ScriptRunner): + pass + + +class OldStyleRunner(ArgsFileRunner, ScriptRunner): + def _get_args_contents(self): + """ + Mimic the argument formatting behaviour of + ActionBase._execute_module(). + """ + return ' '.join( + '%s=%s' % (key, shlex_quote(str(self.args[key]))) + for key in self.args + ) + ' ' # Bug-for-bug :( diff --git a/mitogen-0.2.7/ansible_mitogen/runner.pyc b/mitogen-0.2.7/ansible_mitogen/runner.pyc new file mode 100644 index 000000000..4c4ee0e Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/runner.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/services.py b/mitogen-0.2.7/ansible_mitogen/services.py new file mode 100644 index 000000000..a7c0e46 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/services.py @@ -0,0 +1,537 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +""" +Classes in this file define Mitogen 'services' that run (initially) within the +connection multiplexer process that is forked off the top-level controller +process. + +Once a worker process connects to a multiplexer process +(Connection._connect()), it communicates with these services to establish new +connections, grant access to files by children, and register for notification +when a child has completed a job. +""" + +from __future__ import absolute_import +from __future__ import unicode_literals + +import logging +import os +import os.path +import sys +import threading + +import ansible.constants + +import mitogen +import mitogen.service +import mitogen.utils +import ansible_mitogen.loaders +import ansible_mitogen.module_finder +import ansible_mitogen.target + + +LOG = logging.getLogger(__name__) + +# Force load of plugin to ensure ConfigManager has definitions loaded. Done +# during module import to ensure a single-threaded environment; PluginLoader +# is not thread-safe. +ansible_mitogen.loaders.shell_loader.get('sh') + + +if sys.version_info[0] == 3: + def reraise(tp, value, tb): + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value +else: + exec( + "def reraise(tp, value, tb=None):\n" + " raise tp, value, tb\n" + ) + + +def _get_candidate_temp_dirs(): + try: + # >=2.5 + options = ansible.constants.config.get_plugin_options('shell', 'sh') + remote_tmp = options.get('remote_tmp') or ansible.constants.DEFAULT_REMOTE_TMP + system_tmpdirs = options.get('system_tmpdirs', ('/var/tmp', '/tmp')) + except AttributeError: + # 2.3 + remote_tmp = ansible.constants.DEFAULT_REMOTE_TMP + system_tmpdirs = ('/var/tmp', '/tmp') + + return mitogen.utils.cast([remote_tmp] + list(system_tmpdirs)) + + +def key_from_dict(**kwargs): + """ + Return a unique string representation of a dict as quickly as possible. + Used to generated deduplication keys from a request. + """ + out = [] + stack = [kwargs] + while stack: + obj = stack.pop() + if isinstance(obj, dict): + stack.extend(sorted(obj.items())) + elif isinstance(obj, (list, tuple)): + stack.extend(obj) + else: + out.append(str(obj)) + return ''.join(out) + + +class Error(Exception): + pass + + +class ContextService(mitogen.service.Service): + """ + Used by workers to fetch the single Context instance corresponding to a + connection configuration, creating the matching connection if it does not + exist. + + For connection methods and their parameters, see: + https://mitogen.readthedocs.io/en/latest/api.html#context-factories + + This concentrates connections in the top-level process, which may become a + bottleneck. The bottleneck can be removed using per-CPU connection + processes and arranging for the worker to select one according to a hash of + the connection parameters (sharding). + """ + max_interpreters = int(os.getenv('MITOGEN_MAX_INTERPRETERS', '20')) + + def __init__(self, *args, **kwargs): + super(ContextService, self).__init__(*args, **kwargs) + self._lock = threading.Lock() + #: Records the :meth:`get` result dict for successful calls, returned + #: for identical subsequent calls. Keyed by :meth:`key_from_dict`. + self._response_by_key = {} + #: List of :class:`mitogen.core.Latch` awaiting the result for a + #: particular key. + self._latches_by_key = {} + #: Mapping of :class:`mitogen.core.Context` -> reference count. Each + #: call to :meth:`get` increases this by one. Calls to :meth:`put` + #: decrease it by one. + self._refs_by_context = {} + #: List of contexts in creation order by via= parameter. When + #: :attr:`max_interpreters` is reached, the most recently used context + #: is destroyed to make room for any additional context. + self._lru_by_via = {} + #: :func:`key_from_dict` result by Context. + self._key_by_context = {} + #: Mapping of Context -> parent Context + self._via_by_context = {} + + @mitogen.service.expose(mitogen.service.AllowParents()) + @mitogen.service.arg_spec({ + 'context': mitogen.core.Context + }) + def reset(self, context): + """ + Return a reference, forcing close and discard of the underlying + connection. Used for 'meta: reset_connection' or when some other error + is detected. + """ + LOG.debug('%r.reset(%r)', self, context) + self._lock.acquire() + try: + self._shutdown_unlocked(context) + finally: + self._lock.release() + + @mitogen.service.expose(mitogen.service.AllowParents()) + @mitogen.service.arg_spec({ + 'context': mitogen.core.Context + }) + def put(self, context): + """ + Return a reference, making it eligable for recycling once its reference + count reaches zero. + """ + LOG.debug('%r.put(%r)', self, context) + self._lock.acquire() + try: + if self._refs_by_context.get(context, 0) == 0: + LOG.warning('%r.put(%r): refcount was 0. shutdown_all called?', + self, context) + return + self._refs_by_context[context] -= 1 + finally: + self._lock.release() + + def _produce_response(self, key, response): + """ + Reply to every waiting request matching a configuration key with a + response dictionary, deleting the list of waiters when done. + + :param str key: + Result of :meth:`key_from_dict` + :param dict response: + Response dictionary + :returns: + Number of waiters that were replied to. + """ + self._lock.acquire() + try: + latches = self._latches_by_key.pop(key) + count = len(latches) + for latch in latches: + latch.put(response) + finally: + self._lock.release() + return count + + def _forget_context_unlocked(self, context): + key = self._key_by_context.get(context) + if key is None: + LOG.debug('%r: attempt to forget unknown %r', self, context) + return + + self._response_by_key.pop(key, None) + self._latches_by_key.pop(key, None) + self._key_by_context.pop(context, None) + self._refs_by_context.pop(context, None) + self._via_by_context.pop(context, None) + self._lru_by_via.pop(context, None) + + def _shutdown_unlocked(self, context, lru=None, new_context=None): + """ + Arrange for `context` to be shut down, and optionally add `new_context` + to the LRU list while holding the lock. + """ + LOG.info('%r._shutdown_unlocked(): shutting down %r', self, context) + context.shutdown() + via = self._via_by_context.get(context) + if via: + lru = self._lru_by_via.get(via) + if lru: + if context in lru: + lru.remove(context) + if new_context: + lru.append(new_context) + self._forget_context_unlocked(context) + + def _update_lru_unlocked(self, new_context, spec, via): + """ + Update the LRU ("MRU"?) list associated with the connection described + by `kwargs`, destroying the most recently created context if the list + is full. Finally add `new_context` to the list. + """ + self._via_by_context[new_context] = via + + lru = self._lru_by_via.setdefault(via, []) + if len(lru) < self.max_interpreters: + lru.append(new_context) + return + + for context in reversed(lru): + if self._refs_by_context[context] == 0: + break + else: + LOG.warning('via=%r reached maximum number of interpreters, ' + 'but they are all marked as in-use.', via) + return + + self._shutdown_unlocked(context, lru=lru, new_context=new_context) + + def _update_lru(self, new_context, spec, via): + self._lock.acquire() + try: + self._update_lru_unlocked(new_context, spec, via) + finally: + self._lock.release() + + @mitogen.service.expose(mitogen.service.AllowParents()) + def dump(self): + """ + For testing, return a list of dicts describing every currently + connected context. + """ + return [ + { + 'context_name': context.name, + 'via': getattr(self._via_by_context.get(context), + 'name', None), + 'refs': self._refs_by_context.get(context), + } + for context, key in sorted(self._key_by_context.items(), + key=lambda c_k: c_k[0].context_id) + ] + + @mitogen.service.expose(mitogen.service.AllowParents()) + def shutdown_all(self): + """ + For testing use, arrange for all connections to be shut down. + """ + self._lock.acquire() + try: + for context in list(self._key_by_context): + self._shutdown_unlocked(context) + finally: + self._lock.release() + + def _on_context_disconnect(self, context): + """ + Respond to Context disconnect event by deleting any record of the no + longer reachable context. This method runs in the Broker thread and + must not to block. + """ + self._lock.acquire() + try: + LOG.info('%r: Forgetting %r due to stream disconnect', self, context) + self._forget_context_unlocked(context) + finally: + self._lock.release() + + ALWAYS_PRELOAD = ( + 'ansible.module_utils.basic', + 'ansible.module_utils.json_utils', + 'ansible.release', + 'ansible_mitogen.runner', + 'ansible_mitogen.target', + 'mitogen.fork', + 'mitogen.service', + ) + + def _send_module_forwards(self, context): + self.router.responder.forward_modules(context, self.ALWAYS_PRELOAD) + + _candidate_temp_dirs = None + + def _get_candidate_temp_dirs(self): + """ + Return a list of locations to try to create the single temporary + directory used by the run. This simply caches the (expensive) plugin + load of :func:`_get_candidate_temp_dirs`. + """ + if self._candidate_temp_dirs is None: + self._candidate_temp_dirs = _get_candidate_temp_dirs() + return self._candidate_temp_dirs + + def _connect(self, key, spec, via=None): + """ + Actual connect implementation. Arranges for the Mitogen connection to + be created and enqueues an asynchronous call to start the forked task + parent in the remote context. + + :param key: + Deduplication key representing the connection configuration. + :param spec: + Connection specification. + :returns: + Dict like:: + + { + 'context': mitogen.core.Context or None, + 'via': mitogen.core.Context or None, + 'init_child_result': { + 'fork_context': mitogen.core.Context, + 'home_dir': str or None, + }, + 'msg': str or None + } + + Where `context` is a reference to the newly constructed context, + `init_child_result` is the result of executing + :func:`ansible_mitogen.target.init_child` in that context, `msg` is + an error message and the remaining fields are :data:`None`, or + `msg` is :data:`None` and the remaining fields are set. + """ + try: + method = getattr(self.router, spec['method']) + except AttributeError: + raise Error('unsupported method: %(transport)s' % spec) + + context = method(via=via, unidirectional=True, **spec['kwargs']) + if via and spec.get('enable_lru'): + self._update_lru(context, spec, via) + + # Forget the context when its disconnect event fires. + mitogen.core.listen(context, 'disconnect', + lambda: self._on_context_disconnect(context)) + + self._send_module_forwards(context) + init_child_result = context.call( + ansible_mitogen.target.init_child, + log_level=LOG.getEffectiveLevel(), + candidate_temp_dirs=self._get_candidate_temp_dirs(), + ) + + if os.environ.get('MITOGEN_DUMP_THREAD_STACKS'): + from mitogen import debug + context.call(debug.dump_to_logger) + + self._key_by_context[context] = key + self._refs_by_context[context] = 0 + return { + 'context': context, + 'via': via, + 'init_child_result': init_child_result, + 'msg': None, + } + + def _wait_or_start(self, spec, via=None): + latch = mitogen.core.Latch() + key = key_from_dict(via=via, **spec) + self._lock.acquire() + try: + response = self._response_by_key.get(key) + if response is not None: + self._refs_by_context[response['context']] += 1 + latch.put(response) + return latch + + latches = self._latches_by_key.setdefault(key, []) + first = len(latches) == 0 + latches.append(latch) + finally: + self._lock.release() + + if first: + # I'm the first requestee, so I will create the connection. + try: + response = self._connect(key, spec, via=via) + count = self._produce_response(key, response) + # Only record the response for non-error results. + self._response_by_key[key] = response + # Set the reference count to the number of waiters. + self._refs_by_context[response['context']] += count + except Exception: + self._produce_response(key, sys.exc_info()) + + return latch + + disconnect_msg = ( + 'Channel was disconnected while connection attempt was in progress; ' + 'this may be caused by an abnormal Ansible exit, or due to an ' + 'unreliable target.' + ) + + @mitogen.service.expose(mitogen.service.AllowParents()) + @mitogen.service.arg_spec({ + 'stack': list + }) + def get(self, msg, stack): + """ + Return a Context referring to an established connection with the given + configuration, establishing new connections as necessary. + + :param list stack: + Connection descriptions. Each element is a dict containing 'method' + and 'kwargs' keys describing the Router method and arguments. + Subsequent elements are proxied via the previous. + + :returns dict: + * context: mitogen.parent.Context or None. + * init_child_result: Result of :func:`init_child`. + * msg: StreamError exception text or None. + * method_name: string failing method name. + """ + via = None + for spec in stack: + try: + result = self._wait_or_start(spec, via=via).get() + if isinstance(result, tuple): # exc_info() + reraise(*result) + via = result['context'] + except mitogen.core.ChannelError: + return { + 'context': None, + 'init_child_result': None, + 'method_name': spec['method'], + 'msg': self.disconnect_msg, + } + except mitogen.core.StreamError as e: + return { + 'context': None, + 'init_child_result': None, + 'method_name': spec['method'], + 'msg': str(e), + } + + return result + + +class ModuleDepService(mitogen.service.Service): + """ + Scan a new-style module and produce a cached mapping of module_utils names + to their resolved filesystem paths. + """ + invoker_class = mitogen.service.SerializedInvoker + + def __init__(self, *args, **kwargs): + super(ModuleDepService, self).__init__(*args, **kwargs) + self._cache = {} + + def _get_builtin_names(self, builtin_path, resolved): + return [ + mitogen.core.to_text(fullname) + for fullname, path, is_pkg in resolved + if os.path.abspath(path).startswith(builtin_path) + ] + + def _get_custom_tups(self, builtin_path, resolved): + return [ + (mitogen.core.to_text(fullname), + mitogen.core.to_text(path), + is_pkg) + for fullname, path, is_pkg in resolved + if not os.path.abspath(path).startswith(builtin_path) + ] + + @mitogen.service.expose(policy=mitogen.service.AllowParents()) + @mitogen.service.arg_spec({ + 'module_name': mitogen.core.UnicodeType, + 'module_path': mitogen.core.FsPathTypes, + 'search_path': tuple, + 'builtin_path': mitogen.core.FsPathTypes, + 'context': mitogen.core.Context, + }) + def scan(self, module_name, module_path, search_path, builtin_path, context): + key = (module_name, search_path) + if key not in self._cache: + resolved = ansible_mitogen.module_finder.scan( + module_name=module_name, + module_path=module_path, + search_path=tuple(search_path) + (builtin_path,), + ) + builtin_path = os.path.abspath(builtin_path) + builtin = self._get_builtin_names(builtin_path, resolved) + custom = self._get_custom_tups(builtin_path, resolved) + self._cache[key] = { + 'builtin': builtin, + 'custom': custom, + } + return self._cache[key] diff --git a/mitogen-0.2.7/ansible_mitogen/services.pyc b/mitogen-0.2.7/ansible_mitogen/services.pyc new file mode 100644 index 000000000..7669a6b Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/services.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/strategy.py b/mitogen-0.2.7/ansible_mitogen/strategy.py new file mode 100644 index 000000000..b9211fc --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/strategy.py @@ -0,0 +1,296 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +import os +import signal +import threading + +import mitogen.core +import ansible_mitogen.affinity +import ansible_mitogen.loaders +import ansible_mitogen.mixins +import ansible_mitogen.process + +import ansible +import ansible.executor.process.worker + + +ANSIBLE_VERSION_MIN = '2.3' +ANSIBLE_VERSION_MAX = '2.7' +NEW_VERSION_MSG = ( + "Your Ansible version (%s) is too recent. The most recent version\n" + "supported by Mitogen for Ansible is %s.x. Please check the Mitogen\n" + "release notes to see if a new version is available, otherwise\n" + "subscribe to the corresponding GitHub issue to be notified when\n" + "support becomes available.\n" + "\n" + " https://mitogen.rtfd.io/en/latest/changelog.html\n" + " https://github.com/dw/mitogen/issues/\n" +) +OLD_VERSION_MSG = ( + "Your version of Ansible (%s) is too old. The oldest version supported by " + "Mitogen for Ansible is %s." +) + + +def _assert_supported_release(): + """ + Throw AnsibleError with a descriptive message in case of being loaded into + an unsupported Ansible release. + """ + v = ansible.__version__ + + if v[:len(ANSIBLE_VERSION_MIN)] < ANSIBLE_VERSION_MIN: + raise ansible.errors.AnsibleError( + OLD_VERSION_MSG % (v, ANSIBLE_VERSION_MIN) + ) + + if v[:len(ANSIBLE_VERSION_MAX)] > ANSIBLE_VERSION_MAX: + raise ansible.errors.AnsibleError( + NEW_VERSION_MSG % (ansible.__version__, ANSIBLE_VERSION_MAX) + ) + + +def _patch_awx_callback(): + """ + issue #400: AWX loads a display callback that suffers from thread-safety + issues. Detect the presence of older AWX versions and patch the bug. + """ + # AWX uses sitecustomize.py to force-load this package. If it exists, we're + # running under AWX. + try: + from awx_display_callback.events import EventContext + from awx_display_callback.events import event_context + except ImportError: + return + + if hasattr(EventContext(), '_local'): + # Patched version. + return + + def patch_add_local(self, **kwargs): + tls = vars(self._local) + ctx = tls.setdefault('_ctx', {}) + ctx.update(kwargs) + + EventContext._local = threading.local() + EventContext.add_local = patch_add_local + +_patch_awx_callback() + + +def wrap_action_loader__get(name, *args, **kwargs): + """ + While the mitogen strategy is active, trap action_loader.get() calls, + augmenting any fetched class with ActionModuleMixin, which replaces various + helper methods inherited from ActionBase with implementations that avoid + the use of shell fragments wherever possible. + + This is used instead of static subclassing as it generalizes to third party + action modules outside the Ansible tree. + """ + klass = action_loader__get(name, class_only=True) + if klass: + bases = (ansible_mitogen.mixins.ActionModuleMixin, klass) + adorned_klass = type(str(name), bases, {}) + if kwargs.get('class_only'): + return adorned_klass + return adorned_klass(*args, **kwargs) + + +def wrap_connection_loader__get(name, *args, **kwargs): + """ + While the strategy is active, rewrite connection_loader.get() calls for + some transports into requests for a compatible Mitogen transport. + """ + if name in ('docker', 'kubectl', 'jail', 'local', 'lxc', + 'lxd', 'machinectl', 'setns', 'ssh'): + name = 'mitogen_' + name + return connection_loader__get(name, *args, **kwargs) + + +def wrap_worker__run(*args, **kwargs): + """ + While the strategy is active, rewrite connection_loader.get() calls for + some transports into requests for a compatible Mitogen transport. + """ + # Ignore parent's attempts to murder us when we still need to write + # profiling output. + if mitogen.core._profile_hook.__name__ != '_profile_hook': + signal.signal(signal.SIGTERM, signal.SIG_IGN) + + ansible_mitogen.logging.set_process_name('task') + ansible_mitogen.affinity.policy.assign_worker() + return mitogen.core._profile_hook('WorkerProcess', + lambda: worker__run(*args, **kwargs) + ) + + +class StrategyMixin(object): + """ + This mix-in enhances any built-in strategy by arranging for various Mitogen + services to be initialized in the Ansible top-level process, and for worker + processes to grow support for using those top-level services to communicate + with and execute modules on remote hosts. + + Mitogen: + + A private Broker IO multiplexer thread is created to dispatch IO + between the local Router and any connected streams, including streams + connected to Ansible WorkerProcesses, and SSH commands implementing + connections to remote machines. + + A Router is created that implements message dispatch to any locally + registered handlers, and message routing for remote streams. Router is + the junction point through which WorkerProceses and remote SSH contexts + can communicate. + + Router additionally adds message handlers for a variety of base + services, review the Standard Handles section of the How It Works guide + in the documentation. + + A ContextService is installed as a message handler in the master + process and run on a private thread. It is responsible for accepting + requests to establish new SSH connections from worker processes, and + ensuring precisely one connection exists and is reused for subsequent + playbook steps. The service presently runs in a single thread, so to + begin with, new SSH connections are serialized. + + Finally a mitogen.unix listener is created through which WorkerProcess + can establish a connection back into the master process, in order to + avail of ContextService. A UNIX listener socket is necessary as there + is no more sane mechanism to arrange for IPC between the Router in the + master process, and the corresponding Router in the worker process. + + Ansible: + + PluginLoader monkey patches are installed to catch attempts to create + connection and action plug-ins. + + For connection plug-ins, if the desired method is "local" or "ssh", it + is redirected to the "mitogen" connection plug-in. That plug-in + implements communication via a UNIX socket connection to the top-level + Ansible process, and uses ContextService running in the top-level + process to actually establish and manage the connection. + + For action plug-ins, the original class is looked up as usual, but a + new subclass is created dynamically in order to mix-in + ansible_mitogen.target.ActionModuleMixin, which overrides many of the + methods usually inherited from ActionBase in order to replace them with + pure-Python equivalents that avoid the use of shell. + + In particular, _execute_module() is overridden with an implementation + that uses ansible_mitogen.target.run_module() executed in the target + Context. run_module() implements module execution by importing the + module as if it were a normal Python module, and capturing its output + in the remote process. Since the Mitogen module loader is active in the + remote process, all the heavy lifting of transferring the action module + and its dependencies are automatically handled by Mitogen. + """ + def _install_wrappers(self): + """ + Install our PluginLoader monkey patches and update global variables + with references to the real functions. + """ + global action_loader__get + action_loader__get = ansible_mitogen.loaders.action_loader.get + ansible_mitogen.loaders.action_loader.get = wrap_action_loader__get + + global connection_loader__get + connection_loader__get = ansible_mitogen.loaders.connection_loader.get + ansible_mitogen.loaders.connection_loader.get = wrap_connection_loader__get + + global worker__run + worker__run = ansible.executor.process.worker.WorkerProcess.run + ansible.executor.process.worker.WorkerProcess.run = wrap_worker__run + + def _remove_wrappers(self): + """ + Uninstall the PluginLoader monkey patches. + """ + ansible_mitogen.loaders.action_loader.get = action_loader__get + ansible_mitogen.loaders.connection_loader.get = connection_loader__get + ansible.executor.process.worker.WorkerProcess.run = worker__run + + def _add_plugin_paths(self): + """ + Add the Mitogen plug-in directories to the ModuleLoader path, avoiding + the need for manual configuration. + """ + base_dir = os.path.join(os.path.dirname(__file__), 'plugins') + ansible_mitogen.loaders.connection_loader.add_directory( + os.path.join(base_dir, 'connection') + ) + ansible_mitogen.loaders.action_loader.add_directory( + os.path.join(base_dir, 'action') + ) + + def _queue_task(self, host, task, task_vars, play_context): + """ + Many PluginLoader caches are defective as they are only populated in + the ephemeral WorkerProcess. Touch each plug-in path before forking to + ensure all workers receive a hot cache. + """ + ansible_mitogen.loaders.module_loader.find_plugin( + name=task.action, + mod_type='', + ) + ansible_mitogen.loaders.connection_loader.get( + name=play_context.connection, + class_only=True, + ) + ansible_mitogen.loaders.action_loader.get( + name=task.action, + class_only=True, + ) + + return super(StrategyMixin, self)._queue_task( + host=host, + task=task, + task_vars=task_vars, + play_context=play_context, + ) + + def run(self, iterator, play_context, result=0): + """ + Arrange for a mitogen.master.Router to be available for the duration of + the strategy's real run() method. + """ + _assert_supported_release() + + ansible_mitogen.process.MuxProcess.start() + run = super(StrategyMixin, self).run + self._add_plugin_paths() + self._install_wrappers() + try: + return mitogen.core._profile_hook('Strategy', + lambda: run(iterator, play_context) + ) + finally: + self._remove_wrappers() diff --git a/mitogen-0.2.7/ansible_mitogen/strategy.pyc b/mitogen-0.2.7/ansible_mitogen/strategy.pyc new file mode 100644 index 000000000..0453cc0 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/strategy.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/target.py b/mitogen-0.2.7/ansible_mitogen/target.py new file mode 100644 index 000000000..652b5ad --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/target.py @@ -0,0 +1,777 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +""" +Helper functions intended to be executed on the target. These are entrypoints +for file transfer, module execution and sundry bits like changing file modes. +""" + +import errno +import grp +import operator +import os +import pwd +import re +import signal +import stat +import subprocess +import sys +import tempfile +import traceback +import types + +# Absolute imports for <2.5. +logging = __import__('logging') + +import mitogen.core +import mitogen.fork +import mitogen.parent +import mitogen.service +from mitogen.core import b + +try: + import json +except ImportError: + import simplejson as json + +try: + reduce +except NameError: + # Python 3.x. + from functools import reduce + +try: + BaseException +except NameError: + # Python 2.4 + BaseException = Exception + + +# Ansible since PR #41749 inserts "import __main__" into +# ansible.module_utils.basic. Mitogen's importer will refuse such an import, so +# we must setup a fake "__main__" before that module is ever imported. The +# str() is to cast Unicode to bytes on Python 2.6. +if not sys.modules.get(str('__main__')): + sys.modules[str('__main__')] = types.ModuleType(str('__main__')) + +import ansible.module_utils.json_utils +import ansible_mitogen.runner + + +LOG = logging.getLogger(__name__) + +MAKE_TEMP_FAILED_MSG = ( + u"Unable to find a useable temporary directory. This likely means no\n" + u"system-supplied TMP directory can be written to, or all directories\n" + u"were mounted on 'noexec' filesystems.\n" + u"\n" + u"The following paths were tried:\n" + u" %(paths)s\n" + u"\n" + u"Please check '-vvv' output for a log of individual path errors." +) + +# Python 2.4/2.5 cannot support fork+threads whatsoever, it doesn't even fix up +# interpreter state. So 2.4/2.5 interpreters start .local() contexts for +# isolation instead. Since we don't have any crazy memory sharing problems to +# avoid, there is no virginal fork parent either. The child is started directly +# from the login/become process. In future this will be default everywhere, +# fork is brainwrong from the stone age. +FORK_SUPPORTED = sys.version_info >= (2, 6) + +#: Initialized to an econtext.parent.Context pointing at a pristine fork of +#: the target Python interpreter before it executes any code or imports. +_fork_parent = None + +#: Set by :func:`init_child` to the name of a writeable and executable +#: temporary directory accessible by the active user account. +good_temp_dir = None + + +def subprocess__Popen__close_fds(self, but): + """ + issue #362, #435: subprocess.Popen(close_fds=True) aka. + AnsibleModule.run_command() loops the entire FD space on Python<3.2. + CentOS>5 ships with 1,048,576 FDs by default, resulting in huge (>500ms) + latency starting children. Therefore replace Popen._close_fds on Linux with + a version that is O(fds) rather than O(_SC_OPEN_MAX). + """ + try: + names = os.listdir(u'/proc/self/fd') + except OSError: + # May fail if acting on a container that does not have /proc mounted. + self._original_close_fds(but) + return + + for name in names: + if not name.isdigit(): + continue + + fd = int(name, 10) + if fd > 2 and fd != but: + try: + os.close(fd) + except OSError: + pass + + +if ( + sys.platform.startswith(u'linux') and + sys.version < u'3.0' and + hasattr(subprocess.Popen, u'_close_fds') and + not mitogen.is_master +): + subprocess.Popen._original_close_fds = subprocess.Popen._close_fds + subprocess.Popen._close_fds = subprocess__Popen__close_fds + + +def get_small_file(context, path): + """ + Basic in-memory caching module fetcher. This generates one roundtrip for + every previously unseen file, so it is only a temporary solution. + + :param context: + Context we should direct FileService requests to. For now (and probably + forever) this is just the top-level Mitogen connection manager process. + :param path: + Path to fetch from FileService, must previously have been registered by + a privileged context using the `register` command. + :returns: + Bytestring file data. + """ + pool = mitogen.service.get_or_create_pool(router=context.router) + service = pool.get_service(u'mitogen.service.PushFileService') + return service.get(path) + + +def transfer_file(context, in_path, out_path, sync=False, set_owner=False): + """ + Streamily download a file from the connection multiplexer process in the + controller. + + :param mitogen.core.Context context: + Reference to the context hosting the FileService that will transmit the + file. + :param bytes in_path: + FileService registered name of the input file. + :param bytes out_path: + Name of the output path on the local disk. + :param bool sync: + If :data:`True`, ensure the file content and metadat are fully on disk + before renaming the temporary file over the existing file. This should + ensure in the case of system crash, either the entire old or new file + are visible post-reboot. + :param bool set_owner: + If :data:`True`, look up the metadata username and group on the local + system and file the file owner using :func:`os.fchmod`. + """ + out_path = os.path.abspath(out_path) + fd, tmp_path = tempfile.mkstemp(suffix='.tmp', + prefix='.ansible_mitogen_transfer-', + dir=os.path.dirname(out_path)) + fp = os.fdopen(fd, 'wb', mitogen.core.CHUNK_SIZE) + LOG.debug('transfer_file(%r) temporary file: %s', out_path, tmp_path) + + try: + try: + ok, metadata = mitogen.service.FileService.get( + context=context, + path=in_path, + out_fp=fp, + ) + if not ok: + raise IOError('transfer of %r was interrupted.' % (in_path,)) + + set_file_mode(tmp_path, metadata['mode'], fd=fp.fileno()) + if set_owner: + set_file_owner(tmp_path, metadata['owner'], metadata['group'], + fd=fp.fileno()) + finally: + fp.close() + + if sync: + os.fsync(fp.fileno()) + os.rename(tmp_path, out_path) + except BaseException: + os.unlink(tmp_path) + raise + + os.utime(out_path, (metadata['atime'], metadata['mtime'])) + + +def prune_tree(path): + """ + Like shutil.rmtree(), but log errors rather than discard them, and do not + waste multiple os.stat() calls discovering whether the object can be + deleted, just try deleting it instead. + """ + try: + os.unlink(path) + return + except OSError: + e = sys.exc_info()[1] + if not (os.path.isdir(path) and + e.args[0] in (errno.EPERM, errno.EISDIR)): + LOG.error('prune_tree(%r): %s', path, e) + return + + try: + # Ensure write access for readonly directories. Ignore error in case + # path is on a weird filesystem (e.g. vfat). + os.chmod(path, int('0700', 8)) + except OSError: + e = sys.exc_info()[1] + LOG.warning('prune_tree(%r): %s', path, e) + + try: + for name in os.listdir(path): + if name not in ('.', '..'): + prune_tree(os.path.join(path, name)) + os.rmdir(path) + except OSError: + e = sys.exc_info()[1] + LOG.error('prune_tree(%r): %s', path, e) + + +def is_good_temp_dir(path): + """ + Return :data:`True` if `path` can be used as a temporary directory, logging + any failures that may cause it to be unsuitable. If the directory doesn't + exist, we attempt to create it using :func:`os.makedirs`. + """ + if not os.path.exists(path): + try: + os.makedirs(path, mode=int('0700', 8)) + except OSError: + e = sys.exc_info()[1] + LOG.debug('temp dir %r unusable: did not exist and attempting ' + 'to create it failed: %s', path, e) + return False + + try: + tmp = tempfile.NamedTemporaryFile( + prefix='ansible_mitogen_is_good_temp_dir', + dir=path, + ) + except (OSError, IOError): + e = sys.exc_info()[1] + LOG.debug('temp dir %r unusable: %s', path, e) + return False + + try: + try: + os.chmod(tmp.name, int('0700', 8)) + except OSError: + e = sys.exc_info()[1] + LOG.debug('temp dir %r unusable: chmod failed: %s', path, e) + return False + + try: + # access(.., X_OK) is sufficient to detect noexec. + if not os.access(tmp.name, os.X_OK): + raise OSError('filesystem appears to be mounted noexec') + except OSError: + e = sys.exc_info()[1] + LOG.debug('temp dir %r unusable: %s', path, e) + return False + finally: + tmp.close() + + return True + + +def find_good_temp_dir(candidate_temp_dirs): + """ + Given a list of candidate temp directories extracted from ``ansible.cfg``, + combine it with the Python-builtin list of candidate directories used by + :mod:`tempfile`, then iteratively try each until one is found that is both + writeable and executable. + + :param list candidate_temp_dirs: + List of candidate $variable-expanded and tilde-expanded directory paths + that may be usable as a temporary directory. + """ + paths = [os.path.expandvars(os.path.expanduser(p)) + for p in candidate_temp_dirs] + paths.extend(tempfile._candidate_tempdir_list()) + + for path in paths: + if is_good_temp_dir(path): + LOG.debug('Selected temp directory: %r (from %r)', path, paths) + return path + + raise IOError(MAKE_TEMP_FAILED_MSG % { + 'paths': '\n '.join(paths), + }) + + +@mitogen.core.takes_econtext +def init_child(econtext, log_level, candidate_temp_dirs): + """ + Called by ContextService immediately after connection; arranges for the + (presently) spotless Python interpreter to be forked, where the newly + forked interpreter becomes the parent of any newly forked future + interpreters. + + This is necessary to prevent modules that are executed in-process from + polluting the global interpreter state in a way that effects explicitly + isolated modules. + + :param int log_level: + Logging package level active in the master. + :param list[str] candidate_temp_dirs: + List of $variable-expanded and tilde-expanded directory names to add to + candidate list of temporary directories. + + :returns: + Dict like:: + + { + 'fork_context': mitogen.core.Context or None, + 'good_temp_dir': ... + 'home_dir': str + } + + Where `fork_context` refers to the newly forked 'fork parent' context + the controller will use to start forked jobs, and `home_dir` is the + home directory for the active user account. + """ + # Copying the master's log level causes log messages to be filtered before + # they reach LogForwarder, thus reducing an influx of tiny messges waking + # the connection multiplexer process in the master. + LOG.setLevel(log_level) + logging.getLogger('ansible_mitogen').setLevel(log_level) + + # issue #536: if the json module is available, remove simplejson from the + # importer whitelist to avoid confusing certain Ansible modules. + if json.__name__ == 'json': + econtext.importer.whitelist.remove('simplejson') + + global _fork_parent + if FORK_SUPPORTED: + mitogen.parent.upgrade_router(econtext) + _fork_parent = econtext.router.fork() + + global good_temp_dir + good_temp_dir = find_good_temp_dir(candidate_temp_dirs) + + return { + u'fork_context': _fork_parent, + u'home_dir': mitogen.core.to_text(os.path.expanduser('~')), + u'good_temp_dir': good_temp_dir, + } + + +@mitogen.core.takes_econtext +def spawn_isolated_child(econtext): + """ + For helper functions executed in the fork parent context, arrange for + the context's router to be upgraded as necessary and for a new child to be + prepared. + + The actual fork occurs from the 'virginal fork parent', which does not have + any Ansible modules loaded prior to fork, to avoid conflicts resulting from + custom module_utils paths. + """ + mitogen.parent.upgrade_router(econtext) + if FORK_SUPPORTED: + context = econtext.router.fork() + else: + context = econtext.router.local() + LOG.debug('create_fork_child() -> %r', context) + return context + + +def run_module(kwargs): + """ + Set up the process environment in preparation for running an Ansible + module. This monkey-patches the Ansible libraries in various places to + prevent it from trying to kill the process on completion, and to prevent it + from reading sys.stdin. + """ + runner_name = kwargs.pop('runner_name') + klass = getattr(ansible_mitogen.runner, runner_name) + impl = klass(**mitogen.core.Kwargs(kwargs)) + return impl.run() + + +def _get_async_dir(): + return os.path.expanduser( + os.environ.get('ANSIBLE_ASYNC_DIR', '~/.ansible_async') + ) + + +class AsyncRunner(object): + def __init__(self, job_id, timeout_secs, started_sender, econtext, kwargs): + self.job_id = job_id + self.timeout_secs = timeout_secs + self.started_sender = started_sender + self.econtext = econtext + self.kwargs = kwargs + self._timed_out = False + self._init_path() + + def _init_path(self): + async_dir = _get_async_dir() + if not os.path.exists(async_dir): + os.makedirs(async_dir) + self.path = os.path.join(async_dir, self.job_id) + + def _update(self, dct): + """ + Update an async job status file. + """ + LOG.info('%r._update(%r, %r)', self, self.job_id, dct) + dct.setdefault('ansible_job_id', self.job_id) + dct.setdefault('data', '') + + fp = open(self.path + '.tmp', 'w') + try: + fp.write(json.dumps(dct)) + finally: + fp.close() + os.rename(self.path + '.tmp', self.path) + + def _on_sigalrm(self, signum, frame): + """ + Respond to SIGALRM (job timeout) by updating the job file and killing + the process. + """ + msg = "Job reached maximum time limit of %d seconds." % ( + self.timeout_secs, + ) + self._update({ + "failed": 1, + "finished": 1, + "msg": msg, + }) + self._timed_out = True + self.econtext.broker.shutdown() + + def _install_alarm(self): + signal.signal(signal.SIGALRM, self._on_sigalrm) + signal.alarm(self.timeout_secs) + + def _run_module(self): + kwargs = dict(self.kwargs, **{ + 'detach': True, + 'econtext': self.econtext, + 'emulate_tty': False, + }) + return run_module(kwargs) + + def _parse_result(self, dct): + filtered, warnings = ( + ansible.module_utils.json_utils. + _filter_non_json_lines(dct['stdout']) + ) + result = json.loads(filtered) + result.setdefault('warnings', []).extend(warnings) + result['stderr'] = dct['stderr'] or result.get('stderr', '') + self._update(result) + + def _run(self): + """ + 1. Immediately updates the status file to mark the job as started. + 2. Installs a timer/signal handler to implement the time limit. + 3. Runs as with run_module(), writing the result to the status file. + + :param dict kwargs: + Runner keyword arguments. + :param str job_id: + String job ID. + :param int timeout_secs: + If >0, limit the task's maximum run time. + """ + self._update({ + 'started': 1, + 'finished': 0, + 'pid': os.getpid() + }) + self.started_sender.send(True) + + if self.timeout_secs > 0: + self._install_alarm() + + dct = self._run_module() + if not self._timed_out: + # After SIGALRM fires, there is a window between broker responding + # to shutdown() by killing the process, and work continuing on the + # main thread. If main thread was asleep in at least + # basic.py/select.select(), an EINTR will be raised. We want to + # discard that exception. + try: + self._parse_result(dct) + except Exception: + self._update({ + "failed": 1, + "msg": traceback.format_exc(), + "data": dct['stdout'], # temporary notice only + "stderr": dct['stderr'] + }) + + def run(self): + try: + try: + self._run() + except Exception: + self._update({ + "failed": 1, + "msg": traceback.format_exc(), + }) + finally: + self.econtext.broker.shutdown() + + +@mitogen.core.takes_econtext +def run_module_async(kwargs, job_id, timeout_secs, started_sender, econtext): + """ + Execute a module with its run status and result written to a file, + terminating on the process on completion. This function must run in a child + forked using :func:`create_fork_child`. + + @param mitogen.core.Sender started_sender: + A sender that will receive :data:`True` once the job has reached a + point where its initial job file has been written. This is required to + avoid a race where an overly eager controller can check for a task + before it has reached that point in execution, which is possible at + least on Python 2.4, where forking is not available for async tasks. + """ + arunner = AsyncRunner( + job_id, + timeout_secs, + started_sender, + econtext, + kwargs + ) + arunner.run() + + +def get_user_shell(): + """ + For commands executed directly via an SSH command-line, SSH looks up the + user's shell via getpwuid() and only defaults to /bin/sh if that field is + missing or empty. + """ + try: + pw_shell = pwd.getpwuid(os.geteuid()).pw_shell + except KeyError: + pw_shell = None + + return pw_shell or '/bin/sh' + + +def exec_args(args, in_data='', chdir=None, shell=None, emulate_tty=False): + """ + Run a command in a subprocess, emulating the argument handling behaviour of + SSH. + + :param list[str]: + Argument vector. + :param bytes in_data: + Optional standard input for the command. + :param bool emulate_tty: + If :data:`True`, arrange for stdout and stderr to be merged into the + stdout pipe and for LF to be translated into CRLF, emulating the + behaviour of a TTY. + :return: + (return code, stdout bytes, stderr bytes) + """ + LOG.debug('exec_args(%r, ..., chdir=%r)', args, chdir) + assert isinstance(args, list) + + if emulate_tty: + stderr = subprocess.STDOUT + else: + stderr = subprocess.PIPE + + proc = subprocess.Popen( + args=args, + stdout=subprocess.PIPE, + stderr=stderr, + stdin=subprocess.PIPE, + cwd=chdir, + ) + stdout, stderr = proc.communicate(in_data) + + if emulate_tty: + stdout = stdout.replace(b('\n'), b('\r\n')) + return proc.returncode, stdout, stderr or b('') + + +def exec_command(cmd, in_data='', chdir=None, shell=None, emulate_tty=False): + """ + Run a command in a subprocess, emulating the argument handling behaviour of + SSH. + + :param bytes cmd: + String command line, passed to user's shell. + :param bytes in_data: + Optional standard input for the command. + :return: + (return code, stdout bytes, stderr bytes) + """ + assert isinstance(cmd, mitogen.core.UnicodeType) + return exec_args( + args=[get_user_shell(), '-c', cmd], + in_data=in_data, + chdir=chdir, + shell=shell, + emulate_tty=emulate_tty, + ) + + +def read_path(path): + """ + Fetch the contents of a filesystem `path` as bytes. + """ + return open(path, 'rb').read() + + +def set_file_owner(path, owner, group=None, fd=None): + if owner: + uid = pwd.getpwnam(owner).pw_uid + else: + uid = os.geteuid() + + if group: + gid = grp.getgrnam(group).gr_gid + else: + gid = os.getegid() + + if fd is not None and hasattr(os, 'fchown'): + os.fchown(fd, (uid, gid)) + else: + # Python<2.6 + os.chown(path, (uid, gid)) + + +def write_path(path, s, owner=None, group=None, mode=None, + utimes=None, sync=False): + """ + Writes bytes `s` to a filesystem `path`. + """ + path = os.path.abspath(path) + fd, tmp_path = tempfile.mkstemp(suffix='.tmp', + prefix='.ansible_mitogen_transfer-', + dir=os.path.dirname(path)) + fp = os.fdopen(fd, 'wb', mitogen.core.CHUNK_SIZE) + LOG.debug('write_path(path=%r) temporary file: %s', path, tmp_path) + + try: + try: + if mode: + set_file_mode(tmp_path, mode, fd=fp.fileno()) + if owner or group: + set_file_owner(tmp_path, owner, group, fd=fp.fileno()) + fp.write(s) + finally: + fp.close() + + if sync: + os.fsync(fp.fileno()) + os.rename(tmp_path, path) + except BaseException: + os.unlink(tmp_path) + raise + + if utimes: + os.utime(path, utimes) + + +CHMOD_CLAUSE_PAT = re.compile(r'([uoga]*)([+\-=])([ugo]|[rwx]*)') +CHMOD_MASKS = { + 'u': stat.S_IRWXU, + 'g': stat.S_IRWXG, + 'o': stat.S_IRWXO, + 'a': (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO), +} +CHMOD_BITS = { + 'u': {'r': stat.S_IRUSR, 'w': stat.S_IWUSR, 'x': stat.S_IXUSR}, + 'g': {'r': stat.S_IRGRP, 'w': stat.S_IWGRP, 'x': stat.S_IXGRP}, + 'o': {'r': stat.S_IROTH, 'w': stat.S_IWOTH, 'x': stat.S_IXOTH}, + 'a': { + 'r': (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH), + 'w': (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH), + 'x': (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + } +} + + +def apply_mode_spec(spec, mode): + """ + Given a symbolic file mode change specification in the style of chmod(1) + `spec`, apply changes in the specification to the numeric file mode `mode`. + """ + for clause in mitogen.core.to_text(spec).split(','): + match = CHMOD_CLAUSE_PAT.match(clause) + who, op, perms = match.groups() + for ch in who or 'a': + mask = CHMOD_MASKS[ch] + bits = CHMOD_BITS[ch] + cur_perm_bits = mode & mask + new_perm_bits = reduce(operator.or_, (bits[p] for p in perms), 0) + mode &= ~mask + if op == '=': + mode |= new_perm_bits + elif op == '+': + mode |= new_perm_bits | cur_perm_bits + else: + mode |= cur_perm_bits & ~new_perm_bits + return mode + + +def set_file_mode(path, spec, fd=None): + """ + Update the permissions of a file using the same syntax as chmod(1). + """ + if isinstance(spec, int): + new_mode = spec + elif not mitogen.core.PY3 and isinstance(spec, long): + new_mode = spec + elif spec.isdigit(): + new_mode = int(spec, 8) + else: + mode = os.stat(path).st_mode + new_mode = apply_mode_spec(spec, mode) + + if fd is not None and hasattr(os, 'fchmod'): + os.fchmod(fd, new_mode) + else: + os.chmod(path, new_mode) + + +def file_exists(path): + """ + Return :data:`True` if `path` exists. This is a wrapper function over + :func:`os.path.exists`, since its implementation module varies across + Python versions. + """ + return os.path.exists(path) diff --git a/mitogen-0.2.7/ansible_mitogen/target.pyc b/mitogen-0.2.7/ansible_mitogen/target.pyc new file mode 100644 index 000000000..f6999c9 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/target.pyc differ diff --git a/mitogen-0.2.7/ansible_mitogen/transport_config.py b/mitogen-0.2.7/ansible_mitogen/transport_config.py new file mode 100644 index 000000000..ad1cab3 --- /dev/null +++ b/mitogen-0.2.7/ansible_mitogen/transport_config.py @@ -0,0 +1,636 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import unicode_literals + +""" +Mitogen extends Ansible's target configuration mechanism in several ways that +require some care: + +* Per-task configurables in Ansible like ansible_python_interpreter are + connection-layer configurables in Mitogen. They must be extracted during each + task execution to form the complete connection-layer configuration. + +* Mitogen has extra configurables not supported by Ansible at all, such as + mitogen_ssh_debug_level. These are extracted the same way as + ansible_python_interpreter. + +* Mitogen allows connections to be delegated to other machines. Ansible has no + internal framework for this, and so Mitogen must figure out a delegated + connection configuration all on its own. It cannot reuse much of the Ansible + machinery for building a connection configuration, as that machinery is + deeply spread out and hard-wired to expect Ansible's usual mode of operation. + +For normal and delegate_to connections, Ansible's PlayContext is reused where +possible to maximize compatibility, but for proxy hops, configurations are +built up using the HostVars magic class to call VariableManager.get_vars() +behind the scenes on our behalf. Where Ansible has multiple sources of a +configuration item, for example, ansible_ssh_extra_args, Mitogen must (ideally +perfectly) reproduce how Ansible arrives at its value, without using mechanisms +that are hard-wired or change across Ansible versions. + +That is what this file is for. It exports two spec classes, one that takes all +information from PlayContext, and another that takes (almost) all information +from HostVars. +""" + +import abc +import os +import ansible.utils.shlex +import ansible.constants as C + +from ansible.module_utils.six import with_metaclass + + +import mitogen.core + + +def parse_python_path(s): + """ + Given the string set for ansible_python_interpeter, parse it using shell + syntax and return an appropriate argument vector. + """ + if s: + return ansible.utils.shlex.shlex_split(s) + + +def optional_secret(value): + """ + Wrap `value` in :class:`mitogen.core.Secret` if it is not :data:`None`, + otherwise return :data:`None`. + """ + if value is not None: + return mitogen.core.Secret(value) + + +def first_true(it, default=None): + """ + Return the first truthy element from `it`. + """ + for elem in it: + if elem: + return elem + return default + + +class Spec(with_metaclass(abc.ABCMeta, object)): + """ + A source for variables that comprise a connection configuration. + """ + + @abc.abstractmethod + def transport(self): + """ + The name of the Ansible plug-in implementing the connection. + """ + + @abc.abstractmethod + def inventory_name(self): + """ + The name of the target being connected to as it appears in Ansible's + inventory. + """ + + @abc.abstractmethod + def remote_addr(self): + """ + The network address of the target, or for container and other special + targets, some other unique identifier. + """ + + @abc.abstractmethod + def remote_user(self): + """ + The username of the login account on the target. + """ + + @abc.abstractmethod + def password(self): + """ + The password of the login account on the target. + """ + + @abc.abstractmethod + def become(self): + """ + :data:`True` if privilege escalation should be active. + """ + + @abc.abstractmethod + def become_method(self): + """ + The name of the Ansible become method to use. + """ + + @abc.abstractmethod + def become_user(self): + """ + The username of the target account for become. + """ + + @abc.abstractmethod + def become_pass(self): + """ + The password of the target account for become. + """ + + @abc.abstractmethod + def port(self): + """ + The port of the login service on the target machine. + """ + + @abc.abstractmethod + def python_path(self): + """ + Path to the Python interpreter on the target machine. + """ + + @abc.abstractmethod + def private_key_file(self): + """ + Path to the SSH private key file to use to login. + """ + + @abc.abstractmethod + def ssh_executable(self): + """ + Path to the SSH executable. + """ + + @abc.abstractmethod + def timeout(self): + """ + The generic timeout for all connections. + """ + + @abc.abstractmethod + def ansible_ssh_timeout(self): + """ + The SSH-specific timeout for a connection. + """ + + @abc.abstractmethod + def ssh_args(self): + """ + The list of additional arguments that should be included in an SSH + invocation. + """ + + @abc.abstractmethod + def become_exe(self): + """ + The path to the executable implementing the become method on the remote + machine. + """ + + @abc.abstractmethod + def sudo_args(self): + """ + The list of additional arguments that should be included in a become + invocation. + """ + # TODO: split out into sudo_args/become_args. + + @abc.abstractmethod + def mitogen_via(self): + """ + The value of the mitogen_via= variable for this connection. Indicates + the connection should be established via an intermediary. + """ + + @abc.abstractmethod + def mitogen_kind(self): + """ + The type of container to use with the "setns" transport. + """ + + @abc.abstractmethod + def mitogen_mask_remote_name(self): + """ + Specifies whether to set a fixed "remote_name" field. The remote_name + is the suffix of `argv[0]` for remote interpreters. By default it + includes identifying information from the local process, which may be + undesirable in some circumstances. + """ + + @abc.abstractmethod + def mitogen_docker_path(self): + """ + The path to the "docker" program for the 'docker' transport. + """ + + @abc.abstractmethod + def mitogen_kubectl_path(self): + """ + The path to the "kubectl" program for the 'docker' transport. + """ + + @abc.abstractmethod + def mitogen_lxc_path(self): + """ + The path to the "lxc" program for the 'lxd' transport. + """ + + @abc.abstractmethod + def mitogen_lxc_attach_path(self): + """ + The path to the "lxc-attach" program for the 'lxc' transport. + """ + + @abc.abstractmethod + def mitogen_lxc_info_path(self): + """ + The path to the "lxc-info" program for the 'lxc' transport. + """ + + @abc.abstractmethod + def mitogen_machinectl_path(self): + """ + The path to the "machinectl" program for the 'setns' transport. + """ + + @abc.abstractmethod + def mitogen_ssh_debug_level(self): + """ + The SSH debug level. + """ + + @abc.abstractmethod + def mitogen_ssh_compression(self): + """ + Whether SSH compression is enabled. + """ + + @abc.abstractmethod + def extra_args(self): + """ + Connection-specific arguments. + """ + + +class PlayContextSpec(Spec): + """ + PlayContextSpec takes almost all its information as-is from Ansible's + PlayContext. It is used for normal connections and delegate_to connections, + and should always be accurate. + """ + def __init__(self, connection, play_context, transport, inventory_name): + self._connection = connection + self._play_context = play_context + self._transport = transport + self._inventory_name = inventory_name + + def transport(self): + return self._transport + + def inventory_name(self): + return self._inventory_name + + def remote_addr(self): + return self._play_context.remote_addr + + def remote_user(self): + return self._play_context.remote_user + + def become(self): + return self._play_context.become + + def become_method(self): + return self._play_context.become_method + + def become_user(self): + return self._play_context.become_user + + def become_pass(self): + return optional_secret(self._play_context.become_pass) + + def password(self): + return optional_secret(self._play_context.password) + + def port(self): + return self._play_context.port + + def python_path(self): + s = self._connection.get_task_var('ansible_python_interpreter') + # #511, #536: executor/module_common.py::_get_shebang() hard-wires + # "/usr/bin/python" as the default interpreter path if no other + # interpreter is specified. + return parse_python_path(s or '/usr/bin/python') + + def private_key_file(self): + return self._play_context.private_key_file + + def ssh_executable(self): + return self._play_context.ssh_executable + + def timeout(self): + return self._play_context.timeout + + def ansible_ssh_timeout(self): + return ( + self._connection.get_task_var('ansible_timeout') or + self._connection.get_task_var('ansible_ssh_timeout') or + self.timeout() + ) + + def ssh_args(self): + return [ + mitogen.core.to_text(term) + for s in ( + getattr(self._play_context, 'ssh_args', ''), + getattr(self._play_context, 'ssh_common_args', ''), + getattr(self._play_context, 'ssh_extra_args', '') + ) + for term in ansible.utils.shlex.shlex_split(s or '') + ] + + def become_exe(self): + return self._play_context.become_exe + + def sudo_args(self): + return [ + mitogen.core.to_text(term) + for term in ansible.utils.shlex.shlex_split( + first_true(( + self._play_context.become_flags, + self._play_context.sudo_flags, + # Ansible 2.3. + getattr(C, 'DEFAULT_BECOME_FLAGS', ''), + getattr(C, 'DEFAULT_SUDO_FLAGS', '') + ), default='') + ) + ] + + def mitogen_via(self): + return self._connection.get_task_var('mitogen_via') + + def mitogen_kind(self): + return self._connection.get_task_var('mitogen_kind') + + def mitogen_mask_remote_name(self): + return self._connection.get_task_var('mitogen_mask_remote_name') + + def mitogen_docker_path(self): + return self._connection.get_task_var('mitogen_docker_path') + + def mitogen_kubectl_path(self): + return self._connection.get_task_var('mitogen_kubectl_path') + + def mitogen_lxc_path(self): + return self._connection.get_task_var('mitogen_lxc_path') + + def mitogen_lxc_attach_path(self): + return self._connection.get_task_var('mitogen_lxc_attach_path') + + def mitogen_lxc_info_path(self): + return self._connection.get_task_var('mitogen_lxc_info_path') + + def mitogen_machinectl_path(self): + return self._connection.get_task_var('mitogen_machinectl_path') + + def mitogen_ssh_debug_level(self): + return self._connection.get_task_var('mitogen_ssh_debug_level') + + def mitogen_ssh_compression(self): + return self._connection.get_task_var('mitogen_ssh_compression') + + def extra_args(self): + return self._connection.get_extra_args() + + +class MitogenViaSpec(Spec): + """ + MitogenViaSpec takes most of its information from the HostVars of the + running task. HostVars is a lightweight wrapper around VariableManager, so + it is better to say that VariableManager.get_vars() is the ultimate source + of MitogenViaSpec's information. + + Due to this, mitogen_via= hosts must have all their configuration + information represented as host and group variables. We cannot use any + per-task configuration, as all that data belongs to the real target host. + + Ansible uses all kinds of strange historical logic for calculating + variables, including making their precedence configurable. MitogenViaSpec + must ultimately reimplement all of that logic. It is likely that if you are + having a configruation problem with connection delegation, the answer to + your problem lies in the method implementations below! + """ + def __init__(self, inventory_name, host_vars, become_method, become_user, + play_context): + """ + :param str inventory_name: + The inventory name of the intermediary machine, i.e. not the target + machine. + :param dict host_vars: + The HostVars magic dictionary provided by Ansible in task_vars. + :param str become_method: + If the mitogen_via= spec included a become method, the method it + specifies. + :param str become_user: + If the mitogen_via= spec included a become user, the user it + specifies. + :param PlayContext play_context: + For some global values **only**, the PlayContext used to describe + the real target machine. Values from this object are **strictly + restricted** to values that are Ansible-global, e.g. the passwords + specified interactively. + """ + self._inventory_name = inventory_name + self._host_vars = host_vars + self._become_method = become_method + self._become_user = become_user + # Dangerous! You may find a variable you want in this object, but it's + # almost certainly for the wrong machine! + self._dangerous_play_context = play_context + + def transport(self): + return ( + self._host_vars.get('ansible_connection') or + C.DEFAULT_TRANSPORT + ) + + def inventory_name(self): + return self._inventory_name + + def remote_addr(self): + # play_context.py::MAGIC_VARIABLE_MAPPING + return ( + self._host_vars.get('ansible_ssh_host') or + self._host_vars.get('ansible_host') or + self._inventory_name + ) + + def remote_user(self): + return ( + self._host_vars.get('ansible_ssh_user') or + self._host_vars.get('ansible_user') or + C.DEFAULT_REMOTE_USER + ) + + def become(self): + return bool(self._become_user) + + def become_method(self): + return ( + self._become_method or + self._host_vars.get('ansible_become_method') or + C.DEFAULT_BECOME_METHOD + ) + + def become_user(self): + return self._become_user + + def become_pass(self): + return optional_secret( + self._host_vars.get('ansible_become_password') or + self._host_vars.get('ansible_become_pass') + ) + + def password(self): + return optional_secret( + self._host_vars.get('ansible_ssh_pass') or + self._host_vars.get('ansible_password') + ) + + def port(self): + return ( + self._host_vars.get('ansible_ssh_port') or + self._host_vars.get('ansible_port') or + C.DEFAULT_REMOTE_PORT + ) + + def python_path(self): + s = self._host_vars.get('ansible_python_interpreter') + # #511, #536: executor/module_common.py::_get_shebang() hard-wires + # "/usr/bin/python" as the default interpreter path if no other + # interpreter is specified. + return parse_python_path(s or '/usr/bin/python') + + def private_key_file(self): + # TODO: must come from PlayContext too. + return ( + self._host_vars.get('ansible_ssh_private_key_file') or + self._host_vars.get('ansible_private_key_file') or + C.DEFAULT_PRIVATE_KEY_FILE + ) + + def ssh_executable(self): + return ( + self._host_vars.get('ansible_ssh_executable') or + C.ANSIBLE_SSH_EXECUTABLE + ) + + def timeout(self): + # TODO: must come from PlayContext too. + return C.DEFAULT_TIMEOUT + + def ansible_ssh_timeout(self): + return ( + self._host_vars.get('ansible_timeout') or + self._host_vars.get('ansible_ssh_timeout') or + self.timeout() + ) + + def ssh_args(self): + return [ + mitogen.core.to_text(term) + for s in ( + ( + self._host_vars.get('ansible_ssh_args') or + getattr(C, 'ANSIBLE_SSH_ARGS', None) or + os.environ.get('ANSIBLE_SSH_ARGS') + # TODO: ini entry. older versions. + ), + ( + self._host_vars.get('ansible_ssh_common_args') or + os.environ.get('ANSIBLE_SSH_COMMON_ARGS') + # TODO: ini entry. + ), + ( + self._host_vars.get('ansible_ssh_extra_args') or + os.environ.get('ANSIBLE_SSH_EXTRA_ARGS') + # TODO: ini entry. + ), + ) + for term in ansible.utils.shlex.shlex_split(s) + if s + ] + + def become_exe(self): + return ( + self._host_vars.get('ansible_become_exe') or + C.DEFAULT_BECOME_EXE + ) + + def sudo_args(self): + return [ + mitogen.core.to_text(term) + for s in ( + self._host_vars.get('ansible_sudo_flags') or '', + self._host_vars.get('ansible_become_flags') or '', + ) + for term in ansible.utils.shlex.shlex_split(s) + ] + + def mitogen_via(self): + return self._host_vars.get('mitogen_via') + + def mitogen_kind(self): + return self._host_vars.get('mitogen_kind') + + def mitogen_mask_remote_name(self): + return self._host_vars.get('mitogen_mask_remote_name') + + def mitogen_docker_path(self): + return self._host_vars.get('mitogen_docker_path') + + def mitogen_kubectl_path(self): + return self._host_vars.get('mitogen_kubectl_path') + + def mitogen_lxc_path(self): + return self.host_vars.get('mitogen_lxc_path') + + def mitogen_lxc_attach_path(self): + return self._host_vars.get('mitogen_lxc_attach_path') + + def mitogen_lxc_info_path(self): + return self._host_vars.get('mitogen_lxc_info_path') + + def mitogen_machinectl_path(self): + return self._host_vars.get('mitogen_machinectl_path') + + def mitogen_ssh_debug_level(self): + return self._host_vars.get('mitogen_ssh_debug_level') + + def mitogen_ssh_compression(self): + return self._host_vars.get('mitogen_ssh_compression') + + def extra_args(self): + return [] # TODO diff --git a/mitogen-0.2.7/ansible_mitogen/transport_config.pyc b/mitogen-0.2.7/ansible_mitogen/transport_config.pyc new file mode 100644 index 000000000..149aa39 Binary files /dev/null and b/mitogen-0.2.7/ansible_mitogen/transport_config.pyc differ diff --git a/mitogen-0.2.7/mitogen.egg-info/PKG-INFO b/mitogen-0.2.7/mitogen.egg-info/PKG-INFO new file mode 100644 index 000000000..f63ba91 --- /dev/null +++ b/mitogen-0.2.7/mitogen.egg-info/PKG-INFO @@ -0,0 +1,23 @@ +Metadata-Version: 1.1 +Name: mitogen +Version: 0.2.7 +Summary: Library for writing distributed self-replicating programs. +Home-page: https://github.com/dw/mitogen/ +Author: David Wilson +Author-email: UNKNOWN +License: New BSD +Description: UNKNOWN +Platform: UNKNOWN +Classifier: Environment :: Console +Classifier: Intended Audience :: System Administrators +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: POSIX +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.4 +Classifier: Programming Language :: Python :: 2.5 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Topic :: System :: Distributed Computing +Classifier: Topic :: System :: Systems Administration diff --git a/mitogen-0.2.7/mitogen.egg-info/SOURCES.txt b/mitogen-0.2.7/mitogen.egg-info/SOURCES.txt new file mode 100644 index 000000000..c2abd33 --- /dev/null +++ b/mitogen-0.2.7/mitogen.egg-info/SOURCES.txt @@ -0,0 +1,78 @@ +LICENSE +MANIFEST.in +README.md +setup.cfg +setup.py +ansible_mitogen/__init__.py +ansible_mitogen/affinity.py +ansible_mitogen/connection.py +ansible_mitogen/loaders.py +ansible_mitogen/logging.py +ansible_mitogen/mixins.py +ansible_mitogen/module_finder.py +ansible_mitogen/parsing.py +ansible_mitogen/planner.py +ansible_mitogen/process.py +ansible_mitogen/runner.py +ansible_mitogen/services.py +ansible_mitogen/strategy.py +ansible_mitogen/target.py +ansible_mitogen/transport_config.py +ansible_mitogen/compat/__init__.py +ansible_mitogen/compat/simplejson/__init__.py +ansible_mitogen/compat/simplejson/decoder.py +ansible_mitogen/compat/simplejson/encoder.py +ansible_mitogen/compat/simplejson/scanner.py +ansible_mitogen/plugins/__init__.py +ansible_mitogen/plugins/action/__init__.py +ansible_mitogen/plugins/action/mitogen_get_stack.py +ansible_mitogen/plugins/connection/__init__.py +ansible_mitogen/plugins/connection/mitogen_doas.py +ansible_mitogen/plugins/connection/mitogen_docker.py +ansible_mitogen/plugins/connection/mitogen_jail.py +ansible_mitogen/plugins/connection/mitogen_kubectl.py +ansible_mitogen/plugins/connection/mitogen_local.py +ansible_mitogen/plugins/connection/mitogen_lxc.py +ansible_mitogen/plugins/connection/mitogen_lxd.py +ansible_mitogen/plugins/connection/mitogen_machinectl.py +ansible_mitogen/plugins/connection/mitogen_setns.py +ansible_mitogen/plugins/connection/mitogen_ssh.py +ansible_mitogen/plugins/connection/mitogen_su.py +ansible_mitogen/plugins/connection/mitogen_sudo.py +ansible_mitogen/plugins/strategy/__init__.py +ansible_mitogen/plugins/strategy/mitogen.py +ansible_mitogen/plugins/strategy/mitogen_free.py +ansible_mitogen/plugins/strategy/mitogen_host_pinned.py +ansible_mitogen/plugins/strategy/mitogen_linear.py +mitogen/__init__.py +mitogen/core.py +mitogen/debug.py +mitogen/doas.py +mitogen/docker.py +mitogen/fakessh.py +mitogen/fork.py +mitogen/jail.py +mitogen/kubectl.py +mitogen/lxc.py +mitogen/lxd.py +mitogen/master.py +mitogen/minify.py +mitogen/os_fork.py +mitogen/parent.py +mitogen/profiler.py +mitogen/select.py +mitogen/service.py +mitogen/setns.py +mitogen/ssh.py +mitogen/su.py +mitogen/sudo.py +mitogen/unix.py +mitogen/utils.py +mitogen.egg-info/PKG-INFO +mitogen.egg-info/SOURCES.txt +mitogen.egg-info/dependency_links.txt +mitogen.egg-info/not-zip-safe +mitogen.egg-info/top_level.txt +mitogen/compat/__init__.py +mitogen/compat/pkgutil.py +mitogen/compat/tokenize.py \ No newline at end of file diff --git a/mitogen-0.2.7/mitogen.egg-info/dependency_links.txt b/mitogen-0.2.7/mitogen.egg-info/dependency_links.txt new file mode 100644 index 000000000..8b13789 --- /dev/null +++ b/mitogen-0.2.7/mitogen.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/mitogen-0.2.7/mitogen.egg-info/not-zip-safe b/mitogen-0.2.7/mitogen.egg-info/not-zip-safe new file mode 100644 index 000000000..8b13789 --- /dev/null +++ b/mitogen-0.2.7/mitogen.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/mitogen-0.2.7/mitogen.egg-info/top_level.txt b/mitogen-0.2.7/mitogen.egg-info/top_level.txt new file mode 100644 index 000000000..2360b3f --- /dev/null +++ b/mitogen-0.2.7/mitogen.egg-info/top_level.txt @@ -0,0 +1,2 @@ +ansible_mitogen +mitogen diff --git a/mitogen-0.2.7/mitogen/__init__.py b/mitogen-0.2.7/mitogen/__init__.py new file mode 100644 index 000000000..47fe4d3 --- /dev/null +++ b/mitogen-0.2.7/mitogen/__init__.py @@ -0,0 +1,120 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +""" +On the Mitogen master, this is imported from ``mitogen/__init__.py`` as would +be expected. On the slave, it is built dynamically during startup. +""" + + +#: Library version as a tuple. +__version__ = (0, 2, 7) + + +#: This is :data:`False` in slave contexts. Previously it was used to prevent +#: re-execution of :mod:`__main__` in single file programs, however that now +#: happens automatically. +is_master = True + + +#: This is `0` in a master, otherwise it is the master-assigned ID unique to +#: the slave context used for message routing. +context_id = 0 + + +#: This is :data:`None` in a master, otherwise it is the master-assigned ID +#: unique to the slave's parent context. +parent_id = None + + +#: This is an empty list in a master, otherwise it is a list of parent context +#: IDs ordered from most direct to least direct. +parent_ids = [] + + +import os +_default_profiling = os.environ.get('MITOGEN_PROFILING') is not None +del os + + +def main(log_level='INFO', profiling=_default_profiling): + """ + Convenience decorator primarily useful for writing discardable test + scripts. + + In the master process, when `func` is defined in the :mod:`__main__` + module, arranges for `func(router)` to be invoked immediately, with + :py:class:`mitogen.master.Router` construction and destruction handled just + as in :py:func:`mitogen.utils.run_with_router`. In slaves, this function + does nothing. + + :param str log_level: + Logging package level to configure via + :py:func:`mitogen.utils.log_to_file`. + + :param bool profiling: + If :py:data:`True`, equivalent to setting + :py:attr:`mitogen.master.Router.profiling` prior to router + construction. This causes ``/tmp`` files to be created everywhere at + the end of a successful run with :py:mod:`cProfile` output for every + thread. + + Example: + + :: + + import mitogen + import requests + + def get_url(url): + return requests.get(url).text + + @mitogen.main() + def main(router): + z = router.ssh(hostname='k3') + print(z.call(get_url, 'https://example.org/'))))) + + """ + + def wrapper(func): + if func.__module__ != '__main__': + return func + import mitogen.parent + import mitogen.utils + if profiling: + mitogen.core.enable_profiling() + mitogen.master.Router.profiling = profiling + utils.log_to_file(level=log_level) + return mitogen.core._profile_hook( + 'app.main', + utils.run_with_router, + func, + ) + return wrapper diff --git a/mitogen-0.2.7/mitogen/__init__.pyc b/mitogen-0.2.7/mitogen/__init__.pyc new file mode 100644 index 000000000..dda1609 Binary files /dev/null and b/mitogen-0.2.7/mitogen/__init__.pyc differ diff --git a/mitogen-0.2.7/mitogen/__pycache__/__init__.cpython-37.pyc b/mitogen-0.2.7/mitogen/__pycache__/__init__.cpython-37.pyc new file mode 100644 index 000000000..d7945ea Binary files /dev/null and b/mitogen-0.2.7/mitogen/__pycache__/__init__.cpython-37.pyc differ diff --git a/mitogen-0.2.7/mitogen/__pycache__/core.cpython-37.pyc b/mitogen-0.2.7/mitogen/__pycache__/core.cpython-37.pyc new file mode 100644 index 000000000..fb7492e Binary files /dev/null and b/mitogen-0.2.7/mitogen/__pycache__/core.cpython-37.pyc differ diff --git a/mitogen-0.2.7/mitogen/__pycache__/debug.cpython-37.pyc b/mitogen-0.2.7/mitogen/__pycache__/debug.cpython-37.pyc new file mode 100644 index 000000000..21b0eec Binary files /dev/null and b/mitogen-0.2.7/mitogen/__pycache__/debug.cpython-37.pyc differ diff --git a/mitogen-0.2.7/mitogen/__pycache__/fork.cpython-37.pyc b/mitogen-0.2.7/mitogen/__pycache__/fork.cpython-37.pyc new file mode 100644 index 000000000..a118d91 Binary files /dev/null and b/mitogen-0.2.7/mitogen/__pycache__/fork.cpython-37.pyc differ diff --git a/mitogen-0.2.7/mitogen/__pycache__/master.cpython-37.pyc b/mitogen-0.2.7/mitogen/__pycache__/master.cpython-37.pyc new file mode 100644 index 000000000..4a29a27 Binary files /dev/null and b/mitogen-0.2.7/mitogen/__pycache__/master.cpython-37.pyc differ diff --git a/mitogen-0.2.7/mitogen/__pycache__/minify.cpython-37.pyc b/mitogen-0.2.7/mitogen/__pycache__/minify.cpython-37.pyc new file mode 100644 index 000000000..4f93771 Binary files /dev/null and b/mitogen-0.2.7/mitogen/__pycache__/minify.cpython-37.pyc differ diff --git a/mitogen-0.2.7/mitogen/__pycache__/parent.cpython-37.pyc b/mitogen-0.2.7/mitogen/__pycache__/parent.cpython-37.pyc new file mode 100644 index 000000000..8ac0d87 Binary files /dev/null and b/mitogen-0.2.7/mitogen/__pycache__/parent.cpython-37.pyc differ diff --git a/mitogen-0.2.7/mitogen/__pycache__/select.cpython-37.pyc b/mitogen-0.2.7/mitogen/__pycache__/select.cpython-37.pyc new file mode 100644 index 000000000..603343e Binary files /dev/null and b/mitogen-0.2.7/mitogen/__pycache__/select.cpython-37.pyc differ diff --git a/mitogen-0.2.7/mitogen/__pycache__/service.cpython-37.pyc b/mitogen-0.2.7/mitogen/__pycache__/service.cpython-37.pyc new file mode 100644 index 000000000..e6918dc Binary files /dev/null and b/mitogen-0.2.7/mitogen/__pycache__/service.cpython-37.pyc differ diff --git a/mitogen-0.2.7/mitogen/__pycache__/unix.cpython-37.pyc b/mitogen-0.2.7/mitogen/__pycache__/unix.cpython-37.pyc new file mode 100644 index 000000000..33d1cc2 Binary files /dev/null and b/mitogen-0.2.7/mitogen/__pycache__/unix.cpython-37.pyc differ diff --git a/mitogen-0.2.7/mitogen/__pycache__/utils.cpython-37.pyc b/mitogen-0.2.7/mitogen/__pycache__/utils.cpython-37.pyc new file mode 100644 index 000000000..6042a5b Binary files /dev/null and b/mitogen-0.2.7/mitogen/__pycache__/utils.cpython-37.pyc differ diff --git a/mitogen-0.2.7/mitogen/compat/__init__.py b/mitogen-0.2.7/mitogen/compat/__init__.py new file mode 100644 index 000000000..e69de29 diff --git a/mitogen-0.2.7/mitogen/compat/pkgutil.py b/mitogen-0.2.7/mitogen/compat/pkgutil.py new file mode 100644 index 000000000..28e2aea --- /dev/null +++ b/mitogen-0.2.7/mitogen/compat/pkgutil.py @@ -0,0 +1,593 @@ +"""Utilities to support packages.""" + +# !mitogen: minify_safe + +# NOTE: This module must remain compatible with Python 2.3, as it is shared +# by setuptools for distribution with Python 2.3 and up. + +import os +import sys +import imp +import os.path +from types import ModuleType + +__all__ = [ + 'get_importer', 'iter_importers', 'get_loader', 'find_loader', + 'walk_packages', 'iter_modules', 'get_data', + 'ImpImporter', 'ImpLoader', 'read_code', 'extend_path', +] + +def read_code(stream): + # This helper is needed in order for the PEP 302 emulation to + # correctly handle compiled files + import marshal + + magic = stream.read(4) + if magic != imp.get_magic(): + return None + + stream.read(4) # Skip timestamp + return marshal.load(stream) + + +def simplegeneric(func): + """Make a trivial single-dispatch generic function""" + registry = {} + def wrapper(*args, **kw): + ob = args[0] + try: + cls = ob.__class__ + except AttributeError: + cls = type(ob) + try: + mro = cls.__mro__ + except AttributeError: + try: + class cls(cls, object): + pass + mro = cls.__mro__[1:] + except TypeError: + mro = object, # must be an ExtensionClass or some such :( + for t in mro: + if t in registry: + return registry[t](*args, **kw) + else: + return func(*args, **kw) + try: + wrapper.__name__ = func.__name__ + except (TypeError, AttributeError): + pass # Python 2.3 doesn't allow functions to be renamed + + def register(typ, func=None): + if func is None: + return lambda f: register(typ, f) + registry[typ] = func + return func + + wrapper.__dict__ = func.__dict__ + wrapper.__doc__ = func.__doc__ + wrapper.register = register + return wrapper + + +def walk_packages(path=None, prefix='', onerror=None): + """Yields (module_loader, name, ispkg) for all modules recursively + on path, or, if path is None, all accessible modules. + + 'path' should be either None or a list of paths to look for + modules in. + + 'prefix' is a string to output on the front of every module name + on output. + + Note that this function must import all *packages* (NOT all + modules!) on the given path, in order to access the __path__ + attribute to find submodules. + + 'onerror' is a function which gets called with one argument (the + name of the package which was being imported) if any exception + occurs while trying to import a package. If no onerror function is + supplied, ImportErrors are caught and ignored, while all other + exceptions are propagated, terminating the search. + + Examples: + + # list all modules python can access + walk_packages() + + # list all submodules of ctypes + walk_packages(ctypes.__path__, ctypes.__name__+'.') + """ + + def seen(p, m={}): + if p in m: + return True + m[p] = True + + for importer, name, ispkg in iter_modules(path, prefix): + yield importer, name, ispkg + + if ispkg: + try: + __import__(name) + except ImportError: + if onerror is not None: + onerror(name) + except Exception: + if onerror is not None: + onerror(name) + else: + raise + else: + path = getattr(sys.modules[name], '__path__', None) or [] + + # don't traverse path items we've seen before + path = [p for p in path if not seen(p)] + + for item in walk_packages(path, name+'.', onerror): + yield item + + +def iter_modules(path=None, prefix=''): + """Yields (module_loader, name, ispkg) for all submodules on path, + or, if path is None, all top-level modules on sys.path. + + 'path' should be either None or a list of paths to look for + modules in. + + 'prefix' is a string to output on the front of every module name + on output. + """ + + if path is None: + importers = iter_importers() + else: + importers = map(get_importer, path) + + yielded = {} + for i in importers: + for name, ispkg in iter_importer_modules(i, prefix): + if name not in yielded: + yielded[name] = 1 + yield i, name, ispkg + + +#@simplegeneric +def iter_importer_modules(importer, prefix=''): + if not hasattr(importer, 'iter_modules'): + return [] + return importer.iter_modules(prefix) + +iter_importer_modules = simplegeneric(iter_importer_modules) + + +class ImpImporter: + """PEP 302 Importer that wraps Python's "classic" import algorithm + + ImpImporter(dirname) produces a PEP 302 importer that searches that + directory. ImpImporter(None) produces a PEP 302 importer that searches + the current sys.path, plus any modules that are frozen or built-in. + + Note that ImpImporter does not currently support being used by placement + on sys.meta_path. + """ + + def __init__(self, path=None): + self.path = path + + def find_module(self, fullname, path=None): + # Note: we ignore 'path' argument since it is only used via meta_path + subname = fullname.split(".")[-1] + if subname != fullname and self.path is None: + return None + if self.path is None: + path = None + else: + path = [os.path.realpath(self.path)] + try: + file, filename, etc = imp.find_module(subname, path) + except ImportError: + return None + return ImpLoader(fullname, file, filename, etc) + + def iter_modules(self, prefix=''): + if self.path is None or not os.path.isdir(self.path): + return + + yielded = {} + import inspect + try: + filenames = os.listdir(self.path) + except OSError: + # ignore unreadable directories like import does + filenames = [] + filenames.sort() # handle packages before same-named modules + + for fn in filenames: + modname = inspect.getmodulename(fn) + if modname=='__init__' or modname in yielded: + continue + + path = os.path.join(self.path, fn) + ispkg = False + + if not modname and os.path.isdir(path) and '.' not in fn: + modname = fn + try: + dircontents = os.listdir(path) + except OSError: + # ignore unreadable directories like import does + dircontents = [] + for fn in dircontents: + subname = inspect.getmodulename(fn) + if subname=='__init__': + ispkg = True + break + else: + continue # not a package + + if modname and '.' not in modname: + yielded[modname] = 1 + yield prefix + modname, ispkg + + +class ImpLoader: + """PEP 302 Loader that wraps Python's "classic" import algorithm + """ + code = source = None + + def __init__(self, fullname, file, filename, etc): + self.file = file + self.filename = filename + self.fullname = fullname + self.etc = etc + + def load_module(self, fullname): + self._reopen() + try: + mod = imp.load_module(fullname, self.file, self.filename, self.etc) + finally: + if self.file: + self.file.close() + # Note: we don't set __loader__ because we want the module to look + # normal; i.e. this is just a wrapper for standard import machinery + return mod + + def get_data(self, pathname): + return open(pathname, "rb").read() + + def _reopen(self): + if self.file and self.file.closed: + mod_type = self.etc[2] + if mod_type==imp.PY_SOURCE: + self.file = open(self.filename, 'rU') + elif mod_type in (imp.PY_COMPILED, imp.C_EXTENSION): + self.file = open(self.filename, 'rb') + + def _fix_name(self, fullname): + if fullname is None: + fullname = self.fullname + elif fullname != self.fullname: + raise ImportError("Loader for module %s cannot handle " + "module %s" % (self.fullname, fullname)) + return fullname + + def is_package(self, fullname): + fullname = self._fix_name(fullname) + return self.etc[2]==imp.PKG_DIRECTORY + + def get_code(self, fullname=None): + fullname = self._fix_name(fullname) + if self.code is None: + mod_type = self.etc[2] + if mod_type==imp.PY_SOURCE: + source = self.get_source(fullname) + self.code = compile(source, self.filename, 'exec') + elif mod_type==imp.PY_COMPILED: + self._reopen() + try: + self.code = read_code(self.file) + finally: + self.file.close() + elif mod_type==imp.PKG_DIRECTORY: + self.code = self._get_delegate().get_code() + return self.code + + def get_source(self, fullname=None): + fullname = self._fix_name(fullname) + if self.source is None: + mod_type = self.etc[2] + if mod_type==imp.PY_SOURCE: + self._reopen() + try: + self.source = self.file.read() + finally: + self.file.close() + elif mod_type==imp.PY_COMPILED: + if os.path.exists(self.filename[:-1]): + f = open(self.filename[:-1], 'rU') + self.source = f.read() + f.close() + elif mod_type==imp.PKG_DIRECTORY: + self.source = self._get_delegate().get_source() + return self.source + + + def _get_delegate(self): + return ImpImporter(self.filename).find_module('__init__') + + def get_filename(self, fullname=None): + fullname = self._fix_name(fullname) + mod_type = self.etc[2] + if self.etc[2]==imp.PKG_DIRECTORY: + return self._get_delegate().get_filename() + elif self.etc[2] in (imp.PY_SOURCE, imp.PY_COMPILED, imp.C_EXTENSION): + return self.filename + return None + + +try: + import zipimport + from zipimport import zipimporter + + def iter_zipimport_modules(importer, prefix=''): + dirlist = zipimport._zip_directory_cache[importer.archive].keys() + dirlist.sort() + _prefix = importer.prefix + plen = len(_prefix) + yielded = {} + import inspect + for fn in dirlist: + if not fn.startswith(_prefix): + continue + + fn = fn[plen:].split(os.sep) + + if len(fn)==2 and fn[1].startswith('__init__.py'): + if fn[0] not in yielded: + yielded[fn[0]] = 1 + yield fn[0], True + + if len(fn)!=1: + continue + + modname = inspect.getmodulename(fn[0]) + if modname=='__init__': + continue + + if modname and '.' not in modname and modname not in yielded: + yielded[modname] = 1 + yield prefix + modname, False + + iter_importer_modules.register(zipimporter, iter_zipimport_modules) + +except ImportError: + pass + + +def get_importer(path_item): + """Retrieve a PEP 302 importer for the given path item + + The returned importer is cached in sys.path_importer_cache + if it was newly created by a path hook. + + If there is no importer, a wrapper around the basic import + machinery is returned. This wrapper is never inserted into + the importer cache (None is inserted instead). + + The cache (or part of it) can be cleared manually if a + rescan of sys.path_hooks is necessary. + """ + try: + importer = sys.path_importer_cache[path_item] + except KeyError: + for path_hook in sys.path_hooks: + try: + importer = path_hook(path_item) + break + except ImportError: + pass + else: + importer = None + sys.path_importer_cache.setdefault(path_item, importer) + + if importer is None: + try: + importer = ImpImporter(path_item) + except ImportError: + importer = None + return importer + + +def iter_importers(fullname=""): + """Yield PEP 302 importers for the given module name + + If fullname contains a '.', the importers will be for the package + containing fullname, otherwise they will be importers for sys.meta_path, + sys.path, and Python's "classic" import machinery, in that order. If + the named module is in a package, that package is imported as a side + effect of invoking this function. + + Non PEP 302 mechanisms (e.g. the Windows registry) used by the + standard import machinery to find files in alternative locations + are partially supported, but are searched AFTER sys.path. Normally, + these locations are searched BEFORE sys.path, preventing sys.path + entries from shadowing them. + + For this to cause a visible difference in behaviour, there must + be a module or package name that is accessible via both sys.path + and one of the non PEP 302 file system mechanisms. In this case, + the emulation will find the former version, while the builtin + import mechanism will find the latter. + + Items of the following types can be affected by this discrepancy: + imp.C_EXTENSION, imp.PY_SOURCE, imp.PY_COMPILED, imp.PKG_DIRECTORY + """ + if fullname.startswith('.'): + raise ImportError("Relative module names not supported") + if '.' in fullname: + # Get the containing package's __path__ + pkg = '.'.join(fullname.split('.')[:-1]) + if pkg not in sys.modules: + __import__(pkg) + path = getattr(sys.modules[pkg], '__path__', None) or [] + else: + for importer in sys.meta_path: + yield importer + path = sys.path + for item in path: + yield get_importer(item) + if '.' not in fullname: + yield ImpImporter() + +def get_loader(module_or_name): + """Get a PEP 302 "loader" object for module_or_name + + If the module or package is accessible via the normal import + mechanism, a wrapper around the relevant part of that machinery + is returned. Returns None if the module cannot be found or imported. + If the named module is not already imported, its containing package + (if any) is imported, in order to establish the package __path__. + + This function uses iter_importers(), and is thus subject to the same + limitations regarding platform-specific special import locations such + as the Windows registry. + """ + if module_or_name in sys.modules: + module_or_name = sys.modules[module_or_name] + if isinstance(module_or_name, ModuleType): + module = module_or_name + loader = getattr(module, '__loader__', None) + if loader is not None: + return loader + fullname = module.__name__ + else: + fullname = module_or_name + return find_loader(fullname) + +def find_loader(fullname): + """Find a PEP 302 "loader" object for fullname + + If fullname contains dots, path must be the containing package's __path__. + Returns None if the module cannot be found or imported. This function uses + iter_importers(), and is thus subject to the same limitations regarding + platform-specific special import locations such as the Windows registry. + """ + for importer in iter_importers(fullname): + loader = importer.find_module(fullname) + if loader is not None: + return loader + + return None + + +def extend_path(path, name): + """Extend a package's path. + + Intended use is to place the following code in a package's __init__.py: + + from pkgutil import extend_path + __path__ = extend_path(__path__, __name__) + + This will add to the package's __path__ all subdirectories of + directories on sys.path named after the package. This is useful + if one wants to distribute different parts of a single logical + package as multiple directories. + + It also looks for *.pkg files beginning where * matches the name + argument. This feature is similar to *.pth files (see site.py), + except that it doesn't special-case lines starting with 'import'. + A *.pkg file is trusted at face value: apart from checking for + duplicates, all entries found in a *.pkg file are added to the + path, regardless of whether they are exist the filesystem. (This + is a feature.) + + If the input path is not a list (as is the case for frozen + packages) it is returned unchanged. The input path is not + modified; an extended copy is returned. Items are only appended + to the copy at the end. + + It is assumed that sys.path is a sequence. Items of sys.path that + are not (unicode or 8-bit) strings referring to existing + directories are ignored. Unicode items of sys.path that cause + errors when used as filenames may cause this function to raise an + exception (in line with os.path.isdir() behavior). + """ + + if not isinstance(path, list): + # This could happen e.g. when this is called from inside a + # frozen package. Return the path unchanged in that case. + return path + + pname = os.path.join(*name.split('.')) # Reconstitute as relative path + # Just in case os.extsep != '.' + sname = os.extsep.join(name.split('.')) + sname_pkg = sname + os.extsep + "pkg" + init_py = "__init__" + os.extsep + "py" + + path = path[:] # Start with a copy of the existing path + + for dir in sys.path: + if not isinstance(dir, basestring) or not os.path.isdir(dir): + continue + subdir = os.path.join(dir, pname) + # XXX This may still add duplicate entries to path on + # case-insensitive filesystems + initfile = os.path.join(subdir, init_py) + if subdir not in path and os.path.isfile(initfile): + path.append(subdir) + # XXX Is this the right thing for subpackages like zope.app? + # It looks for a file named "zope.app.pkg" + pkgfile = os.path.join(dir, sname_pkg) + if os.path.isfile(pkgfile): + try: + f = open(pkgfile) + except IOError, msg: + sys.stderr.write("Can't open %s: %s\n" % + (pkgfile, msg)) + else: + for line in f: + line = line.rstrip('\n') + if not line or line.startswith('#'): + continue + path.append(line) # Don't check for existence! + f.close() + + return path + +def get_data(package, resource): + """Get a resource from a package. + + This is a wrapper round the PEP 302 loader get_data API. The package + argument should be the name of a package, in standard module format + (foo.bar). The resource argument should be in the form of a relative + filename, using '/' as the path separator. The parent directory name '..' + is not allowed, and nor is a rooted name (starting with a '/'). + + The function returns a binary string, which is the contents of the + specified resource. + + For packages located in the filesystem, which have already been imported, + this is the rough equivalent of + + d = os.path.dirname(sys.modules[package].__file__) + data = open(os.path.join(d, resource), 'rb').read() + + If the package cannot be located or loaded, or it uses a PEP 302 loader + which does not support get_data(), then None is returned. + """ + + loader = get_loader(package) + if loader is None or not hasattr(loader, 'get_data'): + return None + mod = sys.modules.get(package) or loader.load_module(package) + if mod is None or not hasattr(mod, '__file__'): + return None + + # Modify the resource name to be compatible with the loader.get_data + # signature - an os.path format "filename" starting with the dirname of + # the package's __file__ + parts = resource.split('/') + parts.insert(0, os.path.dirname(mod.__file__)) + resource_name = os.path.join(*parts) + return loader.get_data(resource_name) diff --git a/mitogen-0.2.7/mitogen/compat/tokenize.py b/mitogen-0.2.7/mitogen/compat/tokenize.py new file mode 100644 index 000000000..0473c6a --- /dev/null +++ b/mitogen-0.2.7/mitogen/compat/tokenize.py @@ -0,0 +1,453 @@ +"""Tokenization help for Python programs. + +generate_tokens(readline) is a generator that breaks a stream of +text into Python tokens. It accepts a readline-like method which is called +repeatedly to get the next line of input (or "" for EOF). It generates +5-tuples with these members: + + the token type (see token.py) + the token (a string) + the starting (row, column) indices of the token (a 2-tuple of ints) + the ending (row, column) indices of the token (a 2-tuple of ints) + the original line (string) + +It is designed to match the working of the Python tokenizer exactly, except +that it produces COMMENT tokens for comments and gives type OP for all +operators + +Older entry points + tokenize_loop(readline, tokeneater) + tokenize(readline, tokeneater=printtoken) +are the same, except instead of generating tokens, tokeneater is a callback +function to which the 5 fields described above are passed as 5 arguments, +each time a new token is found.""" + +# !mitogen: minify_safe + +__author__ = 'Ka-Ping Yee ' +__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' + 'Skip Montanaro, Raymond Hettinger') + +from itertools import chain +import string, re +from token import * + +import token +__all__ = [x for x in dir(token) if not x.startswith("_")] +__all__ += ["COMMENT", "tokenize", "generate_tokens", "NL", "untokenize"] +del token + +COMMENT = N_TOKENS +tok_name[COMMENT] = 'COMMENT' +NL = N_TOKENS + 1 +tok_name[NL] = 'NL' +N_TOKENS += 2 + +def group(*choices): return '(' + '|'.join(choices) + ')' +def any(*choices): return group(*choices) + '*' +def maybe(*choices): return group(*choices) + '?' + +Whitespace = r'[ \f\t]*' +Comment = r'#[^\r\n]*' +Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment) +Name = r'[a-zA-Z_]\w*' + +Hexnumber = r'0[xX][\da-fA-F]+[lL]?' +Octnumber = r'(0[oO][0-7]+)|(0[0-7]*)[lL]?' +Binnumber = r'0[bB][01]+[lL]?' +Decnumber = r'[1-9]\d*[lL]?' +Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber) +Exponent = r'[eE][-+]?\d+' +Pointfloat = group(r'\d+\.\d*', r'\.\d+') + maybe(Exponent) +Expfloat = r'\d+' + Exponent +Floatnumber = group(Pointfloat, Expfloat) +Imagnumber = group(r'\d+[jJ]', Floatnumber + r'[jJ]') +Number = group(Imagnumber, Floatnumber, Intnumber) + +# Tail end of ' string. +Single = r"[^'\\]*(?:\\.[^'\\]*)*'" +# Tail end of " string. +Double = r'[^"\\]*(?:\\.[^"\\]*)*"' +# Tail end of ''' string. +Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''" +# Tail end of """ string. +Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""' +Triple = group("[uUbB]?[rR]?'''", '[uUbB]?[rR]?"""') +# Single-line ' or " string. +String = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*'", + r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*"') + +# Because of leftmost-then-longest match semantics, be sure to put the +# longest operators first (e.g., if = came before ==, == would get +# recognized as two instances of =). +Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"<>", r"!=", + r"//=?", + r"[+\-*/%&|^=<>]=?", + r"~") + +Bracket = '[][(){}]' +Special = group(r'\r?\n', r'[:;.,`@]') +Funny = group(Operator, Bracket, Special) + +PlainToken = group(Number, Funny, String, Name) +Token = Ignore + PlainToken + +# First (or only) line of ' or " string. +ContStr = group(r"[uUbB]?[rR]?'[^\n'\\]*(?:\\.[^\n'\\]*)*" + + group("'", r'\\\r?\n'), + r'[uUbB]?[rR]?"[^\n"\\]*(?:\\.[^\n"\\]*)*' + + group('"', r'\\\r?\n')) +PseudoExtras = group(r'\\\r?\n|\Z', Comment, Triple) +PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name) + +tokenprog, pseudoprog, single3prog, double3prog = map( + re.compile, (Token, PseudoToken, Single3, Double3)) +endprogs = {"'": re.compile(Single), '"': re.compile(Double), + "'''": single3prog, '"""': double3prog, + "r'''": single3prog, 'r"""': double3prog, + "u'''": single3prog, 'u"""': double3prog, + "ur'''": single3prog, 'ur"""': double3prog, + "R'''": single3prog, 'R"""': double3prog, + "U'''": single3prog, 'U"""': double3prog, + "uR'''": single3prog, 'uR"""': double3prog, + "Ur'''": single3prog, 'Ur"""': double3prog, + "UR'''": single3prog, 'UR"""': double3prog, + "b'''": single3prog, 'b"""': double3prog, + "br'''": single3prog, 'br"""': double3prog, + "B'''": single3prog, 'B"""': double3prog, + "bR'''": single3prog, 'bR"""': double3prog, + "Br'''": single3prog, 'Br"""': double3prog, + "BR'''": single3prog, 'BR"""': double3prog, + 'r': None, 'R': None, 'u': None, 'U': None, + 'b': None, 'B': None} + +triple_quoted = {} +for t in ("'''", '"""', + "r'''", 'r"""', "R'''", 'R"""', + "u'''", 'u"""', "U'''", 'U"""', + "ur'''", 'ur"""', "Ur'''", 'Ur"""', + "uR'''", 'uR"""', "UR'''", 'UR"""', + "b'''", 'b"""', "B'''", 'B"""', + "br'''", 'br"""', "Br'''", 'Br"""', + "bR'''", 'bR"""', "BR'''", 'BR"""'): + triple_quoted[t] = t +single_quoted = {} +for t in ("'", '"', + "r'", 'r"', "R'", 'R"', + "u'", 'u"', "U'", 'U"', + "ur'", 'ur"', "Ur'", 'Ur"', + "uR'", 'uR"', "UR'", 'UR"', + "b'", 'b"', "B'", 'B"', + "br'", 'br"', "Br'", 'Br"', + "bR'", 'bR"', "BR'", 'BR"' ): + single_quoted[t] = t + +tabsize = 8 + +class TokenError(Exception): pass + +class StopTokenizing(Exception): pass + +def printtoken(type, token, srow_scol, erow_ecol, line): # for testing + srow, scol = srow_scol + erow, ecol = erow_ecol + print("%d,%d-%d,%d:\t%s\t%s" % \ + (srow, scol, erow, ecol, tok_name[type], repr(token))) + +def tokenize(readline, tokeneater=printtoken): + """ + The tokenize() function accepts two parameters: one representing the + input stream, and one providing an output mechanism for tokenize(). + + The first parameter, readline, must be a callable object which provides + the same interface as the readline() method of built-in file objects. + Each call to the function should return one line of input as a string. + + The second parameter, tokeneater, must also be a callable object. It is + called once for each token, with five arguments, corresponding to the + tuples generated by generate_tokens(). + """ + try: + tokenize_loop(readline, tokeneater) + except StopTokenizing: + pass + +# backwards compatible interface +def tokenize_loop(readline, tokeneater): + for token_info in generate_tokens(readline): + tokeneater(*token_info) + +class Untokenizer: + + def __init__(self): + self.tokens = [] + self.prev_row = 1 + self.prev_col = 0 + + def add_whitespace(self, start): + row, col = start + if row < self.prev_row or row == self.prev_row and col < self.prev_col: + raise ValueError("start ({},{}) precedes previous end ({},{})" + .format(row, col, self.prev_row, self.prev_col)) + row_offset = row - self.prev_row + if row_offset: + self.tokens.append("\\\n" * row_offset) + self.prev_col = 0 + col_offset = col - self.prev_col + if col_offset: + self.tokens.append(" " * col_offset) + + def untokenize(self, iterable): + it = iter(iterable) + indents = [] + startline = False + for t in it: + if len(t) == 2: + self.compat(t, it) + break + tok_type, token, start, end, line = t + if tok_type == ENDMARKER: + break + if tok_type == INDENT: + indents.append(token) + continue + elif tok_type == DEDENT: + indents.pop() + self.prev_row, self.prev_col = end + continue + elif tok_type in (NEWLINE, NL): + startline = True + elif startline and indents: + indent = indents[-1] + if start[1] >= len(indent): + self.tokens.append(indent) + self.prev_col = len(indent) + startline = False + self.add_whitespace(start) + self.tokens.append(token) + self.prev_row, self.prev_col = end + if tok_type in (NEWLINE, NL): + self.prev_row += 1 + self.prev_col = 0 + return "".join(self.tokens) + + def compat(self, token, iterable): + indents = [] + toks_append = self.tokens.append + startline = token[0] in (NEWLINE, NL) + prevstring = False + + for tok in chain([token], iterable): + toknum, tokval = tok[:2] + + if toknum in (NAME, NUMBER): + tokval += ' ' + + # Insert a space between two consecutive strings + if toknum == STRING: + if prevstring: + tokval = ' ' + tokval + prevstring = True + else: + prevstring = False + + if toknum == INDENT: + indents.append(tokval) + continue + elif toknum == DEDENT: + indents.pop() + continue + elif toknum in (NEWLINE, NL): + startline = True + elif startline and indents: + toks_append(indents[-1]) + startline = False + toks_append(tokval) + +def untokenize(iterable): + """Transform tokens back into Python source code. + + Each element returned by the iterable must be a token sequence + with at least two elements, a token number and token value. If + only two tokens are passed, the resulting output is poor. + + Round-trip invariant for full input: + Untokenized source will match input source exactly + + Round-trip invariant for limited intput: + # Output text will tokenize the back to the input + t1 = [tok[:2] for tok in generate_tokens(f.readline)] + newcode = untokenize(t1) + readline = iter(newcode.splitlines(1)).next + t2 = [tok[:2] for tok in generate_tokens(readline)] + assert t1 == t2 + """ + ut = Untokenizer() + return ut.untokenize(iterable) + +def generate_tokens(readline): + """ + The generate_tokens() generator requires one argument, readline, which + must be a callable object which provides the same interface as the + readline() method of built-in file objects. Each call to the function + should return one line of input as a string. Alternately, readline + can be a callable function terminating with StopIteration: + readline = open(myfile).next # Example of alternate readline + + The generator produces 5-tuples with these members: the token type; the + token string; a 2-tuple (srow, scol) of ints specifying the row and + column where the token begins in the source; a 2-tuple (erow, ecol) of + ints specifying the row and column where the token ends in the source; + and the line on which the token was found. The line passed is the + logical line; continuation lines are included. + """ + lnum = parenlev = continued = 0 + namechars, numchars = string.ascii_letters + '_', '0123456789' + contstr, needcont = '', 0 + contline = None + indents = [0] + + while 1: # loop over lines in stream + try: + line = readline() + except StopIteration: + line = '' + lnum += 1 + pos, max = 0, len(line) + + if contstr: # continued string + if not line: + raise TokenError("EOF in multi-line string", strstart) + endmatch = endprog.match(line) + if endmatch: + pos = end = endmatch.end(0) + yield (STRING, contstr + line[:end], + strstart, (lnum, end), contline + line) + contstr, needcont = '', 0 + contline = None + elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n': + yield (ERRORTOKEN, contstr + line, + strstart, (lnum, len(line)), contline) + contstr = '' + contline = None + continue + else: + contstr = contstr + line + contline = contline + line + continue + + elif parenlev == 0 and not continued: # new statement + if not line: break + column = 0 + while pos < max: # measure leading whitespace + if line[pos] == ' ': + column += 1 + elif line[pos] == '\t': + column = (column//tabsize + 1)*tabsize + elif line[pos] == '\f': + column = 0 + else: + break + pos += 1 + if pos == max: + break + + if line[pos] in '#\r\n': # skip comments or blank lines + if line[pos] == '#': + comment_token = line[pos:].rstrip('\r\n') + nl_pos = pos + len(comment_token) + yield (COMMENT, comment_token, + (lnum, pos), (lnum, pos + len(comment_token)), line) + yield (NL, line[nl_pos:], + (lnum, nl_pos), (lnum, len(line)), line) + else: + yield ((NL, COMMENT)[line[pos] == '#'], line[pos:], + (lnum, pos), (lnum, len(line)), line) + continue + + if column > indents[-1]: # count indents or dedents + indents.append(column) + yield (INDENT, line[:pos], (lnum, 0), (lnum, pos), line) + while column < indents[-1]: + if column not in indents: + raise IndentationError( + "unindent does not match any outer indentation level", + ("", lnum, pos, line)) + indents = indents[:-1] + yield (DEDENT, '', (lnum, pos), (lnum, pos), line) + + else: # continued statement + if not line: + raise TokenError("EOF in multi-line statement", (lnum, 0)) + continued = 0 + + while pos < max: + pseudomatch = pseudoprog.match(line, pos) + if pseudomatch: # scan for tokens + start, end = pseudomatch.span(1) + spos, epos, pos = (lnum, start), (lnum, end), end + if start == end: + continue + token, initial = line[start:end], line[start] + + if initial in numchars or \ + (initial == '.' and token != '.'): # ordinary number + yield (NUMBER, token, spos, epos, line) + elif initial in '\r\n': + if parenlev > 0: + n = NL + else: + n = NEWLINE + yield (n, token, spos, epos, line) + elif initial == '#': + assert not token.endswith("\n") + yield (COMMENT, token, spos, epos, line) + elif token in triple_quoted: + endprog = endprogs[token] + endmatch = endprog.match(line, pos) + if endmatch: # all on one line + pos = endmatch.end(0) + token = line[start:pos] + yield (STRING, token, spos, (lnum, pos), line) + else: + strstart = (lnum, start) # multiple lines + contstr = line[start:] + contline = line + break + elif initial in single_quoted or \ + token[:2] in single_quoted or \ + token[:3] in single_quoted: + if token[-1] == '\n': # continued string + strstart = (lnum, start) + endprog = (endprogs[initial] or endprogs[token[1]] or + endprogs[token[2]]) + contstr, needcont = line[start:], 1 + contline = line + break + else: # ordinary string + yield (STRING, token, spos, epos, line) + elif initial in namechars: # ordinary name + yield (NAME, token, spos, epos, line) + elif initial == '\\': # continued stmt + continued = 1 + else: + if initial in '([{': + parenlev += 1 + elif initial in ')]}': + parenlev -= 1 + yield (OP, token, spos, epos, line) + else: + yield (ERRORTOKEN, line[pos], + (lnum, pos), (lnum, pos+1), line) + pos += 1 + + for indent in indents[1:]: # pop remaining indent levels + yield (DEDENT, '', (lnum, 0), (lnum, 0), '') + yield (ENDMARKER, '', (lnum, 0), (lnum, 0), '') + +if __name__ == '__main__': # testing + import sys + if len(sys.argv) > 1: + tokenize(open(sys.argv[1]).readline) + else: + tokenize(sys.stdin.readline) diff --git a/mitogen-0.2.7/mitogen/core.py b/mitogen-0.2.7/mitogen/core.py new file mode 100644 index 000000000..ff77bba --- /dev/null +++ b/mitogen-0.2.7/mitogen/core.py @@ -0,0 +1,3409 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +""" +This module implements most package functionality, but remains separate from +non-essential code in order to reduce its size, since it is also serves as the +bootstrap implementation sent to every new slave context. +""" + +import binascii +import collections +import encodings.latin_1 +import errno +import fcntl +import itertools +import linecache +import logging +import os +import pickle as py_pickle +import pstats +import signal +import socket +import struct +import sys +import threading +import time +import traceback +import warnings +import weakref +import zlib + +# Python >3.7 deprecated the imp module. +warnings.filterwarnings('ignore', message='the imp module is deprecated') +import imp + +# Absolute imports for <2.5. +select = __import__('select') + +try: + import cProfile +except ImportError: + cProfile = None + +try: + import thread +except ImportError: + import threading as thread + +try: + import cPickle as pickle +except ImportError: + import pickle + +try: + from cStringIO import StringIO as BytesIO +except ImportError: + from io import BytesIO + +try: + BaseException +except NameError: + BaseException = Exception + +try: + ModuleNotFoundError +except NameError: + ModuleNotFoundError = ImportError + +# TODO: usage of 'import' after setting __name__, but before fixing up +# sys.modules generates a warning. This happens when profiling = True. +warnings.filterwarnings('ignore', + "Parent module 'mitogen' not found while handling absolute import") + +LOG = logging.getLogger('mitogen') +IOLOG = logging.getLogger('mitogen.io') +IOLOG.setLevel(logging.INFO) + +LATIN1_CODEC = encodings.latin_1.Codec() +# str.encode() may take import lock. Deadlock possible if broker calls +# .encode() on behalf of thread currently waiting for module. +UTF8_CODEC = encodings.latin_1.Codec() + +_v = False +_vv = False + +GET_MODULE = 100 +CALL_FUNCTION = 101 +FORWARD_LOG = 102 +ADD_ROUTE = 103 +DEL_ROUTE = 104 +ALLOCATE_ID = 105 +SHUTDOWN = 106 +LOAD_MODULE = 107 +FORWARD_MODULE = 108 +DETACHING = 109 +CALL_SERVICE = 110 + +#: Special value used to signal disconnection or the inability to route a +#: message, when it appears in the `reply_to` field. Usually causes +#: :class:`mitogen.core.ChannelError` to be raised when it is received. +#: +#: It indicates the sender did not know how to process the message, or wishes +#: no further messages to be delivered to it. It is used when: +#: +#: * a remote receiver is disconnected or explicitly closed. +#: * a related message could not be delivered due to no route existing for it. +#: * a router is being torn down, as a sentinel value to notify +#: :meth:`mitogen.core.Router.add_handler` callbacks to clean up. +IS_DEAD = 999 + +try: + BaseException +except NameError: + BaseException = Exception + +PY24 = sys.version_info < (2, 5) +PY3 = sys.version_info > (3,) +if PY3: + b = str.encode + BytesType = bytes + UnicodeType = str + FsPathTypes = (str,) + BufferType = lambda buf, start: memoryview(buf)[start:] + long = int +else: + b = str + BytesType = str + FsPathTypes = (str, unicode) + BufferType = buffer + UnicodeType = unicode + +AnyTextType = (BytesType, UnicodeType) + +try: + next +except NameError: + next = lambda it: it.next() + +# #550: prehistoric WSL did not advertise itself in uname output. +try: + fp = open('/proc/sys/kernel/osrelease') + IS_WSL = 'Microsoft' in fp.read() + fp.close() +except IOError: + IS_WSL = False + + +#: Default size for calls to :meth:`Side.read` or :meth:`Side.write`, and the +#: size of buffers configured by :func:`mitogen.parent.create_socketpair`. This +#: value has many performance implications, 128KiB seems to be a sweet spot. +#: +#: * When set low, large messages cause many :class:`Broker` IO loop +#: iterations, burning CPU and reducing throughput. +#: * When set high, excessive RAM is reserved by the OS for socket buffers (2x +#: per child), and an identically sized temporary userspace buffer is +#: allocated on each read that requires zeroing, and over a particular size +#: may require two system calls to allocate/deallocate. +#: +#: Care must be taken to ensure the underlying kernel object and receiving +#: program support the desired size. For example, +#: +#: * Most UNIXes have TTYs with fixed 2KiB-4KiB buffers, making them unsuitable +#: for efficient IO. +#: * Different UNIXes have varying presets for pipes, which may not be +#: configurable. On recent Linux the default pipe buffer size is 64KiB, but +#: under memory pressure may be as low as 4KiB for unprivileged processes. +#: * When communication is via an intermediary process, its internal buffers +#: effect the speed OS buffers will drain. For example OpenSSH uses 64KiB +#: reads. +#: +#: An ideal :class:`Message` has a size that is a multiple of +#: :data:`CHUNK_SIZE` inclusive of headers, to avoid wasting IO loop iterations +#: writing small trailer chunks. +CHUNK_SIZE = 131072 + +_tls = threading.local() + + +if __name__ == 'mitogen.core': + # When loaded using import mechanism, ExternalContext.main() will not have + # a chance to set the synthetic mitogen global, so just import it here. + import mitogen +else: + # When loaded as __main__, ensure classes and functions gain a __module__ + # attribute consistent with the host process, so that pickling succeeds. + __name__ = 'mitogen.core' + + +class Error(Exception): + """Base for all exceptions raised by Mitogen. + + :param str fmt: + Exception text, or format string if `args` is non-empty. + :param tuple args: + Format string arguments. + """ + def __init__(self, fmt=None, *args): + if args: + fmt %= args + if fmt and not isinstance(fmt, UnicodeType): + fmt = fmt.decode('utf-8') + Exception.__init__(self, fmt) + + +class LatchError(Error): + """Raised when an attempt is made to use a :class:`mitogen.core.Latch` + that has been marked closed.""" + pass + + +class Blob(BytesType): + """A serializable bytes subclass whose content is summarized in repr() + output, making it suitable for logging binary data.""" + def __repr__(self): + return '[blob: %d bytes]' % len(self) + + def __reduce__(self): + return (Blob, (BytesType(self),)) + + +class Secret(UnicodeType): + """A serializable unicode subclass whose content is masked in repr() + output, making it suitable for logging passwords.""" + def __repr__(self): + return '[secret]' + + if not PY3: + # TODO: what is this needed for in 2.x? + def __str__(self): + return UnicodeType(self) + + def __reduce__(self): + return (Secret, (UnicodeType(self),)) + + +class Kwargs(dict): + """ + A serializable dict subclass that indicates its keys should be coerced to + Unicode on Python 3 and bytes on Python<2.6. + + Python 2 produces keyword argument dicts whose keys are bytes, requiring a + helper to ensure compatibility with Python 3 where Unicode is required, + whereas Python 3 produces keyword argument dicts whose keys are Unicode, + requiring a helper for Python 2.4/2.5, where bytes are required. + """ + if PY3: + def __init__(self, dct): + for k, v in dct.items(): + if type(k) is bytes: + self[k.decode()] = v + else: + self[k] = v + elif sys.version_info < (2, 6, 5): + def __init__(self, dct): + for k, v in dct.iteritems(): + if type(k) is unicode: + k, _ = UTF8_CODEC.encode(k) + self[k] = v + + def __repr__(self): + return 'Kwargs(%s)' % (dict.__repr__(self),) + + def __reduce__(self): + return (Kwargs, (dict(self),)) + + +class CallError(Error): + """ + Serializable :class:`Error` subclass raised when :meth:`Context.call() + ` fails. A copy of the traceback from the + external context is appended to the exception message. + """ + def __init__(self, fmt=None, *args): + if not isinstance(fmt, BaseException): + Error.__init__(self, fmt, *args) + else: + e = fmt + cls = e.__class__ + fmt = '%s.%s: %s' % (cls.__module__, cls.__name__, e) + tb = sys.exc_info()[2] + if tb: + fmt += '\n' + fmt += ''.join(traceback.format_tb(tb)) + Error.__init__(self, fmt) + + def __reduce__(self): + return (_unpickle_call_error, (self.args[0],)) + + +def _unpickle_call_error(s): + if not (type(s) is UnicodeType and len(s) < 10000): + raise TypeError('cannot unpickle CallError: bad input') + return CallError(s) + + +class ChannelError(Error): + """Raised when a channel dies or has been closed.""" + remote_msg = 'Channel closed by remote end.' + local_msg = 'Channel closed by local end.' + + +class StreamError(Error): + """Raised when a stream cannot be established.""" + pass + + +class TimeoutError(Error): + """Raised when a timeout occurs on a stream.""" + pass + + +def to_text(o): + """Coerce `o` to Unicode by decoding it from UTF-8 if it is an instance of + :class:`bytes`, otherwise pass it to the :class:`str` constructor. The + returned object is always a plain :class:`str`, any subclass is removed.""" + if isinstance(o, BytesType): + return o.decode('utf-8') + return UnicodeType(o) + + +# Python 2.4 +try: + any +except NameError: + def any(it): + for elem in it: + if elem: + return True + + +def _partition(s, sep, find): + """ + (str|unicode).(partition|rpartition) for Python 2.4/2.5. + """ + idx = find(sep) + if idx != -1: + left = s[0:idx] + return left, sep, s[len(left)+len(sep):] + + +if hasattr(UnicodeType, 'rpartition'): + str_partition = UnicodeType.partition + str_rpartition = UnicodeType.rpartition + bytes_partition = BytesType.partition +else: + def str_partition(s, sep): + return _partition(s, sep, s.find) or (s, u'', u'') + def str_rpartition(s, sep): + return _partition(s, sep, s.rfind) or (u'', u'', s) + def bytes_partition(s, sep): + return _partition(s, sep, s.find) or (s, '', '') + + +def has_parent_authority(msg, _stream=None): + """Policy function for use with :class:`Receiver` and + :meth:`Router.add_handler` that requires incoming messages to originate + from a parent context, or on a :class:`Stream` whose :attr:`auth_id + ` has been set to that of a parent context or the current + context.""" + return (msg.auth_id == mitogen.context_id or + msg.auth_id in mitogen.parent_ids) + + +def listen(obj, name, func): + """ + Arrange for `func(*args, **kwargs)` to be invoked when the named signal is + fired by `obj`. + """ + signals = vars(obj).setdefault('_signals', {}) + signals.setdefault(name, []).append(func) + + +def fire(obj, name, *args, **kwargs): + """ + Arrange for `func(*args, **kwargs)` to be invoked for every function + registered for the named signal on `obj`. + """ + signals = vars(obj).get('_signals', {}) + for func in signals.get(name, ()): + func(*args, **kwargs) + + +def takes_econtext(func): + func.mitogen_takes_econtext = True + return func + + +def takes_router(func): + func.mitogen_takes_router = True + return func + + +def is_blacklisted_import(importer, fullname): + """ + Return :data:`True` if `fullname` is part of a blacklisted package, or if + any packages have been whitelisted and `fullname` is not part of one. + + NB: + - If a package is on both lists, then it is treated as blacklisted. + - If any package is whitelisted, then all non-whitelisted packages are + treated as blacklisted. + """ + return ((not any(fullname.startswith(s) for s in importer.whitelist)) or + (any(fullname.startswith(s) for s in importer.blacklist))) + + +def set_cloexec(fd): + """Set the file descriptor `fd` to automatically close on + :func:`os.execve`. This has no effect on file descriptors inherited across + :func:`os.fork`, they must be explicitly closed through some other means, + such as :func:`mitogen.fork.on_fork`.""" + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + assert fd > 2 + fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) + + +def set_nonblock(fd): + """Set the file descriptor `fd` to non-blocking mode. For most underlying + file types, this causes :func:`os.read` or :func:`os.write` to raise + :class:`OSError` with :data:`errno.EAGAIN` rather than block the thread + when the underlying kernel buffer is exhausted.""" + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + + +def set_block(fd): + """Inverse of :func:`set_nonblock`, i.e. cause `fd` to block the thread + when the underlying kernel buffer is exhausted.""" + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, flags & ~os.O_NONBLOCK) + + +def io_op(func, *args): + """Wrap `func(*args)` that may raise :class:`select.error`, + :class:`IOError`, or :class:`OSError`, trapping UNIX error codes relating + to disconnection and retry events in various subsystems: + + * When a signal is delivered to the process on Python 2, system call retry + is signalled through :data:`errno.EINTR`. The invocation is automatically + restarted. + * When performing IO against a TTY, disconnection of the remote end is + signalled by :data:`errno.EIO`. + * When performing IO against a socket, disconnection of the remote end is + signalled by :data:`errno.ECONNRESET`. + * When performing IO against a pipe, disconnection of the remote end is + signalled by :data:`errno.EPIPE`. + + :returns: + Tuple of `(return_value, disconnect_reason)`, where `return_value` is + the return value of `func(*args)`, and `disconnected` is an exception + instance when disconnection was detected, otherwise :data:`None`. + """ + while True: + try: + return func(*args), None + except (select.error, OSError, IOError): + e = sys.exc_info()[1] + _vv and IOLOG.debug('io_op(%r) -> OSError: %s', func, e) + if e.args[0] == errno.EINTR: + continue + if e.args[0] in (errno.EIO, errno.ECONNRESET, errno.EPIPE): + return None, e + raise + + +class PidfulStreamHandler(logging.StreamHandler): + """A :class:`logging.StreamHandler` subclass used when + :meth:`Router.enable_debug() ` has been + called, or the `debug` parameter was specified during context construction. + Verifies the process ID has not changed on each call to :meth:`emit`, + reopening the associated log file when a change is detected. + + This ensures logging to the per-process output files happens correctly even + when uncooperative third party components call :func:`os.fork`. + """ + #: PID that last opened the log file. + open_pid = None + + #: Output path template. + template = '/tmp/mitogen.%s.%s.log' + + def _reopen(self): + self.acquire() + try: + if self.open_pid == os.getpid(): + return + ts = time.strftime('%Y%m%d_%H%M%S') + path = self.template % (os.getpid(), ts) + self.stream = open(path, 'w', 1) + set_cloexec(self.stream.fileno()) + self.stream.write('Parent PID: %s\n' % (os.getppid(),)) + self.stream.write('Created by:\n\n%s\n' % ( + ''.join(traceback.format_stack()), + )) + self.open_pid = os.getpid() + finally: + self.release() + + def emit(self, record): + if self.open_pid != os.getpid(): + self._reopen() + logging.StreamHandler.emit(self, record) + + +def enable_debug_logging(): + global _v, _vv + _v = True + _vv = True + root = logging.getLogger() + root.setLevel(logging.DEBUG) + IOLOG.setLevel(logging.DEBUG) + handler = PidfulStreamHandler() + handler.formatter = logging.Formatter( + '%(asctime)s %(levelname).1s %(name)s: %(message)s', + '%H:%M:%S' + ) + root.handlers.insert(0, handler) + + +_profile_hook = lambda name, func, *args: func(*args) +_profile_fmt = os.environ.get( + 'MITOGEN_PROFILE_FMT', + '/tmp/mitogen.stats.%(pid)s.%(identity)s.%(now)s.%(ext)s', +) + + +def _profile_hook(name, func, *args): + """ + Call `func(*args)` and return its result. This function is replaced by + :func:`_real_profile_hook` when :func:`enable_profiling` is called. This + interface is obsolete and will be replaced by a signals-based integration + later on. + """ + return func(*args) + + +def _real_profile_hook(name, func, *args): + profiler = cProfile.Profile() + profiler.enable() + try: + return func(*args) + finally: + path = _profile_fmt % { + 'now': int(1e6 * time.time()), + 'identity': name, + 'pid': os.getpid(), + 'ext': '%s' + } + profiler.dump_stats(path % ('pstats',)) + profiler.create_stats() + fp = open(path % ('log',), 'w') + try: + stats = pstats.Stats(profiler, stream=fp) + stats.sort_stats('cumulative') + stats.print_stats() + finally: + fp.close() + + +def enable_profiling(econtext=None): + global _profile_hook + _profile_hook = _real_profile_hook + + +def import_module(modname): + """ + Import `module` and return the attribute named `attr`. + """ + return __import__(modname, None, None, ['']) + + +class Py24Pickler(py_pickle.Pickler): + """ + Exceptions were classic classes until Python 2.5. Sadly for 2.4, cPickle + offers little control over how a classic instance is pickled. Therefore 2.4 + uses a pure-Python pickler, so CallError can be made to look as it does on + newer Pythons. + + This mess will go away once proper serialization exists. + """ + @classmethod + def dumps(cls, obj, protocol): + bio = BytesIO() + self = cls(bio, protocol=protocol) + self.dump(obj) + return bio.getvalue() + + def save_exc_inst(self, obj): + if isinstance(obj, CallError): + func, args = obj.__reduce__() + self.save(func) + self.save(args) + self.write(py_pickle.REDUCE) + else: + py_pickle.Pickler.save_inst(self, obj) + + if PY24: + dispatch = py_pickle.Pickler.dispatch.copy() + dispatch[py_pickle.InstanceType] = save_exc_inst + + +if PY3: + # In 3.x Unpickler is a class exposing find_class as an overridable, but it + # cannot be overridden without subclassing. + class _Unpickler(pickle.Unpickler): + def find_class(self, module, func): + return self.find_global(module, func) + pickle__dumps = pickle.dumps +elif PY24: + # On Python 2.4, we must use a pure-Python pickler. + pickle__dumps = Py24Pickler.dumps + _Unpickler = pickle.Unpickler +else: + pickle__dumps = pickle.dumps + # In 2.x Unpickler is a function exposing a writeable find_global + # attribute. + _Unpickler = pickle.Unpickler + + +class Message(object): + """ + Messages are the fundamental unit of communication, comprising fields from + the :ref:`stream-protocol` header, an optional reference to the receiving + :class:`mitogen.core.Router` for ingress messages, and helper methods for + deserialization and generating replies. + """ + #: Integer target context ID. :class:`Router` delivers messages locally + #: when their :attr:`dst_id` matches :data:`mitogen.context_id`, otherwise + #: they are routed up or downstream. + dst_id = None + + #: Integer source context ID. Used as the target of replies if any are + #: generated. + src_id = None + + #: Context ID under whose authority the message is acting. See + #: :ref:`source-verification`. + auth_id = None + + #: Integer target handle in the destination context. This is one of the + #: :ref:`standard-handles`, or a dynamically generated handle used to + #: receive a one-time reply, such as the return value of a function call. + handle = None + + #: Integer target handle to direct any reply to this message. Used to + #: receive a one-time reply, such as the return value of a function call. + #: :data:`IS_DEAD` has a special meaning when it appears in this field. + reply_to = None + + #: Raw message data bytes. + data = b('') + + _unpickled = object() + + #: The :class:`Router` responsible for routing the message. This is + #: :data:`None` for locally originated messages. + router = None + + #: The :class:`Receiver` over which the message was last received. Part of + #: the :class:`mitogen.select.Select` interface. Defaults to :data:`None`. + receiver = None + + def __init__(self, **kwargs): + """ + Construct a message from from the supplied `kwargs`. :attr:`src_id` and + :attr:`auth_id` are always set to :data:`mitogen.context_id`. + """ + self.src_id = mitogen.context_id + self.auth_id = mitogen.context_id + vars(self).update(kwargs) + assert isinstance(self.data, BytesType) + + def _unpickle_context(self, context_id, name): + return _unpickle_context(context_id, name, router=self.router) + + def _unpickle_sender(self, context_id, dst_handle): + return _unpickle_sender(self.router, context_id, dst_handle) + + def _unpickle_bytes(self, s, encoding): + s, n = LATIN1_CODEC.encode(s) + return s + + def _find_global(self, module, func): + """Return the class implementing `module_name.class_name` or raise + `StreamError` if the module is not whitelisted.""" + if module == __name__: + if func == '_unpickle_call_error' or func == 'CallError': + return _unpickle_call_error + elif func == '_unpickle_sender': + return self._unpickle_sender + elif func == '_unpickle_context': + return self._unpickle_context + elif func == 'Blob': + return Blob + elif func == 'Secret': + return Secret + elif func == 'Kwargs': + return Kwargs + elif module == '_codecs' and func == 'encode': + return self._unpickle_bytes + elif module == '__builtin__' and func == 'bytes': + return BytesType + raise StreamError('cannot unpickle %r/%r', module, func) + + @property + def is_dead(self): + """ + :data:`True` if :attr:`reply_to` is set to the magic value + :data:`IS_DEAD`, indicating the sender considers the channel dead. Dead + messages can be raised in a variety of circumstances, see + :data:`IS_DEAD` for more information. + """ + return self.reply_to == IS_DEAD + + @classmethod + def dead(cls, reason=None, **kwargs): + """ + Syntax helper to construct a dead message. + """ + kwargs['data'], _ = UTF8_CODEC.encode(reason or u'') + return cls(reply_to=IS_DEAD, **kwargs) + + @classmethod + def pickled(cls, obj, **kwargs): + """ + Construct a pickled message, setting :attr:`data` to the serialization + of `obj`, and setting remaining fields using `kwargs`. + + :returns: + The new message. + """ + self = cls(**kwargs) + try: + self.data = pickle__dumps(obj, protocol=2) + except pickle.PicklingError: + e = sys.exc_info()[1] + self.data = pickle__dumps(CallError(e), protocol=2) + return self + + def reply(self, msg, router=None, **kwargs): + """ + Compose a reply to this message and send it using :attr:`router`, or + `router` is :attr:`router` is :data:`None`. + + :param obj: + Either a :class:`Message`, or an object to be serialized in order + to construct a new message. + :param router: + Optional router to use if :attr:`router` is :data:`None`. + :param kwargs: + Optional keyword parameters overriding message fields in the reply. + """ + if not isinstance(msg, Message): + msg = Message.pickled(msg) + msg.dst_id = self.src_id + msg.handle = self.reply_to + vars(msg).update(kwargs) + if msg.handle: + (self.router or router).route(msg) + else: + LOG.debug('Message.reply(): discarding due to zero handle: %r', msg) + + if PY3: + UNPICKLER_KWARGS = {'encoding': 'bytes'} + else: + UNPICKLER_KWARGS = {} + + def _throw_dead(self): + if len(self.data): + raise ChannelError(self.data.decode('utf-8', 'replace')) + elif self.src_id == mitogen.context_id: + raise ChannelError(ChannelError.local_msg) + else: + raise ChannelError(ChannelError.remote_msg) + + def unpickle(self, throw=True, throw_dead=True): + """ + Unpickle :attr:`data`, optionally raising any exceptions present. + + :param bool throw_dead: + If :data:`True`, raise exceptions, otherwise it is the caller's + responsibility. + + :raises CallError: + The serialized data contained CallError exception. + :raises ChannelError: + The `is_dead` field was set. + """ + _vv and IOLOG.debug('%r.unpickle()', self) + if throw_dead and self.is_dead: + self._throw_dead() + + obj = self._unpickled + if obj is Message._unpickled: + fp = BytesIO(self.data) + unpickler = _Unpickler(fp, **self.UNPICKLER_KWARGS) + unpickler.find_global = self._find_global + try: + # Must occur off the broker thread. + obj = unpickler.load() + self._unpickled = obj + except (TypeError, ValueError): + e = sys.exc_info()[1] + raise StreamError('invalid message: %s', e) + + if throw: + if isinstance(obj, CallError): + raise obj + + return obj + + def __repr__(self): + return 'Message(%r, %r, %r, %r, %r, %r..%d)' % ( + self.dst_id, self.src_id, self.auth_id, self.handle, + self.reply_to, (self.data or '')[:50], len(self.data) + ) + + +class Sender(object): + """ + Senders are used to send pickled messages to a handle in another context, + it is the inverse of :class:`mitogen.core.Receiver`. + + Senders may be serialized, making them convenient to wire up data flows. + See :meth:`mitogen.core.Receiver.to_sender` for more information. + + :param Context context: + Context to send messages to. + :param int dst_handle: + Destination handle to send messages to. + """ + def __init__(self, context, dst_handle): + self.context = context + self.dst_handle = dst_handle + + def send(self, data): + """ + Send `data` to the remote end. + """ + _vv and IOLOG.debug('%r.send(%r..)', self, repr(data)[:100]) + self.context.send(Message.pickled(data, handle=self.dst_handle)) + + explicit_close_msg = 'Sender was explicitly closed' + + def close(self): + """ + Send a dead message to the remote, causing :meth:`ChannelError` to be + raised in any waiting thread. + """ + _vv and IOLOG.debug('%r.close()', self) + self.context.send( + Message.dead( + reason=self.explicit_close_msg, + handle=self.dst_handle + ) + ) + + def __repr__(self): + return 'Sender(%r, %r)' % (self.context, self.dst_handle) + + def __reduce__(self): + return _unpickle_sender, (self.context.context_id, self.dst_handle) + + +def _unpickle_sender(router, context_id, dst_handle): + if not (isinstance(router, Router) and + isinstance(context_id, (int, long)) and context_id >= 0 and + isinstance(dst_handle, (int, long)) and dst_handle > 0): + raise TypeError('cannot unpickle Sender: bad input') + return Sender(Context(router, context_id), dst_handle) + + +class Receiver(object): + """ + Receivers maintain a thread-safe queue of messages sent to a handle of this + context from another context. + + :param mitogen.core.Router router: + Router to register the handler on. + + :param int handle: + If not :data:`None`, an explicit handle to register, otherwise an + unused handle is chosen. + + :param bool persist: + If :data:`False`, unregister the handler after one message is received. + Single-message receivers are intended for RPC-like transactions, such + as in the case of :meth:`mitogen.parent.Context.call_async`. + + :param mitogen.core.Context respondent: + Context this receiver is receiving from. If not :data:`None`, arranges + for the receiver to receive a dead message if messages can no longer be + routed to the context due to disconnection, and ignores messages that + did not originate from the respondent context. + """ + #: If not :data:`None`, a reference to a function invoked as + #: `notify(receiver)` when a new message is delivered to this receiver. The + #: function is invoked on the broker thread, therefore it must not block. + #: Used by :class:`mitogen.select.Select` to implement waiting on multiple + #: receivers. + notify = None + + raise_channelerror = True + + def __init__(self, router, handle=None, persist=True, + respondent=None, policy=None, overwrite=False): + self.router = router + #: The handle. + self.handle = handle # Avoid __repr__ crash in add_handler() + self._latch = Latch() # Must exist prior to .add_handler() + self.handle = router.add_handler( + fn=self._on_receive, + handle=handle, + policy=policy, + persist=persist, + respondent=respondent, + overwrite=overwrite, + ) + + def __repr__(self): + return 'Receiver(%r, %r)' % (self.router, self.handle) + + def __enter__(self): + return self + + def __exit__(self, _1, _2, _3): + self.close() + + def to_sender(self): + """ + Return a :class:`Sender` configured to deliver messages to this + receiver. As senders are serializable, this makes it convenient to pass + `(context_id, handle)` pairs around:: + + def deliver_monthly_report(sender): + for line in open('monthly_report.txt'): + sender.send(line) + sender.close() + + @mitogen.main() + def main(router): + remote = router.ssh(hostname='mainframe') + recv = mitogen.core.Receiver(router) + remote.call(deliver_monthly_report, recv.to_sender()) + for msg in recv: + print(msg) + """ + return Sender(self.router.myself(), self.handle) + + def _on_receive(self, msg): + """ + Callback registered for the handle with :class:`Router`; appends data + to the internal queue. + """ + _vv and IOLOG.debug('%r._on_receive(%r)', self, msg) + self._latch.put(msg) + if self.notify: + self.notify(self) + + closed_msg = 'the Receiver has been closed' + + def close(self): + """ + Unregister the receiver's handle from its associated router, and cause + :class:`ChannelError` to be raised in any thread waiting in :meth:`get` + on this receiver. + """ + if self.handle: + self.router.del_handler(self.handle) + self.handle = None + self._latch.close() + + def empty(self): + """ + Return :data:`True` if calling :meth:`get` would block. + + As with :class:`Queue.Queue`, :data:`True` may be returned even though + a subsequent call to :meth:`get` will succeed, since a message may be + posted at any moment between :meth:`empty` and :meth:`get`. + """ + return self._latch.empty() + + def get(self, timeout=None, block=True, throw_dead=True): + """ + Sleep waiting for a message to arrive on this receiver. + + :param float timeout: + If not :data:`None`, specifies a timeout in seconds. + + :raises mitogen.core.ChannelError: + The remote end indicated the channel should be closed, + communication with it was lost, or :meth:`close` was called in the + local process. + + :raises mitogen.core.TimeoutError: + Timeout was reached. + + :returns: + :class:`Message` that was received. + """ + _vv and IOLOG.debug('%r.get(timeout=%r, block=%r)', self, timeout, block) + try: + msg = self._latch.get(timeout=timeout, block=block) + except LatchError: + raise ChannelError(self.closed_msg) + if msg.is_dead and throw_dead: + msg._throw_dead() + return msg + + def __iter__(self): + """ + Yield consecutive :class:`Message` instances delivered to this receiver + until :class:`ChannelError` is raised. + """ + while True: + try: + msg = self.get() + except ChannelError: + return + yield msg + + +class Channel(Sender, Receiver): + """ + A channel inherits from :class:`mitogen.core.Sender` and + `mitogen.core.Receiver` to provide bidirectional functionality. + + This class is incomplete and obsolete, it will be removed in Mitogen 0.3. + Channels were an early attempt at syntax sugar. It is always easier to pass + around unidirectional pairs of senders/receivers, even though the syntax is + baroque: + + .. literalinclude:: ../examples/ping_pong.py + + Since all handles aren't known until after both ends are constructed, for + both ends to communicate through a channel, it is necessary for one end to + retrieve the handle allocated to the other and reconfigure its own channel + to match. Currently this is a manual task. + """ + def __init__(self, router, context, dst_handle, handle=None): + Sender.__init__(self, context, dst_handle) + Receiver.__init__(self, router, handle) + + def close(self): + Receiver.close(self) + Sender.close(self) + + def __repr__(self): + return 'Channel(%s, %s)' % ( + Sender.__repr__(self), + Receiver.__repr__(self) + ) + + +class Importer(object): + """ + Import protocol implementation that fetches modules from the parent + process. + + :param context: Context to communicate via. + """ + # The Mitogen package is handled specially, since the child context must + # construct it manually during startup. + MITOGEN_PKG_CONTENT = [ + 'compat', + 'debug', + 'doas', + 'docker', + 'kubectl', + 'fakessh', + 'fork', + 'jail', + 'lxc', + 'lxd', + 'master', + 'minify', + 'os_fork', + 'parent', + 'select', + 'service', + 'setns', + 'ssh', + 'su', + 'sudo', + 'utils', + ] + + ALWAYS_BLACKLIST = [ + # 2.x generates needless imports for 'builtins', while 3.x does the + # same for '__builtin__'. The correct one is built-in, the other always + # a negative round-trip. + 'builtins', + '__builtin__', + 'thread', + + # org.python.core imported by copy, pickle, xml.sax; breaks Jython, but + # very unlikely to trigger a bug report. + 'org', + ] + + if PY3: + ALWAYS_BLACKLIST += ['cStringIO'] + + def __init__(self, router, context, core_src, whitelist=(), blacklist=()): + self._context = context + self._present = {'mitogen': self.MITOGEN_PKG_CONTENT} + self._lock = threading.Lock() + self.whitelist = list(whitelist) or [''] + self.blacklist = list(blacklist) + self.ALWAYS_BLACKLIST + + # Preserve copies of the original server-supplied whitelist/blacklist + # for later use by children. + self.master_whitelist = self.whitelist[:] + self.master_blacklist = self.blacklist[:] + + # Presence of an entry in this map indicates in-flight GET_MODULE. + self._callbacks = {} + self._cache = {} + if core_src: + self._update_linecache('x/mitogen/core.py', core_src) + self._cache['mitogen.core'] = ( + 'mitogen.core', + None, + 'x/mitogen/core.py', + zlib.compress(core_src, 9), + [], + ) + self._install_handler(router) + + def _update_linecache(self, path, data): + """ + The Python 2.4 linecache module, used to fetch source code for + tracebacks and :func:`inspect.getsource`, does not support PEP-302, + meaning it needs extra help to for Mitogen-loaded modules. Directly + populate its cache if a loaded module belongs to the Mitogen package. + """ + if PY24 and 'mitogen' in path: + linecache.cache[path] = ( + len(data), + 0.0, + [line+'\n' for line in data.splitlines()], + path, + ) + + def _install_handler(self, router): + router.add_handler( + fn=self._on_load_module, + handle=LOAD_MODULE, + policy=has_parent_authority, + ) + + def __repr__(self): + return 'Importer()' + + def builtin_find_module(self, fullname): + # imp.find_module() will always succeed for __main__, because it is a + # built-in module. That means it exists on a special linked list deep + # within the bowels of the interpreter. We must special case it. + if fullname == '__main__': + raise ModuleNotFoundError() + + parent, _, modname = str_rpartition(fullname, '.') + if parent: + path = sys.modules[parent].__path__ + else: + path = None + + fp, pathname, description = imp.find_module(modname, path) + if fp: + fp.close() + + def find_module(self, fullname, path=None): + if hasattr(_tls, 'running'): + return None + + _tls.running = True + try: + _v and LOG.debug('%r.find_module(%r)', self, fullname) + fullname = to_text(fullname) + pkgname, dot, _ = str_rpartition(fullname, '.') + pkg = sys.modules.get(pkgname) + if pkgname and getattr(pkg, '__loader__', None) is not self: + LOG.debug('%r: %r is submodule of a package we did not load', + self, fullname) + return None + + suffix = fullname[len(pkgname+dot):] + if pkgname and suffix not in self._present.get(pkgname, ()): + LOG.debug('%r: master doesn\'t know %r', self, fullname) + return None + + # #114: explicitly whitelisted prefixes override any + # system-installed package. + if self.whitelist != ['']: + if any(fullname.startswith(s) for s in self.whitelist): + return self + + try: + self.builtin_find_module(fullname) + _vv and IOLOG.debug('%r: %r is available locally', + self, fullname) + except ImportError: + _vv and IOLOG.debug('find_module(%r) returning self', fullname) + return self + finally: + del _tls.running + + blacklisted_msg = ( + '%r is present in the Mitogen importer blacklist, therefore this ' + 'context will not attempt to request it from the master, as the ' + 'request will always be refused.' + ) + pkg_resources_msg = ( + 'pkg_resources is prohibited from importing __main__, as it causes ' + 'problems in applications whose main module is not designed to be ' + 're-imported by children.' + ) + absent_msg = ( + 'The Mitogen master process was unable to serve %r. It may be a ' + 'native Python extension, or it may be missing entirely. Check the ' + 'importer debug logs on the master for more information.' + ) + + def _refuse_imports(self, fullname): + if is_blacklisted_import(self, fullname): + raise ModuleNotFoundError(self.blacklisted_msg % (fullname,)) + + f = sys._getframe(2) + requestee = f.f_globals['__name__'] + + if fullname == '__main__' and requestee == 'pkg_resources': + # Anything that imports pkg_resources will eventually cause + # pkg_resources to try and scan __main__ for its __requires__ + # attribute (pkg_resources/__init__.py::_build_master()). This + # breaks any app that is not expecting its __main__ to suddenly be + # sucked over a network and injected into a remote process, like + # py.test. + raise ModuleNotFoundError(self.pkg_resources_msg) + + if fullname == 'pbr': + # It claims to use pkg_resources to read version information, which + # would result in PEP-302 being used, but it actually does direct + # filesystem access. So instead smodge the environment to override + # any version that was defined. This will probably break something + # later. + os.environ['PBR_VERSION'] = '0.0.0' + + def _on_load_module(self, msg): + if msg.is_dead: + return + + tup = msg.unpickle() + fullname = tup[0] + _v and LOG.debug('Importer._on_load_module(%r)', fullname) + + self._lock.acquire() + try: + self._cache[fullname] = tup + if tup[2] is not None and PY24: + self._update_linecache( + path='master:' + tup[2], + data=zlib.decompress(tup[3]) + ) + callbacks = self._callbacks.pop(fullname, []) + finally: + self._lock.release() + + for callback in callbacks: + callback() + + def _request_module(self, fullname, callback): + self._lock.acquire() + try: + present = fullname in self._cache + if not present: + funcs = self._callbacks.get(fullname) + if funcs is not None: + _v and LOG.debug('_request_module(%r): in flight', fullname) + funcs.append(callback) + else: + _v and LOG.debug('_request_module(%r): new request', fullname) + self._callbacks[fullname] = [callback] + self._context.send( + Message(data=b(fullname), handle=GET_MODULE) + ) + finally: + self._lock.release() + + if present: + callback() + + def load_module(self, fullname): + fullname = to_text(fullname) + _v and LOG.debug('Importer.load_module(%r)', fullname) + self._refuse_imports(fullname) + + event = threading.Event() + self._request_module(fullname, event.set) + event.wait() + + ret = self._cache[fullname] + if ret[2] is None: + raise ModuleNotFoundError(self.absent_msg % (fullname,)) + + pkg_present = ret[1] + mod = sys.modules.setdefault(fullname, imp.new_module(fullname)) + mod.__file__ = self.get_filename(fullname) + mod.__loader__ = self + if pkg_present is not None: # it's a package. + mod.__path__ = [] + mod.__package__ = fullname + self._present[fullname] = pkg_present + else: + mod.__package__ = str_rpartition(fullname, '.')[0] or None + + if mod.__package__ and not PY3: + # 2.x requires __package__ to be exactly a string. + mod.__package__, _ = UTF8_CODEC.encode(mod.__package__) + + source = self.get_source(fullname) + try: + code = compile(source, mod.__file__, 'exec', 0, 1) + except SyntaxError: + LOG.exception('while importing %r', fullname) + raise + + if PY3: + exec(code, vars(mod)) + else: + exec('exec code in vars(mod)') + return mod + + def get_filename(self, fullname): + if fullname in self._cache: + path = self._cache[fullname][2] + if path is None: + # If find_loader() returns self but a subsequent master RPC + # reveals the module can't be loaded, and so load_module() + # throws ImportError, on Python 3.x it is still possible for + # the loader to be called to fetch metadata. + raise ModuleNotFoundError(self.absent_msg % (fullname,)) + return u'master:' + self._cache[fullname][2] + + def get_source(self, fullname): + if fullname in self._cache: + compressed = self._cache[fullname][3] + if compressed is None: + raise ModuleNotFoundError(self.absent_msg % (fullname,)) + + source = zlib.decompress(self._cache[fullname][3]) + if PY3: + return to_text(source) + return source + + +class LogHandler(logging.Handler): + def __init__(self, context): + logging.Handler.__init__(self) + self.context = context + self.local = threading.local() + self._buffer = [] + + def uncork(self): + """ + #305: during startup :class:`LogHandler` may be installed before it is + possible to route messages, therefore messages are buffered until + :meth:`uncork` is called by :class:`ExternalContext`. + """ + self._send = self.context.send + for msg in self._buffer: + self._send(msg) + self._buffer = None + + def _send(self, msg): + self._buffer.append(msg) + + def emit(self, rec): + if rec.name == 'mitogen.io' or \ + getattr(self.local, 'in_emit', False): + return + + self.local.in_emit = True + try: + msg = self.format(rec) + encoded = '%s\x00%s\x00%s' % (rec.name, rec.levelno, msg) + if isinstance(encoded, UnicodeType): + # Logging package emits both :( + encoded = encoded.encode('utf-8') + self._send(Message(data=encoded, handle=FORWARD_LOG)) + finally: + self.local.in_emit = False + + +class Side(object): + """ + Represent a single side of a :class:`BasicStream`. This exists to allow + streams implemented using unidirectional (e.g. UNIX pipe) and bidirectional + (e.g. UNIX socket) file descriptors to operate identically. + + :param mitogen.core.Stream stream: + The stream this side is associated with. + + :param int fd: + Underlying file descriptor. + + :param bool keep_alive: + Value for :attr:`keep_alive` + + During construction, the file descriptor has its :data:`os.O_NONBLOCK` flag + enabled using :func:`fcntl.fcntl`. + """ + _fork_refs = weakref.WeakValueDictionary() + + def __init__(self, stream, fd, cloexec=True, keep_alive=True, blocking=False): + #: The :class:`Stream` for which this is a read or write side. + self.stream = stream + #: Integer file descriptor to perform IO on, or :data:`None` if + #: :meth:`close` has been called. + self.fd = fd + self.closed = False + #: If :data:`True`, causes presence of this side in + #: :class:`Broker`'s active reader set to defer shutdown until the + #: side is disconnected. + self.keep_alive = keep_alive + self._fork_refs[id(self)] = self + if cloexec: + set_cloexec(fd) + if not blocking: + set_nonblock(fd) + + def __repr__(self): + return '' % (self.stream, self.fd) + + @classmethod + def _on_fork(cls): + while cls._fork_refs: + _, side = cls._fork_refs.popitem() + _vv and IOLOG.debug('Side._on_fork() closing %r', side) + side.close() + + def close(self): + """ + Call :func:`os.close` on :attr:`fd` if it is not :data:`None`, + then set it to :data:`None`. + """ + if not self.closed: + _vv and IOLOG.debug('%r.close()', self) + self.closed = True + os.close(self.fd) + + def read(self, n=CHUNK_SIZE): + """ + Read up to `n` bytes from the file descriptor, wrapping the underlying + :func:`os.read` call with :func:`io_op` to trap common disconnection + conditions. + + :meth:`read` always behaves as if it is reading from a regular UNIX + file; socket, pipe, and TTY disconnection errors are masked and result + in a 0-sized read like a regular file. + + :returns: + Bytes read, or the empty to string to indicate disconnection was + detected. + """ + if self.closed: + # Refuse to touch the handle after closed, it may have been reused + # by another thread. TODO: synchronize read()/write()/close(). + return b('') + s, disconnected = io_op(os.read, self.fd, n) + if disconnected: + LOG.debug('%r.read(): disconnected: %s', self, disconnected) + return b('') + return s + + def write(self, s): + """ + Write as much of the bytes from `s` as possible to the file descriptor, + wrapping the underlying :func:`os.write` call with :func:`io_op` to + trap common disconnection conditions. + + :returns: + Number of bytes written, or :data:`None` if disconnection was + detected. + """ + if self.closed or self.fd is None: + # Refuse to touch the handle after closed, it may have been reused + # by another thread. + return None + + written, disconnected = io_op(os.write, self.fd, s) + if disconnected: + LOG.debug('%r.write(): disconnected: %s', self, disconnected) + return None + return written + + +class BasicStream(object): + #: A :class:`Side` representing the stream's receive file descriptor. + receive_side = None + + #: A :class:`Side` representing the stream's transmit file descriptor. + transmit_side = None + + def on_receive(self, broker): + """ + Called by :class:`Broker` when the stream's :attr:`receive_side` has + been marked readable using :meth:`Broker.start_receive` and the broker + has detected the associated file descriptor is ready for reading. + + Subclasses must implement this if :meth:`Broker.start_receive` is ever + called on them, and the method must call :meth:`on_disconect` if + reading produces an empty string. + """ + pass + + def on_transmit(self, broker): + """ + Called by :class:`Broker` when the stream's :attr:`transmit_side` + has been marked writeable using :meth:`Broker._start_transmit` and + the broker has detected the associated file descriptor is ready for + writing. + + Subclasses must implement this if :meth:`Broker._start_transmit` is + ever called on them. + """ + pass + + def on_shutdown(self, broker): + """ + Called by :meth:`Broker.shutdown` to allow the stream time to + gracefully shutdown. The base implementation simply called + :meth:`on_disconnect`. + """ + _v and LOG.debug('%r.on_shutdown()', self) + fire(self, 'shutdown') + self.on_disconnect(broker) + + def on_disconnect(self, broker): + """ + Called by :class:`Broker` to force disconnect the stream. The base + implementation simply closes :attr:`receive_side` and + :attr:`transmit_side` and unregisters the stream from the broker. + """ + LOG.debug('%r.on_disconnect()', self) + if self.receive_side: + broker.stop_receive(self) + self.receive_side.close() + if self.transmit_side: + broker._stop_transmit(self) + self.transmit_side.close() + fire(self, 'disconnect') + + +class Stream(BasicStream): + """ + :class:`BasicStream` subclass implementing mitogen's :ref:`stream + protocol `. + """ + #: If not :data:`None`, :class:`Router` stamps this into + #: :attr:`Message.auth_id` of every message received on this stream. + auth_id = None + + #: If not :data:`False`, indicates the stream has :attr:`auth_id` set and + #: its value is the same as :data:`mitogen.context_id` or appears in + #: :data:`mitogen.parent_ids`. + is_privileged = False + + def __init__(self, router, remote_id, **kwargs): + self._router = router + self.remote_id = remote_id + self.name = u'default' + self.sent_modules = set(['mitogen', 'mitogen.core']) + self.construct(**kwargs) + self._input_buf = collections.deque() + self._output_buf = collections.deque() + self._input_buf_len = 0 + self._output_buf_len = 0 + #: Routing records the dst_id of every message arriving from this + #: stream. Any arriving DEL_ROUTE is rebroadcast for any such ID. + self.egress_ids = set() + + def construct(self): + pass + + def _internal_receive(self, broker, buf): + if self._input_buf and self._input_buf_len < 128: + self._input_buf[0] += buf + else: + self._input_buf.append(buf) + + self._input_buf_len += len(buf) + while self._receive_one(broker): + pass + + def on_receive(self, broker): + """Handle the next complete message on the stream. Raise + :class:`StreamError` on failure.""" + _vv and IOLOG.debug('%r.on_receive()', self) + + buf = self.receive_side.read() + if not buf: + return self.on_disconnect(broker) + + self._internal_receive(broker, buf) + + HEADER_FMT = '>hLLLLLL' + HEADER_LEN = struct.calcsize(HEADER_FMT) + HEADER_MAGIC = 0x4d49 # 'MI' + + corrupt_msg = ( + 'Corruption detected: frame signature incorrect. This likely means ' + 'some external process is interfering with the connection. Received:' + '\n\n' + '%r' + ) + + def _receive_one(self, broker): + if self._input_buf_len < self.HEADER_LEN: + return False + + msg = Message() + msg.router = self._router + (magic, msg.dst_id, msg.src_id, msg.auth_id, + msg.handle, msg.reply_to, msg_len) = struct.unpack( + self.HEADER_FMT, + self._input_buf[0][:self.HEADER_LEN], + ) + + if magic != self.HEADER_MAGIC: + LOG.error(self.corrupt_msg, self._input_buf[0][:2048]) + self.on_disconnect(broker) + return False + + if msg_len > self._router.max_message_size: + LOG.error('Maximum message size exceeded (got %d, max %d)', + msg_len, self._router.max_message_size) + self.on_disconnect(broker) + return False + + total_len = msg_len + self.HEADER_LEN + if self._input_buf_len < total_len: + _vv and IOLOG.debug( + '%r: Input too short (want %d, got %d)', + self, msg_len, self._input_buf_len - self.HEADER_LEN + ) + return False + + start = self.HEADER_LEN + prev_start = start + remain = total_len + bits = [] + while remain: + buf = self._input_buf.popleft() + bit = buf[start:remain] + bits.append(bit) + remain -= len(bit) + start + prev_start = start + start = 0 + + msg.data = b('').join(bits) + self._input_buf.appendleft(buf[prev_start+len(bit):]) + self._input_buf_len -= total_len + self._router._async_route(msg, self) + return True + + def pending_bytes(self): + """ + Return the number of bytes queued for transmission on this stream. This + can be used to limit the amount of data buffered in RAM by an otherwise + unlimited consumer. + + For an accurate result, this method should be called from the Broker + thread, for example by using :meth:`Broker.defer_sync`. + """ + return self._output_buf_len + + def on_transmit(self, broker): + """Transmit buffered messages.""" + _vv and IOLOG.debug('%r.on_transmit()', self) + + if self._output_buf: + buf = self._output_buf.popleft() + written = self.transmit_side.write(buf) + if not written: + _v and LOG.debug('%r.on_transmit(): disconnection detected', self) + self.on_disconnect(broker) + return + elif written != len(buf): + self._output_buf.appendleft(BufferType(buf, written)) + + _vv and IOLOG.debug('%r.on_transmit() -> len %d', self, written) + self._output_buf_len -= written + + if not self._output_buf: + broker._stop_transmit(self) + + def _send(self, msg): + _vv and IOLOG.debug('%r._send(%r)', self, msg) + pkt = struct.pack(self.HEADER_FMT, self.HEADER_MAGIC, msg.dst_id, + msg.src_id, msg.auth_id, msg.handle, + msg.reply_to or 0, len(msg.data)) + msg.data + + if not self._output_buf_len: + # Modifying epoll/Kqueue state is expensive, as are needless broker + # loops. Rather than wait for writeability, just write immediately, + # and fall back to the broker loop on error or full buffer. + try: + n = self.transmit_side.write(pkt) + if n: + if n == len(pkt): + return + pkt = pkt[n:] + except OSError: + pass + + self._router.broker._start_transmit(self) + self._output_buf.append(pkt) + self._output_buf_len += len(pkt) + + def send(self, msg): + """Send `data` to `handle`, and tell the broker we have output. May + be called from any thread.""" + self._router.broker.defer(self._send, msg) + + def on_shutdown(self, broker): + """Override BasicStream behaviour of immediately disconnecting.""" + _v and LOG.debug('%r.on_shutdown(%r)', self, broker) + + def accept(self, rfd, wfd): + # TODO: what is this os.dup for? + self.receive_side = Side(self, os.dup(rfd)) + self.transmit_side = Side(self, os.dup(wfd)) + + def __repr__(self): + cls = type(self) + return "%s.%s('%s')" % (cls.__module__, cls.__name__, self.name) + + +class Context(object): + """ + Represent a remote context regardless of the underlying connection method. + Context objects are simple facades that emit messages through an + associated router, and have :ref:`signals` raised against them in response + to various events relating to the context. + + **Note:** This is the somewhat limited core version, used by child + contexts. The master subclass is documented below this one. + + Contexts maintain no internal state and are thread-safe. + + Prefer :meth:`Router.context_by_id` over constructing context objects + explicitly, as that method is deduplicating, and returns the only context + instance :ref:`signals` will be raised on. + + :param Router router: + Router to emit messages through. + :param int context_id: + Context ID. + :param str name: + Context name. + """ + remote_name = None + + def __init__(self, router, context_id, name=None): + self.router = router + self.context_id = context_id + self.name = name + + def __reduce__(self): + name = self.name + if name and not isinstance(name, UnicodeType): + name = UnicodeType(name, 'utf-8') + return _unpickle_context, (self.context_id, name) + + def on_disconnect(self): + _v and LOG.debug('%r.on_disconnect()', self) + fire(self, 'disconnect') + + def send_async(self, msg, persist=False): + """ + Arrange for `msg` to be delivered to this context, with replies + directed to a newly constructed receiver. :attr:`dst_id + ` is set to the target context ID, and :attr:`reply_to + ` is set to the newly constructed receiver's handle. + + :param bool persist: + If :data:`False`, the handler will be unregistered after a single + message has been received. + + :param mitogen.core.Message msg: + The message. + + :returns: + :class:`Receiver` configured to receive any replies sent to the + message's `reply_to` handle. + """ + if self.router.broker._thread == threading.currentThread(): # TODO + raise SystemError('Cannot making blocking call on broker thread') + + receiver = Receiver(self.router, persist=persist, respondent=self) + msg.dst_id = self.context_id + msg.reply_to = receiver.handle + + _v and LOG.debug('%r.send_async(%r)', self, msg) + self.send(msg) + return receiver + + def call_service_async(self, service_name, method_name, **kwargs): + _v and LOG.debug('%r.call_service_async(%r, %r, %r)', + self, service_name, method_name, kwargs) + if isinstance(service_name, BytesType): + service_name = service_name.encode('utf-8') + elif not isinstance(service_name, UnicodeType): + service_name = service_name.name() # Service.name() + tup = (service_name, to_text(method_name), Kwargs(kwargs)) + msg = Message.pickled(tup, handle=CALL_SERVICE) + return self.send_async(msg) + + def send(self, msg): + """ + Arrange for `msg` to be delivered to this context. :attr:`dst_id + ` is set to the target context ID. + + :param Message msg: + Message. + """ + msg.dst_id = self.context_id + self.router.route(msg) + + def call_service(self, service_name, method_name, **kwargs): + recv = self.call_service_async(service_name, method_name, **kwargs) + return recv.get().unpickle() + + def send_await(self, msg, deadline=None): + """ + Like :meth:`send_async`, but expect a single reply (`persist=False`) + delivered within `deadline` seconds. + + :param mitogen.core.Message msg: + The message. + :param float deadline: + If not :data:`None`, seconds before timing out waiting for a reply. + :returns: + Deserialized reply. + :raises TimeoutError: + No message was received and `deadline` passed. + """ + receiver = self.send_async(msg) + response = receiver.get(deadline) + data = response.unpickle() + _vv and IOLOG.debug('%r._send_await() -> %r', self, data) + return data + + def __repr__(self): + return 'Context(%s, %r)' % (self.context_id, self.name) + + +def _unpickle_context(context_id, name, router=None): + if not (isinstance(context_id, (int, long)) and context_id >= 0 and ( + (name is None) or + (isinstance(name, UnicodeType) and len(name) < 100)) + ): + raise TypeError('cannot unpickle Context: bad input') + + if isinstance(router, Router): + return router.context_by_id(context_id, name=name) + return Context(None, context_id, name) # For plain Jane pickle. + + +class Poller(object): + """ + A poller manages OS file descriptors the user is waiting to become + available for IO. The :meth:`poll` method blocks the calling thread + until one or more become ready. The default implementation is based on + :func:`select.poll`. + + Each descriptor has an associated `data` element, which is unique for each + readiness type, and defaults to being the same as the file descriptor. The + :meth:`poll` method yields the data associated with a descriptor, rather + than the descriptor itself, allowing concise loops like:: + + p = Poller() + p.start_receive(conn.fd, data=conn.on_read) + p.start_transmit(conn.fd, data=conn.on_write) + + for callback in p.poll(): + callback() # invoke appropriate bound instance method + + Pollers may be modified while :meth:`poll` is yielding results. Removals + are processed immediately, causing pending events for the descriptor to be + discarded. + + The :meth:`close` method must be called when a poller is discarded to avoid + a resource leak. + + Pollers may only be used by one thread at a time. + """ + SUPPORTED = True + + # This changed from select() to poll() in Mitogen 0.2.4. Since poll() has + # no upper FD limit, it is suitable for use with Latch, which must handle + # FDs larger than select's limit during many-host runs. We want this + # because poll() requires no setup and teardown: just a single system call, + # which is important because Latch.get() creates a Poller on each + # invocation. In a microbenchmark, poll() vs. epoll_ctl() is 30% faster in + # this scenario. If select() must return in future, it is important + # Latch.poller_class is set from parent.py to point to the industrial + # strength poller for the OS, otherwise Latch will fail randomly. + + #: Increments on every poll(). Used to version _rfds and _wfds. + _generation = 1 + + def __init__(self): + self._rfds = {} + self._wfds = {} + + def __repr__(self): + return '%s(%#x)' % (type(self).__name__, id(self)) + + def _update(self, fd): + """ + Required by PollPoller subclass. + """ + pass + + @property + def readers(self): + """ + Return a list of `(fd, data)` tuples for every FD registered for + receive readiness. + """ + return list((fd, data) for fd, (data, gen) in self._rfds.items()) + + @property + def writers(self): + """ + Return a list of `(fd, data)` tuples for every FD registered for + transmit readiness. + """ + return list((fd, data) for fd, (data, gen) in self._wfds.items()) + + def close(self): + """ + Close any underlying OS resource used by the poller. + """ + pass + + def start_receive(self, fd, data=None): + """ + Cause :meth:`poll` to yield `data` when `fd` is readable. + """ + self._rfds[fd] = (data or fd, self._generation) + self._update(fd) + + def stop_receive(self, fd): + """ + Stop yielding readability events for `fd`. + + Redundant calls to :meth:`stop_receive` are silently ignored, this may + change in future. + """ + self._rfds.pop(fd, None) + self._update(fd) + + def start_transmit(self, fd, data=None): + """ + Cause :meth:`poll` to yield `data` when `fd` is writeable. + """ + self._wfds[fd] = (data or fd, self._generation) + self._update(fd) + + def stop_transmit(self, fd): + """ + Stop yielding writeability events for `fd`. + + Redundant calls to :meth:`stop_transmit` are silently ignored, this may + change in future. + """ + self._wfds.pop(fd, None) + self._update(fd) + + def _poll(self, timeout): + (rfds, wfds, _), _ = io_op(select.select, + self._rfds, + self._wfds, + (), timeout + ) + + for fd in rfds: + _vv and IOLOG.debug('%r: POLLIN for %r', self, fd) + data, gen = self._rfds.get(fd, (None, None)) + if gen and gen < self._generation: + yield data + + for fd in wfds: + _vv and IOLOG.debug('%r: POLLOUT for %r', self, fd) + data, gen = self._wfds.get(fd, (None, None)) + if gen and gen < self._generation: + yield data + + if timeout: + timeout *= 1000 + + def poll(self, timeout=None): + """ + Block the calling thread until one or more FDs are ready for IO. + + :param float timeout: + If not :data:`None`, seconds to wait without an event before + returning an empty iterable. + :returns: + Iterable of `data` elements associated with ready FDs. + """ + _vv and IOLOG.debug('%r.poll(%r)', self, timeout) + self._generation += 1 + return self._poll(timeout) + + +class Latch(object): + """ + A latch is a :class:`Queue.Queue`-like object that supports mutation and + waiting from multiple threads, however unlike :class:`Queue.Queue`, + waiting threads always remain interruptible, so CTRL+C always succeeds, and + waits where a timeout is set experience no wake up latency. These + properties are not possible in combination using the built-in threading + primitives available in Python 2.x. + + Latches implement queues using the UNIX self-pipe trick, and a per-thread + :func:`socket.socketpair` that is lazily created the first time any + latch attempts to sleep on a thread, and dynamically associated with the + waiting Latch only for duration of the wait. + + See :ref:`waking-sleeping-threads` for further discussion. + """ + poller_class = Poller + + notify = None + + # The _cls_ prefixes here are to make it crystal clear in the code which + # state mutation isn't covered by :attr:`_lock`. + + #: List of reusable :func:`socket.socketpair` tuples. The list is mutated + #: from multiple threads, the only safe operations are `append()` and + #: `pop()`. + _cls_idle_socketpairs = [] + + #: List of every socket object that must be closed by :meth:`_on_fork`. + #: Inherited descriptors cannot be reused, as the duplicated handles + #: reference the same underlying kernel object in use by the parent. + _cls_all_sockets = [] + + def __init__(self): + self.closed = False + self._lock = threading.Lock() + #: List of unconsumed enqueued items. + self._queue = [] + #: List of `(wsock, cookie)` awaiting an element, where `wsock` is the + #: socketpair's write side, and `cookie` is the string to write. + self._sleeping = [] + #: Number of elements of :attr:`_sleeping` that have already been + #: woken, and have a corresponding element index from :attr:`_queue` + #: assigned to them. + self._waking = 0 + + @classmethod + def _on_fork(cls): + """ + Clean up any files belonging to the parent process after a fork. + """ + cls._cls_idle_socketpairs = [] + while cls._cls_all_sockets: + cls._cls_all_sockets.pop().close() + + def close(self): + """ + Mark the latch as closed, and cause every sleeping thread to be woken, + with :class:`mitogen.core.LatchError` raised in each thread. + """ + self._lock.acquire() + try: + self.closed = True + while self._waking < len(self._sleeping): + wsock, cookie = self._sleeping[self._waking] + self._wake(wsock, cookie) + self._waking += 1 + finally: + self._lock.release() + + def empty(self): + """ + Return :data:`True` if calling :meth:`get` would block. + + As with :class:`Queue.Queue`, :data:`True` may be returned even + though a subsequent call to :meth:`get` will succeed, since a + message may be posted at any moment between :meth:`empty` and + :meth:`get`. + + As with :class:`Queue.Queue`, :data:`False` may be returned even + though a subsequent call to :meth:`get` will block, since another + waiting thread may be woken at any moment between :meth:`empty` and + :meth:`get`. + + :raises LatchError: + The latch has already been marked closed. + """ + self._lock.acquire() + try: + if self.closed: + raise LatchError() + return len(self._queue) == 0 + finally: + self._lock.release() + + def _get_socketpair(self): + """ + Return an unused socketpair, creating one if none exist. + """ + try: + return self._cls_idle_socketpairs.pop() # pop() must be atomic + except IndexError: + rsock, wsock = socket.socketpair() + set_cloexec(rsock.fileno()) + set_cloexec(wsock.fileno()) + self._cls_all_sockets.extend((rsock, wsock)) + return rsock, wsock + + COOKIE_MAGIC, = struct.unpack('L', b('LTCH') * (struct.calcsize('L')//4)) + COOKIE_FMT = '>Qqqq' # #545: id() and get_ident() may exceed long on armhfp. + COOKIE_SIZE = struct.calcsize(COOKIE_FMT) + + def _make_cookie(self): + """ + Return a string encoding the ID of the process, instance and thread. + This disambiguates legitimate wake-ups, accidental writes to the FD, + and buggy internal FD sharing. + """ + return struct.pack(self.COOKIE_FMT, self.COOKIE_MAGIC, + os.getpid(), id(self), thread.get_ident()) + + def get(self, timeout=None, block=True): + """ + Return the next enqueued object, or sleep waiting for one. + + :param float timeout: + If not :data:`None`, specifies a timeout in seconds. + + :param bool block: + If :data:`False`, immediately raise + :class:`mitogen.core.TimeoutError` if the latch is empty. + + :raises mitogen.core.LatchError: + :meth:`close` has been called, and the object is no longer valid. + + :raises mitogen.core.TimeoutError: + Timeout was reached. + + :returns: + The de-queued object. + """ + _vv and IOLOG.debug('%r.get(timeout=%r, block=%r)', + self, timeout, block) + self._lock.acquire() + try: + if self.closed: + raise LatchError() + i = len(self._sleeping) + if len(self._queue) > i: + _vv and IOLOG.debug('%r.get() -> %r', self, self._queue[i]) + return self._queue.pop(i) + if not block: + raise TimeoutError() + rsock, wsock = self._get_socketpair() + cookie = self._make_cookie() + self._sleeping.append((wsock, cookie)) + finally: + self._lock.release() + + poller = self.poller_class() + poller.start_receive(rsock.fileno()) + try: + return self._get_sleep(poller, timeout, block, rsock, wsock, cookie) + finally: + poller.close() + + def _get_sleep(self, poller, timeout, block, rsock, wsock, cookie): + """ + When a result is not immediately available, sleep waiting for + :meth:`put` to write a byte to our socket pair. + """ + _vv and IOLOG.debug( + '%r._get_sleep(timeout=%r, block=%r, rfd=%d, wfd=%d)', + self, timeout, block, rsock.fileno(), wsock.fileno() + ) + + e = None + woken = None + try: + woken = list(poller.poll(timeout)) + except Exception: + e = sys.exc_info()[1] + + self._lock.acquire() + try: + i = self._sleeping.index((wsock, cookie)) + del self._sleeping[i] + if not woken: + raise e or TimeoutError() + + got_cookie = rsock.recv(self.COOKIE_SIZE) + self._cls_idle_socketpairs.append((rsock, wsock)) + + assert cookie == got_cookie, ( + "Cookie incorrect; got %r, expected %r" \ + % (binascii.hexlify(got_cookie), + binascii.hexlify(cookie)) + ) + assert i < self._waking, ( + "Cookie correct, but no queue element assigned." + ) + self._waking -= 1 + if self.closed: + raise LatchError() + _vv and IOLOG.debug('%r.get() wake -> %r', self, self._queue[i]) + return self._queue.pop(i) + finally: + self._lock.release() + + def put(self, obj=None): + """ + Enqueue an object, waking the first thread waiting for a result, if one + exists. + + :param obj: + Object to enqueue. Defaults to :data:`None` as a convenience when + using :class:`Latch` only for synchronization. + :raises mitogen.core.LatchError: + :meth:`close` has been called, and the object is no longer valid. + """ + _vv and IOLOG.debug('%r.put(%r)', self, obj) + self._lock.acquire() + try: + if self.closed: + raise LatchError() + self._queue.append(obj) + + if self._waking < len(self._sleeping): + wsock, cookie = self._sleeping[self._waking] + self._waking += 1 + _vv and IOLOG.debug('%r.put() -> waking wfd=%r', + self, wsock.fileno()) + self._wake(wsock, cookie) + elif self.notify: + self.notify(self) + finally: + self._lock.release() + + def _wake(self, wsock, cookie): + written, disconnected = io_op(os.write, wsock.fileno(), cookie) + assert written == len(cookie) and not disconnected + + def __repr__(self): + return 'Latch(%#x, size=%d, t=%r)' % ( + id(self), + len(self._queue), + threading.currentThread().getName(), + ) + + +class Waker(BasicStream): + """ + :class:`BasicStream` subclass implementing the `UNIX self-pipe trick`_. + Used to wake the multiplexer when another thread needs to modify its state + (via a cross-thread function call). + + .. _UNIX self-pipe trick: https://cr.yp.to/docs/selfpipe.html + """ + broker_ident = None + + def __init__(self, broker): + self._broker = broker + self._lock = threading.Lock() + self._deferred = [] + + rfd, wfd = os.pipe() + self.receive_side = Side(self, rfd) + self.transmit_side = Side(self, wfd) + + def __repr__(self): + return 'Waker(%r rfd=%r, wfd=%r)' % ( + self._broker, + self.receive_side and self.receive_side.fd, + self.transmit_side and self.transmit_side.fd, + ) + + @property + def keep_alive(self): + """ + Prevent immediate Broker shutdown while deferred functions remain. + """ + self._lock.acquire() + try: + return len(self._deferred) + finally: + self._lock.release() + + def on_receive(self, broker): + """ + Drain the pipe and fire callbacks. Since :attr:`_deferred` is + synchronized, :meth:`defer` and :meth:`on_receive` can conspire to + ensure only one byte needs to be pending regardless of queue length. + """ + _vv and IOLOG.debug('%r.on_receive()', self) + self._lock.acquire() + try: + self.receive_side.read(1) + deferred = self._deferred + self._deferred = [] + finally: + self._lock.release() + + for func, args, kwargs in deferred: + try: + func(*args, **kwargs) + except Exception: + LOG.exception('defer() crashed: %r(*%r, **%r)', + func, args, kwargs) + self._broker.shutdown() + + def _wake(self): + """ + Wake the multiplexer by writing a byte. If Broker is midway through + teardown, the FD may already be closed, so ignore EBADF. + """ + try: + self.transmit_side.write(b(' ')) + except OSError: + e = sys.exc_info()[1] + if e.args[0] != errno.EBADF: + raise + + broker_shutdown_msg = ( + "An attempt was made to enqueue a message with a Broker that has " + "already exitted. It is likely your program called Broker.shutdown() " + "too early." + ) + + def defer(self, func, *args, **kwargs): + """ + Arrange for `func()` to execute on the broker thread. This function + returns immediately without waiting the result of `func()`. Use + :meth:`defer_sync` to block until a result is available. + + :raises mitogen.core.Error: + :meth:`defer` was called after :class:`Broker` has begun shutdown. + """ + if thread.get_ident() == self.broker_ident: + _vv and IOLOG.debug('%r.defer() [immediate]', self) + return func(*args, **kwargs) + if self._broker._exitted: + raise Error(self.broker_shutdown_msg) + + _vv and IOLOG.debug('%r.defer() [fd=%r]', self, self.transmit_side.fd) + self._lock.acquire() + try: + if not self._deferred: + self._wake() + self._deferred.append((func, args, kwargs)) + finally: + self._lock.release() + + +class IoLogger(BasicStream): + """ + :class:`BasicStream` subclass that sets up redirection of a standard + UNIX file descriptor back into the Python :mod:`logging` package. + """ + _buf = '' + + def __init__(self, broker, name, dest_fd): + self._broker = broker + self._name = name + self._rsock, self._wsock = socket.socketpair() + os.dup2(self._wsock.fileno(), dest_fd) + set_cloexec(self._wsock.fileno()) + + self._log = logging.getLogger(name) + # #453: prevent accidental log initialization in a child creating a + # feedback loop. + self._log.propagate = False + self._log.handlers = logging.getLogger().handlers[:] + + self.receive_side = Side(self, self._rsock.fileno()) + self.transmit_side = Side(self, dest_fd, cloexec=False, blocking=True) + self._broker.start_receive(self) + + def __repr__(self): + return '' % (self._name,) + + def _log_lines(self): + while self._buf.find('\n') != -1: + line, _, self._buf = str_partition(self._buf, '\n') + self._log.info('%s', line.rstrip('\n')) + + def on_shutdown(self, broker): + """Shut down the write end of the logging socket.""" + _v and LOG.debug('%r.on_shutdown()', self) + if not IS_WSL: + # #333: WSL generates invalid readiness indication on shutdown() + self._wsock.shutdown(socket.SHUT_WR) + self._wsock.close() + self.transmit_side.close() + + def on_receive(self, broker): + _vv and IOLOG.debug('%r.on_receive()', self) + buf = self.receive_side.read() + if not buf: + return self.on_disconnect(broker) + + self._buf += buf.decode('latin1') + self._log_lines() + + +class Router(object): + """ + Route messages between contexts, and invoke local handlers for messages + addressed to this context. :meth:`Router.route() ` straddles the + :class:`Broker` thread and user threads, it is safe to call anywhere. + + **Note:** This is the somewhat limited core version of the Router class + used by child contexts. The master subclass is documented below this one. + """ + context_class = Context + max_message_size = 128 * 1048576 + + #: When :data:`True`, permit children to only communicate with the current + #: context or a parent of the current context. Routing between siblings or + #: children of parents is prohibited, ensuring no communication is possible + #: between intentionally partitioned networks, such as when a program + #: simultaneously manipulates hosts spread across a corporate and a + #: production network, or production networks that are otherwise + #: air-gapped. + #: + #: Sending a prohibited message causes an error to be logged and a dead + #: message to be sent in reply to the errant message, if that message has + #: ``reply_to`` set. + #: + #: The value of :data:`unidirectional` becomes the default for the + #: :meth:`local() ` `unidirectional` + #: parameter. + unidirectional = False + + def __init__(self, broker): + self.broker = broker + listen(broker, 'exit', self._on_broker_exit) + self._setup_logging() + + self._write_lock = threading.Lock() + #: context ID -> Stream; must hold _write_lock to edit or iterate + self._stream_by_id = {} + #: List of contexts to notify of shutdown; must hold _write_lock + self._context_by_id = {} + self._last_handle = itertools.count(1000) + #: handle -> (persistent?, func(msg)) + self._handle_map = {} + #: Context -> set { handle, .. } + self._handles_by_respondent = {} + self.add_handler(self._on_del_route, DEL_ROUTE) + + def __repr__(self): + return 'Router(%r)' % (self.broker,) + + def _setup_logging(self): + """ + This is done in the :class:`Router` constructor for historical reasons. + It must be called before ExternalContext logs its first messages, but + after logging has been setup. It must also be called when any router is + constructed for a consumer app. + """ + # Here seems as good a place as any. + global _v, _vv + _v = logging.getLogger().level <= logging.DEBUG + _vv = IOLOG.level <= logging.DEBUG + + def _on_del_route(self, msg): + """ + Stub :data:`DEL_ROUTE` handler; fires 'disconnect' events on the + corresponding :attr:`_context_by_id` member. This is replaced by + :class:`mitogen.parent.RouteMonitor` in an upgraded context. + """ + LOG.error('%r._on_del_route() %r', self, msg) + if msg.is_dead: + return + + target_id_s, _, name = bytes_partition(msg.data, b(':')) + target_id = int(target_id_s, 10) + context = self._context_by_id.get(target_id) + if context: + fire(context, 'disconnect') + else: + LOG.debug('DEL_ROUTE for unknown ID %r: %r', target_id, msg) + + def _on_stream_disconnect(self, stream): + notify = [] + self._write_lock.acquire() + try: + for context in list(self._context_by_id.values()): + stream_ = self._stream_by_id.get(context.context_id) + if stream_ is stream: + del self._stream_by_id[context.context_id] + notify.append(context) + finally: + self._write_lock.release() + + # Happens outside lock as e.g. RouteMonitor wants the same lock. + for context in notify: + context.on_disconnect() + + broker_exit_msg = 'Broker has exitted' + + def _on_broker_exit(self): + while self._handle_map: + _, (_, func, _, _) = self._handle_map.popitem() + func(Message.dead(self.broker_exit_msg)) + + def myself(self): + """ + Return a :class:`Context` referring to the current process. + """ + return self.context_class( + router=self, + context_id=mitogen.context_id, + name='self', + ) + + def context_by_id(self, context_id, via_id=None, create=True, name=None): + """ + Messy factory/lookup function to find a context by its ID, or construct + it. This will eventually be replaced by a more sensible interface. + """ + context = self._context_by_id.get(context_id) + if context: + return context + + if create and via_id is not None: + via = self.context_by_id(via_id) + else: + via = None + + self._write_lock.acquire() + try: + context = self._context_by_id.get(context_id) + if create and not context: + context = self.context_class(self, context_id, name=name) + context.via = via + self._context_by_id[context_id] = context + finally: + self._write_lock.release() + + return context + + def register(self, context, stream): + """ + Register a newly constructed context and its associated stream, and add + the stream's receive side to the I/O multiplexer. This method remains + public while the design has not yet settled. + """ + _v and LOG.debug('register(%r, %r)', context, stream) + self._write_lock.acquire() + try: + self._stream_by_id[context.context_id] = stream + self._context_by_id[context.context_id] = context + finally: + self._write_lock.release() + + self.broker.start_receive(stream) + listen(stream, 'disconnect', lambda: self._on_stream_disconnect(stream)) + + def stream_by_id(self, dst_id): + """ + Return the :class:`Stream` that should be used to communicate with + `dst_id`. If a specific route for `dst_id` is not known, a reference to + the parent context's stream is returned. + """ + return ( + self._stream_by_id.get(dst_id) or + self._stream_by_id.get(mitogen.parent_id) + ) + + def del_handler(self, handle): + """ + Remove the handle registered for `handle` + + :raises KeyError: + The handle wasn't registered. + """ + _, _, _, respondent = self._handle_map.pop(handle) + if respondent: + self._handles_by_respondent[respondent].discard(handle) + + def add_handler(self, fn, handle=None, persist=True, + policy=None, respondent=None, + overwrite=False): + """ + Invoke `fn(msg)` on the :class:`Broker` thread for each Message sent to + `handle` from this context. Unregister after one invocation if + `persist` is :data:`False`. If `handle` is :data:`None`, a new handle + is allocated and returned. + + :param int handle: + If not :data:`None`, an explicit handle to register, usually one of + the ``mitogen.core.*`` constants. If unspecified, a new unused + handle will be allocated. + + :param bool persist: + If :data:`False`, the handler will be unregistered after a single + message has been received. + + :param Context respondent: + Context that messages to this handle are expected to be sent from. + If specified, arranges for a dead message to be delivered to `fn` + when disconnection of the context is detected. + + In future `respondent` will likely also be used to prevent other + contexts from sending messages to the handle. + + :param function policy: + Function invoked as `policy(msg, stream)` where `msg` is a + :class:`mitogen.core.Message` about to be delivered, and `stream` + is the :class:`mitogen.core.Stream` on which it was received. The + function must return :data:`True`, otherwise an error is logged and + delivery is refused. + + Two built-in policy functions exist: + + * :func:`has_parent_authority`: requires the message arrived from a + parent context, or a context acting with a parent context's + authority (``auth_id``). + + * :func:`mitogen.parent.is_immediate_child`: requires the + message arrived from an immediately connected child, for use in + messaging patterns where either something becomes buggy or + insecure by permitting indirect upstream communication. + + In case of refusal, and the message's ``reply_to`` field is + nonzero, a :class:`mitogen.core.CallError` is delivered to the + sender indicating refusal occurred. + + :param bool overwrite: + If :data:`True`, allow existing handles to be silently overwritten. + + :return: + `handle`, or if `handle` was :data:`None`, the newly allocated + handle. + :raises Error: + Attemp to register handle that was already registered. + """ + handle = handle or next(self._last_handle) + _vv and IOLOG.debug('%r.add_handler(%r, %r, %r)', self, fn, handle, persist) + if handle in self._handle_map and not overwrite: + raise Error(self.duplicate_handle_msg) + + self._handle_map[handle] = persist, fn, policy, respondent + if respondent: + if respondent not in self._handles_by_respondent: + self._handles_by_respondent[respondent] = set() + listen(respondent, 'disconnect', + lambda: self._on_respondent_disconnect(respondent)) + self._handles_by_respondent[respondent].add(handle) + + return handle + + duplicate_handle_msg = 'cannot register a handle that is already exists' + refused_msg = 'refused by policy' + invalid_handle_msg = 'invalid handle' + too_large_msg = 'message too large (max %d bytes)' + respondent_disconnect_msg = 'the respondent Context has disconnected' + broker_shutdown_msg = 'Broker is shutting down' + no_route_msg = 'no route to %r, my ID is %r' + unidirectional_msg = ( + 'routing mode prevents forward of message from context %d via ' + 'context %d' + ) + + def _on_respondent_disconnect(self, context): + for handle in self._handles_by_respondent.pop(context, ()): + _, fn, _, _ = self._handle_map[handle] + fn(Message.dead(self.respondent_disconnect_msg)) + del self._handle_map[handle] + + def on_shutdown(self, broker): + """Called during :meth:`Broker.shutdown`, informs callbacks registered + with :meth:`add_handle_cb` the connection is dead.""" + _v and LOG.debug('%r.on_shutdown(%r)', self, broker) + fire(self, 'shutdown') + for handle, (persist, fn) in self._handle_map.iteritems(): + _v and LOG.debug('%r.on_shutdown(): killing %r: %r', self, handle, fn) + fn(Message.dead(self.broker_shutdown_msg)) + + def _maybe_send_dead(self, msg, reason, *args): + if args: + reason %= args + LOG.debug('%r: %r is dead: %r', self, msg, reason) + if msg.reply_to and not msg.is_dead: + msg.reply(Message.dead(reason=reason), router=self) + + def _invoke(self, msg, stream): + # IOLOG.debug('%r._invoke(%r)', self, msg) + try: + persist, fn, policy, respondent = self._handle_map[msg.handle] + except KeyError: + self._maybe_send_dead(msg, reason=self.invalid_handle_msg) + return + + if respondent and not (msg.is_dead or + msg.src_id == respondent.context_id): + self._maybe_send_dead(msg, 'reply from unexpected context') + return + + if policy and not policy(msg, stream): + self._maybe_send_dead(msg, self.refused_msg) + return + + if not persist: + self.del_handler(msg.handle) + + try: + fn(msg) + except Exception: + LOG.exception('%r._invoke(%r): %r crashed', self, msg, fn) + + def _async_route(self, msg, in_stream=None): + """ + Arrange for `msg` to be forwarded towards its destination. If its + destination is the local context, then arrange for it to be dispatched + using the local handlers. + + This is a lower overhead version of :meth:`route` that may only be + called from the :class:`Broker` thread. + + :param Stream in_stream: + If not :data:`None`, the stream the message arrived on. Used for + performing source route verification, to ensure sensitive messages + such as ``CALL_FUNCTION`` arrive only from trusted contexts. + """ + _vv and IOLOG.debug('%r._async_route(%r, %r)', self, msg, in_stream) + + if len(msg.data) > self.max_message_size: + self._maybe_send_dead(msg, self.too_large_msg % ( + self.max_message_size, + )) + return + + # Perform source verification. + if in_stream: + parent = self._stream_by_id.get(mitogen.parent_id) + expect = self._stream_by_id.get(msg.auth_id, parent) + if in_stream != expect: + LOG.error('%r: bad auth_id: got %r via %r, not %r: %r', + self, msg.auth_id, in_stream, expect, msg) + return + + if msg.src_id != msg.auth_id: + expect = self._stream_by_id.get(msg.src_id, parent) + if in_stream != expect: + LOG.error('%r: bad src_id: got %r via %r, not %r: %r', + self, msg.src_id, in_stream, expect, msg) + return + + if in_stream.auth_id is not None: + msg.auth_id = in_stream.auth_id + + # Maintain a set of IDs the source ever communicated with. + in_stream.egress_ids.add(msg.dst_id) + + if msg.dst_id == mitogen.context_id: + return self._invoke(msg, in_stream) + + out_stream = self._stream_by_id.get(msg.dst_id) + if out_stream is None: + out_stream = self._stream_by_id.get(mitogen.parent_id) + + if out_stream is None: + self._maybe_send_dead(msg, self.no_route_msg, + msg.dst_id, mitogen.context_id) + return + + if in_stream and self.unidirectional and not \ + (in_stream.is_privileged or out_stream.is_privileged): + self._maybe_send_dead(msg, self.unidirectional_msg, + in_stream.remote_id, out_stream.remote_id) + return + + out_stream._send(msg) + + def route(self, msg): + """ + Arrange for the :class:`Message` `msg` to be delivered to its + destination using any relevant downstream context, or if none is found, + by forwarding the message upstream towards the master context. If `msg` + is destined for the local context, it is dispatched using the handles + registered with :meth:`add_handler`. + + This may be called from any thread. + """ + self.broker.defer(self._async_route, msg) + + +class Broker(object): + """ + Responsible for handling I/O multiplexing in a private thread. + + **Note:** This is the somewhat limited core version of the Broker class + used by child contexts. The master subclass is documented below. + """ + poller_class = Poller + _waker = None + _thread = None + + #: Seconds grace to allow :class:`streams ` to shutdown gracefully + #: before force-disconnecting them during :meth:`shutdown`. + shutdown_timeout = 3.0 + + def __init__(self, poller_class=None, activate_compat=True): + self._alive = True + self._exitted = False + self._waker = Waker(self) + #: Arrange for `func(\*args, \**kwargs)` to be executed on the broker + #: thread, or immediately if the current thread is the broker thread. + #: Safe to call from any thread. + self.defer = self._waker.defer + self.poller = self.poller_class() + self.poller.start_receive( + self._waker.receive_side.fd, + (self._waker.receive_side, self._waker.on_receive) + ) + self._thread = threading.Thread( + target=self._broker_main, + name='mitogen.broker' + ) + self._thread.start() + if activate_compat: + self._py24_25_compat() + + def _py24_25_compat(self): + """ + Python 2.4/2.5 have grave difficulties with threads/fork. We + mandatorily quiesce all running threads during fork using a + monkey-patch there. + """ + if sys.version_info < (2, 6): + # import_module() is used to avoid dep scanner. + os_fork = import_module('mitogen.os_fork') + mitogen.os_fork._notice_broker_or_pool(self) + + def start_receive(self, stream): + """ + Mark the :attr:`receive_side ` on `stream` as + ready for reading. Safe to call from any thread. When the associated + file descriptor becomes ready for reading, + :meth:`BasicStream.on_receive` will be called. + """ + _vv and IOLOG.debug('%r.start_receive(%r)', self, stream) + side = stream.receive_side + assert side and side.fd is not None + self.defer(self.poller.start_receive, + side.fd, (side, stream.on_receive)) + + def stop_receive(self, stream): + """ + Mark the :attr:`receive_side ` on `stream` as not + ready for reading. Safe to call from any thread. + """ + _vv and IOLOG.debug('%r.stop_receive(%r)', self, stream) + self.defer(self.poller.stop_receive, stream.receive_side.fd) + + def _start_transmit(self, stream): + """ + Mark the :attr:`transmit_side ` on `stream` as + ready for writing. Must only be called from the Broker thread. When the + associated file descriptor becomes ready for writing, + :meth:`BasicStream.on_transmit` will be called. + """ + _vv and IOLOG.debug('%r._start_transmit(%r)', self, stream) + side = stream.transmit_side + assert side and side.fd is not None + self.poller.start_transmit(side.fd, (side, stream.on_transmit)) + + def _stop_transmit(self, stream): + """ + Mark the :attr:`transmit_side ` on `stream` as not + ready for writing. + """ + _vv and IOLOG.debug('%r._stop_transmit(%r)', self, stream) + self.poller.stop_transmit(stream.transmit_side.fd) + + def keep_alive(self): + """ + Return :data:`True` if any reader's :attr:`Side.keep_alive` attribute + is :data:`True`, or any :class:`Context` is still registered that is + not the master. Used to delay shutdown while some important work is in + progress (e.g. log draining). + """ + it = (side.keep_alive for (_, (side, _)) in self.poller.readers) + return sum(it, 0) + + def defer_sync(self, func): + """ + Arrange for `func()` to execute on :class:`Broker` thread, blocking the + current thread until a result or exception is available. + + :returns: + Return value of `func()`. + """ + latch = Latch() + def wrapper(): + try: + latch.put(func()) + except Exception: + latch.put(sys.exc_info()[1]) + self.defer(wrapper) + res = latch.get() + if isinstance(res, Exception): + raise res + return res + + def _call(self, stream, func): + """ + Call `func(self)`, catching any exception that might occur, logging it, + and force-disconnecting the related `stream`. + """ + try: + func(self) + except Exception: + LOG.exception('%r crashed', stream) + stream.on_disconnect(self) + + def _loop_once(self, timeout=None): + """ + Execute a single :class:`Poller` wait, dispatching any IO events that + caused the wait to complete. + + :param float timeout: + If not :data:`None`, maximum time in seconds to wait for events. + """ + _vv and IOLOG.debug('%r._loop_once(%r, %r)', + self, timeout, self.poller) + #IOLOG.debug('readers =\n%s', pformat(self.poller.readers)) + #IOLOG.debug('writers =\n%s', pformat(self.poller.writers)) + for side, func in self.poller.poll(timeout): + self._call(side.stream, func) + + def _broker_exit(self): + """ + Forcefully call :meth:`Stream.on_disconnect` on any streams that failed + to shut down gracefully, then discard the :class:`Poller`. + """ + for _, (side, _) in self.poller.readers + self.poller.writers: + LOG.debug('_broker_main() force disconnecting %r', side) + side.stream.on_disconnect(self) + + self.poller.close() + + def _broker_shutdown(self): + """ + Invoke :meth:`Stream.on_shutdown` for every active stream, then allow + up to :attr:`shutdown_timeout` seconds for the streams to unregister + themselves, logging an error if any did not unregister during the grace + period. + """ + for _, (side, _) in self.poller.readers + self.poller.writers: + self._call(side.stream, side.stream.on_shutdown) + + deadline = time.time() + self.shutdown_timeout + while self.keep_alive() and time.time() < deadline: + self._loop_once(max(0, deadline - time.time())) + + if self.keep_alive(): + LOG.error('%r: some streams did not close gracefully. ' + 'The most likely cause for this is one or ' + 'more child processes still connected to ' + 'our stdout/stderr pipes.', self) + + def _do_broker_main(self): + """ + Broker thread main function. Dispatches IO events until + :meth:`shutdown` is called. + """ + # For Python 2.4, no way to retrieve ident except on thread. + self._waker.broker_ident = thread.get_ident() + try: + while self._alive: + self._loop_once() + + fire(self, 'shutdown') + self._broker_shutdown() + except Exception: + LOG.exception('_broker_main() crashed') + + self._alive = False # Ensure _alive is consistent on crash. + self._exitted = True + self._broker_exit() + + def _broker_main(self): + _profile_hook('mitogen.broker', self._do_broker_main) + fire(self, 'exit') + + def shutdown(self): + """ + Request broker gracefully disconnect streams and stop. Safe to call + from any thread. + """ + _v and LOG.debug('%r.shutdown()', self) + def _shutdown(): + self._alive = False + if self._alive and not self._exitted: + self.defer(_shutdown) + + def join(self): + """ + Wait for the broker to stop, expected to be called after + :meth:`shutdown`. + """ + self._thread.join() + + def __repr__(self): + return 'Broker(%#x)' % (id(self),) + + +class Dispatcher(object): + """ + Implementation of the :data:`CALL_FUNCTION` handle for a child context. + Listens on the child's main thread for messages sent by + :class:`mitogen.parent.CallChain` and dispatches the function calls they + describe. + + If a :class:`mitogen.parent.CallChain` sending a message is in pipelined + mode, any exception that occurs is recorded, and causes all subsequent + calls with the same `chain_id` to fail with the same exception. + """ + def __init__(self, econtext): + self.econtext = econtext + #: Chain ID -> CallError if prior call failed. + self._error_by_chain_id = {} + self.recv = Receiver(router=econtext.router, + handle=CALL_FUNCTION, + policy=has_parent_authority) + listen(econtext.broker, 'shutdown', self.recv.close) + + @classmethod + @takes_econtext + def forget_chain(cls, chain_id, econtext): + econtext.dispatcher._error_by_chain_id.pop(chain_id, None) + + def _parse_request(self, msg): + data = msg.unpickle(throw=False) + _v and LOG.debug('_dispatch_one(%r)', data) + + chain_id, modname, klass, func, args, kwargs = data + obj = import_module(modname) + if klass: + obj = getattr(obj, klass) + fn = getattr(obj, func) + if getattr(fn, 'mitogen_takes_econtext', None): + kwargs.setdefault('econtext', self.econtext) + if getattr(fn, 'mitogen_takes_router', None): + kwargs.setdefault('router', self.econtext.router) + + return chain_id, fn, args, kwargs + + def _dispatch_one(self, msg): + try: + chain_id, fn, args, kwargs = self._parse_request(msg) + except Exception: + return None, CallError(sys.exc_info()[1]) + + if chain_id in self._error_by_chain_id: + return chain_id, self._error_by_chain_id[chain_id] + + try: + return chain_id, fn(*args, **kwargs) + except Exception: + e = CallError(sys.exc_info()[1]) + if chain_id is not None: + self._error_by_chain_id[chain_id] = e + return chain_id, e + + def _dispatch_calls(self): + for msg in self.recv: + chain_id, ret = self._dispatch_one(msg) + _v and LOG.debug('_dispatch_calls: %r -> %r', msg, ret) + if msg.reply_to: + msg.reply(ret) + elif isinstance(ret, CallError) and chain_id is None: + LOG.error('No-reply function call failed: %s', ret) + + def run(self): + if self.econtext.config.get('on_start'): + self.econtext.config['on_start'](self.econtext) + + _profile_hook('mitogen.child_main', self._dispatch_calls) + + +class ExternalContext(object): + """ + External context implementation. + + .. attribute:: broker + The :class:`mitogen.core.Broker` instance. + + .. attribute:: context + The :class:`mitogen.core.Context` instance. + + .. attribute:: channel + The :class:`mitogen.core.Channel` over which :data:`CALL_FUNCTION` + requests are received. + + .. attribute:: stdout_log + The :class:`mitogen.core.IoLogger` connected to ``stdout``. + + .. attribute:: importer + The :class:`mitogen.core.Importer` instance. + + .. attribute:: stdout_log + The :class:`IoLogger` connected to ``stdout``. + + .. attribute:: stderr_log + The :class:`IoLogger` connected to ``stderr``. + + .. method:: _dispatch_calls + Implementation for the main thread in every child context. + """ + detached = False + + def __init__(self, config): + self.config = config + + def _on_broker_exit(self): + if not self.config['profiling']: + os.kill(os.getpid(), signal.SIGTERM) + + #: On Python >3.4, the global importer lock has been sharded into a + #: per-module lock, meaning there is no guarantee the import statement in + #: service_stub_main will be truly complete before a second thread + #: attempting the same import will see a partially initialized module. + #: Sigh. Therefore serialize execution of the stub itself. + service_stub_lock = threading.Lock() + + def _service_stub_main(self, msg): + self.service_stub_lock.acquire() + try: + import mitogen.service + pool = mitogen.service.get_or_create_pool(router=self.router) + pool._receiver._on_receive(msg) + finally: + self.service_stub_lock.release() + + def _on_call_service_msg(self, msg): + """ + Stub service handler. Start a thread to import the mitogen.service + implementation from, and deliver the message to the newly constructed + pool. This must be done as CALL_SERVICE for e.g. PushFileService may + race with a CALL_FUNCTION blocking the main thread waiting for a result + from that service. + """ + if not msg.is_dead: + th = threading.Thread(target=self._service_stub_main, args=(msg,)) + th.start() + + def _on_shutdown_msg(self, msg): + _v and LOG.debug('_on_shutdown_msg(%r)', msg) + if not msg.is_dead: + self.broker.shutdown() + + def _on_parent_disconnect(self): + if self.detached: + mitogen.parent_ids = [] + mitogen.parent_id = None + LOG.info('Detachment complete') + else: + _v and LOG.debug('%r: parent stream is gone, dying.', self) + self.broker.shutdown() + + def detach(self): + self.detached = True + stream = self.router.stream_by_id(mitogen.parent_id) + if stream: # not double-detach()'d + os.setsid() + self.parent.send_await(Message(handle=DETACHING)) + LOG.info('Detaching from %r; parent is %s', stream, self.parent) + for x in range(20): + pending = self.broker.defer_sync(lambda: stream.pending_bytes()) + if not pending: + break + time.sleep(0.05) + if pending: + LOG.error('Stream had %d bytes after 2000ms', pending) + self.broker.defer(stream.on_disconnect, self.broker) + + def _setup_master(self): + Router.max_message_size = self.config['max_message_size'] + if self.config['profiling']: + enable_profiling() + self.broker = Broker(activate_compat=False) + self.router = Router(self.broker) + self.router.debug = self.config.get('debug', False) + self.router.undirectional = self.config['unidirectional'] + self.router.add_handler( + fn=self._on_shutdown_msg, + handle=SHUTDOWN, + policy=has_parent_authority, + ) + self.router.add_handler( + fn=self._on_call_service_msg, + handle=CALL_SERVICE, + policy=has_parent_authority, + ) + self.master = Context(self.router, 0, 'master') + parent_id = self.config['parent_ids'][0] + if parent_id == 0: + self.parent = self.master + else: + self.parent = Context(self.router, parent_id, 'parent') + + in_fd = self.config.get('in_fd', 100) + out_fd = self.config.get('out_fd', 1) + self.stream = Stream(self.router, parent_id) + self.stream.name = 'parent' + self.stream.accept(in_fd, out_fd) + self.stream.receive_side.keep_alive = False + + listen(self.stream, 'disconnect', self._on_parent_disconnect) + listen(self.broker, 'exit', self._on_broker_exit) + + os.close(in_fd) + + def _reap_first_stage(self): + try: + os.wait() # Reap first stage. + except OSError: + pass # No first stage exists (e.g. fakessh) + + def _setup_logging(self): + self.log_handler = LogHandler(self.master) + root = logging.getLogger() + root.setLevel(self.config['log_level']) + root.handlers = [self.log_handler] + if self.config['debug']: + enable_debug_logging() + + def _setup_importer(self): + importer = self.config.get('importer') + if importer: + importer._install_handler(self.router) + importer._context = self.parent + else: + core_src_fd = self.config.get('core_src_fd', 101) + if core_src_fd: + fp = os.fdopen(core_src_fd, 'rb', 1) + try: + core_src = fp.read() + # Strip "ExternalContext.main()" call from last line. + core_src = b('\n').join(core_src.splitlines()[:-1]) + finally: + fp.close() + else: + core_src = None + + importer = Importer( + self.router, + self.parent, + core_src, + self.config.get('whitelist', ()), + self.config.get('blacklist', ()), + ) + + self.importer = importer + self.router.importer = importer + sys.meta_path.insert(0, self.importer) + + def _setup_package(self): + global mitogen + mitogen = imp.new_module('mitogen') + mitogen.__package__ = 'mitogen' + mitogen.__path__ = [] + mitogen.__loader__ = self.importer + mitogen.main = lambda *args, **kwargs: (lambda func: None) + mitogen.core = sys.modules['__main__'] + mitogen.core.__file__ = 'x/mitogen/core.py' # For inspect.getsource() + mitogen.core.__loader__ = self.importer + sys.modules['mitogen'] = mitogen + sys.modules['mitogen.core'] = mitogen.core + del sys.modules['__main__'] + + def _setup_globals(self): + mitogen.is_master = False + mitogen.__version__ = self.config['version'] + mitogen.context_id = self.config['context_id'] + mitogen.parent_ids = self.config['parent_ids'][:] + mitogen.parent_id = mitogen.parent_ids[0] + + def _nullify_stdio(self): + """ + Open /dev/null to replace stdin, and stdout/stderr temporarily. In case + of odd startup, assume we may be allocated a standard handle. + """ + fd = os.open('/dev/null', os.O_RDWR) + try: + for stdfd in (0, 1, 2): + if fd != stdfd: + os.dup2(fd, stdfd) + finally: + if fd not in (0, 1, 2): + os.close(fd) + + def _setup_stdio(self): + # #481: when stderr is a TTY due to being started via + # tty_create_child()/hybrid_tty_create_child(), and some privilege + # escalation tool like prehistoric versions of sudo exec this process + # over the top of itself, there is nothing left to keep the slave PTY + # open after we replace our stdio. Therefore if stderr is a TTY, keep + # around a permanent dup() to avoid receiving SIGHUP. + try: + if os.isatty(2): + self.reserve_tty_fd = os.dup(2) + set_cloexec(self.reserve_tty_fd) + except OSError: + pass + # When sys.stdout was opened by the runtime, overwriting it will not + # close FD 1. However when forking from a child that previously used + # fdopen(), overwriting it /will/ close FD 1. So we must swallow the + # close before IoLogger overwrites FD 1, otherwise its new FD 1 will be + # clobbered. Additionally, stdout must be replaced with /dev/null prior + # to stdout.close(), since if block buffering was active in the parent, + # any pre-fork buffered data will be flushed on close(), corrupting the + # connection to the parent. + self._nullify_stdio() + sys.stdout.close() + self._nullify_stdio() + + self.stdout_log = IoLogger(self.broker, 'stdout', 1) + self.stderr_log = IoLogger(self.broker, 'stderr', 2) + # Reopen with line buffering. + sys.stdout = os.fdopen(1, 'w', 1) + + def main(self): + self._setup_master() + try: + try: + self._setup_logging() + self._setup_importer() + self._reap_first_stage() + if self.config.get('setup_package', True): + self._setup_package() + self._setup_globals() + if self.config.get('setup_stdio', True): + self._setup_stdio() + + self.dispatcher = Dispatcher(self) + self.router.register(self.parent, self.stream) + self.router._setup_logging() + self.log_handler.uncork() + + sys.executable = os.environ.pop('ARGV0', sys.executable) + _v and LOG.debug('Connected to context %s; my ID is %r', + self.parent, mitogen.context_id) + _v and LOG.debug('pid:%r ppid:%r uid:%r/%r, gid:%r/%r host:%r', + os.getpid(), os.getppid(), os.geteuid(), + os.getuid(), os.getegid(), os.getgid(), + socket.gethostname()) + _v and LOG.debug('Recovered sys.executable: %r', sys.executable) + + self.broker._py24_25_compat() + self.dispatcher.run() + _v and LOG.debug('ExternalContext.main() normal exit') + except KeyboardInterrupt: + LOG.debug('KeyboardInterrupt received, exiting gracefully.') + except BaseException: + LOG.exception('ExternalContext.main() crashed') + raise + finally: + self.broker.shutdown() + self.broker.join() diff --git a/mitogen-0.2.7/mitogen/core.pyc b/mitogen-0.2.7/mitogen/core.pyc new file mode 100644 index 000000000..107eb2f Binary files /dev/null and b/mitogen-0.2.7/mitogen/core.pyc differ diff --git a/mitogen-0.2.7/mitogen/debug.py b/mitogen-0.2.7/mitogen/debug.py new file mode 100644 index 000000000..3d13347 --- /dev/null +++ b/mitogen-0.2.7/mitogen/debug.py @@ -0,0 +1,236 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +""" +Basic signal handler for dumping thread stacks. +""" + +import difflib +import logging +import os +import gc +import signal +import sys +import threading +import time +import traceback + +import mitogen.core +import mitogen.parent + + +LOG = logging.getLogger(__name__) +_last = None + + +def enable_evil_interrupts(): + signal.signal(signal.SIGALRM, (lambda a, b: None)) + signal.setitimer(signal.ITIMER_REAL, 0.01, 0.01) + + +def disable_evil_interrupts(): + signal.setitimer(signal.ITIMER_REAL, 0, 0) + + +def _hex(n): + return '%08x' % n + + +def get_subclasses(klass): + """ + Rather than statically import every interesting subclass, forcing it all to + be transferred and potentially disrupting the debugged environment, + enumerate only those loaded in memory. Also returns the original class. + """ + stack = [klass] + seen = set() + while stack: + klass = stack.pop() + seen.add(klass) + stack.extend(klass.__subclasses__()) + return seen + + +def get_routers(): + return dict( + (_hex(id(router)), router) + for klass in get_subclasses(mitogen.core.Router) + for router in gc.get_referrers(klass) + if isinstance(router, mitogen.core.Router) + ) + + +def get_router_info(): + return { + 'routers': dict( + (id_, { + 'id': id_, + 'streams': len(set(router._stream_by_id.values())), + 'contexts': len(set(router._context_by_id.values())), + 'handles': len(router._handle_map), + }) + for id_, router in get_routers().items() + ) + } + + +def get_stream_info(router_id): + router = get_routers().get(router_id) + return { + 'streams': dict( + (_hex(id(stream)), ({ + 'name': stream.name, + 'remote_id': stream.remote_id, + 'sent_module_count': len(getattr(stream, 'sent_modules', [])), + 'routes': sorted(getattr(stream, 'routes', [])), + 'type': type(stream).__module__, + })) + for via_id, stream in router._stream_by_id.items() + ) + } + + +def format_stacks(): + name_by_id = dict( + (t.ident, t.name) + for t in threading.enumerate() + ) + + l = ['', ''] + for threadId, stack in sys._current_frames().items(): + l += ["# PID %d ThreadID: (%s) %s; %r" % ( + os.getpid(), + name_by_id.get(threadId, ''), + threadId, + stack, + )] + #stack = stack.f_back.f_back + + for filename, lineno, name, line in traceback.extract_stack(stack): + l += [ + 'File: "%s", line %d, in %s' % ( + filename, + lineno, + name + ) + ] + if line: + l += [' ' + line.strip()] + l += [''] + + l += ['', ''] + return '\n'.join(l) + + +def get_snapshot(): + global _last + + s = format_stacks() + snap = s + if _last: + snap += '\n' + diff = list(difflib.unified_diff( + a=_last.splitlines(), + b=s.splitlines(), + fromfile='then', + tofile='now' + )) + + if diff: + snap += '\n'.join(diff) + '\n' + else: + snap += '(no change since last time)\n' + _last = s + return snap + + +def _handler(*_): + fp = open('/dev/tty', 'w', 1) + fp.write(get_snapshot()) + fp.close() + + +def install_handler(): + signal.signal(signal.SIGUSR2, _handler) + + +def _logging_main(secs): + while True: + time.sleep(secs) + LOG.info('PERIODIC THREAD DUMP\n\n%s', get_snapshot()) + + +def dump_to_logger(secs=5): + th = threading.Thread( + target=_logging_main, + kwargs={'secs': secs}, + name='mitogen.debug.dump_to_logger', + ) + th.setDaemon(True) + th.start() + + +class ContextDebugger(object): + @classmethod + @mitogen.core.takes_econtext + def _configure_context(cls, econtext): + mitogen.parent.upgrade_router(econtext) + econtext.debugger = cls(econtext.router) + + def __init__(self, router): + self.router = router + self.router.add_handler( + func=self._on_debug_msg, + handle=mitogen.core.DEBUG, + persist=True, + policy=mitogen.core.has_parent_authority, + ) + mitogen.core.listen(router, 'register', self._on_stream_register) + LOG.debug('Context debugging configured.') + + def _on_stream_register(self, context, stream): + LOG.debug('_on_stream_register: sending configure() to %r', stream) + context.call_async(ContextDebugger._configure_context) + + def _on_debug_msg(self, msg): + if msg != mitogen.core._DEAD: + threading.Thread( + target=self._handle_debug_msg, + name='ContextDebuggerHandler', + args=(msg,) + ).start() + + def _handle_debug_msg(self, msg): + try: + method, args, kwargs = msg.unpickle() + msg.reply(getattr(cls, method)(*args, **kwargs)) + except Exception: + e = sys.exc_info()[1] + msg.reply(mitogen.core.CallError(e)) diff --git a/mitogen-0.2.7/mitogen/debug.pyc b/mitogen-0.2.7/mitogen/debug.pyc new file mode 100644 index 000000000..fc010d0 Binary files /dev/null and b/mitogen-0.2.7/mitogen/debug.pyc differ diff --git a/mitogen-0.2.7/mitogen/doas.py b/mitogen-0.2.7/mitogen/doas.py new file mode 100644 index 000000000..1b687fb --- /dev/null +++ b/mitogen-0.2.7/mitogen/doas.py @@ -0,0 +1,113 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +import logging + +import mitogen.core +import mitogen.parent +from mitogen.core import b + + +LOG = logging.getLogger(__name__) + + +class PasswordError(mitogen.core.StreamError): + pass + + +class Stream(mitogen.parent.Stream): + create_child = staticmethod(mitogen.parent.hybrid_tty_create_child) + child_is_immediate_subprocess = False + + username = 'root' + password = None + doas_path = 'doas' + password_prompt = b('Password:') + incorrect_prompts = ( + b('doas: authentication failed'), + ) + + def construct(self, username=None, password=None, doas_path=None, + password_prompt=None, incorrect_prompts=None, **kwargs): + super(Stream, self).construct(**kwargs) + if username is not None: + self.username = username + if password is not None: + self.password = password + if doas_path is not None: + self.doas_path = doas_path + if password_prompt is not None: + self.password_prompt = password_prompt.lower() + if incorrect_prompts is not None: + self.incorrect_prompts = map(str.lower, incorrect_prompts) + + def _get_name(self): + return u'doas.' + mitogen.core.to_text(self.username) + + def get_boot_command(self): + bits = [self.doas_path, '-u', self.username, '--'] + bits = bits + super(Stream, self).get_boot_command() + LOG.debug('doas command line: %r', bits) + return bits + + password_incorrect_msg = 'doas password is incorrect' + password_required_msg = 'doas password is required' + + def _connect_input_loop(self, it): + password_sent = False + for buf in it: + LOG.debug('%r: received %r', self, buf) + if buf.endswith(self.EC0_MARKER): + self._ec0_received() + return + if any(s in buf.lower() for s in self.incorrect_prompts): + if password_sent: + raise PasswordError(self.password_incorrect_msg) + elif self.password_prompt in buf.lower(): + if self.password is None: + raise PasswordError(self.password_required_msg) + if password_sent: + raise PasswordError(self.password_incorrect_msg) + LOG.debug('sending password') + self.diag_stream.transmit_side.write( + mitogen.core.to_text(self.password + '\n').encode('utf-8') + ) + password_sent = True + raise mitogen.core.StreamError('bootstrap failed') + + def _connect_bootstrap(self): + it = mitogen.parent.iter_read( + fds=[self.receive_side.fd, self.diag_stream.receive_side.fd], + deadline=self.connect_deadline, + ) + try: + self._connect_input_loop(it) + finally: + it.close() diff --git a/mitogen-0.2.7/mitogen/docker.py b/mitogen-0.2.7/mitogen/docker.py new file mode 100644 index 000000000..0c0d40e --- /dev/null +++ b/mitogen-0.2.7/mitogen/docker.py @@ -0,0 +1,81 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +import logging + +import mitogen.core +import mitogen.parent + + +LOG = logging.getLogger(__name__) + + +class Stream(mitogen.parent.Stream): + child_is_immediate_subprocess = False + + container = None + image = None + username = None + docker_path = 'docker' + + # TODO: better way of capturing errors such as "No such container." + create_child_args = { + 'merge_stdio': True + } + + def construct(self, container=None, image=None, + docker_path=None, username=None, + **kwargs): + assert container or image + super(Stream, self).construct(**kwargs) + if container: + self.container = container + if image: + self.image = image + if docker_path: + self.docker_path = docker_path + if username: + self.username = username + + def _get_name(self): + return u'docker.' + (self.container or self.image) + + def get_boot_command(self): + args = ['--interactive'] + if self.username: + args += ['--user=' + self.username] + + bits = [self.docker_path] + if self.container: + bits += ['exec'] + args + [self.container] + elif self.image: + bits += ['run'] + args + ['--rm', self.image] + + return bits + super(Stream, self).get_boot_command() diff --git a/mitogen-0.2.7/mitogen/fakessh.py b/mitogen-0.2.7/mitogen/fakessh.py new file mode 100644 index 000000000..d39a710 --- /dev/null +++ b/mitogen-0.2.7/mitogen/fakessh.py @@ -0,0 +1,461 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +""" +:mod:`mitogen.fakessh` is a stream implementation that starts a subprocess with +its environment modified such that ``PATH`` searches for `ssh` return a Mitogen +implementation of SSH. When invoked, this implementation arranges for the +command line supplied by the caller to be executed in a remote context, reusing +the parent context's (possibly proxied) connection to that remote context. + +This allows tools like `rsync` and `scp` to transparently reuse the connections +and tunnels already established by the host program to connect to a target +machine, without wasteful redundant SSH connection setup, 3-way handshakes, or +firewall hopping configurations, and enables these tools to be used in +impossible scenarios, such as over `sudo` with ``requiretty`` enabled. + +The fake `ssh` command source is written to a temporary file on disk, and +consists of a copy of the :py:mod:`mitogen.core` source code (just like any +other child context), with a line appended to cause it to connect back to the +host process over an FD it inherits. As there is no reliance on an existing +filesystem file, it is possible for child contexts to use fakessh. + +As a consequence of connecting back through an inherited FD, only one SSH +invocation is possible, which is fine for tools like `rsync`, however in future +this restriction will be lifted. + +Sequence: + + 1. ``fakessh`` Context and Stream created by parent context. The stream's + buffer has a :py:func:`_fakessh_main` :py:data:`CALL_FUNCTION + ` enqueued. + 2. Target program (`rsync/scp/sftp`) invoked, which internally executes + `ssh` from ``PATH``. + 3. :py:mod:`mitogen.core` bootstrap begins, recovers the stream FD + inherited via the target program, established itself as the fakessh + context. + 4. :py:func:`_fakessh_main` :py:data:`CALL_FUNCTION + ` is read by fakessh context, + + a. sets up :py:class:`IoPump` for stdio, registers + stdin_handle for local context. + b. Enqueues :py:data:`CALL_FUNCTION ` for + :py:func:`_start_slave` invoked in target context, + + i. the program from the `ssh` command line is started + ii. sets up :py:class:`IoPump` for `ssh` command line process's + stdio pipes + iii. returns `(control_handle, stdin_handle)` to + :py:func:`_fakessh_main` + + 5. :py:func:`_fakessh_main` receives control/stdin handles from from + :py:func:`_start_slave`, + + a. registers remote's stdin_handle with local :py:class:`IoPump`. + b. sends `("start", local_stdin_handle)` to remote's control_handle + c. registers local :py:class:`IoPump` with + :py:class:`mitogen.core.Broker`. + d. loops waiting for `local stdout closed && remote stdout closed` + + 6. :py:func:`_start_slave` control channel receives `("start", stdin_handle)`, + + a. registers remote's stdin_handle with local :py:class:`IoPump` + b. registers local :py:class:`IoPump` with + :py:class:`mitogen.core.Broker`. + c. loops waiting for `local stdout closed && remote stdout closed` +""" + +import getopt +import inspect +import os +import shutil +import socket +import subprocess +import sys +import tempfile +import threading + +import mitogen.core +import mitogen.master +import mitogen.parent + +from mitogen.core import LOG, IOLOG + + +SSH_GETOPTS = ( + "1246ab:c:e:fgi:kl:m:no:p:qstvx" + "ACD:E:F:I:KL:MNO:PQ:R:S:TVw:W:XYy" +) + +_mitogen = None + + +class IoPump(mitogen.core.BasicStream): + _output_buf = '' + _closed = False + + def __init__(self, broker, stdin_fd, stdout_fd): + self._broker = broker + self.receive_side = mitogen.core.Side(self, stdout_fd) + self.transmit_side = mitogen.core.Side(self, stdin_fd) + + def write(self, s): + self._output_buf += s + self._broker._start_transmit(self) + + def close(self): + self._closed = True + # If local process hasn't exitted yet, ensure its write buffer is + # drained before lazily triggering disconnect in on_transmit. + if self.transmit_side.fd is not None: + self._broker._start_transmit(self) + + def on_shutdown(self, broker): + self.close() + + def on_transmit(self, broker): + written = self.transmit_side.write(self._output_buf) + IOLOG.debug('%r.on_transmit() -> len %r', self, written) + if written is None: + self.on_disconnect(broker) + else: + self._output_buf = self._output_buf[written:] + + if not self._output_buf: + broker._stop_transmit(self) + if self._closed: + self.on_disconnect(broker) + + def on_receive(self, broker): + s = self.receive_side.read() + IOLOG.debug('%r.on_receive() -> len %r', self, len(s)) + if s: + mitogen.core.fire(self, 'receive', s) + else: + self.on_disconnect(broker) + + def __repr__(self): + return 'IoPump(%r, %r)' % ( + self.receive_side.fd, + self.transmit_side.fd, + ) + + +class Process(object): + """ + Manages the lifetime and pipe connections of the SSH command running in the + slave. + """ + def __init__(self, router, stdin_fd, stdout_fd, proc=None): + self.router = router + self.stdin_fd = stdin_fd + self.stdout_fd = stdout_fd + self.proc = proc + self.control_handle = router.add_handler(self._on_control) + self.stdin_handle = router.add_handler(self._on_stdin) + self.pump = IoPump(router.broker, stdin_fd, stdout_fd) + self.stdin = None + self.control = None + self.wake_event = threading.Event() + + mitogen.core.listen(self.pump, 'disconnect', self._on_pump_disconnect) + mitogen.core.listen(self.pump, 'receive', self._on_pump_receive) + + if proc: + pmon = mitogen.parent.ProcessMonitor.instance() + pmon.add(proc.pid, self._on_proc_exit) + + def __repr__(self): + return 'Process(%r, %r)' % (self.stdin_fd, self.stdout_fd) + + def _on_proc_exit(self, status): + LOG.debug('%r._on_proc_exit(%r)', self, status) + self.control.put(('exit', status)) + + def _on_stdin(self, msg): + if msg.is_dead: + IOLOG.debug('%r._on_stdin() -> %r', self, data) + self.pump.close() + return + + data = msg.unpickle() + IOLOG.debug('%r._on_stdin() -> len %d', self, len(data)) + self.pump.write(data) + + def _on_control(self, msg): + if not msg.is_dead: + command, arg = msg.unpickle(throw=False) + LOG.debug('%r._on_control(%r, %s)', self, command, arg) + + func = getattr(self, '_on_%s' % (command,), None) + if func: + return func(msg, arg) + + LOG.warning('%r: unknown command %r', self, command) + + def _on_start(self, msg, arg): + dest = mitogen.core.Context(self.router, msg.src_id) + self.control = mitogen.core.Sender(dest, arg[0]) + self.stdin = mitogen.core.Sender(dest, arg[1]) + self.router.broker.start_receive(self.pump) + + def _on_exit(self, msg, arg): + LOG.debug('on_exit: proc = %r', self.proc) + if self.proc: + self.proc.terminate() + else: + self.router.broker.shutdown() + + def _on_pump_receive(self, s): + IOLOG.info('%r._on_pump_receive(len %d)', self, len(s)) + self.stdin.put(s) + + def _on_pump_disconnect(self): + LOG.debug('%r._on_pump_disconnect()', self) + mitogen.core.fire(self, 'disconnect') + self.stdin.close() + self.wake_event.set() + + def start_master(self, stdin, control): + self.stdin = stdin + self.control = control + control.put(('start', (self.control_handle, self.stdin_handle))) + self.router.broker.start_receive(self.pump) + + def wait(self): + while not self.wake_event.isSet(): + # Timeout is used so that sleep is interruptible, as blocking + # variants of libc thread operations cannot be interrupted e.g. via + # KeyboardInterrupt. isSet() test and wait() are separate since in + # <2.7 wait() always returns None. + self.wake_event.wait(0.1) + + +@mitogen.core.takes_router +def _start_slave(src_id, cmdline, router): + """ + This runs in the target context, it is invoked by _fakessh_main running in + the fakessh context immediately after startup. It starts the slave process + (the the point where it has a stdin_handle to target but not stdout_chan to + write to), and waits for main to. + """ + LOG.debug('_start_slave(%r, %r)', router, cmdline) + + proc = subprocess.Popen( + cmdline, + # SSH server always uses user's shell. + shell=True, + # SSH server always executes new commands in the user's HOME. + cwd=os.path.expanduser('~'), + + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + process = Process( + router, + proc.stdin.fileno(), + proc.stdout.fileno(), + proc, + ) + + return process.control_handle, process.stdin_handle + + +# +# SSH client interface. +# + + +def exit(): + _mitogen.broker.shutdown() + + +def die(msg, *args): + if args: + msg %= args + sys.stderr.write('%s\n' % (msg,)) + exit() + + +def parse_args(): + hostname = None + remain = sys.argv[1:] + allopts = [] + restarted = 0 + + while remain and restarted < 2: + opts, args = getopt.getopt(remain, SSH_GETOPTS) + remain = remain[:] # getopt bug! + allopts += opts + if not args: + break + + if not hostname: + hostname = args.pop(0) + remain = remain[remain.index(hostname) + 1:] + + restarted += 1 + + return hostname, allopts, args + + +@mitogen.core.takes_econtext +def _fakessh_main(dest_context_id, econtext): + hostname, opts, args = parse_args() + if not hostname: + die('Missing hostname') + + subsystem = False + for opt, optarg in opts: + if opt == '-s': + subsystem = True + else: + LOG.debug('Warning option %s %s is ignored.', opt, optarg) + + LOG.debug('hostname: %r', hostname) + LOG.debug('opts: %r', opts) + LOG.debug('args: %r', args) + + if subsystem: + die('-s is not yet supported') + + if not args: + die('fakessh: login mode not supported and no command specified') + + dest = mitogen.parent.Context(econtext.router, dest_context_id) + + # Even though SSH receives an argument vector, it still cats the vector + # together before sending to the server, the server just uses /bin/sh -c to + # run the command. We must remain puke-for-puke compatible. + control_handle, stdin_handle = dest.call(_start_slave, + mitogen.context_id, ' '.join(args)) + + LOG.debug('_fakessh_main: received control_handle=%r, stdin_handle=%r', + control_handle, stdin_handle) + + process = Process(econtext.router, 1, 0) + process.start_master( + stdin=mitogen.core.Sender(dest, stdin_handle), + control=mitogen.core.Sender(dest, control_handle), + ) + process.wait() + process.control.put(('exit', None)) + + +def _get_econtext_config(context, sock2): + parent_ids = mitogen.parent_ids[:] + parent_ids.insert(0, mitogen.context_id) + return { + 'context_id': context.context_id, + 'core_src_fd': None, + 'debug': getattr(context.router, 'debug', False), + 'in_fd': sock2.fileno(), + 'log_level': mitogen.parent.get_log_level(), + 'max_message_size': context.router.max_message_size, + 'out_fd': sock2.fileno(), + 'parent_ids': parent_ids, + 'profiling': getattr(context.router, 'profiling', False), + 'unidirectional': getattr(context.router, 'unidirectional', False), + 'setup_stdio': False, + 'version': mitogen.__version__, + } + + +# +# Public API. +# + +@mitogen.core.takes_econtext +@mitogen.core.takes_router +def run(dest, router, args, deadline=None, econtext=None): + """ + Run the command specified by `args` such that ``PATH`` searches for SSH by + the command will cause its attempt to use SSH to execute a remote program + to be redirected to use mitogen to execute that program using the context + `dest` instead. + + :param list args: + Argument vector. + :param mitogen.core.Context dest: + The destination context to execute the SSH command line in. + + :param mitogen.core.Router router: + + :param list[str] args: + Command line arguments for local program, e.g. + ``['rsync', '/tmp', 'remote:/tmp']`` + + :returns: + Exit status of the child process. + """ + if econtext is not None: + mitogen.parent.upgrade_router(econtext) + + context_id = router.allocate_id() + fakessh = mitogen.parent.Context(router, context_id) + fakessh.name = u'fakessh.%d' % (context_id,) + + sock1, sock2 = socket.socketpair() + + stream = mitogen.core.Stream(router, context_id) + stream.name = u'fakessh' + stream.accept(sock1.fileno(), sock1.fileno()) + router.register(fakessh, stream) + + # Held in socket buffer until process is booted. + fakessh.call_async(_fakessh_main, dest.context_id) + + tmp_path = tempfile.mkdtemp(prefix='mitogen_fakessh') + try: + ssh_path = os.path.join(tmp_path, 'ssh') + fp = open(ssh_path, 'w') + try: + fp.write('#!%s\n' % (mitogen.parent.get_sys_executable(),)) + fp.write(inspect.getsource(mitogen.core)) + fp.write('\n') + fp.write('ExternalContext(%r).main()\n' % ( + _get_econtext_config(context, sock2), + )) + finally: + fp.close() + + os.chmod(ssh_path, int('0755', 8)) + env = os.environ.copy() + env.update({ + 'PATH': '%s:%s' % (tmp_path, env.get('PATH', '')), + 'ARGV0': mitogen.parent.get_sys_executable(), + 'SSH_PATH': ssh_path, + }) + + proc = subprocess.Popen(args, env=env) + return proc.wait() + finally: + shutil.rmtree(tmp_path) diff --git a/mitogen-0.2.7/mitogen/fork.py b/mitogen-0.2.7/mitogen/fork.py new file mode 100644 index 000000000..d6685d7 --- /dev/null +++ b/mitogen-0.2.7/mitogen/fork.py @@ -0,0 +1,223 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +import logging +import os +import random +import sys +import threading +import traceback + +import mitogen.core +import mitogen.parent + + +LOG = logging.getLogger('mitogen') + +# Python 2.4/2.5 cannot support fork+threads whatsoever, it doesn't even fix up +# interpreter state. So 2.4/2.5 interpreters start .local() contexts for +# isolation instead. Since we don't have any crazy memory sharing problems to +# avoid, there is no virginal fork parent either. The child is started directly +# from the login/become process. In future this will be default everywhere, +# fork is brainwrong from the stone age. +FORK_SUPPORTED = sys.version_info >= (2, 6) + + +class Error(mitogen.core.StreamError): + pass + + +def fixup_prngs(): + """ + Add 256 bits of /dev/urandom to OpenSSL's PRNG in the child, and re-seed + the random package with the same data. + """ + s = os.urandom(256 // 8) + random.seed(s) + if 'ssl' in sys.modules: + sys.modules['ssl'].RAND_add(s, 75.0) + + +def reset_logging_framework(): + """ + After fork, ensure any logging.Handler locks are recreated, as a variety of + threads in the parent may have been using the logging package at the moment + of fork. + + It is not possible to solve this problem in general; see + https://github.com/dw/mitogen/issues/150 for a full discussion. + """ + logging._lock = threading.RLock() + + # The root logger does not appear in the loggerDict. + for name in [None] + list(logging.Logger.manager.loggerDict): + for handler in logging.getLogger(name).handlers: + handler.createLock() + + root = logging.getLogger() + root.handlers = [ + handler + for handler in root.handlers + if not isinstance(handler, mitogen.core.LogHandler) + ] + + +def on_fork(): + """ + Should be called by any program integrating Mitogen each time the process + is forked, in the context of the new child. + """ + reset_logging_framework() # Must be first! + fixup_prngs() + mitogen.core.Latch._on_fork() + mitogen.core.Side._on_fork() + mitogen.core.ExternalContext.service_stub_lock = threading.Lock() + + mitogen__service = sys.modules.get('mitogen.service') + if mitogen__service: + mitogen__service._pool_lock = threading.Lock() + + +def handle_child_crash(): + """ + Respond to _child_main() crashing by ensuring the relevant exception is + logged to /dev/tty. + """ + tty = open('/dev/tty', 'wb') + tty.write('\n\nFORKED CHILD PID %d CRASHED\n%s\n\n' % ( + os.getpid(), + traceback.format_exc(), + )) + tty.close() + os._exit(1) + + +class Stream(mitogen.parent.Stream): + child_is_immediate_subprocess = True + + #: Reference to the importer, if any, recovered from the parent. + importer = None + + #: User-supplied function for cleaning up child process state. + on_fork = None + + python_version_msg = ( + "The mitogen.fork method is not supported on Python versions " + "prior to 2.6, since those versions made no attempt to repair " + "critical interpreter state following a fork. Please use the " + "local() method instead." + ) + + def construct(self, old_router, max_message_size, on_fork=None, + debug=False, profiling=False, unidirectional=False, + on_start=None): + if not FORK_SUPPORTED: + raise Error(self.python_version_msg) + + # fork method only supports a tiny subset of options. + super(Stream, self).construct(max_message_size=max_message_size, + debug=debug, profiling=profiling, + unidirectional=False) + self.on_fork = on_fork + self.on_start = on_start + + responder = getattr(old_router, 'responder', None) + if isinstance(responder, mitogen.parent.ModuleForwarder): + self.importer = responder.importer + + name_prefix = u'fork' + + def start_child(self): + parentfp, childfp = mitogen.parent.create_socketpair() + self.pid = os.fork() + if self.pid: + childfp.close() + # Decouple the socket from the lifetime of the Python socket object. + fd = os.dup(parentfp.fileno()) + parentfp.close() + return self.pid, fd, None + else: + parentfp.close() + self._wrap_child_main(childfp) + + def _wrap_child_main(self, childfp): + try: + self._child_main(childfp) + except BaseException: + handle_child_crash() + + def _child_main(self, childfp): + on_fork() + if self.on_fork: + self.on_fork() + mitogen.core.set_block(childfp.fileno()) + + # Expected by the ExternalContext.main(). + os.dup2(childfp.fileno(), 1) + os.dup2(childfp.fileno(), 100) + + # Overwritten by ExternalContext.main(); we must replace the + # parent-inherited descriptors that were closed by Side._on_fork() to + # avoid ExternalContext.main() accidentally allocating new files over + # the standard handles. + os.dup2(childfp.fileno(), 0) + + # Avoid corrupting the stream on fork crash by dupping /dev/null over + # stderr. Instead, handle_child_crash() uses /dev/tty to log errors. + devnull = os.open('/dev/null', os.O_WRONLY) + if devnull != 2: + os.dup2(devnull, 2) + os.close(devnull) + + # If we're unlucky, childfp.fileno() may coincidentally be one of our + # desired FDs. In that case closing it breaks ExternalContext.main(). + if childfp.fileno() not in (0, 1, 100): + childfp.close() + + config = self.get_econtext_config() + config['core_src_fd'] = None + config['importer'] = self.importer + config['setup_package'] = False + if self.on_start: + config['on_start'] = self.on_start + + try: + try: + mitogen.core.ExternalContext(config).main() + except Exception: + # TODO: report exception somehow. + os._exit(72) + finally: + # Don't trigger atexit handlers, they were copied from the parent. + os._exit(0) + + def _connect_bootstrap(self): + # None required. + pass diff --git a/mitogen-0.2.7/mitogen/fork.pyc b/mitogen-0.2.7/mitogen/fork.pyc new file mode 100644 index 000000000..0b11617 Binary files /dev/null and b/mitogen-0.2.7/mitogen/fork.pyc differ diff --git a/mitogen-0.2.7/mitogen/jail.py b/mitogen-0.2.7/mitogen/jail.py new file mode 100644 index 000000000..6e0ac68 --- /dev/null +++ b/mitogen-0.2.7/mitogen/jail.py @@ -0,0 +1,65 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +import logging + +import mitogen.core +import mitogen.parent + + +LOG = logging.getLogger(__name__) + + +class Stream(mitogen.parent.Stream): + child_is_immediate_subprocess = False + create_child_args = { + 'merge_stdio': True + } + + container = None + username = None + jexec_path = '/usr/sbin/jexec' + + def construct(self, container, jexec_path=None, username=None, **kwargs): + super(Stream, self).construct(**kwargs) + self.container = container + self.username = username + if jexec_path: + self.jexec_path = jexec_path + + def _get_name(self): + return u'jail.' + self.container + + def get_boot_command(self): + bits = [self.jexec_path] + if self.username: + bits += ['-U', self.username] + bits += [self.container] + return bits + super(Stream, self).get_boot_command() diff --git a/mitogen-0.2.7/mitogen/kubectl.py b/mitogen-0.2.7/mitogen/kubectl.py new file mode 100644 index 000000000..ef626e1 --- /dev/null +++ b/mitogen-0.2.7/mitogen/kubectl.py @@ -0,0 +1,65 @@ +# Copyright 2018, Yannig Perre +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +import logging + +import mitogen.core +import mitogen.parent + + +LOG = logging.getLogger(__name__) + + +class Stream(mitogen.parent.Stream): + child_is_immediate_subprocess = True + + pod = None + kubectl_path = 'kubectl' + kubectl_args = None + + # TODO: better way of capturing errors such as "No such container." + create_child_args = { + 'merge_stdio': True + } + + def construct(self, pod, kubectl_path=None, kubectl_args=None, **kwargs): + super(Stream, self).construct(**kwargs) + assert pod + self.pod = pod + if kubectl_path: + self.kubectl_path = kubectl_path + self.kubectl_args = kubectl_args or [] + + def _get_name(self): + return u'kubectl.%s%s' % (self.pod, self.kubectl_args) + + def get_boot_command(self): + bits = [self.kubectl_path] + self.kubectl_args + ['exec', '-it', self.pod] + return bits + ["--"] + super(Stream, self).get_boot_command() diff --git a/mitogen-0.2.7/mitogen/lxc.py b/mitogen-0.2.7/mitogen/lxc.py new file mode 100644 index 000000000..879d19a --- /dev/null +++ b/mitogen-0.2.7/mitogen/lxc.py @@ -0,0 +1,75 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +import logging + +import mitogen.core +import mitogen.parent + + +LOG = logging.getLogger(__name__) + + +class Stream(mitogen.parent.Stream): + child_is_immediate_subprocess = False + create_child_args = { + # If lxc-attach finds any of stdin, stdout, stderr connected to a TTY, + # to prevent input injection it creates a proxy pty, forcing all IO to + # be buffered in <4KiB chunks. So ensure stderr is also routed to the + # socketpair. + 'merge_stdio': True + } + + container = None + lxc_attach_path = 'lxc-attach' + + eof_error_hint = ( + 'Note: many versions of LXC do not report program execution failure ' + 'meaningfully. Please check the host logs (/var/log) for more ' + 'information.' + ) + + def construct(self, container, lxc_attach_path=None, **kwargs): + super(Stream, self).construct(**kwargs) + self.container = container + if lxc_attach_path: + self.lxc_attach_path = lxc_attach_path + + def _get_name(self): + return u'lxc.' + self.container + + def get_boot_command(self): + bits = [ + self.lxc_attach_path, + '--clear-env', + '--name', self.container, + '--', + ] + return bits + super(Stream, self).get_boot_command() diff --git a/mitogen-0.2.7/mitogen/lxd.py b/mitogen-0.2.7/mitogen/lxd.py new file mode 100644 index 000000000..faea256 --- /dev/null +++ b/mitogen-0.2.7/mitogen/lxd.py @@ -0,0 +1,77 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +import logging + +import mitogen.core +import mitogen.parent + + +LOG = logging.getLogger(__name__) + + +class Stream(mitogen.parent.Stream): + child_is_immediate_subprocess = False + create_child_args = { + # If lxc finds any of stdin, stdout, stderr connected to a TTY, to + # prevent input injection it creates a proxy pty, forcing all IO to be + # buffered in <4KiB chunks. So ensure stderr is also routed to the + # socketpair. + 'merge_stdio': True + } + + container = None + lxc_path = 'lxc' + python_path = 'python' + + eof_error_hint = ( + 'Note: many versions of LXC do not report program execution failure ' + 'meaningfully. Please check the host logs (/var/log) for more ' + 'information.' + ) + + def construct(self, container, lxc_path=None, **kwargs): + super(Stream, self).construct(**kwargs) + self.container = container + if lxc_path: + self.lxc_path = lxc_path + + def _get_name(self): + return u'lxd.' + self.container + + def get_boot_command(self): + bits = [ + self.lxc_path, + 'exec', + '--mode=noninteractive', + self.container, + '--', + ] + return bits + super(Stream, self).get_boot_command() diff --git a/mitogen-0.2.7/mitogen/master.py b/mitogen-0.2.7/mitogen/master.py new file mode 100644 index 000000000..1396f4e --- /dev/null +++ b/mitogen-0.2.7/mitogen/master.py @@ -0,0 +1,1173 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +""" +This module implements functionality required by master processes, such as +starting new contexts via SSH. Its size is also restricted, since it must +be sent to any context that will be used to establish additional child +contexts. +""" + +import dis +import imp +import inspect +import itertools +import logging +import os +import pkgutil +import re +import string +import sys +import time +import threading +import types +import zlib + +if not hasattr(pkgutil, 'find_loader'): + # find_loader() was new in >=2.5, but the modern pkgutil.py syntax has + # been kept intentionally 2.3 compatible so we can reuse it. + from mitogen.compat import pkgutil + +import mitogen +import mitogen.core +import mitogen.minify +import mitogen.parent + +from mitogen.core import b +from mitogen.core import IOLOG +from mitogen.core import LOG +from mitogen.core import str_partition +from mitogen.core import str_rpartition +from mitogen.core import to_text + +imap = getattr(itertools, 'imap', map) +izip = getattr(itertools, 'izip', zip) + +try: + any +except NameError: + from mitogen.core import any + +try: + next +except NameError: + from mitogen.core import next + + +RLOG = logging.getLogger('mitogen.ctx') + + +def _stdlib_paths(): + """Return a set of paths from which Python imports the standard library. + """ + attr_candidates = [ + 'prefix', + 'real_prefix', # virtualenv: only set inside a virtual environment. + 'base_prefix', # venv: always set, equal to prefix if outside. + ] + prefixes = (getattr(sys, a) for a in attr_candidates if hasattr(sys, a)) + version = 'python%s.%s' % sys.version_info[0:2] + return set(os.path.abspath(os.path.join(p, 'lib', version)) + for p in prefixes) + + +def is_stdlib_name(modname): + """Return :data:`True` if `modname` appears to come from the standard + library. + """ + if imp.is_builtin(modname) != 0: + return True + + module = sys.modules.get(modname) + if module is None: + return False + + # six installs crap with no __file__ + modpath = os.path.abspath(getattr(module, '__file__', '')) + return is_stdlib_path(modpath) + + +_STDLIB_PATHS = _stdlib_paths() + + +def is_stdlib_path(path): + return any( + os.path.commonprefix((libpath, path)) == libpath + and 'site-packages' not in path + and 'dist-packages' not in path + for libpath in _STDLIB_PATHS + ) + + +def get_child_modules(path): + """Return the suffixes of submodules directly neated beneath of the package + directory at `path`. + + :param str path: + Path to the module's source code on disk, or some PEP-302-recognized + equivalent. Usually this is the module's ``__file__`` attribute, but + is specified explicitly to avoid loading the module. + + :return: + List of submodule name suffixes. + """ + it = pkgutil.iter_modules([os.path.dirname(path)]) + return [to_text(name) for _, name, _ in it] + + +def _get_core_source(): + """ + Master version of parent.get_core_source(). + """ + source = inspect.getsource(mitogen.core) + return mitogen.minify.minimize_source(source) + + +if mitogen.is_master: + # TODO: find a less surprising way of installing this. + mitogen.parent._get_core_source = _get_core_source + + +LOAD_CONST = dis.opname.index('LOAD_CONST') +IMPORT_NAME = dis.opname.index('IMPORT_NAME') + + +def _getarg(nextb, c): + if c >= dis.HAVE_ARGUMENT: + return nextb() | (nextb() << 8) + + +if sys.version_info < (3, 0): + def iter_opcodes(co): + # Yield `(op, oparg)` tuples from the code object `co`. + ordit = imap(ord, co.co_code) + nextb = ordit.next + return ((c, _getarg(nextb, c)) for c in ordit) +elif sys.version_info < (3, 6): + def iter_opcodes(co): + # Yield `(op, oparg)` tuples from the code object `co`. + ordit = iter(co.co_code) + nextb = ordit.__next__ + return ((c, _getarg(nextb, c)) for c in ordit) +else: + def iter_opcodes(co): + # Yield `(op, oparg)` tuples from the code object `co`. + ordit = iter(co.co_code) + nextb = ordit.__next__ + # https://github.com/abarnert/cpython/blob/c095a32f/Python/wordcode.md + return ((c, nextb()) for c in ordit) + + +def scan_code_imports(co): + """ + Given a code object `co`, scan its bytecode yielding any ``IMPORT_NAME`` + and associated prior ``LOAD_CONST`` instructions representing an `Import` + statement or `ImportFrom` statement. + + :return: + Generator producing `(level, modname, namelist)` tuples, where: + + * `level`: -1 for normal import, 0, for absolute import, and >0 for + relative import. + * `modname`: Name of module to import, or from where `namelist` names + are imported. + * `namelist`: for `ImportFrom`, the list of names to be imported from + `modname`. + """ + opit = iter_opcodes(co) + opit, opit2, opit3 = itertools.tee(opit, 3) + + try: + next(opit2) + next(opit3) + next(opit3) + except StopIteration: + return + + if sys.version_info >= (2, 5): + for oparg1, oparg2, (op3, arg3) in izip(opit, opit2, opit3): + if op3 == IMPORT_NAME: + op2, arg2 = oparg2 + op1, arg1 = oparg1 + if op1 == op2 == LOAD_CONST: + yield (co.co_consts[arg1], + co.co_names[arg3], + co.co_consts[arg2] or ()) + else: + # Python 2.4 did not yet have 'level', so stack format differs. + for oparg1, (op2, arg2) in izip(opit, opit2): + if op2 == IMPORT_NAME: + op1, arg1 = oparg1 + if op1 == LOAD_CONST: + yield (-1, co.co_names[arg2], co.co_consts[arg1] or ()) + + +class ThreadWatcher(object): + """ + Manage threads that wait for another thread to shut down, before invoking + `on_join()` for each associated ThreadWatcher. + + In CPython it seems possible to use this method to ensure a non-main thread + is signalled when the main thread has exited, using a third thread as a + proxy. + """ + #: Protects remaining _cls_* members. + _cls_lock = threading.Lock() + + #: PID of the process that last modified the class data. If the PID + #: changes, it means the thread watch dict refers to threads that no longer + #: exist in the current process (since it forked), and so must be reset. + _cls_pid = None + + #: Map watched Thread -> list of ThreadWatcher instances. + _cls_instances_by_target = {} + + #: Map watched Thread -> watcher Thread for each watched thread. + _cls_thread_by_target = {} + + @classmethod + def _reset(cls): + """If we have forked since the watch dictionaries were initialized, all + that has is garbage, so clear it.""" + if os.getpid() != cls._cls_pid: + cls._cls_pid = os.getpid() + cls._cls_instances_by_target.clear() + cls._cls_thread_by_target.clear() + + def __init__(self, target, on_join): + self.target = target + self.on_join = on_join + + @classmethod + def _watch(cls, target): + target.join() + for watcher in cls._cls_instances_by_target[target]: + watcher.on_join() + + def install(self): + self._cls_lock.acquire() + try: + self._reset() + lst = self._cls_instances_by_target.setdefault(self.target, []) + lst.append(self) + if self.target not in self._cls_thread_by_target: + self._cls_thread_by_target[self.target] = threading.Thread( + name='mitogen.master.join_thread_async', + target=self._watch, + args=(self.target,) + ) + self._cls_thread_by_target[self.target].start() + finally: + self._cls_lock.release() + + def remove(self): + self._cls_lock.acquire() + try: + self._reset() + lst = self._cls_instances_by_target.get(self.target, []) + if self in lst: + lst.remove(self) + finally: + self._cls_lock.release() + + @classmethod + def watch(cls, target, on_join): + watcher = cls(target, on_join) + watcher.install() + return watcher + + +class LogForwarder(object): + """ + Install a :data:`mitogen.core.FORWARD_LOG` handler that delivers forwarded + log events into the local logging framework. This is used by the master's + :class:`Router`. + + The forwarded :class:`logging.LogRecord` objects are delivered to loggers + under ``mitogen.ctx.*`` corresponding to their + :attr:`mitogen.core.Context.name`, with the message prefixed with the + logger name used in the child. The records include some extra attributes: + + * ``mitogen_message``: Unicode original message without the logger name + prepended. + * ``mitogen_context``: :class:`mitogen.parent.Context` reference to the + source context. + * ``mitogen_name``: Original logger name. + + :param mitogen.master.Router router: + Router to install the handler on. + """ + def __init__(self, router): + self._router = router + self._cache = {} + router.add_handler( + fn=self._on_forward_log, + handle=mitogen.core.FORWARD_LOG, + ) + + def _on_forward_log(self, msg): + if msg.is_dead: + return + + logger = self._cache.get(msg.src_id) + if logger is None: + context = self._router.context_by_id(msg.src_id) + if context is None: + LOG.error('%s: dropping log from unknown context ID %d', + self, msg.src_id) + return + + name = '%s.%s' % (RLOG.name, context.name) + self._cache[msg.src_id] = logger = logging.getLogger(name) + + name, level_s, s = msg.data.decode('latin1').split('\x00', 2) + + # See logging.Handler.makeRecord() + record = logging.LogRecord( + name=logger.name, + level=int(level_s), + pathname='(unknown file)', + lineno=0, + msg=('%s: %s' % (name, s)), + args=(), + exc_info=None, + ) + record.mitogen_message = s + record.mitogen_context = self._router.context_by_id(msg.src_id) + record.mitogen_name = name + logger.handle(record) + + def __repr__(self): + return 'LogForwarder(%r)' % (self._router,) + + +class ModuleFinder(object): + """ + Given the name of a loaded module, make a best-effort attempt at finding + related modules likely needed by a child context requesting the original + module. + """ + def __init__(self): + #: Import machinery is expensive, keep :py:meth`:get_module_source` + #: results around. + self._found_cache = {} + + #: Avoid repeated dependency scanning, which is expensive. + self._related_cache = {} + + def __repr__(self): + return 'ModuleFinder()' + + def _looks_like_script(self, path): + """ + Return :data:`True` if the (possibly extensionless) file at `path` + resembles a Python script. For now we simply verify the file contains + ASCII text. + """ + fp = open(path, 'rb') + try: + sample = fp.read(512).decode('latin-1') + return not set(sample).difference(string.printable) + finally: + fp.close() + + def _py_filename(self, path): + if not path: + return None + + if path[-4:] in ('.pyc', '.pyo'): + path = path.rstrip('co') + + if path.endswith('.py'): + return path + + if os.path.exists(path) and self._looks_like_script(path): + return path + + def _get_main_module_defective_python_3x(self, fullname): + """ + Recent versions of Python 3.x introduced an incomplete notion of + importer specs, and in doing so created permanent asymmetry in the + :mod:`pkgutil` interface handling for the `__main__` module. Therefore + we must handle `__main__` specially. + """ + if fullname != '__main__': + return None + + mod = sys.modules.get(fullname) + if not mod: + return None + + path = getattr(mod, '__file__', None) + if not (os.path.exists(path) and self._looks_like_script(path)): + return None + + fp = open(path, 'rb') + try: + source = fp.read() + finally: + fp.close() + + return path, source, False + + def _get_module_via_pkgutil(self, fullname): + """ + Attempt to fetch source code via pkgutil. In an ideal world, this would + be the only required implementation of get_module(). + """ + try: + # Pre-'import spec' this returned None, in Python3.6 it raises + # ImportError. + loader = pkgutil.find_loader(fullname) + except ImportError: + e = sys.exc_info()[1] + LOG.debug('%r._get_module_via_pkgutil(%r): %s', + self, fullname, e) + return None + + IOLOG.debug('%r._get_module_via_pkgutil(%r) -> %r', + self, fullname, loader) + if not loader: + return + + try: + path = self._py_filename(loader.get_filename(fullname)) + source = loader.get_source(fullname) + is_pkg = loader.is_package(fullname) + except (AttributeError, ImportError): + # - Per PEP-302, get_source() and is_package() are optional, + # calling them may throw AttributeError. + # - get_filename() may throw ImportError if pkgutil.find_loader() + # picks a "parent" package's loader for some crap that's been + # stuffed in sys.modules, for example in the case of urllib3: + # "loader for urllib3.contrib.pyopenssl cannot handle + # requests.packages.urllib3.contrib.pyopenssl" + e = sys.exc_info()[1] + LOG.debug('%r: loading %r using %r failed: %s', + self, fullname, loader, e) + return + + if path is None or source is None: + return + + if isinstance(source, mitogen.core.UnicodeType): + # get_source() returns "string" according to PEP-302, which was + # reinterpreted for Python 3 to mean a Unicode string. + source = source.encode('utf-8') + + return path, source, is_pkg + + def _get_module_via_sys_modules(self, fullname): + """ + Attempt to fetch source code via sys.modules. This is specifically to + support __main__, but it may catch a few more cases. + """ + module = sys.modules.get(fullname) + LOG.debug('_get_module_via_sys_modules(%r) -> %r', fullname, module) + if not isinstance(module, types.ModuleType): + LOG.debug('sys.modules[%r] absent or not a regular module', + fullname) + return + + path = self._py_filename(getattr(module, '__file__', '')) + if not path: + return + + is_pkg = hasattr(module, '__path__') + try: + source = inspect.getsource(module) + except IOError: + # Work around inspect.getsourcelines() bug for 0-byte __init__.py + # files. + if not is_pkg: + raise + source = '\n' + + if isinstance(source, mitogen.core.UnicodeType): + # get_source() returns "string" according to PEP-302, which was + # reinterpreted for Python 3 to mean a Unicode string. + source = source.encode('utf-8') + + return path, source, is_pkg + + def _get_module_via_parent_enumeration(self, fullname): + """ + Attempt to fetch source code by examining the module's (hopefully less + insane) parent package. Required for older versions of + ansible.compat.six and plumbum.colors. + """ + if fullname not in sys.modules: + # Don't attempt this unless a module really exists in sys.modules, + # else we could return junk. + return + + pkgname, _, modname = str_rpartition(to_text(fullname), u'.') + pkg = sys.modules.get(pkgname) + if pkg is None or not hasattr(pkg, '__file__'): + return + + pkg_path = os.path.dirname(pkg.__file__) + try: + fp, path, ext = imp.find_module(modname, [pkg_path]) + try: + path = self._py_filename(path) + if not path: + fp.close() + return + + source = fp.read() + finally: + if fp: + fp.close() + + if isinstance(source, mitogen.core.UnicodeType): + # get_source() returns "string" according to PEP-302, which was + # reinterpreted for Python 3 to mean a Unicode string. + source = source.encode('utf-8') + return path, source, False + except ImportError: + e = sys.exc_info()[1] + LOG.debug('imp.find_module(%r, %r) -> %s', modname, [pkg_path], e) + + def add_source_override(self, fullname, path, source, is_pkg): + """ + Explicitly install a source cache entry, preventing usual lookup + methods from being used. + + Beware the value of `path` is critical when `is_pkg` is specified, + since it directs where submodules are searched for. + + :param str fullname: + Name of the module to override. + :param str path: + Module's path as it will appear in the cache. + :param bytes source: + Module source code as a bytestring. + :param bool is_pkg: + :data:`True` if the module is a package. + """ + self._found_cache[fullname] = (path, source, is_pkg) + + get_module_methods = [ + _get_main_module_defective_python_3x, + _get_module_via_pkgutil, + _get_module_via_sys_modules, + _get_module_via_parent_enumeration, + ] + + def get_module_source(self, fullname): + """Given the name of a loaded module `fullname`, attempt to find its + source code. + + :returns: + Tuple of `(module path, source text, is package?)`, or :data:`None` + if the source cannot be found. + """ + tup = self._found_cache.get(fullname) + if tup: + return tup + + for method in self.get_module_methods: + tup = method(self, fullname) + if tup: + #LOG.debug('%r returned %r', method, tup) + break + else: + tup = None, None, None + LOG.debug('get_module_source(%r): cannot find source', fullname) + + self._found_cache[fullname] = tup + return tup + + def resolve_relpath(self, fullname, level): + """Given an ImportFrom AST node, guess the prefix that should be tacked + on to an alias name to produce a canonical name. `fullname` is the name + of the module in which the ImportFrom appears. + """ + mod = sys.modules.get(fullname, None) + if hasattr(mod, '__path__'): + fullname += '.__init__' + + if level == 0 or not fullname: + return '' + + bits = fullname.split('.') + if len(bits) <= level: + # This would be an ImportError in real code. + return '' + + return '.'.join(bits[:-level]) + '.' + + def generate_parent_names(self, fullname): + while '.' in fullname: + fullname, _, _ = str_rpartition(to_text(fullname), u'.') + yield fullname + + def find_related_imports(self, fullname): + """ + Return a list of non-stdlib modules that are directly imported by + `fullname`, plus their parents. + + The list is determined by retrieving the source code of + `fullname`, compiling it, and examining all IMPORT_NAME ops. + + :param fullname: Fully qualified name of an _already imported_ module + for which source code can be retrieved + :type fullname: str + """ + related = self._related_cache.get(fullname) + if related is not None: + return related + + modpath, src, _ = self.get_module_source(fullname) + if src is None: + return [] + + maybe_names = list(self.generate_parent_names(fullname)) + + co = compile(src, modpath, 'exec') + for level, modname, namelist in scan_code_imports(co): + if level == -1: + modnames = [modname, '%s.%s' % (fullname, modname)] + else: + modnames = [ + '%s%s' % (self.resolve_relpath(fullname, level), modname) + ] + + maybe_names.extend(modnames) + maybe_names.extend( + '%s.%s' % (mname, name) + for mname in modnames + for name in namelist + ) + + return self._related_cache.setdefault(fullname, sorted( + set( + mitogen.core.to_text(name) + for name in maybe_names + if sys.modules.get(name) is not None + and not is_stdlib_name(name) + and u'six.moves' not in name # TODO: crap + ) + )) + + def find_related(self, fullname): + """ + Return a list of non-stdlib modules that are imported directly or + indirectly by `fullname`, plus their parents. + + This method is like :py:meth:`find_related_imports`, but also + recursively searches any modules which are imported by `fullname`. + + :param fullname: Fully qualified name of an _already imported_ module + for which source code can be retrieved + :type fullname: str + """ + stack = [fullname] + found = set() + + while stack: + name = stack.pop(0) + names = self.find_related_imports(name) + stack.extend(set(names).difference(set(found).union(stack))) + found.update(names) + + found.discard(fullname) + return sorted(found) + + +class ModuleResponder(object): + def __init__(self, router): + self._router = router + self._finder = ModuleFinder() + self._cache = {} # fullname -> pickled + self.blacklist = [] + self.whitelist = [''] + + #: Context -> set([fullname, ..]) + self._forwarded_by_context = {} + + #: Number of GET_MODULE messages received. + self.get_module_count = 0 + #: Total time spent in uncached GET_MODULE. + self.get_module_secs = 0.0 + #: Total time spent minifying modules. + self.minify_secs = 0.0 + #: Number of successful LOAD_MODULE messages sent. + self.good_load_module_count = 0 + #: Total bytes in successful LOAD_MODULE payloads. + self.good_load_module_size = 0 + #: Number of negative LOAD_MODULE messages sent. + self.bad_load_module_count = 0 + + router.add_handler( + fn=self._on_get_module, + handle=mitogen.core.GET_MODULE, + ) + + def __repr__(self): + return 'ModuleResponder(%r)' % (self._router,) + + def add_source_override(self, fullname, path, source, is_pkg): + """ + See :meth:`ModuleFinder.add_source_override. + """ + self._finder.add_source_override(fullname, path, source, is_pkg) + + MAIN_RE = re.compile(b(r'^if\s+__name__\s*==\s*.__main__.\s*:'), re.M) + main_guard_msg = ( + "A child context attempted to import __main__, however the main " + "module present in the master process lacks an execution guard. " + "Update %r to prevent unintended execution, using a guard like:\n" + "\n" + " if __name__ == '__main__':\n" + " # your code here.\n" + ) + + def whitelist_prefix(self, fullname): + if self.whitelist == ['']: + self.whitelist = ['mitogen'] + self.whitelist.append(fullname) + + def blacklist_prefix(self, fullname): + self.blacklist.append(fullname) + + def neutralize_main(self, path, src): + """Given the source for the __main__ module, try to find where it + begins conditional execution based on a "if __name__ == '__main__'" + guard, and remove any code after that point.""" + match = self.MAIN_RE.search(src) + if match: + return src[:match.start()] + + if b('mitogen.main(') in src: + return src + + LOG.error(self.main_guard_msg, path) + raise ImportError('refused') + + def _make_negative_response(self, fullname): + return (fullname, None, None, None, ()) + + minify_safe_re = re.compile(b(r'\s+#\s*!mitogen:\s*minify_safe')) + + def _build_tuple(self, fullname): + if fullname in self._cache: + return self._cache[fullname] + + if mitogen.core.is_blacklisted_import(self, fullname): + raise ImportError('blacklisted') + + path, source, is_pkg = self._finder.get_module_source(fullname) + if path and is_stdlib_path(path): + # Prevent loading of 2.x<->3.x stdlib modules! This costs one + # RTT per hit, so a client-side solution is also required. + LOG.debug('%r: refusing to serve stdlib module %r', + self, fullname) + tup = self._make_negative_response(fullname) + self._cache[fullname] = tup + return tup + + if source is None: + # TODO: make this .warning() or similar again once importer has its + # own logging category. + LOG.debug('_build_tuple(%r): could not locate source', fullname) + tup = self._make_negative_response(fullname) + self._cache[fullname] = tup + return tup + + if self.minify_safe_re.search(source): + # If the module contains a magic marker, it's safe to minify. + t0 = time.time() + source = mitogen.minify.minimize_source(source).encode('utf-8') + self.minify_secs += time.time() - t0 + + if is_pkg: + pkg_present = get_child_modules(path) + LOG.debug('_build_tuple(%r, %r) -> %r', + path, fullname, pkg_present) + else: + pkg_present = None + + if fullname == '__main__': + source = self.neutralize_main(path, source) + compressed = mitogen.core.Blob(zlib.compress(source, 9)) + related = [ + to_text(name) + for name in self._finder.find_related(fullname) + if not mitogen.core.is_blacklisted_import(self, name) + ] + # 0:fullname 1:pkg_present 2:path 3:compressed 4:related + tup = ( + to_text(fullname), + pkg_present, + to_text(path), + compressed, + related + ) + self._cache[fullname] = tup + return tup + + def _send_load_module(self, stream, fullname): + if fullname not in stream.sent_modules: + tup = self._build_tuple(fullname) + msg = mitogen.core.Message.pickled( + tup, + dst_id=stream.remote_id, + handle=mitogen.core.LOAD_MODULE, + ) + LOG.debug('%s: sending module %s (%.2f KiB)', + stream.name, fullname, len(msg.data) / 1024.0) + self._router._async_route(msg) + stream.sent_modules.add(fullname) + if tup[2] is not None: + self.good_load_module_count += 1 + self.good_load_module_size += len(msg.data) + else: + self.bad_load_module_count += 1 + + def _send_module_load_failed(self, stream, fullname): + self.bad_load_module_count += 1 + stream.send( + mitogen.core.Message.pickled( + self._make_negative_response(fullname), + dst_id=stream.remote_id, + handle=mitogen.core.LOAD_MODULE, + ) + ) + + def _send_module_and_related(self, stream, fullname): + if fullname in stream.sent_modules: + return + + try: + tup = self._build_tuple(fullname) + for name in tup[4]: # related + parent, _, _ = str_partition(name, '.') + if parent != fullname and parent not in stream.sent_modules: + # Parent hasn't been sent, so don't load submodule yet. + continue + + self._send_load_module(stream, name) + self._send_load_module(stream, fullname) + except Exception: + LOG.debug('While importing %r', fullname, exc_info=True) + self._send_module_load_failed(stream, fullname) + + def _on_get_module(self, msg): + if msg.is_dead: + return + + stream = self._router.stream_by_id(msg.src_id) + if stream is None: + return + + fullname = msg.data.decode() + LOG.debug('%s requested module %s', stream.name, fullname) + self.get_module_count += 1 + if fullname in stream.sent_modules: + LOG.warning('_on_get_module(): dup request for %r from %r', + fullname, stream) + + t0 = time.time() + try: + self._send_module_and_related(stream, fullname) + finally: + self.get_module_secs += time.time() - t0 + + def _send_forward_module(self, stream, context, fullname): + if stream.remote_id != context.context_id: + stream.send( + mitogen.core.Message( + data=b('%s\x00%s' % (context.context_id, fullname)), + handle=mitogen.core.FORWARD_MODULE, + dst_id=stream.remote_id, + ) + ) + + def _forward_one_module(self, context, fullname): + forwarded = self._forwarded_by_context.get(context) + if forwarded is None: + forwarded = set() + self._forwarded_by_context[context] = forwarded + + if fullname in forwarded: + return + + path = [] + while fullname: + path.append(fullname) + fullname, _, _ = str_rpartition(fullname, u'.') + + stream = self._router.stream_by_id(context.context_id) + if stream is None: + LOG.debug('%r: dropping forward of %s to no longer existent ' + '%r', self, path[0], context) + return + + for fullname in reversed(path): + self._send_module_and_related(stream, fullname) + self._send_forward_module(stream, context, fullname) + + def _forward_modules(self, context, fullnames): + IOLOG.debug('%r._forward_modules(%r, %r)', self, context, fullnames) + for fullname in fullnames: + self._forward_one_module(context, mitogen.core.to_text(fullname)) + + def forward_modules(self, context, fullnames): + self._router.broker.defer(self._forward_modules, context, fullnames) + + +class Broker(mitogen.core.Broker): + """ + .. note:: + + You may construct as many brokers as desired, and use the same broker + for multiple routers, however usually only one broker need exist. + Multiple brokers may be useful when dealing with sets of children with + differing lifetimes. For example, a subscription service where + non-payment results in termination for one customer. + + :param bool install_watcher: + If :data:`True`, an additional thread is started to monitor the + lifetime of the main thread, triggering :meth:`shutdown` + automatically in case the user forgets to call it, or their code + crashed. + + You should not rely on this functionality in your program, it is only + intended as a fail-safe and to simplify the API for new users. In + particular, alternative Python implementations may not be able to + support watching the main thread. + """ + shutdown_timeout = 5.0 + _watcher = None + poller_class = mitogen.parent.PREFERRED_POLLER + + def __init__(self, install_watcher=True): + if install_watcher: + self._watcher = ThreadWatcher.watch( + target=threading.currentThread(), + on_join=self.shutdown, + ) + super(Broker, self).__init__() + + def shutdown(self): + super(Broker, self).shutdown() + if self._watcher: + self._watcher.remove() + + +class Router(mitogen.parent.Router): + """ + Extend :class:`mitogen.core.Router` with functionality useful to masters, + and child contexts who later become masters. Currently when this class is + required, the target context's router is upgraded at runtime. + + .. note:: + + You may construct as many routers as desired, and use the same broker + for multiple routers, however usually only one broker and router need + exist. Multiple routers may be useful when dealing with separate trust + domains, for example, manipulating infrastructure belonging to separate + customers or projects. + + :param mitogen.master.Broker broker: + Broker to use. If not specified, a private :class:`Broker` is created. + + :param int max_message_size: + Override the maximum message size this router is willing to receive or + transmit. Any value set here is automatically inherited by any children + created by the router. + + This has a liberal default of 128 MiB, but may be set much lower. + Beware that setting it below 64KiB may encourage unexpected failures as + parents and children can no longer route large Python modules that may + be required by your application. + """ + + broker_class = Broker + + #: When :data:`True`, cause the broker thread and any subsequent broker and + #: main threads existing in any child to write + #: ``/tmp/mitogen.stats...log`` containing a + #: :mod:`cProfile` dump on graceful exit. Must be set prior to construction + #: of any :class:`Broker`, e.g. via:: + #: + #: mitogen.master.Router.profiling = True + profiling = os.environ.get('MITOGEN_PROFILING') is not None + + def __init__(self, broker=None, max_message_size=None): + if broker is None: + broker = self.broker_class() + if max_message_size: + self.max_message_size = max_message_size + super(Router, self).__init__(broker) + self.upgrade() + + def upgrade(self): + self.id_allocator = IdAllocator(self) + self.responder = ModuleResponder(self) + self.log_forwarder = LogForwarder(self) + self.route_monitor = mitogen.parent.RouteMonitor(router=self) + self.add_handler( # TODO: cutpaste. + fn=self._on_detaching, + handle=mitogen.core.DETACHING, + persist=True, + ) + + def _on_broker_exit(self): + super(Router, self)._on_broker_exit() + dct = self.get_stats() + dct['self'] = self + dct['minify_ms'] = 1000 * dct['minify_secs'] + dct['get_module_ms'] = 1000 * dct['get_module_secs'] + dct['good_load_module_size_kb'] = dct['good_load_module_size'] / 1024.0 + dct['good_load_module_size_avg'] = ( + ( + dct['good_load_module_size'] / + (float(dct['good_load_module_count']) or 1.0) + ) / 1024.0 + ) + + LOG.debug( + '%(self)r: stats: ' + '%(get_module_count)d module requests in ' + '%(get_module_ms)d ms, ' + '%(good_load_module_count)d sent ' + '(%(minify_ms)d ms minify time), ' + '%(bad_load_module_count)d negative responses. ' + 'Sent %(good_load_module_size_kb).01f kb total, ' + '%(good_load_module_size_avg).01f kb avg.' + % dct + ) + + def get_stats(self): + """ + Return performance data for the module responder. + + :returns: + + Dict containing keys: + + * `get_module_count`: Integer count of + :data:`mitogen.core.GET_MODULE` messages received. + * `get_module_secs`: Floating point total seconds spent servicing + :data:`mitogen.core.GET_MODULE` requests. + * `good_load_module_count`: Integer count of successful + :data:`mitogen.core.LOAD_MODULE` messages sent. + * `good_load_module_size`: Integer total bytes sent in + :data:`mitogen.core.LOAD_MODULE` message payloads. + * `bad_load_module_count`: Integer count of negative + :data:`mitogen.core.LOAD_MODULE` messages sent. + * `minify_secs`: CPU seconds spent minifying modules marked + minify-safe. + """ + return { + 'get_module_count': self.responder.get_module_count, + 'get_module_secs': self.responder.get_module_secs, + 'good_load_module_count': self.responder.good_load_module_count, + 'good_load_module_size': self.responder.good_load_module_size, + 'bad_load_module_count': self.responder.bad_load_module_count, + 'minify_secs': self.responder.minify_secs, + } + + def enable_debug(self): + """ + Cause this context and any descendant child contexts to write debug + logs to ``/tmp/mitogen..log``. + """ + mitogen.core.enable_debug_logging() + self.debug = True + + def __enter__(self): + return self + + def __exit__(self, e_type, e_val, tb): + self.broker.shutdown() + self.broker.join() + + def disconnect_stream(self, stream): + self.broker.defer(stream.on_disconnect, self.broker) + + def disconnect_all(self): + for stream in self._stream_by_id.values(): + self.disconnect_stream(stream) + + +class IdAllocator(object): + def __init__(self, router): + self.router = router + self.next_id = 1 + self.lock = threading.Lock() + router.add_handler( + fn=self.on_allocate_id, + handle=mitogen.core.ALLOCATE_ID, + ) + + def __repr__(self): + return 'IdAllocator(%r)' % (self.router,) + + BLOCK_SIZE = 1000 + + def allocate(self): + """ + Arrange for a unique context ID to be allocated and associated with a + route leading to the active context. In masters, the ID is generated + directly, in children it is forwarded to the master via a + :data:`mitogen.core.ALLOCATE_ID` message. + """ + self.lock.acquire() + try: + id_ = self.next_id + self.next_id += 1 + return id_ + finally: + self.lock.release() + + def allocate_block(self): + self.lock.acquire() + try: + id_ = self.next_id + self.next_id += self.BLOCK_SIZE + end_id = id_ + self.BLOCK_SIZE + LOG.debug('%r: allocating [%d..%d)', self, id_, end_id) + return id_, end_id + finally: + self.lock.release() + + def on_allocate_id(self, msg): + if msg.is_dead: + return + + id_, last_id = self.allocate_block() + requestee = self.router.context_by_id(msg.src_id) + LOG.debug('%r: allocating [%r..%r) to %r', + self, id_, last_id, requestee) + msg.reply((id_, last_id)) diff --git a/mitogen-0.2.7/mitogen/master.pyc b/mitogen-0.2.7/mitogen/master.pyc new file mode 100644 index 000000000..2c2627f Binary files /dev/null and b/mitogen-0.2.7/mitogen/master.pyc differ diff --git a/mitogen-0.2.7/mitogen/minify.py b/mitogen-0.2.7/mitogen/minify.py new file mode 100644 index 000000000..dc9f517 --- /dev/null +++ b/mitogen-0.2.7/mitogen/minify.py @@ -0,0 +1,139 @@ +# Copyright 2017, Alex Willmer +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +import sys + +try: + from io import StringIO +except ImportError: + from StringIO import StringIO + +import mitogen.core + +if sys.version_info < (2, 7, 11): + from mitogen.compat import tokenize +else: + import tokenize + + +def minimize_source(source): + """Remove comments and docstrings from Python `source`, preserving line + numbers and syntax of empty blocks. + + :param str source: + The source to minimize. + + :returns str: + The minimized source. + """ + source = mitogen.core.to_text(source) + tokens = tokenize.generate_tokens(StringIO(source).readline) + tokens = strip_comments(tokens) + tokens = strip_docstrings(tokens) + tokens = reindent(tokens) + return tokenize.untokenize(tokens) + + +def strip_comments(tokens): + """Drop comment tokens from a `tokenize` stream. + + Comments on lines 1-2 are kept, to preserve hashbang and encoding. + Trailing whitespace is remove from all lines. + """ + prev_typ = None + prev_end_col = 0 + for typ, tok, (start_row, start_col), (end_row, end_col), line in tokens: + if typ in (tokenize.NL, tokenize.NEWLINE): + if prev_typ in (tokenize.NL, tokenize.NEWLINE): + start_col = 0 + else: + start_col = prev_end_col + end_col = start_col + 1 + elif typ == tokenize.COMMENT and start_row > 2: + continue + prev_typ = typ + prev_end_col = end_col + yield typ, tok, (start_row, start_col), (end_row, end_col), line + + +def strip_docstrings(tokens): + """Replace docstring tokens with NL tokens in a `tokenize` stream. + + Any STRING token not part of an expression is deemed a docstring. + Indented docstrings are not yet recognised. + """ + stack = [] + state = 'wait_string' + for t in tokens: + typ = t[0] + if state == 'wait_string': + if typ in (tokenize.NL, tokenize.COMMENT): + yield t + elif typ in (tokenize.DEDENT, tokenize.INDENT, tokenize.STRING): + stack.append(t) + elif typ == tokenize.NEWLINE: + stack.append(t) + start_line, end_line = stack[0][2][0], stack[-1][3][0]+1 + for i in range(start_line, end_line): + yield tokenize.NL, '\n', (i, 0), (i,1), '\n' + for t in stack: + if t[0] in (tokenize.DEDENT, tokenize.INDENT): + yield t[0], t[1], (i+1, t[2][1]), (i+1, t[3][1]), t[4] + del stack[:] + else: + stack.append(t) + for t in stack: yield t + del stack[:] + state = 'wait_newline' + elif state == 'wait_newline': + if typ == tokenize.NEWLINE: + state = 'wait_string' + yield t + + +def reindent(tokens, indent=' '): + """Replace existing indentation in a token steam, with `indent`. + """ + old_levels = [] + old_level = 0 + new_level = 0 + for typ, tok, (start_row, start_col), (end_row, end_col), line in tokens: + if typ == tokenize.INDENT: + old_levels.append(old_level) + old_level = len(tok) + new_level += 1 + tok = indent * new_level + elif typ == tokenize.DEDENT: + old_level = old_levels.pop() + new_level -= 1 + start_col = max(0, start_col - old_level + new_level) + if start_row == end_row: + end_col = start_col + len(tok) + yield typ, tok, (start_row, start_col), (end_row, end_col), line diff --git a/mitogen-0.2.7/mitogen/minify.pyc b/mitogen-0.2.7/mitogen/minify.pyc new file mode 100644 index 000000000..5ab7bb1 Binary files /dev/null and b/mitogen-0.2.7/mitogen/minify.pyc differ diff --git a/mitogen-0.2.7/mitogen/os_fork.py b/mitogen-0.2.7/mitogen/os_fork.py new file mode 100644 index 000000000..b27cfd5 --- /dev/null +++ b/mitogen-0.2.7/mitogen/os_fork.py @@ -0,0 +1,183 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +""" +Support for operating in a mixed threading/forking environment. +""" + +import os +import socket +import sys +import weakref + +import mitogen.core + + +# List of weakrefs. On Python 2.4, mitogen.core registers its Broker on this +# list and mitogen.service registers its Pool too. +_brokers = weakref.WeakKeyDictionary() +_pools = weakref.WeakKeyDictionary() + + +def _notice_broker_or_pool(obj): + """ + Used by :mod:`mitogen.core` and :mod:`mitogen.service` to automatically + register every broker and pool on Python 2.4/2.5. + """ + if isinstance(obj, mitogen.core.Broker): + _brokers[obj] = True + else: + _pools[obj] = True + + +def wrap_os__fork(): + corker = Corker( + brokers=list(_brokers), + pools=list(_pools), + ) + try: + corker.cork() + return os__fork() + finally: + corker.uncork() + + +# If Python 2.4/2.5 where threading state is not fixed up, subprocess.Popen() +# may still deadlock due to the broker thread. In this case, pause os.fork() so +# that all active threads are paused during fork. +if sys.version_info < (2, 6): + os__fork = os.fork + os.fork = wrap_os__fork + + +class Corker(object): + """ + Arrange for :class:`mitogen.core.Broker` and optionally + :class:`mitogen.service.Pool` to be temporarily "corked" while fork + operations may occur. + + In a mixed threading/forking environment, it is critical no threads are + active at the moment of fork, as they could hold mutexes whose state is + unrecoverably snapshotted in the locked state in the fork child, causing + deadlocks at random future moments. + + To ensure a target thread has all locks dropped, it is made to write a + large string to a socket with a small buffer that has :data:`os.O_NONBLOCK` + disabled. CPython will drop the GIL and enter the ``write()`` system call, + where it will block until the socket buffer is drained, or the write side + is closed. + + :class:`mitogen.core.Poller` is used to ensure the thread really has + blocked outside any Python locks, by checking if the socket buffer has + started to fill. + + Since this necessarily involves posting a message to every existent thread + and verifying acknowledgement, it will never be a fast operation. + + This does not yet handle the case of corking being initiated from within a + thread that is also a cork target. + + :param brokers: + Sequence of :class:`mitogen.core.Broker` instances to cork. + :param pools: + Sequence of :class:`mitogen.core.Pool` instances to cork. + """ + def __init__(self, brokers=(), pools=()): + self.brokers = brokers + self.pools = pools + + def _do_cork(self, s, wsock): + try: + try: + while True: + # at least EINTR is possible. Do our best to keep handling + # outside the GIL in this case using sendall(). + wsock.sendall(s) + except socket.error: + pass + finally: + wsock.close() + + def _cork_one(self, s, obj): + """ + Construct a socketpair, saving one side of it, and passing the other to + `obj` to be written to by one of its threads. + """ + rsock, wsock = mitogen.parent.create_socketpair(size=4096) + mitogen.core.set_cloexec(rsock.fileno()) + mitogen.core.set_cloexec(wsock.fileno()) + mitogen.core.set_block(wsock) # gevent + self._rsocks.append(rsock) + obj.defer(self._do_cork, s, wsock) + + def _verify_one(self, rsock): + """ + Pause until the socket `rsock` indicates readability, due to + :meth:`_do_cork` triggering a blocking write on another thread. + """ + poller = mitogen.core.Poller() + poller.start_receive(rsock.fileno()) + try: + while True: + for fd in poller.poll(): + return + finally: + poller.close() + + def cork(self): + """ + Arrange for any associated brokers and pools to be paused with no locks + held. This will not return until each thread acknowledges it has ceased + execution. + """ + s = mitogen.core.b('CORK') * ((128 // 4) * 1024) + self._rsocks = [] + + # Pools must be paused first, as existing work may require the + # participation of a broker in order to complete. + for pool in self.pools: + if not pool.closed: + for x in range(pool.size): + self._cork_one(s, pool) + + for broker in self.brokers: + if broker._alive: + self._cork_one(s, broker) + + # Pause until we can detect every thread has entered write(). + for rsock in self._rsocks: + self._verify_one(rsock) + + def uncork(self): + """ + Arrange for paused threads to resume operation. + """ + for rsock in self._rsocks: + rsock.close() diff --git a/mitogen-0.2.7/mitogen/parent.py b/mitogen-0.2.7/mitogen/parent.py new file mode 100644 index 000000000..3d02bc4 --- /dev/null +++ b/mitogen-0.2.7/mitogen/parent.py @@ -0,0 +1,2330 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +""" +This module defines functionality common to master and parent processes. It is +sent to any child context that is due to become a parent, due to recursive +connection. +""" + +import codecs +import errno +import fcntl +import getpass +import inspect +import logging +import os +import signal +import socket +import struct +import subprocess +import sys +import termios +import textwrap +import threading +import time +import zlib + +# Absolute imports for <2.5. +select = __import__('select') + +try: + import thread +except ImportError: + import threading as thread + +import mitogen.core +from mitogen.core import b +from mitogen.core import bytes_partition +from mitogen.core import LOG +from mitogen.core import IOLOG + +try: + next +except NameError: + # Python 2.4/2.5 + from mitogen.core import next + + +itervalues = getattr(dict, 'itervalues', dict.values) + +if mitogen.core.PY3: + xrange = range + closure_attr = '__closure__' + IM_SELF_ATTR = '__self__' +else: + closure_attr = 'func_closure' + IM_SELF_ATTR = 'im_self' + + +try: + SC_OPEN_MAX = os.sysconf('SC_OPEN_MAX') +except ValueError: + SC_OPEN_MAX = 1024 + +OPENPTY_MSG = ( + "Failed to create a PTY: %s. It is likely the maximum number of PTYs has " + "been reached. Consider increasing the 'kern.tty.ptmx_max' sysctl on OS " + "X, the 'kernel.pty.max' sysctl on Linux, or modifying your configuration " + "to avoid PTY use." +) + +SYS_EXECUTABLE_MSG = ( + "The Python sys.executable variable is unset, indicating Python was " + "unable to determine its original program name. Unless explicitly " + "configured otherwise, child contexts will be started using " + "'/usr/bin/python'" +) +_sys_executable_warning_logged = False + + +def _ioctl_cast(n): + """ + Linux ioctl() request parameter is unsigned, whereas on BSD/Darwin it is + signed. Until 2.5 Python exclusively implemented the BSD behaviour, + preventing use of large unsigned int requests like the TTY layer uses + below. So on 2.4, we cast our unsigned to look like signed for Python. + """ + if sys.version_info < (2, 5): + n, = struct.unpack('i', struct.pack('I', n)) + return n + + +# If not :data:`None`, called prior to exec() of any new child process. Used by +# :func:`mitogen.utils.reset_affinity` to allow the child to be freely +# scheduled. +_preexec_hook = None + +# Get PTY number; asm-generic/ioctls.h +LINUX_TIOCGPTN = _ioctl_cast(2147767344) + +# Lock/unlock PTY; asm-generic/ioctls.h +LINUX_TIOCSPTLCK = _ioctl_cast(1074025521) + +IS_LINUX = os.uname()[0] == 'Linux' + +SIGNAL_BY_NUM = dict( + (getattr(signal, name), name) + for name in sorted(vars(signal), reverse=True) + if name.startswith('SIG') and not name.startswith('SIG_') +) + + +def get_log_level(): + return (LOG.level or logging.getLogger().level or logging.INFO) + + +def get_sys_executable(): + """ + Return :data:`sys.executable` if it is set, otherwise return + ``"/usr/bin/python"`` and log a warning. + """ + if sys.executable: + return sys.executable + + global _sys_executable_warning_logged + if not _sys_executable_warning_logged: + LOG.warn(SYS_EXECUTABLE_MSG) + _sys_executable_warning_logged = True + + return '/usr/bin/python' + + +_core_source_lock = threading.Lock() +_core_source_partial = None + + +def _get_core_source(): + """ + In non-masters, simply fetch the cached mitogen.core source code via the + import mechanism. In masters, this function is replaced with a version that + performs minification directly. + """ + return inspect.getsource(mitogen.core) + + +def get_core_source_partial(): + """ + _get_core_source() is expensive, even with @lru_cache in minify.py, threads + can enter it simultaneously causing severe slowdowns. + """ + global _core_source_partial + + if _core_source_partial is None: + _core_source_lock.acquire() + try: + if _core_source_partial is None: + _core_source_partial = PartialZlib( + _get_core_source().encode('utf-8') + ) + finally: + _core_source_lock.release() + + return _core_source_partial + + +def get_default_remote_name(): + """ + Return the default name appearing in argv[0] of remote machines. + """ + s = u'%s@%s:%d' + s %= (getpass.getuser(), socket.gethostname(), os.getpid()) + # In mixed UNIX/Windows environments, the username may contain slashes. + return s.translate({ + ord(u'\\'): ord(u'_'), + ord(u'/'): ord(u'_') + }) + + +def is_immediate_child(msg, stream): + """ + Handler policy that requires messages to arrive only from immediately + connected children. + """ + return msg.src_id == stream.remote_id + + +def flags(names): + """Return the result of ORing a set of (space separated) :py:mod:`termios` + module constants together.""" + return sum(getattr(termios, name, 0) + for name in names.split()) + + +def cfmakeraw(tflags): + """Given a list returned by :py:func:`termios.tcgetattr`, return a list + modified in a manner similar to the `cfmakeraw()` C library function, but + additionally disabling local echo.""" + # BSD: https://github.com/freebsd/freebsd/blob/master/lib/libc/gen/termios.c#L162 + # Linux: https://github.com/lattera/glibc/blob/master/termios/cfmakeraw.c#L20 + iflag, oflag, cflag, lflag, ispeed, ospeed, cc = tflags + iflag &= ~flags('IMAXBEL IXOFF INPCK BRKINT PARMRK ISTRIP INLCR ICRNL IXON IGNPAR') + iflag &= ~flags('IGNBRK BRKINT PARMRK') + oflag &= ~flags('OPOST') + lflag &= ~flags('ECHO ECHOE ECHOK ECHONL ICANON ISIG IEXTEN NOFLSH TOSTOP PENDIN') + cflag &= ~flags('CSIZE PARENB') + cflag |= flags('CS8 CREAD') + return [iflag, oflag, cflag, lflag, ispeed, ospeed, cc] + + +def disable_echo(fd): + old = termios.tcgetattr(fd) + new = cfmakeraw(old) + flags = getattr(termios, 'TCSASOFT', 0) + if not mitogen.core.IS_WSL: + # issue #319: Windows Subsystem for Linux as of July 2018 throws EINVAL + # if TCSAFLUSH is specified. + flags |= termios.TCSAFLUSH + termios.tcsetattr(fd, flags, new) + + +def close_nonstandard_fds(): + for fd in xrange(3, SC_OPEN_MAX): + try: + os.close(fd) + except OSError: + pass + + +def create_socketpair(size=None): + """ + Create a :func:`socket.socketpair` to use for use as a child process's UNIX + stdio channels. As socket pairs are bidirectional, they are economical on + file descriptor usage as the same descriptor can be used for ``stdin`` and + ``stdout``. As they are sockets their buffers are tunable, allowing large + buffers to be configured in order to improve throughput for file transfers + and reduce :class:`mitogen.core.Broker` IO loop iterations. + """ + parentfp, childfp = socket.socketpair() + parentfp.setsockopt(socket.SOL_SOCKET, + socket.SO_SNDBUF, + size or mitogen.core.CHUNK_SIZE) + childfp.setsockopt(socket.SOL_SOCKET, + socket.SO_RCVBUF, + size or mitogen.core.CHUNK_SIZE) + return parentfp, childfp + + +def detach_popen(**kwargs): + """ + Use :class:`subprocess.Popen` to construct a child process, then hack the + Popen so that it forgets the child it created, allowing it to survive a + call to Popen.__del__. + + If the child process is not detached, there is a race between it exitting + and __del__ being called. If it exits before __del__ runs, then __del__'s + call to :func:`os.waitpid` will capture the one and only exit event + delivered to this process, causing later 'legitimate' calls to fail with + ECHILD. + + :param list close_on_error: + Array of integer file descriptors to close on exception. + :returns: + Process ID of the new child. + """ + # This allows Popen() to be used for e.g. graceful post-fork error + # handling, without tying the surrounding code into managing a Popen + # object, which isn't possible for at least :mod:`mitogen.fork`. This + # should be replaced by a swappable helper class in a future version. + real_preexec_fn = kwargs.pop('preexec_fn', None) + def preexec_fn(): + if _preexec_hook: + _preexec_hook() + if real_preexec_fn: + real_preexec_fn() + proc = subprocess.Popen(preexec_fn=preexec_fn, **kwargs) + proc._child_created = False + return proc.pid + + +def create_child(args, merge_stdio=False, stderr_pipe=False, preexec_fn=None): + """ + Create a child process whose stdin/stdout is connected to a socket. + + :param args: + Argument vector for execv() call. + :param bool merge_stdio: + If :data:`True`, arrange for `stderr` to be connected to the `stdout` + socketpair, rather than inherited from the parent process. This may be + necessary to ensure that not TTY is connected to any stdio handle, for + instance when using LXC. + :param bool stderr_pipe: + If :data:`True` and `merge_stdio` is :data:`False`, arrange for + `stderr` to be connected to a separate pipe, to allow any ongoing debug + logs generated by e.g. SSH to be outpu as the session progresses, + without interfering with `stdout`. + :returns: + `(pid, socket_obj, :data:`None` or pipe_fd)` + """ + parentfp, childfp = create_socketpair() + # When running under a monkey patches-enabled gevent, the socket module + # yields file descriptors who already have O_NONBLOCK, which is + # persisted across fork, totally breaking Python. Therefore, drop + # O_NONBLOCK from Python's future stdin fd. + mitogen.core.set_block(childfp.fileno()) + + stderr_r = None + extra = {} + if merge_stdio: + extra = {'stderr': childfp} + elif stderr_pipe: + stderr_r, stderr_w = os.pipe() + mitogen.core.set_cloexec(stderr_r) + mitogen.core.set_cloexec(stderr_w) + extra = {'stderr': stderr_w} + + try: + pid = detach_popen( + args=args, + stdin=childfp, + stdout=childfp, + close_fds=True, + preexec_fn=preexec_fn, + **extra + ) + except Exception: + childfp.close() + parentfp.close() + if stderr_pipe: + os.close(stderr_r) + os.close(stderr_w) + raise + + if stderr_pipe: + os.close(stderr_w) + childfp.close() + # Decouple the socket from the lifetime of the Python socket object. + fd = os.dup(parentfp.fileno()) + parentfp.close() + + LOG.debug('create_child() child %d fd %d, parent %d, cmd: %s', + pid, fd, os.getpid(), Argv(args)) + return pid, fd, stderr_r + + +def _acquire_controlling_tty(): + os.setsid() + if sys.platform in ('linux', 'linux2'): + # On Linux, the controlling tty becomes the first tty opened by a + # process lacking any prior tty. + os.close(os.open(os.ttyname(2), os.O_RDWR)) + if hasattr(termios, 'TIOCSCTTY') and not mitogen.core.IS_WSL: + # #550: prehistoric WSL does not like TIOCSCTTY. + # On BSD an explicit ioctl is required. For some inexplicable reason, + # Python 2.6 on Travis also requires it. + fcntl.ioctl(2, termios.TIOCSCTTY) + + +def _linux_broken_devpts_openpty(): + """ + #462: On broken Linux hosts with mismatched configuration (e.g. old + /etc/fstab template installed), /dev/pts may be mounted without the gid= + mount option, causing new slave devices to be created with the group ID of + the calling process. This upsets glibc, whose openpty() is required by + specification to produce a slave owned by a special group ID (which is + always the 'tty' group). + + Glibc attempts to use "pt_chown" to fix ownership. If that fails, it + chown()s the PTY directly, which fails due to non-root, causing openpty() + to fail with EPERM ("Operation not permitted"). Since we don't need the + magical TTY group to run sudo and su, open the PTY ourselves in this case. + """ + master_fd = None + try: + # Opening /dev/ptmx causes a PTY pair to be allocated, and the + # corresponding slave /dev/pts/* device to be created, owned by UID/GID + # matching this process. + master_fd = os.open('/dev/ptmx', os.O_RDWR) + # Clear the lock bit from the PTY. This a prehistoric feature from a + # time when slave device files were persistent. + fcntl.ioctl(master_fd, LINUX_TIOCSPTLCK, struct.pack('i', 0)) + # Since v4.13 TIOCGPTPEER exists to open the slave in one step, but we + # must support older kernels. Ask for the PTY number. + pty_num_s = fcntl.ioctl(master_fd, LINUX_TIOCGPTN, + struct.pack('i', 0)) + pty_num, = struct.unpack('i', pty_num_s) + pty_name = '/dev/pts/%d' % (pty_num,) + # Now open it with O_NOCTTY to ensure it doesn't change our controlling + # TTY. Otherwise when we close the FD we get killed by the kernel, and + # the child we spawn that should really attach to it will get EPERM + # during _acquire_controlling_tty(). + slave_fd = os.open(pty_name, os.O_RDWR|os.O_NOCTTY) + return master_fd, slave_fd + except OSError: + if master_fd is not None: + os.close(master_fd) + e = sys.exc_info()[1] + raise mitogen.core.StreamError(OPENPTY_MSG, e) + + +def openpty(): + """ + Call :func:`os.openpty`, raising a descriptive error if the call fails. + + :raises mitogen.core.StreamError: + Creating a PTY failed. + :returns: + See :func`os.openpty`. + """ + try: + return os.openpty() + except OSError: + e = sys.exc_info()[1] + if IS_LINUX and e.args[0] == errno.EPERM: + return _linux_broken_devpts_openpty() + raise mitogen.core.StreamError(OPENPTY_MSG, e) + + +def tty_create_child(args): + """ + Return a file descriptor connected to the master end of a pseudo-terminal, + whose slave end is connected to stdin/stdout/stderr of a new child process. + The child is created such that the pseudo-terminal becomes its controlling + TTY, ensuring access to /dev/tty returns a new file descriptor open on the + slave end. + + :param list args: + :py:func:`os.execl` argument list. + + :returns: + `(pid, tty_fd, None)` + """ + master_fd, slave_fd = openpty() + try: + mitogen.core.set_block(slave_fd) + disable_echo(master_fd) + disable_echo(slave_fd) + + pid = detach_popen( + args=args, + stdin=slave_fd, + stdout=slave_fd, + stderr=slave_fd, + preexec_fn=_acquire_controlling_tty, + close_fds=True, + ) + except Exception: + os.close(master_fd) + os.close(slave_fd) + raise + + os.close(slave_fd) + LOG.debug('tty_create_child() child %d fd %d, parent %d, cmd: %s', + pid, master_fd, os.getpid(), Argv(args)) + return pid, master_fd, None + + +def hybrid_tty_create_child(args): + """ + Like :func:`tty_create_child`, except attach stdin/stdout to a socketpair + like :func:`create_child`, but leave stderr and the controlling TTY + attached to a TTY. + + :param list args: + :py:func:`os.execl` argument list. + + :returns: + `(pid, socketpair_fd, tty_fd)` + """ + master_fd, slave_fd = openpty() + + try: + disable_echo(master_fd) + disable_echo(slave_fd) + mitogen.core.set_block(slave_fd) + + parentfp, childfp = create_socketpair() + try: + mitogen.core.set_block(childfp) + pid = detach_popen( + args=args, + stdin=childfp, + stdout=childfp, + stderr=slave_fd, + preexec_fn=_acquire_controlling_tty, + close_fds=True, + ) + except Exception: + parentfp.close() + childfp.close() + raise + except Exception: + os.close(master_fd) + os.close(slave_fd) + raise + + os.close(slave_fd) + childfp.close() + # Decouple the socket from the lifetime of the Python socket object. + stdio_fd = os.dup(parentfp.fileno()) + parentfp.close() + + LOG.debug('hybrid_tty_create_child() pid=%d stdio=%d, tty=%d, cmd: %s', + pid, stdio_fd, master_fd, Argv(args)) + return pid, stdio_fd, master_fd + + +def write_all(fd, s, deadline=None): + """Arrange for all of bytestring `s` to be written to the file descriptor + `fd`. + + :param int fd: + File descriptor to write to. + :param bytes s: + Bytestring to write to file descriptor. + :param float deadline: + If not :data:`None`, absolute UNIX timestamp after which timeout should + occur. + + :raises mitogen.core.TimeoutError: + Bytestring could not be written entirely before deadline was exceeded. + :raises mitogen.parent.EofError: + Stream indicated EOF, suggesting the child process has exitted. + :raises mitogen.core.StreamError: + File descriptor was disconnected before write could complete. + """ + timeout = None + written = 0 + poller = PREFERRED_POLLER() + poller.start_transmit(fd) + + try: + while written < len(s): + if deadline is not None: + timeout = max(0, deadline - time.time()) + if timeout == 0: + raise mitogen.core.TimeoutError('write timed out') + + if mitogen.core.PY3: + window = memoryview(s)[written:] + else: + window = buffer(s, written) + + for fd in poller.poll(timeout): + n, disconnected = mitogen.core.io_op(os.write, fd, window) + if disconnected: + raise EofError('EOF on stream during write') + + written += n + finally: + poller.close() + + +class PartialZlib(object): + """ + Because the mitogen.core source has a line appended to it during bootstrap, + it must be recompressed for each connection. This is not a problem for a + small number of connections, but it amounts to 30 seconds CPU time by the + time 500 targets are in use. + + For that reason, build a compressor containing mitogen.core and flush as + much of it as possible into an initial buffer. Then to append the custom + line, clone the compressor and compress just that line. + + A full compression costs ~6ms on a modern machine, this method costs ~35 + usec. + """ + def __init__(self, s): + self.s = s + if sys.version_info > (2, 5): + self._compressor = zlib.compressobj(9) + self._out = self._compressor.compress(s) + self._out += self._compressor.flush(zlib.Z_SYNC_FLUSH) + else: + self._compressor = None + + def append(self, s): + """ + Append the bytestring `s` to the compressor state and return the + final compressed output. + """ + if self._compressor is None: + return zlib.compress(self.s + s, 9) + else: + compressor = self._compressor.copy() + out = self._out + out += compressor.compress(s) + return out + compressor.flush() + + +class IteratingRead(object): + def __init__(self, fds, deadline=None): + self.deadline = deadline + self.timeout = None + self.poller = PREFERRED_POLLER() + for fd in fds: + self.poller.start_receive(fd) + + self.bits = [] + self.timeout = None + + def close(self): + self.poller.close() + + def __iter__(self): + return self + + def next(self): + while self.poller.readers: + if self.deadline is not None: + self.timeout = max(0, self.deadline - time.time()) + if self.timeout == 0: + break + + for fd in self.poller.poll(self.timeout): + s, disconnected = mitogen.core.io_op(os.read, fd, 4096) + if disconnected or not s: + LOG.debug('iter_read(%r) -> disconnected: %s', + fd, disconnected) + self.poller.stop_receive(fd) + else: + IOLOG.debug('iter_read(%r) -> %r', fd, s) + self.bits.append(s) + return s + + if not self.poller.readers: + raise EofError(u'EOF on stream; last 300 bytes received: %r' % + (b('').join(self.bits)[-300:].decode('latin1'),)) + + raise mitogen.core.TimeoutError('read timed out') + + __next__ = next + + +def iter_read(fds, deadline=None): + """Return a generator that arranges for up to 4096-byte chunks to be read + at a time from the file descriptor `fd` until the generator is destroyed. + + :param int fd: + File descriptor to read from. + :param float deadline: + If not :data:`None`, an absolute UNIX timestamp after which timeout + should occur. + + :raises mitogen.core.TimeoutError: + Attempt to read beyond deadline. + :raises mitogen.parent.EofError: + All streams indicated EOF, suggesting the child process has exitted. + :raises mitogen.core.StreamError: + Attempt to read past end of file. + """ + return IteratingRead(fds=fds, deadline=deadline) + + +def discard_until(fd, s, deadline): + """Read chunks from `fd` until one is encountered that ends with `s`. This + is used to skip output produced by ``/etc/profile``, ``/etc/motd`` and + mandatory SSH banners while waiting for :attr:`Stream.EC0_MARKER` to + appear, indicating the first stage is ready to receive the compressed + :mod:`mitogen.core` source. + + :param int fd: + File descriptor to read from. + :param bytes s: + Marker string to discard until encountered. + :param float deadline: + Absolute UNIX timestamp after which timeout should occur. + + :raises mitogen.core.TimeoutError: + Attempt to read beyond deadline. + :raises mitogen.parent.EofError: + All streams indicated EOF, suggesting the child process has exitted. + :raises mitogen.core.StreamError: + Attempt to read past end of file. + """ + it = iter_read([fd], deadline) + try: + for buf in it: + if IOLOG.level == logging.DEBUG: + for line in buf.splitlines(): + IOLOG.debug('discard_until: discarding %r', line) + if buf.endswith(s): + return + finally: + it.close() # ensure Poller.close() is called. + + +def _upgrade_broker(broker): + """ + Extract the poller state from Broker and replace it with the industrial + strength poller for this OS. Must run on the Broker thread. + """ + # This function is deadly! The act of calling start_receive() generates log + # messages which must be silenced as the upgrade progresses, otherwise the + # poller state will change as it is copied, resulting in write fds that are + # lost. (Due to LogHandler->Router->Stream->Broker->Poller, where Stream + # only calls start_transmit() when transitioning from empty to non-empty + # buffer. If the start_transmit() is lost, writes from the child hang + # permanently). + root = logging.getLogger() + old_level = root.level + root.setLevel(logging.CRITICAL) + + old = broker.poller + new = PREFERRED_POLLER() + for fd, data in old.readers: + new.start_receive(fd, data) + for fd, data in old.writers: + new.start_transmit(fd, data) + + old.close() + broker.poller = new + root.setLevel(old_level) + LOG.debug('replaced %r with %r (new: %d readers, %d writers; ' + 'old: %d readers, %d writers)', old, new, + len(new.readers), len(new.writers), + len(old.readers), len(old.writers)) + + +@mitogen.core.takes_econtext +def upgrade_router(econtext): + if not isinstance(econtext.router, Router): # TODO + econtext.broker.defer(_upgrade_broker, econtext.broker) + econtext.router.__class__ = Router # TODO + econtext.router.upgrade( + importer=econtext.importer, + parent=econtext.parent, + ) + + +def stream_by_method_name(name): + """ + Given the name of a Mitogen connection method, import its implementation + module and return its Stream subclass. + """ + if name == u'local': + name = u'parent' + module = mitogen.core.import_module(u'mitogen.' + name) + return module.Stream + + +@mitogen.core.takes_econtext +def _proxy_connect(name, method_name, kwargs, econtext): + """ + Implements the target portion of Router._proxy_connect() by upgrading the + local context to a parent if it was not already, then calling back into + Router._connect() using the arguments passed to the parent's + Router.connect(). + + :returns: + Dict containing: + * ``id``: :data:`None`, or integer new context ID. + * ``name``: :data:`None`, or string name attribute of new Context. + * ``msg``: :data:`None`, or StreamError exception text. + """ + upgrade_router(econtext) + + try: + context = econtext.router._connect( + klass=stream_by_method_name(method_name), + name=name, + **kwargs + ) + except mitogen.core.StreamError: + return { + u'id': None, + u'name': None, + u'msg': 'error occurred on host %s: %s' % ( + socket.gethostname(), + sys.exc_info()[1], + ), + } + + return { + u'id': context.context_id, + u'name': context.name, + u'msg': None, + } + + +def wstatus_to_str(status): + """ + Parse and format a :func:`os.waitpid` exit status. + """ + if os.WIFEXITED(status): + return 'exited with return code %d' % (os.WEXITSTATUS(status),) + if os.WIFSIGNALED(status): + n = os.WTERMSIG(status) + return 'exited due to signal %d (%s)' % (n, SIGNAL_BY_NUM.get(n)) + if os.WIFSTOPPED(status): + n = os.WSTOPSIG(status) + return 'stopped due to signal %d (%s)' % (n, SIGNAL_BY_NUM.get(n)) + return 'unknown wait status (%d)' % (status,) + + +class EofError(mitogen.core.StreamError): + """ + Raised by :func:`iter_read` and :func:`write_all` when EOF is detected by + the child process. + """ + # inherits from StreamError to maintain compatibility. + pass + + +class Argv(object): + """ + Wrapper to defer argv formatting when debug logging is disabled. + """ + def __init__(self, argv): + self.argv = argv + + must_escape = frozenset('\\$"`!') + must_escape_or_space = must_escape | frozenset(' ') + + def escape(self, x): + if not self.must_escape_or_space.intersection(x): + return x + + s = '"' + for c in x: + if c in self.must_escape: + s += '\\' + s += c + s += '"' + return s + + def __str__(self): + return ' '.join(map(self.escape, self.argv)) + + +class CallSpec(object): + """ + Wrapper to defer call argument formatting when debug logging is disabled. + """ + def __init__(self, func, args, kwargs): + self.func = func + self.args = args + self.kwargs = kwargs + + def _get_name(self): + bits = [self.func.__module__] + if inspect.ismethod(self.func): + im_self = getattr(self.func, IM_SELF_ATTR) + bits.append(getattr(im_self, '__name__', None) or + getattr(type(im_self), '__name__', None)) + bits.append(self.func.__name__) + return u'.'.join(bits) + + def _get_args(self): + return u', '.join(repr(a) for a in self.args) + + def _get_kwargs(self): + s = u'' + if self.kwargs: + s = u', '.join('%s=%r' % (k, v) for k, v in self.kwargs.items()) + if self.args: + s = u', ' + s + return s + + def __repr__(self): + return '%s(%s%s)' % ( + self._get_name(), + self._get_args(), + self._get_kwargs(), + ) + + +class PollPoller(mitogen.core.Poller): + """ + Poller based on the POSIX poll(2) interface. Not available on some versions + of OS X, otherwise it is the preferred poller for small FD counts. + """ + SUPPORTED = hasattr(select, 'poll') + _repr = 'PollPoller()' + + def __init__(self): + super(PollPoller, self).__init__() + self._pollobj = select.poll() + + # TODO: no proof we dont need writemask too + _readmask = ( + getattr(select, 'POLLIN', 0) | + getattr(select, 'POLLHUP', 0) + ) + + def _update(self, fd): + mask = (((fd in self._rfds) and self._readmask) | + ((fd in self._wfds) and select.POLLOUT)) + if mask: + self._pollobj.register(fd, mask) + else: + try: + self._pollobj.unregister(fd) + except KeyError: + pass + + def _poll(self, timeout): + if timeout: + timeout *= 1000 + + events, _ = mitogen.core.io_op(self._pollobj.poll, timeout) + for fd, event in events: + if event & self._readmask: + IOLOG.debug('%r: POLLIN|POLLHUP for %r', self, fd) + data, gen = self._rfds.get(fd, (None, None)) + if gen and gen < self._generation: + yield data + if event & select.POLLOUT: + IOLOG.debug('%r: POLLOUT for %r', self, fd) + data, gen = self._wfds.get(fd, (None, None)) + if gen and gen < self._generation: + yield data + + +class KqueuePoller(mitogen.core.Poller): + """ + Poller based on the FreeBSD/Darwin kqueue(2) interface. + """ + SUPPORTED = hasattr(select, 'kqueue') + _repr = 'KqueuePoller()' + + def __init__(self): + super(KqueuePoller, self).__init__() + self._kqueue = select.kqueue() + self._changelist = [] + + def close(self): + super(KqueuePoller, self).close() + self._kqueue.close() + + def _control(self, fd, filters, flags): + mitogen.core._vv and IOLOG.debug( + '%r._control(%r, %r, %r)', self, fd, filters, flags) + # TODO: at shutdown it is currently possible for KQ_EV_ADD/KQ_EV_DEL + # pairs to be pending after the associated file descriptor has already + # been closed. Fixing this requires maintaining extra state, or perhaps + # making fd closure the poller's responsibility. In the meantime, + # simply apply changes immediately. + # self._changelist.append(select.kevent(fd, filters, flags)) + changelist = [select.kevent(fd, filters, flags)] + events, _ = mitogen.core.io_op(self._kqueue.control, changelist, 0, 0) + assert not events + + def start_receive(self, fd, data=None): + mitogen.core._vv and IOLOG.debug('%r.start_receive(%r, %r)', + self, fd, data) + if fd not in self._rfds: + self._control(fd, select.KQ_FILTER_READ, select.KQ_EV_ADD) + self._rfds[fd] = (data or fd, self._generation) + + def stop_receive(self, fd): + mitogen.core._vv and IOLOG.debug('%r.stop_receive(%r)', self, fd) + if fd in self._rfds: + self._control(fd, select.KQ_FILTER_READ, select.KQ_EV_DELETE) + del self._rfds[fd] + + def start_transmit(self, fd, data=None): + mitogen.core._vv and IOLOG.debug('%r.start_transmit(%r, %r)', + self, fd, data) + if fd not in self._wfds: + self._control(fd, select.KQ_FILTER_WRITE, select.KQ_EV_ADD) + self._wfds[fd] = (data or fd, self._generation) + + def stop_transmit(self, fd): + mitogen.core._vv and IOLOG.debug('%r.stop_transmit(%r)', self, fd) + if fd in self._wfds: + self._control(fd, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE) + del self._wfds[fd] + + def _poll(self, timeout): + changelist = self._changelist + self._changelist = [] + events, _ = mitogen.core.io_op(self._kqueue.control, + changelist, 32, timeout) + for event in events: + fd = event.ident + if event.flags & select.KQ_EV_ERROR: + LOG.debug('ignoring stale event for fd %r: errno=%d: %s', + fd, event.data, errno.errorcode.get(event.data)) + elif event.filter == select.KQ_FILTER_READ: + data, gen = self._rfds.get(fd, (None, None)) + # Events can still be read for an already-discarded fd. + if gen and gen < self._generation: + mitogen.core._vv and IOLOG.debug('%r: POLLIN: %r', self, fd) + yield data + elif event.filter == select.KQ_FILTER_WRITE and fd in self._wfds: + data, gen = self._wfds.get(fd, (None, None)) + if gen and gen < self._generation: + mitogen.core._vv and IOLOG.debug('%r: POLLOUT: %r', self, fd) + yield data + + +class EpollPoller(mitogen.core.Poller): + """ + Poller based on the Linux epoll(2) interface. + """ + SUPPORTED = hasattr(select, 'epoll') + _repr = 'EpollPoller()' + + def __init__(self): + super(EpollPoller, self).__init__() + self._epoll = select.epoll(32) + self._registered_fds = set() + + def close(self): + super(EpollPoller, self).close() + self._epoll.close() + + def _control(self, fd): + mitogen.core._vv and IOLOG.debug('%r._control(%r)', self, fd) + mask = (((fd in self._rfds) and select.EPOLLIN) | + ((fd in self._wfds) and select.EPOLLOUT)) + if mask: + if fd in self._registered_fds: + self._epoll.modify(fd, mask) + else: + self._epoll.register(fd, mask) + self._registered_fds.add(fd) + elif fd in self._registered_fds: + self._epoll.unregister(fd) + self._registered_fds.remove(fd) + + def start_receive(self, fd, data=None): + mitogen.core._vv and IOLOG.debug('%r.start_receive(%r, %r)', + self, fd, data) + self._rfds[fd] = (data or fd, self._generation) + self._control(fd) + + def stop_receive(self, fd): + mitogen.core._vv and IOLOG.debug('%r.stop_receive(%r)', self, fd) + self._rfds.pop(fd, None) + self._control(fd) + + def start_transmit(self, fd, data=None): + mitogen.core._vv and IOLOG.debug('%r.start_transmit(%r, %r)', + self, fd, data) + self._wfds[fd] = (data or fd, self._generation) + self._control(fd) + + def stop_transmit(self, fd): + mitogen.core._vv and IOLOG.debug('%r.stop_transmit(%r)', self, fd) + self._wfds.pop(fd, None) + self._control(fd) + + _inmask = (getattr(select, 'EPOLLIN', 0) | + getattr(select, 'EPOLLHUP', 0)) + + def _poll(self, timeout): + the_timeout = -1 + if timeout is not None: + the_timeout = timeout + + events, _ = mitogen.core.io_op(self._epoll.poll, the_timeout, 32) + for fd, event in events: + if event & self._inmask: + data, gen = self._rfds.get(fd, (None, None)) + if gen and gen < self._generation: + # Events can still be read for an already-discarded fd. + mitogen.core._vv and IOLOG.debug('%r: POLLIN: %r', self, fd) + yield data + if event & select.EPOLLOUT: + data, gen = self._wfds.get(fd, (None, None)) + if gen and gen < self._generation: + mitogen.core._vv and IOLOG.debug('%r: POLLOUT: %r', self, fd) + yield data + + +# 2.4 and 2.5 only had select.select() and select.poll(). +for _klass in mitogen.core.Poller, PollPoller, KqueuePoller, EpollPoller: + if _klass.SUPPORTED: + PREFERRED_POLLER = _klass + +# For apps that start threads dynamically, it's possible Latch will also get +# very high-numbered wait fds when there are many connections, and so select() +# becomes useless there too. So swap in our favourite poller. +if PollPoller.SUPPORTED: + mitogen.core.Latch.poller_class = PollPoller +else: + mitogen.core.Latch.poller_class = PREFERRED_POLLER + + +class DiagLogStream(mitogen.core.BasicStream): + """ + For "hybrid TTY/socketpair" mode, after a connection has been setup, a + spare TTY file descriptor will exist that cannot be closed, and to which + SSH or sudo may continue writing log messages. + + The descriptor cannot be closed since the UNIX TTY layer will send a + termination signal to any processes whose controlling TTY is the TTY that + has been closed. + + DiagLogStream takes over this descriptor and creates corresponding log + messages for anything written to it. + """ + + def __init__(self, fd, stream): + self.receive_side = mitogen.core.Side(self, fd) + self.transmit_side = self.receive_side + self.stream = stream + self.buf = '' + + def __repr__(self): + return "mitogen.parent.DiagLogStream(fd=%r, '%s')" % ( + self.receive_side.fd, + self.stream.name, + ) + + def on_receive(self, broker): + """ + This handler is only called after the stream is registered with the IO + loop, the descriptor is manually read/written by _connect_bootstrap() + prior to that. + """ + buf = self.receive_side.read() + if not buf: + return self.on_disconnect(broker) + + self.buf += buf.decode('utf-8', 'replace') + while u'\n' in self.buf: + lines = self.buf.split('\n') + self.buf = lines[-1] + for line in lines[:-1]: + LOG.debug('%s: %s', self.stream.name, line.rstrip()) + + +class Stream(mitogen.core.Stream): + """ + Base for streams capable of starting new slaves. + """ + #: The path to the remote Python interpreter. + python_path = get_sys_executable() + + #: Maximum time to wait for a connection attempt. + connect_timeout = 30.0 + + #: Derived from :py:attr:`connect_timeout`; absolute floating point + #: UNIX timestamp after which the connection attempt should be abandoned. + connect_deadline = None + + #: True to cause context to write verbose /tmp/mitogen..log. + debug = False + + #: True to cause context to write /tmp/mitogen.stats...log. + profiling = False + + #: Set to the child's PID by connect(). + pid = None + + #: Passed via Router wrapper methods, must eventually be passed to + #: ExternalContext.main(). + max_message_size = None + + #: If :attr:`create_child` supplied a diag_fd, references the corresponding + #: :class:`DiagLogStream`, allowing it to be disconnected when this stream + #: is disconnected. Set to :data:`None` if no `diag_fd` was present. + diag_stream = None + + #: Function with the semantics of :func:`create_child` used to create the + #: child process. + create_child = staticmethod(create_child) + + #: Dictionary of extra kwargs passed to :attr:`create_child`. + create_child_args = {} + + #: :data:`True` if the remote has indicated that it intends to detach, and + #: should not be killed on disconnect. + detached = False + + #: If :data:`True`, indicates the child should not be killed during + #: graceful detachment, as it the actual process implementing the child + #: context. In all other cases, the subprocess is SSH, sudo, or a similar + #: tool that should be reminded to quit during disconnection. + child_is_immediate_subprocess = True + + #: Prefix given to default names generated by :meth:`connect`. + name_prefix = u'local' + + _reaped = False + + def __init__(self, *args, **kwargs): + super(Stream, self).__init__(*args, **kwargs) + self.sent_modules = set(['mitogen', 'mitogen.core']) + + def construct(self, max_message_size, remote_name=None, python_path=None, + debug=False, connect_timeout=None, profiling=False, + unidirectional=False, old_router=None, **kwargs): + """Get the named context running on the local machine, creating it if + it does not exist.""" + super(Stream, self).construct(**kwargs) + self.max_message_size = max_message_size + if python_path: + self.python_path = python_path + if connect_timeout: + self.connect_timeout = connect_timeout + if remote_name is None: + remote_name = get_default_remote_name() + if '/' in remote_name or '\\' in remote_name: + raise ValueError('remote_name= cannot contain slashes') + self.remote_name = remote_name + self.debug = debug + self.profiling = profiling + self.unidirectional = unidirectional + self.max_message_size = max_message_size + self.connect_deadline = time.time() + self.connect_timeout + + def on_shutdown(self, broker): + """Request the slave gracefully shut itself down.""" + LOG.debug('%r closing CALL_FUNCTION channel', self) + self._send( + mitogen.core.Message( + src_id=mitogen.context_id, + dst_id=self.remote_id, + handle=mitogen.core.SHUTDOWN, + ) + ) + + def _reap_child(self): + """ + Reap the child process during disconnection. + """ + if self.detached and self.child_is_immediate_subprocess: + LOG.debug('%r: immediate child is detached, won\'t reap it', self) + return + + if self.profiling: + LOG.info('%r: wont kill child because profiling=True', self) + return + + if self._reaped: + # on_disconnect() may be invoked more than once, for example, if + # there is still a pending message to be sent after the first + # on_disconnect() call. + return + + try: + pid, status = os.waitpid(self.pid, os.WNOHANG) + except OSError: + e = sys.exc_info()[1] + if e.args[0] == errno.ECHILD: + LOG.warn('%r: waitpid(%r) produced ECHILD', self, self.pid) + return + raise + + self._reaped = True + if pid: + LOG.debug('%r: PID %d %s', self, pid, wstatus_to_str(status)) + return + + if not self._router.profiling: + # For processes like sudo we cannot actually send sudo a signal, + # because it is setuid, so this is best-effort only. + LOG.debug('%r: child process still alive, sending SIGTERM', self) + try: + os.kill(self.pid, signal.SIGTERM) + except OSError: + e = sys.exc_info()[1] + if e.args[0] != errno.EPERM: + raise + + def on_disconnect(self, broker): + super(Stream, self).on_disconnect(broker) + if self.diag_stream is not None: + self.diag_stream.on_disconnect(broker) + self._reap_child() + + # Minimised, gzipped, base64'd and passed to 'python -c'. It forks, dups + # file descriptor 0 as 100, creates a pipe, then execs a new interpreter + # with a custom argv. + # * Optimized for minimum byte count after minification & compression. + # * 'CONTEXT_NAME' and 'PREAMBLE_COMPRESSED_LEN' are substituted with + # their respective values. + # * CONTEXT_NAME must be prefixed with the name of the Python binary in + # order to allow virtualenvs to detect their install prefix. + # * For Darwin, OS X installs a craptacular argv0-introspecting Python + # version switcher as /usr/bin/python. Override attempts to call it + # with an explicit call to python2.7 + # + # Locals: + # R: read side of interpreter stdin. + # W: write side of interpreter stdin. + # r: read side of core_src FD. + # w: write side of core_src FD. + # C: the decompressed core source. + + # Final os.close(2) to avoid --py-debug build from corrupting stream with + # "[1234 refs]" during exit. + @staticmethod + def _first_stage(): + R,W=os.pipe() + r,w=os.pipe() + if os.fork(): + os.dup2(0,100) + os.dup2(R,0) + os.dup2(r,101) + os.close(R) + os.close(r) + os.close(W) + os.close(w) + if sys.platform == 'darwin' and sys.executable == '/usr/bin/python': + sys.executable += sys.version[:3] + os.environ['ARGV0']=sys.executable + os.execl(sys.executable,sys.executable+'(mitogen:CONTEXT_NAME)') + os.write(1,'MITO000\n'.encode()) + C=_(os.fdopen(0,'rb').read(PREAMBLE_COMPRESSED_LEN),'zip') + fp=os.fdopen(W,'wb',0) + fp.write(C) + fp.close() + fp=os.fdopen(w,'wb',0) + fp.write(C) + fp.close() + os.write(1,'MITO001\n'.encode()) + os.close(2) + + def get_python_argv(self): + """ + Return the initial argument vector elements necessary to invoke Python, + by returning a 1-element list containing :attr:`python_path` if it is a + string, or simply returning it if it is already a list. + + This allows emulation of existing tools where the Python invocation may + be set to e.g. `['/usr/bin/env', 'python']`. + """ + if isinstance(self.python_path, list): + return self.python_path + return [self.python_path] + + def get_boot_command(self): + source = inspect.getsource(self._first_stage) + source = textwrap.dedent('\n'.join(source.strip().split('\n')[2:])) + source = source.replace(' ', '\t') + source = source.replace('CONTEXT_NAME', self.remote_name) + preamble_compressed = self.get_preamble() + source = source.replace('PREAMBLE_COMPRESSED_LEN', + str(len(preamble_compressed))) + compressed = zlib.compress(source.encode(), 9) + encoded = codecs.encode(compressed, 'base64').replace(b('\n'), b('')) + # We can't use bytes.decode() in 3.x since it was restricted to always + # return unicode, so codecs.decode() is used instead. In 3.x + # codecs.decode() requires a bytes object. Since we must be compatible + # with 2.4 (no bytes literal), an extra .encode() either returns the + # same str (2.x) or an equivalent bytes (3.x). + return self.get_python_argv() + [ + '-c', + 'import codecs,os,sys;_=codecs.decode;' + 'exec(_(_("%s".encode(),"base64"),"zip"))' % (encoded.decode(),) + ] + + def get_econtext_config(self): + assert self.max_message_size is not None + parent_ids = mitogen.parent_ids[:] + parent_ids.insert(0, mitogen.context_id) + return { + 'parent_ids': parent_ids, + 'context_id': self.remote_id, + 'debug': self.debug, + 'profiling': self.profiling, + 'unidirectional': self.unidirectional, + 'log_level': get_log_level(), + 'whitelist': self._router.get_module_whitelist(), + 'blacklist': self._router.get_module_blacklist(), + 'max_message_size': self.max_message_size, + 'version': mitogen.__version__, + } + + def get_preamble(self): + suffix = ( + '\nExternalContext(%r).main()\n' %\ + (self.get_econtext_config(),) + ) + partial = get_core_source_partial() + return partial.append(suffix.encode('utf-8')) + + def start_child(self): + args = self.get_boot_command() + try: + return self.create_child(args, **self.create_child_args) + except OSError: + e = sys.exc_info()[1] + msg = 'Child start failed: %s. Command was: %s' % (e, Argv(args)) + raise mitogen.core.StreamError(msg) + + eof_error_hint = None + + def _adorn_eof_error(self, e): + """ + Used by subclasses to provide additional information in the case of a + failed connection. + """ + if self.eof_error_hint: + e.args = ('%s\n\n%s' % (e.args[0], self.eof_error_hint),) + + def _get_name(self): + """ + Called by :meth:`connect` after :attr:`pid` is known. Subclasses can + override it to specify a default stream name, or set + :attr:`name_prefix` to generate a default format. + """ + return u'%s.%s' % (self.name_prefix, self.pid) + + def connect(self): + LOG.debug('%r.connect()', self) + self.pid, fd, diag_fd = self.start_child() + self.name = self._get_name() + self.receive_side = mitogen.core.Side(self, fd) + self.transmit_side = mitogen.core.Side(self, os.dup(fd)) + if diag_fd is not None: + self.diag_stream = DiagLogStream(diag_fd, self) + else: + self.diag_stream = None + + LOG.debug('%r.connect(): pid:%r stdin:%r, stdout:%r, diag:%r', + self, self.pid, self.receive_side.fd, self.transmit_side.fd, + self.diag_stream and self.diag_stream.receive_side.fd) + + try: + self._connect_bootstrap() + except EofError: + self.on_disconnect(self._router.broker) + e = sys.exc_info()[1] + self._adorn_eof_error(e) + raise + except Exception: + self.on_disconnect(self._router.broker) + self._reap_child() + raise + + #: Sentinel value emitted by the first stage to indicate it is ready to + #: receive the compressed bootstrap. For :mod:`mitogen.ssh` this must have + #: length of at least `max(len('password'), len('debug1:'))` + EC0_MARKER = mitogen.core.b('MITO000\n') + EC1_MARKER = mitogen.core.b('MITO001\n') + + def _ec0_received(self): + LOG.debug('%r._ec0_received()', self) + write_all(self.transmit_side.fd, self.get_preamble()) + discard_until(self.receive_side.fd, self.EC1_MARKER, + self.connect_deadline) + if self.diag_stream: + self._router.broker.start_receive(self.diag_stream) + + def _connect_bootstrap(self): + discard_until(self.receive_side.fd, self.EC0_MARKER, + self.connect_deadline) + self._ec0_received() + + +class ChildIdAllocator(object): + def __init__(self, router): + self.router = router + self.lock = threading.Lock() + self.it = iter(xrange(0)) + + def allocate(self): + self.lock.acquire() + try: + for id_ in self.it: + return id_ + + master = mitogen.core.Context(self.router, 0) + start, end = master.send_await( + mitogen.core.Message(dst_id=0, handle=mitogen.core.ALLOCATE_ID) + ) + self.it = iter(xrange(start, end)) + finally: + self.lock.release() + + return self.allocate() + + +class CallChain(object): + """ + Deliver :data:`mitogen.core.CALL_FUNCTION` messages to a target context, + optionally threading related calls so an exception in an earlier call + cancels subsequent calls. + + :param mitogen.core.Context context: + Target context. + :param bool pipelined: + Enable pipelining. + + :meth:`call`, :meth:`call_no_reply` and :meth:`call_async` + normally issue calls and produce responses with no memory of prior + exceptions. If a call made with :meth:`call_no_reply` fails, the exception + is logged to the target context's logging framework. + + **Pipelining** + + When pipelining is enabled, if an exception occurs during a call, + subsequent calls made by the same :class:`CallChain` fail with the same + exception, including those already in-flight on the network, and no further + calls execute until :meth:`reset` is invoked. + + No exception is logged for calls made with :meth:`call_no_reply`, instead + the exception is saved and reported as the result of subsequent + :meth:`call` or :meth:`call_async` calls. + + Sequences of asynchronous calls can be made without wasting network + round-trips to discover if prior calls succeed, and chains originating from + multiple unrelated source contexts may overlap concurrently at a target + context without interference. + + In this example, 4 calls complete in one round-trip:: + + chain = mitogen.parent.CallChain(context, pipelined=True) + chain.call_no_reply(os.mkdir, '/tmp/foo') + + # If previous mkdir() failed, this never runs: + chain.call_no_reply(os.mkdir, '/tmp/foo/bar') + + # If either mkdir() failed, this never runs, and the exception is + # asynchronously delivered to the receiver. + recv = chain.call_async(subprocess.check_output, '/tmp/foo') + + # If anything so far failed, this never runs, and raises the exception. + chain.call(do_something) + + # If this code was executed, the exception would also be raised. + if recv.get().unpickle() == 'baz': + pass + + When pipelining is enabled, :meth:`reset` must be invoked to ensure any + exception is discarded, otherwise unbounded memory usage is possible in + long-running programs. The context manager protocol is supported to ensure + :meth:`reset` is always invoked:: + + with mitogen.parent.CallChain(context, pipelined=True) as chain: + chain.call_no_reply(...) + chain.call_no_reply(...) + chain.call_no_reply(...) + chain.call(...) + + # chain.reset() automatically invoked. + """ + def __init__(self, context, pipelined=False): + self.context = context + if pipelined: + self.chain_id = self.make_chain_id() + else: + self.chain_id = None + + @classmethod + def make_chain_id(cls): + return '%s-%s-%x-%x' % ( + socket.gethostname(), + os.getpid(), + thread.get_ident(), + int(1e6 * time.time()), + ) + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, self.context) + + def __enter__(self): + return self + + def __exit__(self, _1, _2, _3): + self.reset() + + def reset(self): + """ + Instruct the target to forget any related exception. + """ + if not self.chain_id: + return + + saved, self.chain_id = self.chain_id, None + try: + self.call_no_reply(mitogen.core.Dispatcher.forget_chain, saved) + finally: + self.chain_id = saved + + closures_msg = ( + 'Mitogen cannot invoke closures, as doing so would require ' + 'serializing arbitrary program state, and no universal ' + 'method exists to recover a reference to them.' + ) + + lambda_msg = ( + 'Mitogen cannot invoke anonymous functions, as no universal method ' + 'exists to recover a reference to an anonymous function.' + ) + + method_msg = ( + 'Mitogen cannot invoke instance methods, as doing so would require ' + 'serializing arbitrary program state.' + ) + + def make_msg(self, fn, *args, **kwargs): + if getattr(fn, closure_attr, None) is not None: + raise TypeError(self.closures_msg) + if fn.__name__ == '': + raise TypeError(self.lambda_msg) + + if inspect.ismethod(fn): + im_self = getattr(fn, IM_SELF_ATTR) + if not inspect.isclass(im_self): + raise TypeError(self.method_msg) + klass = mitogen.core.to_text(im_self.__name__) + else: + klass = None + + tup = ( + self.chain_id, + mitogen.core.to_text(fn.__module__), + klass, + mitogen.core.to_text(fn.__name__), + args, + mitogen.core.Kwargs(kwargs) + ) + return mitogen.core.Message.pickled(tup, + handle=mitogen.core.CALL_FUNCTION) + + def call_no_reply(self, fn, *args, **kwargs): + """ + Like :meth:`call_async`, but do not wait for a return value, and inform + the target context no reply is expected. If the call fails and + pipelining is disabled, the exception will be logged to the target + context's logging framework. + """ + LOG.debug('%r.call_no_reply(): %r', self, CallSpec(fn, args, kwargs)) + self.context.send(self.make_msg(fn, *args, **kwargs)) + + def call_async(self, fn, *args, **kwargs): + """ + Arrange for `fn(*args, **kwargs)` to be invoked on the context's main + thread. + + :param fn: + A free function in module scope or a class method of a class + directly reachable from module scope: + + .. code-block:: python + + # mymodule.py + + def my_func(): + '''A free function reachable as mymodule.my_func''' + + class MyClass: + @classmethod + def my_classmethod(cls): + '''Reachable as mymodule.MyClass.my_classmethod''' + + def my_instancemethod(self): + '''Unreachable: requires a class instance!''' + + class MyEmbeddedClass: + @classmethod + def my_classmethod(cls): + '''Not directly reachable from module scope!''' + + :param tuple args: + Function arguments, if any. See :ref:`serialization-rules` for + permitted types. + :param dict kwargs: + Function keyword arguments, if any. See :ref:`serialization-rules` + for permitted types. + :returns: + :class:`mitogen.core.Receiver` configured to receive the result of + the invocation: + + .. code-block:: python + + recv = context.call_async(os.check_output, 'ls /tmp/') + try: + # Prints output once it is received. + msg = recv.get() + print(msg.unpickle()) + except mitogen.core.CallError, e: + print('Call failed:', str(e)) + + Asynchronous calls may be dispatched in parallel to multiple + contexts and consumed as they complete using + :class:`mitogen.select.Select`. + """ + LOG.debug('%r.call_async(): %r', self, CallSpec(fn, args, kwargs)) + return self.context.send_async(self.make_msg(fn, *args, **kwargs)) + + def call(self, fn, *args, **kwargs): + """ + Like :meth:`call_async`, but block until the return value is available. + Equivalent to:: + + call_async(fn, *args, **kwargs).get().unpickle() + + :returns: + The function's return value. + :raises mitogen.core.CallError: + An exception was raised in the remote context during execution. + """ + receiver = self.call_async(fn, *args, **kwargs) + return receiver.get().unpickle(throw_dead=False) + + +class Context(mitogen.core.Context): + """ + Extend :class:`mitogen.core.Context` with functionality useful to masters, + and child contexts who later become parents. Currently when this class is + required, the target context's router is upgraded at runtime. + """ + #: A :class:`CallChain` instance constructed by default, with pipelining + #: disabled. :meth:`call`, :meth:`call_async` and :meth:`call_no_reply` use + #: this instance. + call_chain_class = CallChain + + via = None + + def __init__(self, *args, **kwargs): + super(Context, self).__init__(*args, **kwargs) + self.default_call_chain = self.call_chain_class(self) + + def __ne__(self, other): + return not (self == other) + + def __eq__(self, other): + return (isinstance(other, mitogen.core.Context) and + (other.context_id == self.context_id) and + (other.router == self.router)) + + def __hash__(self): + return hash((self.router, self.context_id)) + + def call_async(self, fn, *args, **kwargs): + """ + See :meth:`CallChain.call_async`. + """ + return self.default_call_chain.call_async(fn, *args, **kwargs) + + def call(self, fn, *args, **kwargs): + """ + See :meth:`CallChain.call`. + """ + return self.default_call_chain.call(fn, *args, **kwargs) + + def call_no_reply(self, fn, *args, **kwargs): + """ + See :meth:`CallChain.call_no_reply`. + """ + self.default_call_chain.call_no_reply(fn, *args, **kwargs) + + def shutdown(self, wait=False): + """ + Arrange for the context to receive a ``SHUTDOWN`` message, triggering + graceful shutdown. + + Due to a lack of support for timers, no attempt is made yet to force + terminate a hung context using this method. This will be fixed shortly. + + :param bool wait: + If :data:`True`, block the calling thread until the context has + completely terminated. + + :returns: + If `wait` is :data:`False`, returns a :class:`mitogen.core.Latch` + whose :meth:`get() ` method returns + :data:`None` when shutdown completes. The `timeout` parameter may + be used to implement graceful timeouts. + """ + LOG.debug('%r.shutdown() sending SHUTDOWN', self) + latch = mitogen.core.Latch() + mitogen.core.listen(self, 'disconnect', lambda: latch.put(None)) + self.send( + mitogen.core.Message( + handle=mitogen.core.SHUTDOWN, + ) + ) + + if wait: + latch.get() + else: + return latch + + +class RouteMonitor(object): + """ + Generate and respond to :data:`mitogen.core.ADD_ROUTE` and + :data:`mitogen.core.DEL_ROUTE` messages sent to the local context by + maintaining a table of available routes, and propagating messages towards + parents and siblings as appropriate. + + :class:`RouteMonitor` is responsible for generating routing messages for + directly attached children. It learns of new children via + :meth:`notice_stream` called by :class:`Router`, and subscribes to their + ``disconnect`` event to learn when they disappear. + + In children, constructing this class overwrites the stub + :data:`mitogen.core.DEL_ROUTE` handler installed by + :class:`mitogen.core.ExternalContext`, which is expected behaviour when a + child is beging upgraded in preparation to become a parent of children of + its own. + + By virtue of only being active while responding to messages from a handler, + RouteMonitor lives entirely on the broker thread, so its data requires no + locking. + + :param Router router: + Router to install handlers on. + :param Context parent: + :data:`None` in the master process, or reference to the parent context + we should propagate route updates towards. + """ + def __init__(self, router, parent=None): + self.router = router + self.parent = parent + #: Mapping of Stream instance to integer context IDs reachable via the + #: stream; used to cleanup routes during disconnection. + self._routes_by_stream = {} + self.router.add_handler( + fn=self._on_add_route, + handle=mitogen.core.ADD_ROUTE, + persist=True, + policy=is_immediate_child, + overwrite=True, + ) + self.router.add_handler( + fn=self._on_del_route, + handle=mitogen.core.DEL_ROUTE, + persist=True, + policy=is_immediate_child, + overwrite=True, + ) + + def __repr__(self): + return 'RouteMonitor()' + + def _send_one(self, stream, handle, target_id, name): + """ + Compose and send an update message on a stream. + + :param mitogen.core.Stream stream: + Stream to send it on. + :param int handle: + :data:`mitogen.core.ADD_ROUTE` or :data:`mitogen.core.DEL_ROUTE` + :param int target_id: + ID of the connecting or disconnecting context. + :param str name: + Context name or :data:`None`. + """ + if not stream: + # We may not have a stream during shutdown. + return + + data = str(target_id) + if name: + data = '%s:%s' % (target_id, name) + stream.send( + mitogen.core.Message( + handle=handle, + data=data.encode('utf-8'), + dst_id=stream.remote_id, + ) + ) + + def _propagate_up(self, handle, target_id, name=None): + """ + In a non-master context, propagate an update towards the master. + + :param int handle: + :data:`mitogen.core.ADD_ROUTE` or :data:`mitogen.core.DEL_ROUTE` + :param int target_id: + ID of the connecting or disconnecting context. + :param str name: + For :data:`mitogen.core.ADD_ROUTE`, the name of the new context + assigned by its parent. This is used by parents to assign the + :attr:`mitogen.core.Context.name` attribute. + """ + if self.parent: + stream = self.router.stream_by_id(self.parent.context_id) + self._send_one(stream, handle, target_id, name) + + def _propagate_down(self, handle, target_id): + """ + For DEL_ROUTE, we additionally want to broadcast the message to any + stream that has ever communicated with the disconnecting ID, so + core.py's :meth:`mitogen.core.Router._on_del_route` can turn the + message into a disconnect event. + + :param int handle: + :data:`mitogen.core.ADD_ROUTE` or :data:`mitogen.core.DEL_ROUTE` + :param int target_id: + ID of the connecting or disconnecting context. + """ + for stream in self.router.get_streams(): + if target_id in stream.egress_ids and ( + (self.parent is None) or + (self.parent.context_id != stream.remote_id) + ): + self._send_one(stream, mitogen.core.DEL_ROUTE, target_id, None) + + def notice_stream(self, stream): + """ + When this parent is responsible for a new directly connected child + stream, we're also responsible for broadcasting DEL_ROUTE upstream + if/when that child disconnects. + """ + self._routes_by_stream[stream] = set([stream.remote_id]) + self._propagate_up(mitogen.core.ADD_ROUTE, stream.remote_id, + stream.name) + mitogen.core.listen( + obj=stream, + name='disconnect', + func=lambda: self._on_stream_disconnect(stream), + ) + + def get_routes(self, stream): + """ + Return the set of context IDs reachable on a stream. + + :param mitogen.core.Stream stream: + :returns: set([int]) + """ + return self._routes_by_stream.get(stream) or set() + + def _on_stream_disconnect(self, stream): + """ + Respond to disconnection of a local stream by propagating DEL_ROUTE for + any contexts we know were attached to it. + """ + # During a stream crash it is possible for disconnect signal to fire + # twice, in which case ignore the second instance. + routes = self._routes_by_stream.pop(stream, None) + if routes is None: + return + + LOG.debug('%r: %r is gone; propagating DEL_ROUTE for %r', + self, stream, routes) + for target_id in routes: + self.router.del_route(target_id) + self._propagate_up(mitogen.core.DEL_ROUTE, target_id) + self._propagate_down(mitogen.core.DEL_ROUTE, target_id) + + context = self.router.context_by_id(target_id, create=False) + if context: + mitogen.core.fire(context, 'disconnect') + + def _on_add_route(self, msg): + """ + Respond to :data:`mitogen.core.ADD_ROUTE` by validating the source of + the message, updating the local table, and propagating the message + upwards. + """ + if msg.is_dead: + return + + target_id_s, _, target_name = bytes_partition(msg.data, b(':')) + target_name = target_name.decode() + target_id = int(target_id_s) + self.router.context_by_id(target_id).name = target_name + stream = self.router.stream_by_id(msg.auth_id) + current = self.router.stream_by_id(target_id) + if current and current.remote_id != mitogen.parent_id: + LOG.error('Cannot add duplicate route to %r via %r, ' + 'already have existing route via %r', + target_id, stream, current) + return + + LOG.debug('Adding route to %d via %r', target_id, stream) + self._routes_by_stream[stream].add(target_id) + self.router.add_route(target_id, stream) + self._propagate_up(mitogen.core.ADD_ROUTE, target_id, target_name) + + def _on_del_route(self, msg): + """ + Respond to :data:`mitogen.core.DEL_ROUTE` by validating the source of + the message, updating the local table, propagating the message + upwards, and downwards towards any stream that every had a message + forwarded from it towards the disconnecting context. + """ + if msg.is_dead: + return + + target_id = int(msg.data) + registered_stream = self.router.stream_by_id(target_id) + if registered_stream is None: + return + + stream = self.router.stream_by_id(msg.auth_id) + if registered_stream != stream: + LOG.error('%r: received DEL_ROUTE for %d from %r, expected %r', + self, target_id, stream, registered_stream) + return + + context = self.router.context_by_id(target_id, create=False) + if context: + LOG.debug('%r: firing local disconnect for %r', self, context) + mitogen.core.fire(context, 'disconnect') + + LOG.debug('%r: deleting route to %d via %r', self, target_id, stream) + routes = self._routes_by_stream.get(stream) + if routes: + routes.discard(target_id) + + self.router.del_route(target_id) + if stream.remote_id != mitogen.parent_id: + self._propagate_up(mitogen.core.DEL_ROUTE, target_id) + self._propagate_down(mitogen.core.DEL_ROUTE, target_id) + + +class Router(mitogen.core.Router): + context_class = Context + debug = False + profiling = False + + id_allocator = None + responder = None + log_forwarder = None + route_monitor = None + + def upgrade(self, importer, parent): + LOG.debug('%r.upgrade()', self) + self.id_allocator = ChildIdAllocator(router=self) + self.responder = ModuleForwarder( + router=self, + parent_context=parent, + importer=importer, + ) + self.route_monitor = RouteMonitor(self, parent) + self.add_handler( + fn=self._on_detaching, + handle=mitogen.core.DETACHING, + persist=True, + ) + + def _on_detaching(self, msg): + if msg.is_dead: + return + stream = self.stream_by_id(msg.src_id) + if stream.remote_id != msg.src_id or stream.detached: + LOG.warning('bad DETACHING received on %r: %r', stream, msg) + return + LOG.debug('%r: marking as detached', stream) + stream.detached = True + msg.reply(None) + + def get_streams(self): + """ + Return a snapshot of all streams in existence at time of call. + """ + self._write_lock.acquire() + try: + return itervalues(self._stream_by_id) + finally: + self._write_lock.release() + + def add_route(self, target_id, stream): + """ + Arrange for messages whose `dst_id` is `target_id` to be forwarded on + the directly connected stream for `via_id`. This method is called + automatically in response to :data:`mitogen.core.ADD_ROUTE` messages, + but remains public while the design has not yet settled, and situations + may arise where routing is not fully automatic. + """ + LOG.debug('%r.add_route(%r, %r)', self, target_id, stream) + assert isinstance(target_id, int) + assert isinstance(stream, Stream) + + self._write_lock.acquire() + try: + self._stream_by_id[target_id] = stream + finally: + self._write_lock.release() + + def del_route(self, target_id): + LOG.debug('%r.del_route(%r)', self, target_id) + # DEL_ROUTE may be sent by a parent if it knows this context sent + # messages to a peer that has now disconnected, to let us raise + # 'disconnect' event on the appropriate Context instance. In that case, + # we won't a matching _stream_by_id entry for the disappearing route, + # so don't raise an error for a missing key here. + self._write_lock.acquire() + try: + self._stream_by_id.pop(target_id, None) + finally: + self._write_lock.release() + + def get_module_blacklist(self): + if mitogen.context_id == 0: + return self.responder.blacklist + return self.importer.master_blacklist + + def get_module_whitelist(self): + if mitogen.context_id == 0: + return self.responder.whitelist + return self.importer.master_whitelist + + def allocate_id(self): + return self.id_allocator.allocate() + + connection_timeout_msg = u"Connection timed out." + + def _connect(self, klass, name=None, **kwargs): + context_id = self.allocate_id() + context = self.context_class(self, context_id) + kwargs['old_router'] = self + kwargs['max_message_size'] = self.max_message_size + stream = klass(self, context_id, **kwargs) + if name is not None: + stream.name = name + try: + stream.connect() + except mitogen.core.TimeoutError: + raise mitogen.core.StreamError(self.connection_timeout_msg) + context.name = stream.name + self.route_monitor.notice_stream(stream) + self.register(context, stream) + return context + + def connect(self, method_name, name=None, **kwargs): + klass = stream_by_method_name(method_name) + kwargs.setdefault(u'debug', self.debug) + kwargs.setdefault(u'profiling', self.profiling) + kwargs.setdefault(u'unidirectional', self.unidirectional) + + via = kwargs.pop(u'via', None) + if via is not None: + return self.proxy_connect(via, method_name, name=name, + **mitogen.core.Kwargs(kwargs)) + return self._connect(klass, name=name, + **mitogen.core.Kwargs(kwargs)) + + def proxy_connect(self, via_context, method_name, name=None, **kwargs): + resp = via_context.call(_proxy_connect, + name=name, + method_name=method_name, + kwargs=mitogen.core.Kwargs(kwargs), + ) + if resp['msg'] is not None: + raise mitogen.core.StreamError(resp['msg']) + + name = u'%s.%s' % (via_context.name, resp['name']) + context = self.context_class(self, resp['id'], name=name) + context.via = via_context + self._write_lock.acquire() + try: + self._context_by_id[context.context_id] = context + finally: + self._write_lock.release() + return context + + def doas(self, **kwargs): + return self.connect(u'doas', **kwargs) + + def docker(self, **kwargs): + return self.connect(u'docker', **kwargs) + + def kubectl(self, **kwargs): + return self.connect(u'kubectl', **kwargs) + + def fork(self, **kwargs): + return self.connect(u'fork', **kwargs) + + def jail(self, **kwargs): + return self.connect(u'jail', **kwargs) + + def local(self, **kwargs): + return self.connect(u'local', **kwargs) + + def lxc(self, **kwargs): + return self.connect(u'lxc', **kwargs) + + def lxd(self, **kwargs): + return self.connect(u'lxd', **kwargs) + + def setns(self, **kwargs): + return self.connect(u'setns', **kwargs) + + def su(self, **kwargs): + return self.connect(u'su', **kwargs) + + def sudo(self, **kwargs): + return self.connect(u'sudo', **kwargs) + + def ssh(self, **kwargs): + return self.connect(u'ssh', **kwargs) + + +class ProcessMonitor(object): + """ + Install a :data:`signal.SIGCHLD` handler that generates callbacks when a + specific child process has exitted. This class is obsolete, do not use. + """ + def __init__(self): + # pid -> callback() + self.callback_by_pid = {} + signal.signal(signal.SIGCHLD, self._on_sigchld) + + def _on_sigchld(self, _signum, _frame): + for pid, callback in self.callback_by_pid.items(): + pid, status = os.waitpid(pid, os.WNOHANG) + if pid: + callback(status) + del self.callback_by_pid[pid] + + def add(self, pid, callback): + """ + Add a callback function to be notified of the exit status of a process. + + :param int pid: + Process ID to be notified of. + + :param callback: + Function invoked as `callback(status)`, where `status` is the raw + exit status of the child process. + """ + self.callback_by_pid[pid] = callback + + _instance = None + + @classmethod + def instance(cls): + if cls._instance is None: + cls._instance = cls() + return cls._instance + + +class ModuleForwarder(object): + """ + Respond to GET_MODULE requests in a slave by forwarding the request to our + parent context, or satisfying the request from our local Importer cache. + """ + def __init__(self, router, parent_context, importer): + self.router = router + self.parent_context = parent_context + self.importer = importer + router.add_handler( + fn=self._on_forward_module, + handle=mitogen.core.FORWARD_MODULE, + persist=True, + policy=mitogen.core.has_parent_authority, + ) + router.add_handler( + fn=self._on_get_module, + handle=mitogen.core.GET_MODULE, + persist=True, + policy=is_immediate_child, + ) + + def __repr__(self): + return 'ModuleForwarder(%r)' % (self.router,) + + def _on_forward_module(self, msg): + if msg.is_dead: + return + + context_id_s, _, fullname = bytes_partition(msg.data, b('\x00')) + fullname = mitogen.core.to_text(fullname) + context_id = int(context_id_s) + stream = self.router.stream_by_id(context_id) + if stream.remote_id == mitogen.parent_id: + LOG.error('%r: dropping FORWARD_MODULE(%d, %r): no route to child', + self, context_id, fullname) + return + + if fullname in stream.sent_modules: + return + + LOG.debug('%r._on_forward_module() sending %r to %r via %r', + self, fullname, context_id, stream.remote_id) + self._send_module_and_related(stream, fullname) + if stream.remote_id != context_id: + stream._send( + mitogen.core.Message( + data=msg.data, + handle=mitogen.core.FORWARD_MODULE, + dst_id=stream.remote_id, + ) + ) + + def _on_get_module(self, msg): + LOG.debug('%r._on_get_module(%r)', self, msg) + if msg.is_dead: + return + + fullname = msg.data.decode('utf-8') + callback = lambda: self._on_cache_callback(msg, fullname) + self.importer._request_module(fullname, callback) + + def _on_cache_callback(self, msg, fullname): + LOG.debug('%r._on_get_module(): sending %r', self, fullname) + stream = self.router.stream_by_id(msg.src_id) + self._send_module_and_related(stream, fullname) + + def _send_module_and_related(self, stream, fullname): + tup = self.importer._cache[fullname] + for related in tup[4]: + rtup = self.importer._cache.get(related) + if rtup: + self._send_one_module(stream, rtup) + else: + LOG.debug('%r._send_module_and_related(%r): absent: %r', + self, fullname, related) + + self._send_one_module(stream, tup) + + def _send_one_module(self, stream, tup): + if tup[0] not in stream.sent_modules: + stream.sent_modules.add(tup[0]) + self.router._async_route( + mitogen.core.Message.pickled( + tup, + dst_id=stream.remote_id, + handle=mitogen.core.LOAD_MODULE, + ) + ) diff --git a/mitogen-0.2.7/mitogen/parent.pyc b/mitogen-0.2.7/mitogen/parent.pyc new file mode 100644 index 000000000..3e65f72 Binary files /dev/null and b/mitogen-0.2.7/mitogen/parent.pyc differ diff --git a/mitogen-0.2.7/mitogen/profiler.py b/mitogen-0.2.7/mitogen/profiler.py new file mode 100644 index 000000000..74bbdb2 --- /dev/null +++ b/mitogen-0.2.7/mitogen/profiler.py @@ -0,0 +1,166 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +"""mitogen.profiler + Record and report cProfile statistics from a run. Creates one aggregated + output file, one aggregate containing only workers, and one for the + top-level process. + +Usage: + mitogen.profiler record [args ..] + mitogen.profiler report [sort_mode] + mitogen.profiler stat [args ..] + +Mode: + record: Record a trace. + report: Report on a previously recorded trace. + stat: Record and report in a single step. + +Where: + dest_path: Filesystem prefix to write .pstats files to. + sort_mode: Sorting mode; defaults to "cumulative". See: + https://docs.python.org/2/library/profile.html#pstats.Stats.sort_stats + +Example: + mitogen.profiler record /tmp/mypatch ansible-playbook foo.yml + mitogen.profiler dump /tmp/mypatch-worker.pstats +""" + +from __future__ import print_function +import os +import pstats +import cProfile +import shutil +import subprocess +import sys +import tempfile +import time + +import mitogen.core + + +def try_merge(stats, path): + try: + stats.add(path) + return True + except Exception as e: + print('Failed. Race? Will retry. %s' % (e,)) + return False + + +def merge_stats(outpath, inpaths): + first, rest = inpaths[0], inpaths[1:] + for x in range(5): + try: + stats = pstats.Stats(first) + except EOFError: + time.sleep(0.2) + continue + + print("Writing %r..." % (outpath,)) + for path in rest: + #print("Merging %r into %r.." % (os.path.basename(path), outpath)) + for x in range(5): + if try_merge(stats, path): + break + time.sleep(0.2) + + stats.dump_stats(outpath) + + +def generate_stats(outpath, tmpdir): + print('Generating stats..') + all_paths = [] + paths_by_ident = {} + + for name in os.listdir(tmpdir): + if name.endswith('-dump.pstats'): + ident, _, pid = name.partition('-') + path = os.path.join(tmpdir, name) + all_paths.append(path) + paths_by_ident.setdefault(ident, []).append(path) + + merge_stats('%s-all.pstat' % (outpath,), all_paths) + for ident, paths in paths_by_ident.items(): + merge_stats('%s-%s.pstat' % (outpath, ident), paths) + + +def do_record(tmpdir, path, *args): + env = os.environ.copy() + fmt = '%(identity)s-%(pid)s.%(now)s-dump.%(ext)s' + env['MITOGEN_PROFILING'] = '1' + env['MITOGEN_PROFILE_FMT'] = os.path.join(tmpdir, fmt) + rc = subprocess.call(args, env=env) + generate_stats(path, tmpdir) + return rc + + +def do_report(tmpdir, path, sort='cumulative'): + stats = pstats.Stats(path).sort_stats(sort) + stats.print_stats(100) + + +def do_stat(tmpdir, sort, *args): + valid_sorts = pstats.Stats.sort_arg_dict_default + if sort not in valid_sorts: + sys.stderr.write('Invalid sort %r, must be one of %s\n' % + (sort, ', '.join(sorted(valid_sorts)))) + sys.exit(1) + + outfile = os.path.join(tmpdir, 'combined') + do_record(tmpdir, outfile, *args) + aggs = ('app.main', 'mitogen.broker', 'mitogen.child_main', + 'mitogen.service.pool', 'Strategy', 'WorkerProcess', + 'all') + for agg in aggs: + path = '%s-%s.pstat' % (outfile, agg) + if os.path.exists(path): + print() + print() + print('------ Aggregation %r ------' % (agg,)) + print() + do_report(tmpdir, path, sort) + print() + + +def main(): + if len(sys.argv) < 2 or sys.argv[1] not in ('record', 'report', 'stat'): + sys.stderr.write(__doc__) + sys.exit(1) + + func = globals()['do_' + sys.argv[1]] + tmpdir = tempfile.mkdtemp(prefix='mitogen.profiler') + try: + sys.exit(func(tmpdir, *sys.argv[2:]) or 0) + finally: + shutil.rmtree(tmpdir) + +if __name__ == '__main__': + main() diff --git a/mitogen-0.2.7/mitogen/select.py b/mitogen-0.2.7/mitogen/select.py new file mode 100644 index 000000000..51aebc2 --- /dev/null +++ b/mitogen-0.2.7/mitogen/select.py @@ -0,0 +1,333 @@ +# Copyright 2019, David Wilson +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +# !mitogen: minify_safe + +import mitogen.core + + +class Error(mitogen.core.Error): + pass + + +class Event(object): + """ + Represents one selected event. + """ + #: The first Receiver or Latch the event traversed. + source = None + + #: The :class:`mitogen.core.Message` delivered to a receiver, or the object + #: posted to a latch. + data = None + + +class Select(object): + """ + Support scatter/gather asynchronous calls and waiting on multiple + :class:`receivers `, + :class:`channels `, + :class:`latches `, and + :class:`sub-selects