run update

Shared/lib/python3.4/site-packages/stem/util/__init__.py

@@ -1,10 +1,14 @@
-# Copyright 2011-2015, Damian Johnson and The Tor Project
+# Copyright 2011-2018, Damian Johnson and The Tor Project
 # See LICENSE for licensing information
 
 """
 Utility functions used by the stem library.
 """
 
+import datetime
+
+import stem.prereq
+
 __all__ = [
   'conf',
   'connection',
@@ -17,4 +21,135 @@ __all__ = [
   'term',
   'test_tools',
   'tor_tools',
+  'datetime_to_unix',
 ]
+
+# Beginning with Stem 1.7 we take attribute types into account when hashing
+# and checking equality. That is to say, if two Stem classes' attributes are
+# the same but use different types we no longer consider them to be equal.
+# For example...
+#
+#   s1 = Schedule(classes = ['Math', 'Art', 'PE'])
+#   s2 = Schedule(classes = ('Math', 'Art', 'PE'))
+#
+# Prior to Stem 1.7 s1 and s2 would be equal, but afterward unless Stem's
+# construcotr normalizes the types they won't.
+#
+# This change in behavior is the right thing to do but carries some risk, so
+# we provide the following constant to revert to legacy behavior. If you find
+# yourself using it them please let me know (https://www.atagar.com/contact/)
+# since this flag will go away in the future.
+
+HASH_TYPES = True
+
+
+def _hash_value(val):
+  if not HASH_TYPES:
+    my_hash = 0
+  else:
+    # TODO: I hate doing this but until Python 2.x support is dropped we
+    # can't readily be strict about bytes vs unicode for attributes. This
+    # is because test assertions often use strings, and normalizing this
+    # would require wrapping most with to_unicode() calls.
+    #
+    # This hack will go away when we drop Python 2.x support.
+
+    if _is_str(val):
+      my_hash = hash('str')
+    else:
+      # Hashing common builtins (ints, bools, etc) provide consistant values but many others vary their value on interpreter invokation.
+
+      my_hash = hash(str(type(val)))
+
+  if isinstance(val, (tuple, list)):
+    for v in val:
+      my_hash = (my_hash * 1024) + hash(v)
+  elif isinstance(val, dict):
+    for k in sorted(val.keys()):
+      my_hash = (my_hash * 2048) + (hash(k) * 1024) + hash(val[k])
+  else:
+    my_hash += hash(val)
+
+  return my_hash
+
+
+def _is_str(val):
+  """
+  Check if a value is a string. This will be removed when we no longer provide
+  backward compatibility for the Python 2.x series.
+
+  :param object val: value to be checked
+
+  :returns: **True** if the value is some form of string (unicode or bytes),
+    and **False** otherwise
+  """
+
+  if stem.prereq.is_python_3():
+    return isinstance(val, (bytes, str))
+  else:
+    return isinstance(val, (bytes, unicode))
+
+
+def _is_int(val):
+  """
+  Check if a value is an integer. This will be removed when we no longer
+  provide backward compatibility for the Python 2.x series.
+
+  :param object val: value to be checked
+
+  :returns: **True** if the value is some form of integer (int or long),
+    and **False** otherwise
+  """
+
+  if stem.prereq.is_python_3():
+    return isinstance(val, int)
+  else:
+    return isinstance(val, (int, long))
+
+
+def datetime_to_unix(timestamp):
+  """
+  Converts a utc datetime object to a unix timestamp.
+
+  .. versionadded:: 1.5.0
+
+  :param datetime timestamp: timestamp to be converted
+
+  :returns: **float** for the unix timestamp of the given datetime object
+  """
+
+  if stem.prereq._is_python_26():
+    delta = (timestamp - datetime.datetime(1970, 1, 1))
+    return delta.days * 86400 + delta.seconds
+  else:
+    return (timestamp - datetime.datetime(1970, 1, 1)).total_seconds()
+
+
+def _hash_attr(obj, *attributes, **kwargs):
+  """
+  Provide a hash value for the given set of attributes.
+
+  :param Object obj: object to be hashed
+  :param list attributes: attribute names to take into account
+  :param bool cache: persists hash in a '_cached_hash' object attribute
+  :param class parent: include parent's hash value
+  """
+
+  is_cached = kwargs.get('cache', False)
+  parent_class = kwargs.get('parent', None)
+  cached_hash = getattr(obj, '_cached_hash', None)
+
+  if is_cached and cached_hash is not None:
+    return cached_hash
+
+  my_hash = parent_class.__hash__(obj) if parent_class else 0
+  my_hash = my_hash * 1024 + hash(str(type(obj)))
+
+  for attr in attributes:
+    val = getattr(obj, attr)
+    my_hash = my_hash * 1024 + _hash_value(val)
+
+  if is_cached:
+    setattr(obj, '_cached_hash', my_hash)
+
+  return my_hash

Shared/lib/python3.4/site-packages/stem/util/conf.py

@@ -1,4 +1,4 @@
-# Copyright 2011-2015, Damian Johnson and The Tor Project
+# Copyright 2011-2018, Damian Johnson and The Tor Project
 # See LICENSE for licensing information
 
 """
@@ -161,6 +161,8 @@ import inspect
 import os
 import threading
 
+import stem.prereq
+
 from stem.util import log
 
 try:
@@ -213,9 +215,8 @@ def config_dict(handle, conf_mappings, handler = None):
   For more information about how we convert types see our
   :func:`~stem.util.conf.Config.get` method.
 
-  **The dictionary you get from this is manged by the
-  :class:`~stem.util.conf.Config` class and should be treated as being
-  read-only.**
+  **The dictionary you get from this is manged by the Config class and should
+  be treated as being read-only.**
 
   :param str handle: unique identifier for a config instance
   :param dict conf_mappings: config key/value mappings used as our defaults
@@ -274,15 +275,15 @@ def uses_settings(handle, path, lazy_load = True):
 
   config = get_config(handle)
 
-  if not lazy_load and not config.get('settings_loaded', False):
+  if not lazy_load and not config._settings_loaded:
     config.load(path)
-    config.set('settings_loaded', 'true')
+    config._settings_loaded = True
 
   def decorator(func):
     def wrapped(*args, **kwargs):
-      if lazy_load and not config.get('settings_loaded', False):
+      if lazy_load and not config._settings_loaded:
         config.load(path)
-        config.set('settings_loaded', 'true')
+        config._settings_loaded = True
 
       if 'config' in inspect.getargspec(func).args:
         return func(*args, config = config, **kwargs)
@@ -446,11 +447,14 @@ class Config(object):
     #
     # Information for what values fail to load and why are reported to
     # 'stem.util.log'.
+
+    .. versionchanged:: 1.7.0
+       Class can now be used as a dictionary.
   """
 
   def __init__(self):
     self._path = None        # location we last loaded from or saved to
-    self._contents = {}      # configuration key/value pairs
+    self._contents = OrderedDict()  # configuration key/value pairs
     self._listeners = []     # functors to be notified of config changes
 
     # used for accessing _contents
@@ -459,7 +463,10 @@ class Config(object):
     # keys that have been requested (used to provide unused config contents)
     self._requested_keys = set()
 
-  def load(self, path = None):
+    # flag to support lazy loading in uses_settings()
+    self._settings_loaded = False
+
+  def load(self, path = None, commenting = True):
     """
     Reads in the contents of the given path, adding its configuration values
     to our current contents. If the path is a directory then this loads each
@@ -468,8 +475,16 @@ class Config(object):
     .. versionchanged:: 1.3.0
        Added support for directories.
 
+    .. versionchanged:: 1.3.0
+       Added the **commenting** argument.
+
+    .. versionchanged:: 1.6.0
+       Avoid loading vim swap files.
+
     :param str path: file or directory path to be loaded, this uses the last
       loaded path if not provided
+    :param bool commenting: ignore line content after a '#' if **True**, read
+      otherwise
 
     :raises:
       * **IOError** if we fail to read the file (it doesn't exist, insufficient
@@ -485,6 +500,9 @@ class Config(object):
     if os.path.isdir(self._path):
       for root, dirnames, filenames in os.walk(self._path):
         for filename in filenames:
+          if filename.endswith('.swp'):
+            continue  # vim swap file
+
           self.load(os.path.join(root, filename))
 
       return
@@ -497,7 +515,7 @@ class Config(object):
         line = read_contents.pop(0)
 
         # strips any commenting or excess whitespace
-        comment_start = line.find('#')
+        comment_start = line.find('#') if commenting else -1
 
         if comment_start != -1:
           line = line[:comment_start]
@@ -506,14 +524,10 @@ class Config(object):
 
         # parse the key/value pair
         if line:
-          try:
+          if ' ' in line:
             key, value = line.split(' ', 1)
-            value = value.strip()
-          except ValueError:
-            log.debug("Config entry '%s' is expected to be of the format 'Key Value', defaulting to '%s' -> ''" % (line, line))
-            key, value = line, ''
-
-          if not value:
+            self.set(key, value.strip(), False)
+          else:
             # this might be a multi-line entry, try processing it as such
             multiline_buffer = []
 
@@ -523,10 +537,9 @@ class Config(object):
               multiline_buffer.append(content)
 
             if multiline_buffer:
-              self.set(key, '\n'.join(multiline_buffer), False)
-              continue
-
-          self.set(key, value, False)
+              self.set(line, '\n'.join(multiline_buffer), False)
+            else:
+              self.set(line, '', False)  # default to a key => '' mapping
 
   def save(self, path = None):
     """
@@ -535,7 +548,9 @@ class Config(object):
 
     :param str path: location to be saved to
 
-    :raises: **ValueError** if no path was provided and we've never been provided one
+    :raises:
+      * **IOError** if we fail to save the file (insufficient permissions, etc)
+      * **ValueError** if no path was provided and we've never been provided one
     """
 
     if path:
@@ -544,8 +559,11 @@ class Config(object):
       raise ValueError('Unable to save configuration: no path provided')
 
     with self._contents_lock:
+      if not os.path.exists(os.path.dirname(self._path)):
+        os.makedirs(os.path.dirname(self._path))
+
       with open(self._path, 'w') as output_file:
-        for entry_key in sorted(self.keys()):
+        for entry_key in self.keys():
           for entry_value in self.get_value(entry_key, multiple = True):
             # check for multi line entries
             if '\n' in entry_value:
@@ -612,6 +630,9 @@ class Config(object):
     Appends the given key/value configuration mapping, behaving the same as if
     we'd loaded this from a configuration file.
 
+    .. versionchanged:: 1.5.0
+       Allow removal of values by overwriting with a **None** value.
+
     :param str key: key for the configuration mapping
     :param str,list value: value we're setting the mapping to
     :param bool overwrite: replaces the previous value if **True**, otherwise
@@ -619,7 +640,14 @@ class Config(object):
     """
 
     with self._contents_lock:
-      if isinstance(value, str):
+      unicode_type = str if stem.prereq.is_python_3() else unicode
+
+      if value is None:
+        if overwrite and key in self._contents:
+          del self._contents[key]
+        else:
+          pass  # no value so this is a no-op
+      elif isinstance(value, (bytes, unicode_type)):
         if not overwrite and key in self._contents:
           self._contents[key].append(value)
         else:
@@ -636,7 +664,7 @@ class Config(object):
         for listener in self._listeners:
           listener(self, key)
       else:
-        raise ValueError("Config.set() only accepts str, list, or tuple. Provided value was a '%s'" % type(value))
+        raise ValueError("Config.set() only accepts str (bytes or unicode), list, or tuple. Provided value was a '%s'" % type(value))
 
   def get(self, key, default = None):
     """
@@ -743,3 +771,7 @@ class Config(object):
         message_id = 'stem.util.conf.missing_config_key_%s' % key
         log.log_once(message_id, log.TRACE, "config entry '%s' not found, defaulting to '%s'" % (key, default))
         return default
+
+  def __getitem__(self, key):
+    with self._contents_lock:
+      return self._contents[key]

Shared/lib/python3.4/site-packages/stem/util/connection.py

@@ -1,4 +1,4 @@
-# Copyright 2012-2015, Damian Johnson and The Tor Project
+# Copyright 2012-2018, Damian Johnson and The Tor Project
 # See LICENSE for licensing information
 
 """
@@ -17,6 +17,8 @@ Connection and networking based utility functions.
   is_valid_port - checks if something is a valid representation for a port
   is_private_address - checks if an IPv4 address belongs to a private range or not
 
+  address_to_int - provides an integer representation of an IP address
+
   expand_ipv6_address - provides an IPv6 address with its collapsed portions expanded
   get_mask_ipv4 - provides the mask representation for a given number of bits
   get_mask_ipv6 - provides the IPv6 mask representation for a given number of bits
@@ -26,9 +28,17 @@ Connection and networking based utility functions.
   Method for resolving a process' connections.
 
   .. versionadded:: 1.1.0
+
   .. versionchanged:: 1.4.0
      Added **NETSTAT_WINDOWS**.
 
+  .. versionchanged:: 1.6.0
+     Added **BSD_FSTAT**.
+
+  .. deprecated:: 1.6.0
+     The SOCKSTAT connection resolver is proving to be unreliable
+     (:trac:`23057`), and will be dropped in the 2.0.0 release unless fixed.
+
   ====================  ===========
   Resolver              Description
   ====================  ===========
@@ -37,9 +47,10 @@ Connection and networking based utility functions.
   **NETSTAT_WINDOWS**   netstat command under Windows
   **SS**                ss command
   **LSOF**              lsof command
-  **SOCKSTAT**          sockstat command under *nix
+  **SOCKSTAT**          sockstat command under \*nix
   **BSD_SOCKSTAT**      sockstat command under FreeBSD
   **BSD_PROCSTAT**      procstat command under FreeBSD
+  **BSD_FSTAT**         fstat command under OpenBSD
   ====================  ===========
 """
 
@@ -50,11 +61,11 @@ import os
 import platform
 import re
 
+import stem.util
 import stem.util.proc
 import stem.util.system
 
-from stem import str_type
-from stem.util import conf, enum, log
+from stem.util import conf, enum, log, str_tools
 
 # Connection resolution is risky to log about since it's highly likely to
 # contain sensitive information. That said, it's also difficult to get right in
@@ -71,17 +82,10 @@ Resolver = enum.Enum(
   ('LSOF', 'lsof'),
   ('SOCKSTAT', 'sockstat'),
   ('BSD_SOCKSTAT', 'sockstat (bsd)'),
-  ('BSD_PROCSTAT', 'procstat (bsd)')
+  ('BSD_PROCSTAT', 'procstat (bsd)'),
+  ('BSD_FSTAT', 'fstat (bsd)')
 )
 
-Connection = collections.namedtuple('Connection', [
-  'local_address',
-  'local_port',
-  'remote_address',
-  'remote_port',
-  'protocol',
-])
-
 FULL_IPv4_MASK = '255.255.255.255'
 FULL_IPv6_MASK = 'FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF:FFFF'
 
@@ -92,8 +96,8 @@ PORT_USES = None  # port number => description
 RESOLVER_COMMAND = {
   Resolver.PROC: '',
 
-  # -n = prevents dns lookups, -p = include process
-  Resolver.NETSTAT: 'netstat -np',
+  # -n = prevents dns lookups, -p = include process, -W = don't crop addresses (needed for ipv6)
+  Resolver.NETSTAT: 'netstat -npW',
 
   # -a = show all TCP/UDP connections, -n = numeric addresses and ports, -o = include pid
   Resolver.NETSTAT_WINDOWS: 'netstat -ano',
@@ -112,62 +116,97 @@ RESOLVER_COMMAND = {
 
   # -f <pid> = process pid
   Resolver.BSD_PROCSTAT: 'procstat -f {pid}',
+
+  # -p <pid> = process pid
+  Resolver.BSD_FSTAT: 'fstat -p {pid}',
 }
 
 RESOLVER_FILTER = {
   Resolver.PROC: '',
 
   # tcp        0    586 192.168.0.1:44284       38.229.79.2:443         ESTABLISHED 15843/tor
-  Resolver.NETSTAT: '^{protocol}\s+.*\s+{local_address}:{local_port}\s+{remote_address}:{remote_port}\s+ESTABLISHED\s+{pid}/{name}\s*$',
+  Resolver.NETSTAT: '^{protocol}\s+.*\s+{local}\s+{remote}\s+ESTABLISHED\s+{pid}/{name}\s*$',
 
   # tcp        586 192.168.0.1:44284       38.229.79.2:443         ESTABLISHED 15843
-  Resolver.NETSTAT_WINDOWS: '^\s*{protocol}\s+{local_address}:{local_port}\s+{remote_address}:{remote_port}\s+ESTABLISHED\s+{pid}\s*$',
+  Resolver.NETSTAT_WINDOWS: '^\s*{protocol}\s+{local}\s+{remote}\s+ESTABLISHED\s+{pid}\s*$',
 
   # tcp    ESTAB      0      0           192.168.0.20:44415       38.229.79.2:443    users:(("tor",15843,9))
-  Resolver.SS: '^{protocol}\s+ESTAB\s+.*\s+{local_address}:{local_port}\s+{remote_address}:{remote_port}\s+users:\(\("{name}",{pid},[0-9]+\)\)$',
+  Resolver.SS: '^{protocol}\s+ESTAB\s+.*\s+{local}\s+{remote}\s+users:\(\("{name}",(?:pid=)?{pid},(?:fd=)?[0-9]+\)\)$',
 
   # tor  3873  atagar  45u  IPv4  40994  0t0  TCP 10.243.55.20:45724->194.154.227.109:9001 (ESTABLISHED)
-  Resolver.LSOF: '^{name}\s+{pid}\s+.*\s+{protocol}\s+{local_address}:{local_port}->{remote_address}:{remote_port} \(ESTABLISHED\)$',
+  Resolver.LSOF: '^{name}\s+{pid}\s+.*\s+{protocol}\s+{local}->{remote} \(ESTABLISHED\)$',
 
   # atagar   tor                  15843    tcp4   192.168.0.20:44092        68.169.35.102:443         ESTABLISHED
-  Resolver.SOCKSTAT: '^\S+\s+{name}\s+{pid}\s+{protocol}4\s+{local_address}:{local_port}\s+{remote_address}:{remote_port}\s+ESTABLISHED$',
+  Resolver.SOCKSTAT: '^\S+\s+{name}\s+{pid}\s+{protocol}4\s+{local}\s+{remote}\s+ESTABLISHED$',
 
   # _tor     tor        4397  12 tcp4   172.27.72.202:54011   127.0.0.1:9001
-  Resolver.BSD_SOCKSTAT: '^\S+\s+{name}\s+{pid}\s+\S+\s+{protocol}4\s+{local_address}:{local_port}\s+{remote_address}:{remote_port}$',
+  Resolver.BSD_SOCKSTAT: '^\S+\s+{name}\s+{pid}\s+\S+\s+{protocol}4\s+{local}\s+{remote}$',
 
   # 3561 tor                 4 s - rw---n--   2       0 TCP 10.0.0.2:9050 10.0.0.1:22370
-  Resolver.BSD_PROCSTAT: '^\s*{pid}\s+{name}\s+.*\s+{protocol}\s+{local_address}:{local_port}\s+{remote_address}:{remote_port}$',
+  Resolver.BSD_PROCSTAT: '^\s*{pid}\s+{name}\s+.*\s+{protocol}\s+{local}\s+{remote}$',
+
+  # _tor     tor        15843   20* internet stream tcp 0x0 192.168.1.100:36174 --> 4.3.2.1:443
+  Resolver.BSD_FSTAT: '^\S+\s+{name}\s+{pid}\s+.*\s+{protocol}\s+\S+\s+{local}\s+[-<]-[->]\s+{remote}$',
 }
 
 
-def get_connections(resolver, process_pid = None, process_name = None):
+class Connection(collections.namedtuple('Connection', ['local_address', 'local_port', 'remote_address', 'remote_port', 'protocol', 'is_ipv6'])):
+  """
+  Network connection information.
+
+  .. versionchanged:: 1.5.0
+     Added the **is_ipv6** attribute.
+
+  :var str local_address: ip address the connection originates from
+  :var int local_port: port the connection originates from
+  :var str remote_address: destionation ip address
+  :var int remote_port: destination port
+  :var str protocol: protocol of the connection ('tcp', 'udp', etc)
+  :var bool is_ipv6: addresses are ipv6 if true, and ipv4 otherwise
+  """
+
+
+def get_connections(resolver = None, process_pid = None, process_name = None):
   """
   Retrieves a list of the current connections for a given process. This
-  provides a list of Connection instances, which have five attributes...
-
-    * **local_address** (str)
-    * **local_port** (int)
-    * **remote_address** (str)
-    * **remote_port** (int)
-    * **protocol** (str, generally either 'tcp' or 'udp')
+  provides a list of :class:`~stem.util.connection.Connection`. Note that
+  addresses may be IPv4 *or* IPv6 depending on what the platform supports.
 
   .. versionadded:: 1.1.0
 
-  :param Resolver resolver: method of connection resolution to use
+  .. versionchanged:: 1.5.0
+     Made our resolver argument optional.
+
+  .. versionchanged:: 1.5.0
+     IPv6 support when resolving via proc, netstat, lsof, or ss.
+
+  :param Resolver resolver: method of connection resolution to use, if not
+    provided then one is picked from among those that should likely be
+    available for the system
   :param int process_pid: pid of the process to retrieve
   :param str process_name: name of the process to retrieve
 
-  :returns: **list** of Connection instances
+  :returns: **list** of :class:`~stem.util.connection.Connection` instances
 
   :raises:
-    * **ValueError** if using **Resolver.PROC** or **Resolver.BSD_PROCSTAT**
-      and the process_pid wasn't provided
+    * **ValueError** if neither a process_pid nor process_name is provided
 
     * **IOError** if no connections are available or resolution fails
       (generally they're indistinguishable). The common causes are the
       command being unavailable or permissions.
   """
 
+  if not resolver:
+    available_resolvers = system_resolvers()
+
+    if available_resolvers:
+      resolver = available_resolvers[0]
+    else:
+      raise IOError('Unable to determine a connection resolver')
+
+  if not process_pid and not process_name:
+    raise ValueError('You must provide a pid or process name to provide connections for')
+
   def _log(msg):
     if LOG_CONNECTION_RESOLUTION:
       log.debug(msg)
@@ -181,14 +220,20 @@ def get_connections(resolver, process_pid = None, process_name = None):
     except ValueError:
       raise ValueError('Process pid was non-numeric: %s' % process_pid)
 
-  if process_pid is None and process_name and resolver == Resolver.NETSTAT_WINDOWS:
-    process_pid = stem.util.system.pid_by_name(process_name)
+  if process_pid is None:
+    all_pids = stem.util.system.pid_by_name(process_name, True)
 
-  if process_pid is None and resolver in (Resolver.NETSTAT_WINDOWS, Resolver.PROC, Resolver.BSD_PROCSTAT):
-    raise ValueError('%s resolution requires a pid' % resolver)
+    if len(all_pids) == 0:
+      if resolver in (Resolver.NETSTAT_WINDOWS, Resolver.PROC, Resolver.BSD_PROCSTAT):
+        raise IOError("Unable to determine the pid of '%s'. %s requires the pid to provide the connections." % (process_name, resolver))
+    elif len(all_pids) == 1:
+      process_pid = all_pids[0]
+    else:
+      if resolver in (Resolver.NETSTAT_WINDOWS, Resolver.PROC, Resolver.BSD_PROCSTAT):
+        raise IOError("There's multiple processes named '%s'. %s requires a single pid to provide the connections." % (process_name, resolver))
 
   if resolver == Resolver.PROC:
-    return [Connection(*conn) for conn in stem.util.proc.connections(process_pid)]
+    return stem.util.proc.connections(pid = process_pid)
 
   resolver_command = RESOLVER_COMMAND[resolver].format(pid = process_pid)
 
@@ -199,10 +244,8 @@ def get_connections(resolver, process_pid = None, process_name = None):
 
   resolver_regex_str = RESOLVER_FILTER[resolver].format(
     protocol = '(?P<protocol>\S+)',
-    local_address = '(?P<local_address>[0-9.]+)',
-    local_port = '(?P<local_port>[0-9]+)',
-    remote_address = '(?P<remote_address>[0-9.]+)',
-    remote_port = '(?P<remote_port>[0-9]+)',
+    local = '(?P<local>[\[\]0-9a-f.:]+)',
+    remote = '(?P<remote>[\[\]0-9a-f.:]+)',
     pid = process_pid if process_pid else '[0-9]*',
     name = process_name if process_name else '\S*',
   )
@@ -213,28 +256,41 @@ def get_connections(resolver, process_pid = None, process_name = None):
   connections = []
   resolver_regex = re.compile(resolver_regex_str)
 
+  def _parse_address_str(addr_type, addr_str, line):
+    addr, port = addr_str.rsplit(':', 1)
+
+    if not is_valid_ipv4_address(addr) and not is_valid_ipv6_address(addr, allow_brackets = True):
+      _log('Invalid %s address (%s): %s' % (addr_type, addr, line))
+      return None, None
+    elif not is_valid_port(port):
+      _log('Invalid %s port (%s): %s' % (addr_type, port, line))
+      return None, None
+    else:
+      _log('Valid %s:%s: %s' % (addr, port, line))
+      return addr.lstrip('[').rstrip(']'), int(port)
+
   for line in results:
     match = resolver_regex.match(line)
 
     if match:
       attr = match.groupdict()
-      local_addr = attr['local_address']
-      local_port = int(attr['local_port'])
-      remote_addr = attr['remote_address']
-      remote_port = int(attr['remote_port'])
+
+      local_addr, local_port = _parse_address_str('local', attr['local'], line)
+      remote_addr, remote_port = _parse_address_str('remote', attr['remote'], line)
+
+      if not (local_addr and local_port and remote_addr and remote_port):
+        continue  # missing or malformed field
+
       protocol = attr['protocol'].lower()
 
-      if remote_addr == '0.0.0.0':
-        continue  # procstat response for unestablished connections
+      if protocol == 'tcp6':
+        protocol = 'tcp'
 
-      if not (is_valid_ipv4_address(local_addr) and is_valid_ipv4_address(remote_addr)):
-        _log('Invalid address (%s or %s): %s' % (local_addr, remote_addr, line))
-      elif not (is_valid_port(local_port) and is_valid_port(remote_port)):
-        _log('Invalid port (%s or %s): %s' % (local_port, remote_port, line))
-      elif protocol not in ('tcp', 'udp'):
+      if protocol not in ('tcp', 'udp'):
         _log('Unrecognized protocol (%s): %s' % (protocol, line))
+        continue
 
-      conn = Connection(local_addr, local_port, remote_addr, remote_port, protocol)
+      conn = Connection(local_addr, local_port, remote_addr, remote_port, protocol, is_valid_ipv6_address(local_addr))
       connections.append(conn)
       _log(str(conn))
 
@@ -261,6 +317,7 @@ def system_resolvers(system = None):
 
   :returns: **list** of :data:`~stem.util.connection.Resolver` instances available on this platform
   """
+
   if system is None:
     if stem.util.system.is_gentoo():
       system = 'Gentoo'
@@ -269,8 +326,10 @@ def system_resolvers(system = None):
 
   if system == 'Windows':
     resolvers = [Resolver.NETSTAT_WINDOWS]
-  elif system in ('Darwin', 'OpenBSD'):
+  elif system == 'Darwin':
     resolvers = [Resolver.LSOF]
+  elif system == 'OpenBSD':
+    resolvers = [Resolver.BSD_FSTAT]
   elif system == 'FreeBSD':
     # Netstat is available, but lacks a '-p' equivalent so we can't associate
     # the results to processes. The platform also has a ss command, but it
@@ -349,7 +408,9 @@ def is_valid_ipv4_address(address):
   :returns: **True** if input is a valid IPv4 address, **False** otherwise
   """
 
-  if not isinstance(address, (bytes, str_type)):
+  if isinstance(address, bytes):
+    address = str_tools._to_unicode(address)
+  elif not stem.util._is_str(address):
     return False
 
   # checks if theres four period separated values
@@ -377,10 +438,31 @@ def is_valid_ipv6_address(address, allow_brackets = False):
   :returns: **True** if input is a valid IPv6 address, **False** otherwise
   """
 
+  if isinstance(address, bytes):
+    address = str_tools._to_unicode(address)
+  elif not stem.util._is_str(address):
+    return False
+
   if allow_brackets:
     if address.startswith('[') and address.endswith(']'):
       address = address[1:-1]
 
+  if address.count('.') == 3:
+    # Likely an ipv4-mapped portion. Check that its vaild, then replace with a
+    # filler.
+
+    ipv4_start = address.rfind(':', 0, address.find('.')) + 1
+    ipv4_end = address.find(':', ipv4_start + 1)
+
+    if ipv4_end == -1:
+      ipv4_end = None  # don't crop the last character
+
+    if not is_valid_ipv4_address(address[ipv4_start:ipv4_end]):
+      return False
+
+    addr_comp = [address[:ipv4_start - 1] if ipv4_start != 0 else None, 'ff:ff', address[ipv4_end + 1:] if ipv4_end else None]
+    address = ':'.join(filter(None, addr_comp))
+
   # addresses are made up of eight colon separated groups of four hex digits
   # with leading zeros being optional
   # https://en.wikipedia.org/wiki/IPv6#Address_format
@@ -469,6 +551,24 @@ def is_private_address(address):
   return False
 
 
+def address_to_int(address):
+  """
+  Provides an integer representation of a IPv4 or IPv6 address that can be used
+  for sorting.
+
+  .. versionadded:: 1.5.0
+
+  :param str address: IPv4 or IPv6 address
+
+  :returns: **int** representation of the address
+  """
+
+  # TODO: Could be neat to also use this for serialization if we also had an
+  # int_to_address() function.
+
+  return int(_address_to_binary(address), 2)
+
+
 def expand_ipv6_address(address):
   """
   Expands abbreviated IPv6 addresses to their full colon separated hex format.
@@ -482,6 +582,9 @@ def expand_ipv6_address(address):
     >>> expand_ipv6_address('::')
     '0000:0000:0000:0000:0000:0000:0000:0000'
 
+    >>> expand_ipv6_address('::ffff:5.9.158.75')
+    '0000:0000:0000:0000:0000:ffff:0509:9e4b'
+
   :param str address: IPv6 address to be expanded
 
   :raises: **ValueError** if the address can't be expanded due to being malformed
@@ -490,6 +593,25 @@ def expand_ipv6_address(address):
   if not is_valid_ipv6_address(address):
     raise ValueError("'%s' isn't a valid IPv6 address" % address)
 
+  # expand ipv4-mapped portions of addresses
+  if address.count('.') == 3:
+    ipv4_start = address.rfind(':', 0, address.find('.')) + 1
+    ipv4_end = address.find(':', ipv4_start + 1)
+
+    if ipv4_end == -1:
+      ipv4_end = None  # don't crop the last character
+
+    # Converts ipv4 address to its hex ipv6 representation. For instance...
+    #
+    #   '5.9.158.75' => '0509:9e4b'
+
+    ipv4_bin = _address_to_binary(address[ipv4_start:ipv4_end])
+    groupings = [ipv4_bin[16 * i:16 * (i + 1)] for i in range(2)]
+    ipv6_snippet = ':'.join(['%04x' % int(group, 2) for group in groupings])
+
+    addr_comp = [address[:ipv4_start - 1] if ipv4_start != 0 else None, ipv6_snippet, address[ipv4_end + 1:] if ipv4_end else None]
+    address = ':'.join(filter(None, addr_comp))
+
   # expands collapsed groupings, there can only be a single '::' in a valid
   # address
   if '::' in address:
@@ -577,7 +699,7 @@ def _get_masked_bits(mask):
     raise ValueError("'%s' is an invalid subnet mask" % mask)
 
   # converts octets to binary representation
-  mask_bin = _get_address_binary(mask)
+  mask_bin = _address_to_binary(mask)
   mask_match = re.match('^(1*)(0*)$', mask_bin)
 
   if mask_match:
@@ -599,7 +721,7 @@ def _get_binary(value, bits):
   return ''.join([str((value >> y) & 1) for y in range(bits - 1, -1, -1)])
 
 
-def _get_address_binary(address):
+def _address_to_binary(address):
   """
   Provides the binary value for an IPv4 or IPv6 address.
 
@@ -644,6 +766,7 @@ def _cryptovariables_equal(x, y):
     _hmac_sha256(CRYPTOVARIABLE_EQUALITY_COMPARISON_NONCE, x) ==
     _hmac_sha256(CRYPTOVARIABLE_EQUALITY_COMPARISON_NONCE, y))
 
+
 # TODO: drop with stem 2.x
 # We renamed our methods to drop a redundant 'get_*' prefix, so alias the old
 # names for backward compatability.

Shared/lib/python3.4/site-packages/stem/util/enum.py

@@ -1,4 +1,4 @@
-# Copyright 2011-2015, Damian Johnson and The Tor Project
+# Copyright 2011-2018, Damian Johnson and The Tor Project
 # See LICENSE for licensing information
 
 """
@@ -40,7 +40,7 @@ constructed as simple type listings...
     +- __iter__ - iterator over our enum keys
 """
 
-from stem import str_type
+import stem.util
 
 
 def UppercaseEnum(*args):
@@ -76,7 +76,7 @@ class Enum(object):
     keys, values = [], []
 
     for entry in args:
-      if isinstance(entry, (bytes, str_type)):
+      if stem.util._is_str(entry):
         key, val = entry, _to_camel_case(entry)
       elif isinstance(entry, tuple) and len(entry) == 2:
         key, val = entry

Shared/lib/python3.4/site-packages/stem/util/log.py

@@ -1,4 +1,4 @@
-# Copyright 2011-2015, Damian Johnson and The Tor Project
+# Copyright 2011-2018, Damian Johnson and The Tor Project
 # See LICENSE for licensing information
 
 """
@@ -88,9 +88,13 @@ DEDUPLICATION_MESSAGE_IDS = set()
 
 
 class _NullHandler(logging.Handler):
+  def __init__(self):
+    logging.Handler.__init__(self, level = logging.FATAL + 5)  # disable logging
+
   def emit(self, record):
     pass
 
+
 if not LOGGER.handlers:
   LOGGER.addHandler(_NullHandler())
 
@@ -99,7 +103,7 @@ def get_logger():
   """
   Provides the stem logger.
 
-  :return: **logging.Logger** for stem
+  :returns: **logging.Logger** for stem
   """
 
   return LOGGER
@@ -118,6 +122,22 @@ def logging_level(runlevel):
     return logging.FATAL + 5
 
 
+def is_tracing():
+  """
+  Checks if we're logging at the trace runlevel.
+
+  .. versionadded:: 1.6.0
+
+  :returns: **True** if we're logging at the trace runlevel and **False** otherwise
+  """
+
+  for handler in get_logger().handlers:
+    if handler.level <= logging_level(TRACE):
+      return True
+
+  return False
+
+
 def escape(message):
   """
   Escapes specific sequences for logging (newlines, tabs, carriage returns). If
@@ -199,8 +219,8 @@ class LogBuffer(logging.Handler):
   Basic log handler that listens for stem events and stores them so they can be
   read later. Log entries are cleared as they are read.
 
-    .. versionchanged:: 1.4.0
-       Added the yield_records argument.
+  .. versionchanged:: 1.4.0
+     Added the yield_records argument.
   """
 
   def __init__(self, runlevel, yield_records = False):

Shared/lib/python3.4/site-packages/stem/util/ports.cfg

@@ -310,4 +310,5 @@ port 19638 => Ensim
 port 23399 => Skype
 port 30301 => BitTorrent
 port 33434 => traceroute
+port 50002 => Electrum Bitcoin SSL
 

Shared/lib/python3.4/site-packages/stem/util/proc.py

@@ -1,4 +1,4 @@
-# Copyright 2011-2015, Damian Johnson and The Tor Project
+# Copyright 2011-2018, Damian Johnson and The Tor Project
 # See LICENSE for licensing information
 
 """
@@ -54,14 +54,23 @@ import socket
 import sys
 import time
 
+import stem.prereq
+import stem.util.connection
 import stem.util.enum
+import stem.util.str_tools
 
 from stem.util import log
 
 try:
-  # added in python 3.2
-  from functools import lru_cache
+  # unavailable on windows (#19823)
+  import pwd
+  IS_PWD_AVAILABLE = True
 except ImportError:
+  IS_PWD_AVAILABLE = False
+
+if stem.prereq._is_lru_cache_available():
+  from functools import lru_cache
+else:
   from stem.util.lru_cache import lru_cache
 
 # os.sysconf is only defined on unix
@@ -70,6 +79,9 @@ try:
 except AttributeError:
   CLOCK_TICKS = None
 
+IS_LITTLE_ENDIAN = sys.byteorder == 'little'
+ENCODED_ADDR = {}  # cache of encoded ips to their decoded version
+
 Stat = stem.util.enum.Enum(
   ('COMMAND', 'command'), ('CPU_UTIME', 'utime'),
   ('CPU_STIME', 'stime'), ('START_TIME', 'start time')
@@ -324,38 +336,110 @@ def file_descriptors_used(pid):
     raise IOError('Unable to check number of file descriptors used: %s' % exc)
 
 
-def connections(pid):
+def connections(pid = None, user = None):
   """
-  Queries connection related information from the proc contents. This provides
-  similar results to netstat, lsof, sockstat, and other connection resolution
-  utilities (though the lookup is far quicker).
+  Queries connections from the proc contents. This matches netstat, lsof, and
+  friends but is much faster. If no **pid** or **user** are provided this
+  provides all present connections.
 
-  :param int pid: process id of the process to be queried
+  :param int pid: pid to provide connections for
+  :param str user: username to look up connections for
 
-  :returns: A listing of connection tuples of the form **[(local_ipAddr1,
-    local_port1, foreign_ipAddr1, foreign_port1, protocol), ...]** (addresses
-    and protocols are strings and ports are ints)
+  :returns: **list** of :class:`~stem.util.connection.Connection` instances
 
   :raises: **IOError** if it can't be determined
   """
 
+  start_time, conn = time.time(), []
+
+  if pid:
+    parameter = 'connections for pid %s' % pid
+
+    try:
+      pid = int(pid)
+
+      if pid < 0:
+        raise IOError("Process pids can't be negative: %s" % pid)
+    except (ValueError, TypeError):
+      raise IOError('Process pid was non-numeric: %s' % pid)
+  elif user:
+    parameter = 'connections for user %s' % user
+  else:
+    parameter = 'all connections'
+
   try:
-    pid = int(pid)
+    if not IS_PWD_AVAILABLE:
+      raise IOError("This requires python's pwd module, which is unavailable on Windows.")
 
-    if pid < 0:
-      raise IOError("Process pids can't be negative: %s" % pid)
-  except (ValueError, TypeError):
-    raise IOError('Process pid was non-numeric: %s' % pid)
+    inodes = _inodes_for_sockets(pid) if pid else set()
+    process_uid = stem.util.str_tools._to_bytes(str(pwd.getpwnam(user).pw_uid)) if user else None
 
-  if pid == 0:
-    return []
+    for proc_file_path in ('/proc/net/tcp', '/proc/net/tcp6', '/proc/net/udp', '/proc/net/udp6'):
+      if proc_file_path.endswith('6') and not os.path.exists(proc_file_path):
+        continue  # ipv6 proc contents are optional
 
-  # fetches the inode numbers for socket file descriptors
+      protocol = proc_file_path[10:].rstrip('6')  # 'tcp' or 'udp'
+      is_ipv6 = proc_file_path.endswith('6')
 
-  start_time, parameter = time.time(), 'process connections'
-  inodes = []
+      try:
+        with open(proc_file_path, 'rb') as proc_file:
+          proc_file.readline()  # skip the first line
 
-  for fd in os.listdir('/proc/%s/fd' % pid):
+          for line in proc_file:
+            _, l_dst, r_dst, status, _, _, _, uid, _, inode = line.split()[:10]
+
+            if inodes and inode not in inodes:
+              continue
+            elif process_uid and uid != process_uid:
+              continue
+            elif protocol == 'tcp' and status != b'01':
+              continue  # skip tcp connections that aren't yet established
+
+            div = l_dst.find(b':')
+            l_addr = _unpack_addr(l_dst[:div])
+            l_port = int(l_dst[div + 1:], 16)
+
+            div = r_dst.find(b':')
+            r_addr = _unpack_addr(r_dst[:div])
+            r_port = int(r_dst[div + 1:], 16)
+
+            if r_addr == '0.0.0.0' or r_addr == '0000:0000:0000:0000:0000:0000':
+              continue  # no address
+            elif l_port == 0 or r_port == 0:
+              continue  # no port
+
+            conn.append(stem.util.connection.Connection(l_addr, l_port, r_addr, r_port, protocol, is_ipv6))
+      except IOError as exc:
+        raise IOError("unable to read '%s': %s" % (proc_file_path, exc))
+      except Exception as exc:
+        raise IOError("unable to parse '%s': %s" % (proc_file_path, exc))
+
+    _log_runtime(parameter, '/proc/net/[tcp|udp]', start_time)
+    return conn
+  except IOError as exc:
+    _log_failure(parameter, exc)
+    raise
+
+
+def _inodes_for_sockets(pid):
+  """
+  Provides inodes in use by a process for its sockets.
+
+  :param int pid: process id of the process to be queried
+
+  :returns: **set** with inodes for its sockets
+
+  :raises: **IOError** if it can't be determined
+  """
+
+  inodes = set()
+
+  try:
+    fd_contents = os.listdir('/proc/%s/fd' % pid)
+  except OSError as exc:
+    raise IOError('Unable to read our file descriptors: %s' % exc)
+
+  for fd in fd_contents:
     fd_path = '/proc/%s/fd/%s' % (pid, fd)
 
     try:
@@ -364,57 +448,18 @@ def connections(pid):
       fd_name = os.readlink(fd_path)
 
       if fd_name.startswith('socket:['):
-        inodes.append(fd_name[8:-1])
+        inodes.add(stem.util.str_tools._to_bytes(fd_name[8:-1]))
     except OSError as exc:
       if not os.path.exists(fd_path):
         continue  # descriptors may shift while we're in the middle of iterating over them
 
       # most likely couldn't be read due to permissions
-      exc = IOError('unable to determine file descriptor destination (%s): %s' % (exc, fd_path))
-      _log_failure(parameter, exc)
-      raise exc
+      raise IOError('unable to determine file descriptor destination (%s): %s' % (exc, fd_path))
 
-  if not inodes:
-    # unable to fetch any connections for this process
-    return []
-
-  # check for the connection information from the /proc/net contents
-
-  conn = []
-
-  for proc_file_path in ('/proc/net/tcp', '/proc/net/udp'):
-    try:
-      proc_file = open(proc_file_path)
-      proc_file.readline()  # skip the first line
-
-      for line in proc_file:
-        _, l_addr, f_addr, status, _, _, _, _, _, inode = line.split()[:10]
-
-        if inode in inodes:
-          # if a tcp connection, skip if it isn't yet established
-          if proc_file_path.endswith('/tcp') and status != '01':
-            continue
-
-          local_ip, local_port = _decode_proc_address_encoding(l_addr)
-          foreign_ip, foreign_port = _decode_proc_address_encoding(f_addr)
-          protocol = proc_file_path[10:]
-          conn.append((local_ip, local_port, foreign_ip, foreign_port, protocol))
-
-      proc_file.close()
-    except IOError as exc:
-      exc = IOError("unable to read '%s': %s" % (proc_file_path, exc))
-      _log_failure(parameter, exc)
-      raise exc
-    except Exception as exc:
-      exc = IOError("unable to parse '%s': %s" % (proc_file_path, exc))
-      _log_failure(parameter, exc)
-      raise exc
-
-  _log_runtime(parameter, '/proc/net/[tcp|udp]', start_time)
-  return conn
+  return inodes
 
 
-def _decode_proc_address_encoding(addr):
+def _unpack_addr(addr):
   """
   Translates an address entry in the /proc/net/* contents to a human readable
   form (`reference <http://linuxdevcenter.com/pub/a/linux/2000/11/16/LinuxAdmin.html>`_,
@@ -422,35 +467,40 @@ def _decode_proc_address_encoding(addr):
 
   ::
 
-    "0500000A:0016" -> ("10.0.0.5", 22)
+    "0500000A" -> "10.0.0.5"
+    "F804012A4A5190010000000002000000" -> "2a01:4f8:190:514a::2"
 
   :param str addr: proc address entry to be decoded
 
-  :returns: **tuple** of the form **(addr, port)**, with addr as a string and port an int
+  :returns: **str** of the decoded address
   """
 
-  ip, port = addr.split(':')
+  if addr not in ENCODED_ADDR:
+    if len(addr) == 8:
+      # IPv4 address
+      decoded = base64.b16decode(addr)[::-1] if IS_LITTLE_ENDIAN else base64.b16decode(addr)
+      ENCODED_ADDR[addr] = socket.inet_ntop(socket.AF_INET, decoded)
+    else:
+      # IPv6 address
 
-  # the port is represented as a two-byte hexadecimal number
-  port = int(port, 16)
+      if IS_LITTLE_ENDIAN:
+        # Group into eight characters, then invert in pairs...
+        #
+        #   https://trac.torproject.org/projects/tor/ticket/18079#comment:24
 
-  if sys.version_info >= (3,):
-    ip = ip.encode('ascii')
+        inverted = []
 
-  # The IPv4 address portion is a little-endian four-byte hexadecimal number.
-  # That is, the least significant byte is listed first, so we need to reverse
-  # the order of the bytes to convert it to an IP address.
-  #
-  # This needs to account for the endian ordering as per...
-  # http://code.google.com/p/psutil/issues/detail?id=201
-  # https://trac.torproject.org/projects/tor/ticket/4777
+        for i in range(4):
+          grouping = addr[8 * i:8 * (i + 1)]
+          inverted += [grouping[2 * i:2 * (i + 1)] for i in range(4)][::-1]
 
-  if sys.byteorder == 'little':
-    ip = socket.inet_ntop(socket.AF_INET, base64.b16decode(ip)[::-1])
-  else:
-    ip = socket.inet_ntop(socket.AF_INET, base64.b16decode(ip))
+        encoded = b''.join(inverted)
+      else:
+        encoded = addr
 
-  return (ip, port)
+      ENCODED_ADDR[addr] = stem.util.connection.expand_ipv6_address(socket.inet_ntop(socket.AF_INET6, base64.b16decode(encoded)))
+
+  return ENCODED_ADDR[addr]
 
 
 def _is_float(*value):
@@ -508,7 +558,7 @@ def _get_lines(file_path, line_prefixes, parameter):
       return results
   except IOError as exc:
     _log_failure(parameter, exc)
-    raise exc
+    raise
 
 
 def _log_runtime(parameter, proc_location, start_time):
@@ -534,6 +584,7 @@ def _log_failure(parameter, exc):
 
   log.debug('proc call failed (%s): %s' % (parameter, exc))
 
+
 # TODO: drop with stem 2.x
 # We renamed our methods to drop a redundant 'get_*' prefix, so alias the old
 # names for backward compatability.

Shared/lib/python3.4/site-packages/stem/util/str_tools.py

@@ -1,4 +1,4 @@
-# Copyright 2012-2015, Damian Johnson and The Tor Project
+# Copyright 2012-2018, Damian Johnson and The Tor Project
 # See LICENSE for licensing information
 
 """
@@ -27,10 +27,9 @@ import re
 import sys
 
 import stem.prereq
+import stem.util
 import stem.util.enum
 
-from stem import str_type
-
 # label conversion tuples of the form...
 # (bits / bytes / seconds, short label, long label)
 
@@ -75,13 +74,13 @@ if stem.prereq.is_python_3():
       return msg
 else:
   def _to_bytes_impl(msg):
-    if msg is not None and isinstance(msg, str_type):
+    if msg is not None and isinstance(msg, unicode):
       return codecs.latin_1_encode(msg, 'replace')[0]
     else:
       return msg
 
   def _to_unicode_impl(msg):
-    if msg is not None and not isinstance(msg, str_type):
+    if msg is not None and not isinstance(msg, unicode):
       return msg.decode('utf-8', 'replace')
     else:
       return msg
@@ -117,6 +116,22 @@ def _to_unicode(msg):
   return _to_unicode_impl(msg)
 
 
+def _to_int(msg):
+  """
+  Serializes a string to a number.
+
+  :param str msg: string to be serialized
+
+  :returns: **int** representation of the string
+  """
+
+  if stem.prereq.is_python_3() and isinstance(msg, bytes):
+    # iterating over bytes in python3 provides ints rather than characters
+    return sum([pow(256, (len(msg) - i - 1)) * c for (i, c) in enumerate(msg)])
+  else:
+    return sum([pow(256, (len(msg) - i - 1)) * ord(c) for (i, c) in enumerate(msg)])
+
+
 def _to_camel_case(label, divider = '_', joiner = ' '):
   """
   Converts the given string to camel case, ie:
@@ -145,6 +160,24 @@ def _to_camel_case(label, divider = '_', joiner = ' '):
   return joiner.join(words)
 
 
+def _split_by_length(msg, size):
+  """
+  Splits a string into a list of strings up to the given size.
+
+  ::
+
+    >>> _split_by_length('hello', 2)
+    ['he', 'll', 'o']
+
+  :param str msg: string to split
+  :param int size: number of characters to chunk into
+
+  :returns: **list** with chunked string components
+  """
+
+  return [msg[i:i + size] for i in range(0, len(msg), size)]
+
+
 # This needs to be defined after _to_camel_case() to avoid a circular
 # dependency with the enum module.
 
@@ -210,6 +243,9 @@ def crop(msg, size, min_word_length = 4, min_crop = 0, ending = Ending.ELLIPSE,
   # ellipse, and cropping words requires an extra space for hyphens
 
   if ending == Ending.ELLIPSE:
+    if size < 3:
+      return ('', msg) if get_remainder else ''
+
     size -= 3
   elif min_word_length and ending == Ending.HYPHEN:
     min_word_length += 1
@@ -262,7 +298,7 @@ def crop(msg, size, min_word_length = 4, min_crop = 0, ending = Ending.ELLIPSE,
   return (return_msg, remainder) if get_remainder else return_msg
 
 
-def size_label(byte_count, decimal = 0, is_long = False, is_bytes = True):
+def size_label(byte_count, decimal = 0, is_long = False, is_bytes = True, round = False):
   """
   Converts a number of bytes into a human readable label in its most
   significant units. For instance, 7500 bytes would return "7 KB". If the
@@ -281,18 +317,22 @@ def size_label(byte_count, decimal = 0, is_long = False, is_bytes = True):
     >>> size_label(1050, 3, True)
     '1.025 Kilobytes'
 
+  .. versionchanged:: 1.6.0
+     Added round argument.
+
   :param int byte_count: number of bytes to be converted
   :param int decimal: number of decimal digits to be included
   :param bool is_long: expands units label
   :param bool is_bytes: provides units in bytes if **True**, bits otherwise
+  :param bool round: rounds normally if **True**, otherwise rounds down
 
   :returns: **str** with human readable representation of the size
   """
 
   if is_bytes:
-    return _get_label(SIZE_UNITS_BYTES, byte_count, decimal, is_long)
+    return _get_label(SIZE_UNITS_BYTES, byte_count, decimal, is_long, round)
   else:
-    return _get_label(SIZE_UNITS_BITS, byte_count, decimal, is_long)
+    return _get_label(SIZE_UNITS_BITS, byte_count, decimal, is_long, round)
 
 
 def time_label(seconds, decimal = 0, is_long = False):
@@ -456,7 +496,7 @@ def _parse_timestamp(entry):
   :raises: **ValueError** if the timestamp is malformed
   """
 
-  if not isinstance(entry, (str, str_type)):
+  if not stem.util._is_str(entry):
     raise ValueError('parse_timestamp() input must be a str, got a %s' % type(entry))
 
   try:
@@ -482,7 +522,7 @@ def _parse_iso_timestamp(entry):
   :raises: **ValueError** if the timestamp is malformed
   """
 
-  if not isinstance(entry, (str, str_type)):
+  if not stem.util._is_str(entry):
     raise ValueError('parse_iso_timestamp() input must be a str, got a %s' % type(entry))
 
   # based after suggestions from...
@@ -496,7 +536,7 @@ def _parse_iso_timestamp(entry):
   if len(microseconds) != 6 or not microseconds.isdigit():
     raise ValueError("timestamp's microseconds should be six digits")
 
-  if timestamp_str[10] == 'T':
+  if len(timestamp_str) > 10 and timestamp_str[10] == 'T':
     timestamp_str = timestamp_str[:10] + ' ' + timestamp_str[11:]
   else:
     raise ValueError("timestamp didn't contain delimeter 'T' between date and time")
@@ -505,7 +545,7 @@ def _parse_iso_timestamp(entry):
   return timestamp + datetime.timedelta(microseconds = int(microseconds))
 
 
-def _get_label(units, count, decimal, is_long):
+def _get_label(units, count, decimal, is_long, round = False):
   """
   Provides label corresponding to units of the highest significance in the
   provided set. This rounds down (ie, integer truncation after visible units).
@@ -515,6 +555,7 @@ def _get_label(units, count, decimal, is_long):
   :param int count: number of base units being converted
   :param int decimal: decimal precision of label
   :param bool is_long: uses the long label if **True**, short label otherwise
+  :param bool round: rounds normally if **True**, otherwise rounds down
   """
 
   # formatted string for the requested number of digits
@@ -529,10 +570,12 @@ def _get_label(units, count, decimal, is_long):
 
   for count_per_unit, short_label, long_label in units:
     if count >= count_per_unit:
-      # Rounding down with a '%f' is a little clunky. Reducing the count so
-      # it'll divide evenly as the rounded down value.
+      if not round:
+        # Rounding down with a '%f' is a little clunky. Reducing the count so
+        # it'll divide evenly as the rounded down value.
+
+        count -= count % (count_per_unit / (10 ** decimal))
 
-      count -= count % (count_per_unit / (10 ** decimal))
       count_label = label_format % (count / count_per_unit)
 
       if is_long:
@@ -548,6 +591,7 @@ def _get_label(units, count, decimal, is_long):
       else:
         return count_label + short_label
 
+
 # TODO: drop with stem 2.x
 # We renamed our methods to drop a redundant 'get_*' prefix, so alias the old
 # names for backward compatability.

Shared/lib/python3.4/site-packages/stem/util/system.py

@@ -1,4 +1,4 @@
-# Copyright 2011-2015, Damian Johnson and The Tor Project
+# Copyright 2011-2018, Damian Johnson and The Tor Project
 # See LICENSE for licensing information
 
 """
@@ -10,6 +10,10 @@ best-effort, providing **None** if the lookup fails.
    Dropped the get_* prefix from several function names. The old names still
    work, but are deprecated aliases.
 
+.. versionchanged:: 1.5.0
+   Added the **SYSTEM_CALL_TIME** global, which tracks total time spent making
+   system commands.
+
 **Module Overview:**
 
 ::
@@ -17,16 +21,19 @@ best-effort, providing **None** if the lookup fails.
   is_windows - checks if we're running on windows
   is_mac - checks if we're running on a mac
   is_gentoo - checks if we're running on gentoo
+  is_slackware - checks if we're running on slackware
   is_bsd - checks if we're running on the bsd family of operating systems
 
   is_available - determines if a command is available on this system
   is_running - determines if a given process is running
+  size_of - provides the memory usage of an object
   call - runs the given system command and provides back the results
 
   name_by_pid - gets the name for a process by the given pid
   pid_by_name - gets the pid for a process by the given name
   pid_by_port - gets the pid for a process listening to a given port
   pid_by_open_file - gets the pid for the process with an open file
+  pids_by_user - provides processes owned by a user
   cwd - provides the current working directory for a given process
   user - provides the user a process is running under
   start_time - provides the unix timestamp when the process started
@@ -40,25 +47,63 @@ best-effort, providing **None** if the lookup fails.
 
   get_process_name - provides our process' name
   set_process_name - changes our process' name
+
+.. data:: Status (enum)
+
+  State of a subprocess.
+
+  .. versionadded:: 1.6.0
+
+  ====================  ===========
+  Status                Description
+  ====================  ===========
+  PENDING               not yet started
+  RUNNING               currently being performed
+  DONE                  completed successfully
+  FAILED                failed with an exception
+  ====================  ===========
 """
 
+import collections
 import ctypes
 import ctypes.util
-import distutils.spawn
+import itertools
 import mimetypes
+import multiprocessing
 import os
 import platform
 import re
 import subprocess
+import sys
 import tarfile
+import threading
 import time
 
+import stem.prereq
+import stem.util
+import stem.util.enum
 import stem.util.proc
 import stem.util.str_tools
 
-from stem import UNDEFINED, str_type
+from stem import UNDEFINED
 from stem.util import log
 
+State = stem.util.enum.UppercaseEnum(
+  'PENDING',
+  'RUNNING',
+  'DONE',
+  'FAILED',
+)
+
+SIZE_RECURSES = {
+  tuple: iter,
+  list: iter,
+  collections.deque: iter,
+  dict: lambda d: itertools.chain.from_iterable(d.items()),
+  set: iter,
+  frozenset: iter,
+}
+
 # Mapping of commands to if they're available or not.
 
 CMD_AVAILABLE_CACHE = {}
@@ -84,6 +129,8 @@ GET_PID_BY_PORT_NETSTAT = 'netstat -npltu'
 GET_PID_BY_PORT_SOCKSTAT = 'sockstat -4l -P tcp -p %s'
 GET_PID_BY_PORT_LSOF = 'lsof -wnP -iTCP -sTCP:LISTEN'
 GET_PID_BY_FILE_LSOF = 'lsof -tw %s'
+GET_PIDS_BY_USER_LINUX = 'ps -o pid -u %s'
+GET_PIDS_BY_USER_BSD = 'ps -o pid -U %s'
 GET_CWD_PWDX = 'pwdx %s'
 GET_CWD_LSOF = 'lsof -a -p %s -d cwd -Fn'
 GET_BSD_JAIL_ID_PS = 'ps -p %s -o jid'
@@ -125,6 +172,143 @@ _PROCESS_NAME = None
 
 _MAX_NAME_LENGTH = -1
 
+# Tracks total time spent shelling out to other commands like 'ps' and
+# 'netstat', so we can account for it as part of our cpu time along with
+# os.times().
+
+SYSTEM_CALL_TIME = 0.0
+SYSTEM_CALL_TIME_LOCK = threading.RLock()
+
+
+class CallError(OSError):
+  """
+  Error response when making a system call. This is an **OSError** subclass
+  with additional information about the process. Depending on the nature of the
+  error not all of these attributes will be available.
+
+  :var str msg: exception string
+  :var str command: command that was ran
+  :var int exit_status: exit code of the process
+  :var float runtime: time the command took to run
+  :var str stdout: stdout of the process
+  :var str stderr: stderr of the process
+  """
+
+  def __init__(self, msg, command, exit_status, runtime, stdout, stderr):
+    self.msg = msg
+    self.command = command
+    self.exit_status = exit_status
+    self.runtime = runtime
+    self.stdout = stdout
+    self.stderr = stderr
+
+  def __str__(self):
+    return self.msg
+
+
+class CallTimeoutError(CallError):
+  """
+  Error response when making a system call that has timed out.
+
+  .. versionadded:: 1.6.0
+
+  :var float timeout: time we waited
+  """
+
+  def __init__(self, msg, command, exit_status, runtime, stdout, stderr, timeout):
+    super(CallTimeoutError, self).__init__(msg, command, exit_status, runtime, stdout, stderr)
+    self.timeout = timeout
+
+
+class DaemonTask(object):
+  """
+  Invokes the given function in a subprocess, returning the value.
+
+  .. versionadded:: 1.6.0
+
+  :var function runner: function to be invoked by the subprocess
+  :var tuple args: arguments to provide to the subprocess
+  :var int priority: subprocess nice priority
+
+  :var stem.util.system.State status: state of the subprocess
+  :var float runtime: seconds subprocess took to complete
+  :var object result: return value of subprocess if successful
+  :var exception error: exception raised by subprocess if it failed
+  """
+
+  def __init__(self, runner, args = None, priority = 15, start = False):
+    self.runner = runner
+    self.args = args
+    self.priority = priority
+
+    self.status = State.PENDING
+    self.runtime = None
+    self.result = None
+    self.error = None
+
+    self._process = None
+    self._pipe = None
+
+    if start:
+      self.run()
+
+  def run(self):
+    """
+    Invokes the task if it hasn't already been started. If it has this is a
+    no-op.
+    """
+
+    if self.status == State.PENDING:
+      self._pipe, child_pipe = multiprocessing.Pipe()
+      self._process = multiprocessing.Process(target = DaemonTask._run_wrapper, args = (child_pipe, self.priority, self.runner, self.args))
+      self._process.start()
+      self.status = State.RUNNING
+
+  def join(self):
+    """
+    Provides the result of the daemon task. If still running this blocks until
+    the task is completed.
+
+    :returns: response of the function we ran
+
+    :raises: exception raised by the function if it failed with one
+    """
+
+    if self.status == State.PENDING:
+      self.run()
+
+    if self.status == State.RUNNING:
+      self._process.join()
+      response = self._pipe.recv()
+
+      self.status = response[0]
+      self.runtime = response[1]
+
+      if self.status == State.DONE:
+        self.result = response[2]
+      elif self.status == State.FAILED:
+        self.error = response[2]
+
+    if self.status == State.DONE:
+      return self.result
+    elif self.status == State.FAILED:
+      raise self.error
+    else:
+      raise RuntimeError('BUG: unexpected status from daemon task, %s' % self.status)
+
+  @staticmethod
+  def _run_wrapper(conn, priority, runner, args):
+    start_time = time.time()
+    os.nice(priority)
+
+    try:
+      result = runner(*args) if args else runner()
+      conn.send((State.DONE, time.time() - start_time, result))
+    except Exception as exc:
+      conn.send((State.FAILED, time.time() - start_time, exc))
+    finally:
+      conn.close()
+
 
 def is_windows():
   """
@@ -156,6 +340,16 @@ def is_gentoo():
   return os.path.exists('/etc/gentoo-release')
 
 
+def is_slackware():
+  """
+  Checks if we are running on a Slackware system.
+
+  :returns: **bool** to indicate if we're on a Slackware system
+  """
+
+  return os.path.exists('/etc/slackware-version')
+
+
 def is_bsd():
   """
   Checks if we are within the BSD family of operating systems. This currently
@@ -164,7 +358,7 @@ def is_bsd():
   :returns: **bool** to indicate if we're on a BSD OS
   """
 
-  return platform.system() in ('Darwin', 'FreeBSD', 'OpenBSD')
+  return platform.system() in ('Darwin', 'FreeBSD', 'OpenBSD', 'NetBSD')
 
 
 def is_available(command, cached=True):
@@ -188,27 +382,49 @@ def is_available(command, cached=True):
     command = command.split(' ')[0]
 
   if command in SHELL_COMMANDS:
-    # we can't actually look it up, so hope the shell really provides it...
-
-    return True
+    return True  # we can't actually look it up, so hope the shell really provides it...
   elif cached and command in CMD_AVAILABLE_CACHE:
     return CMD_AVAILABLE_CACHE[command]
-  else:
-    cmd_exists = distutils.spawn.find_executable(command) is not None
-    CMD_AVAILABLE_CACHE[command] = cmd_exists
-    return cmd_exists
+  elif 'PATH' not in os.environ:
+    return False  # lacking a path will cause find_executable() to internally fail
+
+  cmd_exists = False
+
+  for path in os.environ['PATH'].split(os.pathsep):
+    cmd_path = os.path.join(path, command)
+
+    if is_windows():
+      cmd_path += '.exe'
+
+    if os.path.exists(cmd_path) and os.access(cmd_path, os.X_OK):
+      cmd_exists = True
+      break
+
+  CMD_AVAILABLE_CACHE[command] = cmd_exists
+  return cmd_exists
 
 
 def is_running(command):
   """
-  Checks for if a process with a given name is running or not.
+  Checks for if a process with a given name or pid is running.
 
-  :param str command: process name to be checked
+  .. versionchanged:: 1.6.0
+     Added support for list and pid arguments.
+
+  :param str,list,int command: process name if a str, multiple process names if
+    a list, or pid if an int to be checked
 
   :returns: **True** if the process is running, **False** if it's not among ps
     results, and **None** if ps can't be queried
   """
 
+  if isinstance(command, int):
+    try:
+      os.kill(command, 0)
+      return True
+    except OSError:
+      return False
+
   # Linux and the BSD families have different variants of ps. Guess based on
   # the is_bsd() check which to try first, then fall back to the other.
   #
@@ -236,12 +452,63 @@ def is_running(command):
       command_listing = call(secondary_resolver, None)
 
     if command_listing:
-      command_listing = map(str_type.strip, command_listing)
-      return command in command_listing
+      command_listing = [c.strip() for c in command_listing]
+
+      if stem.util._is_str(command):
+        command = [command]
+
+      for cmd in command:
+        if cmd in command_listing:
+          return True
+
+      return False
 
   return None
 
 
+def size_of(obj, exclude = None):
+  """
+  Provides the `approximate memory usage of an object
+  <https://code.activestate.com/recipes/577504/>`_. This can recurse tuples,
+  lists, deques, dicts, and sets. To teach this function to inspect additional
+  object types expand SIZE_RECURSES...
+
+  ::
+
+    stem.util.system.SIZE_RECURSES[SomeClass] = SomeClass.get_elements
+
+  .. versionadded:: 1.6.0
+
+  :param object obj: object to provide the size of
+  :param set exclude: object ids to exclude from size estimation
+
+  :returns: **int** with the size of the object in bytes
+
+  :raises: **NotImplementedError** if using PyPy
+  """
+
+  if stem.prereq.is_pypy():
+    raise NotImplementedError('PyPy does not implement sys.getsizeof()')
+
+  if exclude is None:
+    exclude = set()
+  elif id(obj) in exclude:
+    return 0
+
+  try:
+    size = sys.getsizeof(obj)
+  except TypeError:
+    size = sys.getsizeof(0)  # estimate if object lacks a __sizeof__
+
+  exclude.add(id(obj))
+
+  if type(obj) in SIZE_RECURSES:
+    for entry in SIZE_RECURSES[type(obj)](obj):
+      size += size_of(entry, exclude)
+
+  return size
+
+
 def name_by_pid(pid):
   """
   Attempts to determine the name a given process is running under (not
@@ -614,6 +881,38 @@ def pid_by_open_file(path):
   return None  # all queries failed
 
 
+def pids_by_user(user):
+  """
+  Provides processes owned by a given user.
+
+  .. versionadded:: 1.5.0
+
+  :param str user: user to look up processes for
+
+  :returns: **list** with the process ids, **None** if it can't be determined
+  """
+
+  # example output:
+  #   atagar@odin:~$ ps -o pid -u avahi
+  #     PID
+  #     914
+  #     915
+
+  if is_available('ps'):
+    if is_bsd():
+      results = call(GET_PIDS_BY_USER_BSD % user, None)
+    else:
+      results = call(GET_PIDS_BY_USER_LINUX % user, None)
+
+    if results:
+      try:
+        return list(map(int, results[1:]))
+      except ValueError:
+        pass
+
+  return None
+
+
 def cwd(pid):
   """
   Provides the working directory of the given process.
@@ -668,8 +967,8 @@ def cwd(pid):
   if is_available('lsof'):
     results = call(GET_CWD_LSOF % pid, [])
 
-    if len(results) == 2 and results[1].startswith('n/'):
-      lsof_result = results[1][1:].strip()
+    if len(results) >= 2 and results[-1].startswith('n/'):
+      lsof_result = results[-1][1:].strip()
 
       # If we lack read permissions for the cwd then it returns...
       # p2683
@@ -765,7 +1064,7 @@ def tail(target, lines = None):
   """
 
   if isinstance(target, str):
-    with open(target) as target_file:
+    with open(target, 'rb') as target_file:
       for line in tail(target_file, lines):
         yield line
 
@@ -777,13 +1076,13 @@ def tail(target, lines = None):
   target.seek(0, 2)  # go to the end of the file
   block_end_byte = target.tell()
   block_number = -1
-  content = ''
+  content = b''
 
   while (lines is None or lines > 0) and block_end_byte > 0:
     if (block_end_byte - BLOCK_SIZE > 0):
       # read the last block we haven't yet read
       target.seek(block_number * BLOCK_SIZE, 2)
-      content, completed_lines = (target.read(BLOCK_SIZE) + content).split('\n', 1)
+      content, completed_lines = (target.read(BLOCK_SIZE) + content).split(b'\n', 1)
     else:
       # reached the start of the file, just read what's left
       target.seek(0, 0)
@@ -794,7 +1093,7 @@ def tail(target, lines = None):
         if lines is not None:
           lines -= 1
 
-        yield line
+        yield stem.util.str_tools._to_unicode(line)
 
     block_end_byte -= BLOCK_SIZE
     block_number -= 1
@@ -951,63 +1250,105 @@ def files_with_suffix(base_path, suffix):
           yield os.path.join(root, filename)
 
 
-def call(command, default = UNDEFINED, ignore_exit_status = False):
+def call(command, default = UNDEFINED, ignore_exit_status = False, timeout = None, cwd = None, env = None):
   """
+  call(command, default = UNDEFINED, ignore_exit_status = False)
+
   Issues a command in a subprocess, blocking until completion and returning the
   results. This is not actually ran in a shell so pipes and other shell syntax
   are not permitted.
 
+  .. versionchanged:: 1.5.0
+     Providing additional information upon failure by raising a CallError. This
+     is a subclass of OSError, providing backward compatibility.
+
+  .. versionchanged:: 1.5.0
+     Added env argument.
+
+  .. versionchanged:: 1.6.0
+     Added timeout and cwd arguments.
+
   :param str,list command: command to be issued
   :param object default: response if the query fails
   :param bool ignore_exit_status: reports failure if our command's exit status
     was non-zero
+  :param float timeout: maximum seconds to wait, blocks indefinitely if
+    **None**
+  :param dict env: environment variables
 
   :returns: **list** with the lines of output from the command
 
-  :raises: **OSError** if this fails and no default was provided
+  :raises:
+    * **CallError** if this fails and no default was provided
+    * **CallTimeoutError** if the timeout is reached without a default
   """
 
+  # TODO: in stem 2.x return a struct with stdout, stderr, and runtime instead
+
+  global SYSTEM_CALL_TIME
+
   if isinstance(command, str):
     command_list = command.split(' ')
   else:
-    command_list = command
+    command_list = list(map(str, command))
+
+  exit_status, runtime, stdout, stderr = None, None, None, None
+  start_time = time.time()
 
   try:
     is_shell_command = command_list[0] in SHELL_COMMANDS
 
-    start_time = time.time()
-    process = subprocess.Popen(command_list, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = is_shell_command)
+    process = subprocess.Popen(command_list, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = is_shell_command, cwd = cwd, env = env)
+
+    if timeout:
+      while process.poll() is None:
+        if time.time() - start_time > timeout:
+          raise CallTimeoutError("Process didn't finish after %0.1f seconds" % timeout, ' '.join(command_list), None, timeout, '', '', timeout)
+
+        time.sleep(0.001)
 
     stdout, stderr = process.communicate()
     stdout, stderr = stdout.strip(), stderr.strip()
     runtime = time.time() - start_time
 
     log.debug('System call: %s (runtime: %0.2f)' % (command, runtime))
-    trace_prefix = 'Received from system (%s)' % command
 
-    if stdout and stderr:
-      log.trace(trace_prefix + ', stdout:\n%s\nstderr:\n%s' % (stdout, stderr))
-    elif stdout:
-      log.trace(trace_prefix + ', stdout:\n%s' % stdout)
-    elif stderr:
-      log.trace(trace_prefix + ', stderr:\n%s' % stderr)
+    if log.is_tracing():
+      trace_prefix = 'Received from system (%s)' % command
 
-    exit_code = process.poll()
+      if stdout and stderr:
+        log.trace(trace_prefix + ', stdout:\n%s\nstderr:\n%s' % (stdout, stderr))
+      elif stdout:
+        log.trace(trace_prefix + ', stdout:\n%s' % stdout)
+      elif stderr:
+        log.trace(trace_prefix + ', stderr:\n%s' % stderr)
 
-    if not ignore_exit_status and exit_code != 0:
-      raise OSError('%s returned exit status %i' % (command, exit_code))
+    exit_status = process.poll()
+
+    if not ignore_exit_status and exit_status != 0:
+      raise OSError('%s returned exit status %i' % (command, exit_status))
 
     if stdout:
       return stdout.decode('utf-8', 'replace').splitlines()
     else:
       return []
+  except CallTimeoutError:
+    log.debug('System call (timeout): %s (after %0.4fs)' % (command, timeout))
+
+    if default != UNDEFINED:
+      return default
+    else:
+      raise
   except OSError as exc:
     log.debug('System call (failed): %s (error: %s)' % (command, exc))
 
     if default != UNDEFINED:
       return default
     else:
-      raise exc
+      raise CallError(str(exc), ' '.join(command_list), exit_status, runtime, stdout, stderr)
+  finally:
+    with SYSTEM_CALL_TIME_LOCK:
+      SYSTEM_CALL_TIME += time.time() - start_time
 
 
 def get_process_name():
@@ -1150,7 +1491,7 @@ def _set_proc_title(process_name):
 
   libc = ctypes.CDLL(ctypes.util.find_library('c'))
   name_buffer = ctypes.create_string_buffer(len(process_name) + 1)
-  name_buffer.value = process_name
+  name_buffer.value = process_name.encode()
 
   try:
     libc.setproctitle(ctypes.byref(name_buffer))

Shared/lib/python3.4/site-packages/stem/util/term.py

@@ -1,4 +1,4 @@
-# Copyright 2011-2015, Damian Johnson and The Tor Project
+# Copyright 2011-2018, Damian Johnson and The Tor Project
 # See LICENSE for licensing information
 
 """
@@ -8,12 +8,13 @@ Utilities for working with the terminal.
 
 ::
 
+  encoding - provides the ANSI escape sequence for a terminal attribute
   format - wrap text with ANSI for the given colors or attributes
 
 .. data:: Color (enum)
 .. data:: BgColor (enum)
 
-  Enumerations for foreground or background terminal color.
+  Foreground or background terminal colors.
 
   =========== ===========
   Color       Description
@@ -30,15 +31,19 @@ Utilities for working with the terminal.
 
 .. data:: Attr (enum)
 
-  Enumerations of terminal text attributes.
+  Terminal text attributes.
+
+  .. versionchanged:: 1.5.0
+     Added the LINES attribute.
 
   =================== ===========
   Attr                Description
   =================== ===========
   **BOLD**            heavy typeface
-  **HILIGHT**         inverted foreground and background
+  **HIGHLIGHT**       inverted foreground and background
   **UNDERLINE**       underlined text
   **READLINE_ESCAPE** wrap encodings in `RL_PROMPT_START_IGNORE and RL_PROMPT_END_IGNORE sequences <https://stackoverflow.com/questions/9468435/look-how-to-fix-column-calculation-in-python-readline-if-use-color-prompt>`_
+  **LINES**           formats lines individually
   =================== ===========
 """
 
@@ -54,17 +59,52 @@ DISABLE_COLOR_SUPPORT = False
 
 Color = stem.util.enum.Enum(*TERM_COLORS)
 BgColor = stem.util.enum.Enum(*['BG_' + color for color in TERM_COLORS])
-Attr = stem.util.enum.Enum('BOLD', 'UNDERLINE', 'HILIGHT', 'READLINE_ESCAPE')
+Attr = stem.util.enum.Enum('BOLD', 'UNDERLINE', 'HIGHLIGHT', 'READLINE_ESCAPE', 'LINES')
 
 # mappings of terminal attribute enums to their ANSI escape encoding
 FG_ENCODING = dict([(list(Color)[i], str(30 + i)) for i in range(8)])
 BG_ENCODING = dict([(list(BgColor)[i], str(40 + i)) for i in range(8)])
-ATTR_ENCODING = {Attr.BOLD: '1', Attr.UNDERLINE: '4', Attr.HILIGHT: '7'}
+ATTR_ENCODING = {Attr.BOLD: '1', Attr.UNDERLINE: '4', Attr.HIGHLIGHT: '7'}
 
 CSI = '\x1B[%sm'
 RESET = CSI % '0'
 
 
+def encoding(*attrs):
+  """
+  Provides the ANSI escape sequence for these terminal color or attributes.
+
+  .. versionadded:: 1.5.0
+
+  :param list attr: :data:`~stem.util.terminal.Color`,
+    :data:`~stem.util.terminal.BgColor`, or :data:`~stem.util.terminal.Attr` to
+    provide an ecoding for
+
+  :returns: **str** of the ANSI escape sequence, **None** no attributes are
+    recognized
+  """
+
+  term_encodings = []
+
+  for attr in attrs:
+    # TODO: Account for an earlier misspelled attribute. This should be dropped
+    # in Stem. 2.0.x.
+
+    if attr == 'HILIGHT':
+      attr = 'HIGHLIGHT'
+
+    attr = stem.util.str_tools._to_camel_case(attr)
+    term_encoding = FG_ENCODING.get(attr, None)
+    term_encoding = BG_ENCODING.get(attr, term_encoding)
+    term_encoding = ATTR_ENCODING.get(attr, term_encoding)
+
+    if term_encoding:
+      term_encodings.append(term_encoding)
+
+  if term_encodings:
+    return CSI % ';'.join(term_encodings)
+
+
 def format(msg, *attr):
   """
   Simple terminal text formatting using `ANSI escape sequences
@@ -75,38 +115,39 @@ def format(msg, *attr):
   * `termcolor <https://pypi.python.org/pypi/termcolor>`_
   * `colorama <https://pypi.python.org/pypi/colorama>`_
 
+  .. versionchanged:: 1.6.0
+     Normalized return value to be unicode to better support python 2/3
+     compatibility.
+
   :param str msg: string to be formatted
   :param str attr: text attributes, this can be :data:`~stem.util.term.Color`,
     :data:`~stem.util.term.BgColor`, or :data:`~stem.util.term.Attr` enums
     and are case insensitive (so strings like 'red' are fine)
 
-  :returns: **str** wrapped with ANSI escape encodings, starting with the given
+  :returns: **unicode** wrapped with ANSI escape encodings, starting with the given
     attributes and ending with a reset
   """
 
+  msg = stem.util.str_tools._to_unicode(msg)
+
   if DISABLE_COLOR_SUPPORT:
     return msg
 
+  if Attr.LINES in attr:
+    attr = list(attr)
+    attr.remove(Attr.LINES)
+    lines = [format(line, *attr) for line in msg.split('\n')]
+    return '\n'.join(lines)
+
   # if we have reset sequences in the message then apply our attributes
   # after each of them
 
   if RESET in msg:
     return ''.join([format(comp, *attr) for comp in msg.split(RESET)])
 
-  encodings = []
-
-  for text_attr in attr:
-    text_attr, encoding = stem.util.str_tools._to_camel_case(text_attr), None
-    encoding = FG_ENCODING.get(text_attr, encoding)
-    encoding = BG_ENCODING.get(text_attr, encoding)
-    encoding = ATTR_ENCODING.get(text_attr, encoding)
-
-    if encoding:
-      encodings.append(encoding)
-
-  if encodings:
-    prefix, suffix = CSI % ';'.join(encodings), RESET
+  prefix, suffix = encoding(*attr), RESET
 
+  if prefix:
     if Attr.READLINE_ESCAPE in attr:
       prefix = '\001%s\002' % prefix
       suffix = '\001%s\002' % suffix

Shared/lib/python3.4/site-packages/stem/util/test_tools.py

@@ -1,46 +1,329 @@
-# Copyright 2015, Damian Johnson and The Tor Project
+# Copyright 2015-2018, Damian Johnson and The Tor Project
 # See LICENSE for licensing information
 
 """
 Helper functions for testing.
 
+Our **stylistic_issues**, **pyflakes_issues**, and **type_check_issues**
+respect a 'exclude_paths' in our test config, excluding any absolute paths
+matching those regexes. Issue strings can start or end with an asterisk
+to match just against the prefix or suffix. For instance...
+
+::
+
+  exclude_paths .*/stem/test/data/.*
+
 .. versionadded:: 1.2.0
 
 ::
 
+  TimedTestRunner - test runner that tracks test runtimes
+  test_runtimes - provides runtime of tests excuted through TimedTestRunners
   clean_orphaned_pyc - delete *.pyc files without corresponding *.py
 
   is_pyflakes_available - checks if pyflakes is available
-  is_pep8_available - checks if pep8 is available
+  is_pycodestyle_available - checks if pycodestyle is available
 
-  stylistic_issues - checks for PEP8 and other stylistic issues
   pyflakes_issues - static checks for problems via pyflakes
+  stylistic_issues - checks for PEP8 and other stylistic issues
 """
 
 import collections
 import linecache
+import multiprocessing
 import os
 import re
+import threading
+import time
+import traceback
+import unittest
 
+import stem.prereq
 import stem.util.conf
+import stem.util.enum
 import stem.util.system
 
 CONFIG = stem.util.conf.config_dict('test', {
-  'pep8.ignore': [],
+  'pep8.ignore': [],  # TODO: drop with stem 2.x, legacy alias for pycodestyle.ignore
+  'pycodestyle.ignore': [],
   'pyflakes.ignore': [],
   'exclude_paths': [],
 })
 
-Issue = collections.namedtuple('Issue', [
-  'line_number',
-  'message',
-  'line',
-])
+TEST_RUNTIMES = {}
+ASYNC_TESTS = {}
+
+AsyncStatus = stem.util.enum.UppercaseEnum('PENDING', 'RUNNING', 'FINISHED')
+AsyncResult = collections.namedtuple('AsyncResult', 'type msg')
+
+# TODO: Providing a copy of SkipTest that works with python 2.6. This will be
+# dropped when we remove python 2.6 support.
+
+if stem.prereq._is_python_26():
+  class SkipTest(Exception):
+    'Notes that the test was skipped.'
+else:
+  SkipTest = unittest.case.SkipTest
+
+
+def assert_equal(expected, actual, msg = None):
+  """
+  Function form of a TestCase's assertEqual.
+
+  .. versionadded:: 1.6.0
+
+  :param object expected: expected value
+  :param object actual: actual value
+  :param str msg: message if assertion fails
+
+  :raises: **AssertionError** if values aren't equal
+  """
+
+  if expected != actual:
+    raise AssertionError("Expected '%s' but was '%s'" % (expected, actual) if msg is None else msg)
+
+
+def assert_in(expected, actual, msg = None):
+  """
+  Asserts that a given value is within this content.
+
+  .. versionadded:: 1.6.0
+
+  :param object expected: expected value
+  :param object actual: actual value
+  :param str msg: message if assertion fails
+
+  :raises: **AssertionError** if the expected value isn't in the actual
+  """
+
+  if expected not in actual:
+    raise AssertionError("Expected '%s' to be within '%s'" % (expected, actual) if msg is None else msg)
+
+
+def skip(msg):
+  """
+  Function form of a TestCase's skipTest.
+
+  .. versionadded:: 1.6.0
+
+  :param str msg: reason test is being skipped
+
+  :raises: **unittest.case.SkipTest** for this reason
+  """
+
+  raise SkipTest(msg)
+
+
+def asynchronous(func):
+  test = stem.util.test_tools.AsyncTest(func)
+  ASYNC_TESTS[test.name] = test
+  return test.method
+
+
+class AsyncTest(object):
+  """
+  Test that's run asychronously. These are functions (no self reference)
+  performed like the following...
+
+  ::
+
+    class MyTest(unittest.TestCase):
+      @staticmethod
+      def run_tests():
+        MyTest.test_addition = stem.util.test_tools.AsyncTest(MyTest.test_addition).method
+
+      @staticmethod
+      def test_addition():
+        if 1 + 1 != 2:
+          raise AssertionError('tisk, tisk')
+
+    MyTest.run()
+
+  .. versionadded:: 1.6.0
+  """
+
+  def __init__(self, runner, args = None, threaded = False):
+    self.name = '%s.%s' % (runner.__module__, runner.__name__)
+
+    self._runner = runner
+    self._runner_args = args
+    self._threaded = threaded
+
+    self.method = lambda test: self.result(test)  # method that can be mixed into TestCases
+
+    self._process = None
+    self._process_pipe = None
+    self._process_lock = threading.RLock()
+
+    self._result = None
+    self._status = AsyncStatus.PENDING
+
+  def run(self, *runner_args, **kwargs):
+    if stem.prereq._is_python_26():
+      return  # not supported under python 2.6
+
+    def _wrapper(conn, runner, args):
+      os.nice(12)
+
+      try:
+        runner(*args) if args else runner()
+        conn.send(AsyncResult('success', None))
+      except AssertionError as exc:
+        conn.send(AsyncResult('failure', str(exc)))
+      except SkipTest as exc:
+        conn.send(AsyncResult('skipped', str(exc)))
+      except:
+        conn.send(AsyncResult('error', traceback.format_exc()))
+      finally:
+        conn.close()
+
+    with self._process_lock:
+      if self._status == AsyncStatus.PENDING:
+        if runner_args:
+          self._runner_args = runner_args
+
+        if 'threaded' in kwargs:
+          self._threaded = kwargs['threaded']
+
+        self._process_pipe, child_pipe = multiprocessing.Pipe()
+
+        if self._threaded:
+          self._process = threading.Thread(
+            target = _wrapper,
+            args = (child_pipe, self._runner, self._runner_args),
+            name = 'Background test of %s' % self.name,
+          )
+
+          self._process.setDaemon(True)
+        else:
+          self._process = multiprocessing.Process(target = _wrapper, args = (child_pipe, self._runner, self._runner_args))
+
+        self._process.start()
+        self._status = AsyncStatus.RUNNING
+
+  def pid(self):
+    with self._process_lock:
+      return self._process.pid if (self._process and not self._threaded) else None
+
+  def join(self):
+    self.result(None)
+
+  def result(self, test):
+    if stem.prereq._is_python_26():
+      return  # not supported under python 2.6
+
+    with self._process_lock:
+      if self._status == AsyncStatus.PENDING:
+        self.run()
+
+      if self._status == AsyncStatus.RUNNING:
+        self._result = self._process_pipe.recv()
+        self._process.join()
+        self._status = AsyncStatus.FINISHED
+
+      if test and self._result.type == 'failure':
+        test.fail(self._result.msg)
+      elif test and self._result.type == 'error':
+        test.fail(self._result.msg)
+      elif test and self._result.type == 'skipped':
+        test.skipTest(self._result.msg)
+
+
+class Issue(collections.namedtuple('Issue', ['line_number', 'message', 'line'])):
+  """
+  Issue encountered by pyflakes or pycodestyle.
+
+  :var int line_number: line number the issue occured on
+  :var str message: description of the issue
+  :var str line: content of the line the issue is about
+  """
+
+
+class TimedTestRunner(unittest.TextTestRunner):
+  """
+  Test runner that tracks the runtime of individual tests. When tests are run
+  with this their runtimes are made available through
+  :func:`stem.util.test_tools.test_runtimes`.
+
+  .. versionadded:: 1.6.0
+  """
+
+  def run(self, test):
+    for t in test._tests:
+      original_type = type(t)
+
+      class _TestWrapper(original_type):
+        def run(self, result = None):
+          start_time = time.time()
+          result = super(type(self), self).run(result)
+          TEST_RUNTIMES[self.id()] = time.time() - start_time
+          return result
+
+        # TODO: remove and drop unnecessary 'returns' when dropping python 2.6
+        # support
+
+        def skipTest(self, message):
+          if not stem.prereq._is_python_26():
+            return super(original_type, self).skipTest(message)
+
+        # TODO: remove when dropping python 2.6 support
+
+        def assertItemsEqual(self, expected, actual):
+          if stem.prereq._is_python_26():
+            self.assertEqual(set(expected), set(actual))
+          else:
+            return super(original_type, self).assertItemsEqual(expected, actual)
+
+        def assertRaisesWith(self, exc_type, exc_msg, func, *args, **kwargs):
+          """
+          Asserts the given invokation raises the expected excepiton. This is
+          similar to unittest's assertRaises and assertRaisesRegexp, but checks
+          for an exact match.
+
+          This method is **not** being vended to external users and may be
+          changed without notice. If you want this method to be part of our
+          vended API then please let us know.
+          """
+
+          return self.assertRaisesRegexp(exc_type, '^%s$' % re.escape(exc_msg), func, *args, **kwargs)
+
+        def assertRaisesRegexp(self, exc_type, exc_msg, func, *args, **kwargs):
+          if stem.prereq._is_python_26():
+            try:
+              func(*args, **kwargs)
+              self.fail('Expected a %s to be raised but nothing was' % exc_type)
+            except exc_type as exc:
+              self.assertTrue(re.search(exc_msg, str(exc), re.MULTILINE))
+          else:
+            return super(original_type, self).assertRaisesRegexp(exc_type, exc_msg, func, *args, **kwargs)
+
+        def id(self):
+          return '%s.%s.%s' % (original_type.__module__, original_type.__name__, self._testMethodName)
+
+        def __str__(self):
+          return '%s (%s.%s)' % (self._testMethodName, original_type.__module__, original_type.__name__)
+
+      t.__class__ = _TestWrapper
+
+    return super(TimedTestRunner, self).run(test)
+
+
+def test_runtimes():
+  """
+  Provides the runtimes of tests executed through TimedTestRunners.
+
+  :returns: **dict** of fully qualified test names to floats for the runtime in
+    seconds
+
+  .. versionadded:: 1.6.0
+  """
+
+  return dict(TEST_RUNTIMES)
 
 
 def clean_orphaned_pyc(paths):
   """
-  Deletes any file with a *.pyc extention without a corresponding *.py. This
+  Deletes any file with a \*.pyc extention without a corresponding \*.py. This
   helps to address a common gotcha when deleting python files...
 
   * You delete module 'foo.py' and run the tests to ensure that you haven't
@@ -90,50 +373,46 @@ def is_pyflakes_available():
   :returns: **True** if we can use pyflakes and **False** otherwise
   """
 
-  try:
-    import pyflakes.api
-    import pyflakes.reporter
-    return True
-  except ImportError:
-    return False
+  return _module_exists('pyflakes.api') and _module_exists('pyflakes.reporter')
 
 
-def is_pep8_available():
+def is_pycodestyle_available():
   """
-  Checks if pep8 is availalbe.
+  Checks if pycodestyle is availalbe.
 
-  :returns: **True** if we can use pep8 and **False** otherwise
+  :returns: **True** if we can use pycodestyle and **False** otherwise
   """
 
-  try:
-    import pep8
-
-    if not hasattr(pep8, 'BaseReport'):
-      raise ImportError()
-
-    return True
-  except ImportError:
+  if _module_exists('pycodestyle'):
+    import pycodestyle
+  elif _module_exists('pep8'):
+    import pep8 as pycodestyle
+  else:
     return False
 
+  return hasattr(pycodestyle, 'BaseReport')
 
-def stylistic_issues(paths, check_two_space_indents = False, check_newlines = False, check_trailing_whitespace = False, check_exception_keyword = False, prefer_single_quotes = False):
+
+def stylistic_issues(paths, check_newlines = False, check_exception_keyword = False, prefer_single_quotes = False):
   """
   Checks for stylistic issues that are an issue according to the parts of PEP8
-  we conform to. You can suppress PEP8 issues by making a 'test' configuration
-  that sets 'pep8.ignore'.
+  we conform to. You can suppress pycodestyle issues by making a 'test'
+  configuration that sets 'pycodestyle.ignore'.
 
   For example, with a 'test/settings.cfg' of...
 
   ::
 
-    # PEP8 compliance issues that we're ignoreing...
+    # pycodestyle compliance issues that we're ignoreing...
     #
     # * E111 and E121 four space indentations
     # * E501 line is over 79 characters
 
-    pep8.ignore E111
-    pep8.ignore E121
-    pep8.ignore E501
+    pycodestyle.ignore E111
+    pycodestyle.ignore E121
+    pycodestyle.ignore E501
+
+    pycodestyle.ignore run_tests.py => E402: import stem.util.enum
 
   ... you can then run tests with...
 
@@ -146,9 +425,6 @@ def stylistic_issues(paths, check_two_space_indents = False, check_newlines = Fa
 
     issues = stylistic_issues('my_project')
 
-  If a 'exclude_paths' was set in our test config then we exclude any absolute
-  paths matching those regexes.
-
   .. versionchanged:: 1.3.0
      Renamed from get_stylistic_issues() to stylistic_issues(). The old name
      still works as an alias, but will be dropped in Stem version 2.0.0.
@@ -160,89 +436,106 @@ def stylistic_issues(paths, check_two_space_indents = False, check_newlines = Fa
   .. versionchanged:: 1.4.0
      Added the prefer_single_quotes option.
 
+  .. versionchanged:: 1.6.0
+     Changed 'pycodestyle.ignore' code snippets to only need to match against
+     the prefix.
+
   :param list paths: paths to search for stylistic issues
-  :param bool check_two_space_indents: check for two space indentations and
-    that no tabs snuck in
   :param bool check_newlines: check that we have standard newlines (\\n), not
     windows (\\r\\n) nor classic mac (\\r)
-  :param bool check_trailing_whitespace: check that our lines don't end with
-    trailing whitespace
   :param bool check_exception_keyword: checks that we're using 'as' for
     exceptions rather than a comma
   :param bool prefer_single_quotes: standardize on using single rather than
     double quotes for strings, when reasonable
 
-  :returns: **dict** of the form ``path => [(line_number, message)...]``
+  :returns: dict of paths list of :class:`stem.util.test_tools.Issue` instances
   """
 
   issues = {}
 
-  if is_pep8_available():
-    import pep8
+  ignore_rules = []
+  ignore_for_file = []
 
-    class StyleReport(pep8.BaseReport):
-      def __init__(self, options):
-        super(StyleReport, self).__init__(options)
+  for rule in CONFIG['pycodestyle.ignore'] + CONFIG['pep8.ignore']:
+    if '=>' in rule:
+      path, rule_entry = rule.split('=>', 1)
+
+      if ':' in rule_entry:
+        rule, code = rule_entry.split(':', 1)
+        ignore_for_file.append((path.strip(), rule.strip(), code.strip()))
+    else:
+      ignore_rules.append(rule)
+
+  def is_ignored(path, rule, code):
+    for ignored_path, ignored_rule, ignored_code in ignore_for_file:
+      if path.endswith(ignored_path) and ignored_rule == rule and code.strip().startswith(ignored_code):
+        return True
+
+    return False
+
+  if is_pycodestyle_available():
+    if _module_exists('pep8'):
+      import pep8 as pycodestyle
+    else:
+      import pycodestyle
+
+    class StyleReport(pycodestyle.BaseReport):
+      def init_file(self, filename, lines, expected, line_offset):
+        super(StyleReport, self).init_file(filename, lines, expected, line_offset)
+
+        if not check_newlines and not check_exception_keyword and not prefer_single_quotes:
+          return
+
+        is_block_comment = False
+
+        for index, line in enumerate(lines):
+          content = line.split('#', 1)[0].strip()
+
+          if check_newlines and '\r' in line:
+            issues.setdefault(filename, []).append(Issue(index + 1, 'contains a windows newline', line))
+
+          if not content:
+            continue  # blank line
+
+          if '"""' in content:
+            is_block_comment = not is_block_comment
+
+          if check_exception_keyword and content.startswith('except') and content.endswith(', exc:'):
+            # Python 2.6 - 2.7 supports two forms for exceptions...
+            #
+            #   except ValueError, exc:
+            #   except ValueError as exc:
+            #
+            # The former is the old method and no longer supported in python 3
+            # going forward.
+
+            # TODO: This check only works if the exception variable is called
+            # 'exc'. We should generalize this via a regex so other names work
+            # too.
+
+            issues.setdefault(filename, []).append(Issue(index + 1, "except clause should use 'as', not comma", line))
+
+          if prefer_single_quotes and not is_block_comment:
+            if '"' in content and "'" not in content and '"""' not in content and not content.endswith('\\'):
+              # Checking if the line already has any single quotes since that
+              # usually means double quotes are preferable for the content (for
+              # instance "I'm hungry"). Also checking for '\' at the end since
+              # that can indicate a multi-line string.
+
+              issues.setdefault(filename, []).append(Issue(index + 1, 'use single rather than double quotes', line))
 
       def error(self, line_number, offset, text, check):
         code = super(StyleReport, self).error(line_number, offset, text, check)
 
         if code:
-          issues.setdefault(self.filename, []).append(Issue(line_number, '%s %s' % (code, text), text))
+          line = linecache.getline(self.filename, line_number)
 
-    style_checker = pep8.StyleGuide(ignore = CONFIG['pep8.ignore'], reporter = StyleReport)
+          if not is_ignored(self.filename, code, line):
+            issues.setdefault(self.filename, []).append(Issue(line_number, text, line))
+
+    style_checker = pycodestyle.StyleGuide(ignore = ignore_rules, reporter = StyleReport)
     style_checker.check_files(list(_python_files(paths)))
 
-  if check_two_space_indents or check_newlines or check_trailing_whitespace or check_exception_keyword:
-    for path in _python_files(paths):
-      with open(path) as f:
-        file_contents = f.read()
-
-      lines = file_contents.split('\n')
-      is_block_comment = False
-
-      for index, line in enumerate(lines):
-        whitespace, content = re.match('^(\s*)(.*)$', line).groups()
-
-        # TODO: This does not check that block indentations are two spaces
-        # because differentiating source from string blocks ("""foo""") is more
-        # of a pita than I want to deal with right now.
-
-        if '"""' in content:
-          is_block_comment = not is_block_comment
-
-        if check_two_space_indents and '\t' in whitespace:
-          issues.setdefault(path, []).append(Issue(index + 1, 'indentation has a tab', line))
-        elif check_newlines and '\r' in content:
-          issues.setdefault(path, []).append(Issue(index + 1, 'contains a windows newline', line))
-        elif check_trailing_whitespace and content != content.rstrip():
-          issues.setdefault(path, []).append(Issue(index + 1, 'line has trailing whitespace', line))
-        elif check_exception_keyword and content.lstrip().startswith('except') and content.endswith(', exc:'):
-          # Python 2.6 - 2.7 supports two forms for exceptions...
-          #
-          #   except ValueError, exc:
-          #   except ValueError as exc:
-          #
-          # The former is the old method and no longer supported in python 3
-          # going forward.
-
-          # TODO: This check only works if the exception variable is called
-          # 'exc'. We should generalize this via a regex so other names work
-          # too.
-
-          issues.setdefault(path, []).append(Issue(index + 1, "except clause should use 'as', not comma", line))
-
-        if prefer_single_quotes and line and not is_block_comment:
-          content = line.strip().split('#', 1)[0]
-
-          if '"' in content and "'" not in content and '"""' not in content and not content.endswith('\\'):
-            # Checking if the line already has any single quotes since that
-            # usually means double quotes are preferable for the content (for
-            # instance "I'm hungry"). Also checking for '\' at the end since
-            # that can indicate a multi-line string.
-
-            issues.setdefault(path, []).append(Issue(index + 1, "use single rather than double quotes", line))
-
   return issues
 
 
@@ -254,10 +547,7 @@ def pyflakes_issues(paths):
   ::
 
     pyflakes.ignore stem/util/test_tools.py => 'pyflakes' imported but unused
-    pyflakes.ignore stem/util/test_tools.py => 'pep8' imported but unused
-
-  If a 'exclude_paths' was set in our test config then we exclude any absolute
-  paths matching those regexes.
+    pyflakes.ignore stem/util/test_tools.py => 'pycodestyle' imported but unused
 
   .. versionchanged:: 1.3.0
      Renamed from get_pyflakes_issues() to pyflakes_issues(). The old name
@@ -267,9 +557,12 @@ def pyflakes_issues(paths):
      Changing tuples in return value to be namedtuple instances, and adding the
      line that had the issue.
 
+  .. versionchanged:: 1.5.0
+     Support matching against prefix or suffix issue strings.
+
   :param list paths: paths to search for problems
 
-  :returns: dict of the form ``path => [(line_number, message)...]``
+  :returns: dict of paths list of :class:`stem.util.test_tools.Issue` instances
   """
 
   issues = {}
@@ -300,15 +593,24 @@ def pyflakes_issues(paths):
         # path ends with any of them.
 
         for ignored_path, ignored_issues in self._ignored_issues.items():
-          if path.endswith(ignored_path) and issue in ignored_issues:
-            return True
+          if path.endswith(ignored_path):
+            if issue in ignored_issues:
+              return True
+
+            for prefix in [i[:1] for i in ignored_issues if i.endswith('*')]:
+              if issue.startswith(prefix):
+                return True
+
+            for suffix in [i[1:] for i in ignored_issues if i.startswith('*')]:
+              if issue.endswith(suffix):
+                return True
 
         return False
 
       def _register_issue(self, path, line_number, issue, line):
         if not self._is_ignored(path, issue):
           if path and line_number and not line:
-            line = linecache.getline(path, line_number)
+            line = linecache.getline(path, line_number).strip()
 
           issues.setdefault(path, []).append(Issue(line_number, issue, line))
 
@@ -320,6 +622,22 @@ def pyflakes_issues(paths):
   return issues
 
 
+def _module_exists(module_name):
+  """
+  Checks if a module exists.
+
+  :param str module_name: module to check existance of
+
+  :returns: **True** if module exists and **False** otherwise
+  """
+
+  try:
+    __import__(module_name)
+    return True
+  except ImportError:
+    return False
+
+
 def _python_files(paths):
   for path in paths:
     for file_path in stem.util.system.files_with_suffix(path, '.py'):
@@ -333,9 +651,12 @@ def _python_files(paths):
       if not skip:
         yield file_path
 
+
 # TODO: drop with stem 2.x
 # We renamed our methods to drop a redundant 'get_*' prefix, so alias the old
-# names for backward compatability.
+# names for backward compatability, and account for pep8 being renamed to
+# pycodestyle.
 
 get_stylistic_issues = stylistic_issues
 get_pyflakes_issues = pyflakes_issues
+is_pep8_available = is_pycodestyle_available

Shared/lib/python3.4/site-packages/stem/util/tor_tools.py

@@ -1,4 +1,4 @@
-# Copyright 2012-2015, Damian Johnson and The Tor Project
+# Copyright 2012-2018, Damian Johnson and The Tor Project
 # See LICENSE for licensing information
 
 """
@@ -21,6 +21,8 @@ Miscellaneous utility functions for working with tor.
 
 import re
 
+import stem.util.str_tools
+
 # The control-spec defines the following as...
 #
 #   Fingerprint = "$" 40*HEXDIG
@@ -54,6 +56,9 @@ def is_valid_fingerprint(entry, check_prefix = False):
   :returns: **True** if the string could be a relay fingerprint, **False** otherwise
   """
 
+  if isinstance(entry, bytes):
+    entry = stem.util.str_tools._to_unicode(entry)
+
   try:
     if check_prefix:
       if not entry or entry[0] != '$':
@@ -75,6 +80,9 @@ def is_valid_nickname(entry):
   :returns: **True** if the string could be a nickname, **False** otherwise
   """
 
+  if isinstance(entry, bytes):
+    entry = stem.util.str_tools._to_unicode(entry)
+
   try:
     return bool(NICKNAME_PATTERN.match(entry))
   except TypeError:
@@ -88,6 +96,9 @@ def is_valid_circuit_id(entry):
   :returns: **True** if the string could be a circuit id, **False** otherwise
   """
 
+  if isinstance(entry, bytes):
+    entry = stem.util.str_tools._to_unicode(entry)
+
   try:
     return bool(CIRC_ID_PATTERN.match(entry))
   except TypeError:
@@ -124,6 +135,9 @@ def is_valid_hidden_service_address(entry):
   :returns: **True** if the string could be a hidden service address, **False** otherwise
   """
 
+  if isinstance(entry, bytes):
+    entry = stem.util.str_tools._to_unicode(entry)
+
   try:
     return bool(HS_ADDRESS_PATTERN.match(entry))
   except TypeError: