Psutil 3 Upgrade

pyfarm/agent/entrypoints/development.py

@@ -32,14 +32,12 @@
 from random import choice, randint, random
 from textwrap import dedent
 
-import psutil
 import requests
 
-from pyfarm.core.utility import convert
 from pyfarm.agent.config import config
 from pyfarm.agent.logger import getLogger
-from pyfarm.agent.sysinfo import memory
 from pyfarm.agent.utility import dumps
+from pyfarm.agent.sysinfo import memory
 
 
 logger = getLogger("agent.cmd")
@@ -48,7 +46,6 @@
 
 
 def fake_render():
-    process = psutil.Process()
     memory_used_at_start = memory.used_ram()
 
     logger.info("sys.argv: %r", sys.argv)

pyfarm/agent/sysinfo/network.py

@@ -34,12 +34,12 @@
 
 import socket
 
-import netifaces
+import psutil
 from netaddr import IPSet, IPNetwork, IPAddress
 
 from pyfarm.agent.logger import getLogger
 
-logger = getLogger("agent.dns")
+logger = getLogger("agent.network")
 
 IP_PRIVATE = IPSet([
     IPNetwork("10.0.0.0/8"),
@@ -66,18 +66,28 @@ def mac_addresses(long_addresses=False, as_integers=False):
         When ``True`` convert all mac addresses to integers.
     """
     results = set()
-    for ifaces in map(netifaces.ifaddresses, netifaces.interfaces()):
-        for entry in ifaces.get(netifaces.AF_LINK, []):
-            mac = entry.get("addr", "")
-
-            if all([mac, not long_addresses, len(mac) == 17]) \
-               or all([long_addresses, mac, len(mac) >= 17]):
-                mac_as_int = int("0x" + mac.replace(":", ""), 0)
-                if mac_as_int != 0:
-                    if as_integers:
-                        results.add(mac_as_int)
-                    else:
-                        results.add(mac)
+    net_if_addrs = psutil.net_if_addrs()
+
+    for name, nics in net_if_addrs.items():
+        for nic in nics:
+            if nic.family != psutil.AF_LINK:
+                continue
+
+            if not long_addresses and len(nic.address) > 17:
+                continue
+
+            int_mac = int(
+                "0x" + nic.address.replace(":", "").replace("-", ""), 0)
+
+            # We don't want to include mac addresses like 00:00:00:00:00:00.
+            if int_mac == 0:
+                continue
+
+            mac = nic.address
+            if as_integers:
+                mac = int_mac
+
+            results.add(mac)
 
     return tuple(results)
 
@@ -171,23 +181,29 @@ def hostname(trust_name_from_ips=True):
 def addresses(private_only=True):
     """Returns a tuple of all non-local ip addresses."""
     results = set()
+    net_if_addrs = psutil.net_if_addrs()
 
-    for interface in netifaces.interfaces():
-        addrinfo = netifaces.ifaddresses(interface)
-        for address in addrinfo.get(socket.AF_INET, []):
-            addr = address.get("addr")
+    for name, nics in net_if_addrs.items():
+        for nic in nics:
+            if nic.address is None:
+                continue
+
+            elif nic.family == socket.AF_INET:
+                if not private_only:
+                    results.add(nic.address)
+                    continue
 
-            if addr is not None:
-                # Make sure that what we're getting out of
-                # netifaces is something we can use.
                 try:
-                    ip = IPAddress(addr)
+                    if IPAddress(nic.address) in IP_PRIVATE:
+                        results.add(nic.address)
                 except ValueError:  # pragma: no cover
                     logger.error(
-                        "Could not convert %s to a valid IP object" % addr)
-                else:
-                    if ip in IP_PRIVATE or not private_only:
-                        results.add(addr)
+                        "Could not convert %s to a valid IP object",
+                        nic.address)
+
+            # TODO: Support the IPv6 address family.
+            else:
+                logger.debug("Unsupported address family: %s", nic.family)
 
     if not addresses:  # pragma: no cover
         logger.error("No addresses could be found")
@@ -198,16 +214,19 @@ def addresses(private_only=True):
 def interfaces():
     """Returns the names of all valid network interface names"""
     results = set()
+    net_if_addrs = psutil.net_if_addrs()
 
-    for name in netifaces.interfaces():
-        # only add network interfaces which have IPv4
-        addresses = netifaces.ifaddresses(name)
+    for name, nics in net_if_addrs.items():
+        for nic in nics:
+            if nic.address is None:
+                continue
 
-        if socket.AF_INET not in addresses:  # pragma: no cover
-            continue
+            elif nic.family == socket.AF_INET:
+                results.add(name)
 
-        if any(addr.get("addr") for addr in addresses[socket.AF_INET]):
-            results.add(name)
+            elif nic.family == socket.AF_INET6:
+                logger.warning(
+                    "IPv6 not yet supported in interfaces() for %s", name)
 
     if not results:  # pragma: no cover
         logger.warning("Failed to find any interfaces")

setup.py

@@ -27,8 +27,7 @@
     "pyfarm.core>=0.9.3",
     # "PyOpenSSL", "service_identity",  # required for full SSL support
     "netaddr", "twisted", "ntplib", "requests!=2.4.0", "treq",
-    "voluptuous", "jinja2", "psutil>=2.1.0",
-    "netifaces>=0.10.2", "pyasn1"]
+    "voluptuous", "jinja2", "psutil>=3.2.2,<4.0.0", "pyasn1"]
 
 if "READTHEDOCS" in os.environ:
     install_requires += ["sphinxcontrib-httpdomain", "sphinx"]

tests/test_agent/test_sysinfo.py

@@ -56,8 +56,8 @@
 except ImportError:  # pragma: no cover
     getuid = NotImplemented
 
-import netifaces
 from mock import Mock, patch
+from netaddr import IPAddress
 from twisted.internet import reactor
 from twisted.internet.defer import inlineCallbacks, succeed
 
@@ -187,23 +187,52 @@ def test_hostname_trust_dns_mappings(self):
                 "Failed to find a hostname matching %s "
                 "options were %s" % (correct_hostname, reverse_hostnames))
 
-    def test_addresses(self):
+    def test_addresses_length(self):
         self.assertGreaterEqual(len(network.addresses(private_only=False)), 1)
 
     def test_addresses_type(self):
         self.assertIsInstance(network.addresses(), tuple)
 
+    def test_addresses_ipv4(self):
+        addresses = network.addresses()
+        net_if_addrs = psutil.net_if_addrs()
+
+        expected = set()
+        for name, nics in net_if_addrs.items():
+            for nic in nics:
+                if nic.family == socket.AF_INET and nic.address:
+                    if IPAddress(nic.address) in network.IP_PRIVATE:
+                        expected.add(nic.address)
+
+        self.assertEqual(expected, set(addresses))
+
     def test_interfaces(self):
         names = list(network.interfaces())
 
         # We assume all hosts have at least the loopback interface.
         self.assertGreaterEqual(len(names), 1)
-        self.assertEqual(isinstance(names, list), True)
-        self.assertTrue(
-            all(name in netifaces.interfaces() for name in names))
 
-        addresses = map(netifaces.ifaddresses, names)
-        self.assertEqual(all(socket.AF_INET in i for i in addresses), True)
+        for name in names:
+            for psutil_name, nics in psutil.net_if_addrs().items():
+                if name != psutil_name:
+                    continue
+                for nic in nics:
+                    if (nic.family in (socket.AF_INET, socket.AF_INET6)
+                        and nic.address):
+                        break
+                else:
+                    self.fail("Failed to locate nic entry for %s" % name)
+            break
+        else:
+            self.fail("Failed to locate nic")
+
+    def test_mac_addresses_does_not_include_loopback(self):
+        for mac in network.mac_addresses(as_integers=True):
+            self.assertNotEqual(mac, 0)
+
+    def test_mac_addresses_does_not_include_long_addresses(self):
+        for mac in network.mac_addresses(as_integers=False):
+            self.assertLessEqual(len(mac), 17)
 
 
 class TestCPU(TestCase):