diff --git a/pyfarm/agent/entrypoints/development.py b/pyfarm/agent/entrypoints/development.py index 3124f392..b9e4e57c 100644 --- a/pyfarm/agent/entrypoints/development.py +++ b/pyfarm/agent/entrypoints/development.py @@ -32,14 +32,12 @@ from random import choice, randint, random from textwrap import dedent -import psutil import requests -from pyfarm.core.utility import convert from pyfarm.agent.config import config from pyfarm.agent.logger import getLogger -from pyfarm.agent.sysinfo import memory from pyfarm.agent.utility import dumps +from pyfarm.agent.sysinfo import memory logger = getLogger("agent.cmd") @@ -48,7 +46,6 @@ def fake_render(): - process = psutil.Process() memory_used_at_start = memory.used_ram() logger.info("sys.argv: %r", sys.argv) diff --git a/pyfarm/agent/sysinfo/network.py b/pyfarm/agent/sysinfo/network.py index 532ebe70..819ff647 100644 --- a/pyfarm/agent/sysinfo/network.py +++ b/pyfarm/agent/sysinfo/network.py @@ -34,12 +34,12 @@ import socket -import netifaces +import psutil from netaddr import IPSet, IPNetwork, IPAddress from pyfarm.agent.logger import getLogger -logger = getLogger("agent.dns") +logger = getLogger("agent.network") IP_PRIVATE = IPSet([ IPNetwork("10.0.0.0/8"), @@ -66,18 +66,28 @@ def mac_addresses(long_addresses=False, as_integers=False): When ``True`` convert all mac addresses to integers. """ results = set() - for ifaces in map(netifaces.ifaddresses, netifaces.interfaces()): - for entry in ifaces.get(netifaces.AF_LINK, []): - mac = entry.get("addr", "") - - if all([mac, not long_addresses, len(mac) == 17]) \ - or all([long_addresses, mac, len(mac) >= 17]): - mac_as_int = int("0x" + mac.replace(":", ""), 0) - if mac_as_int != 0: - if as_integers: - results.add(mac_as_int) - else: - results.add(mac) + net_if_addrs = psutil.net_if_addrs() + + for name, nics in net_if_addrs.items(): + for nic in nics: + if nic.family != psutil.AF_LINK: + continue + + if not long_addresses and len(nic.address) > 17: + continue + + int_mac = int( + "0x" + nic.address.replace(":", "").replace("-", ""), 0) + + # We don't want to include mac addresses like 00:00:00:00:00:00. + if int_mac == 0: + continue + + mac = nic.address + if as_integers: + mac = int_mac + + results.add(mac) return tuple(results) @@ -171,23 +181,29 @@ def hostname(trust_name_from_ips=True): def addresses(private_only=True): """Returns a tuple of all non-local ip addresses.""" results = set() + net_if_addrs = psutil.net_if_addrs() - for interface in netifaces.interfaces(): - addrinfo = netifaces.ifaddresses(interface) - for address in addrinfo.get(socket.AF_INET, []): - addr = address.get("addr") + for name, nics in net_if_addrs.items(): + for nic in nics: + if nic.address is None: + continue + + elif nic.family == socket.AF_INET: + if not private_only: + results.add(nic.address) + continue - if addr is not None: - # Make sure that what we're getting out of - # netifaces is something we can use. try: - ip = IPAddress(addr) + if IPAddress(nic.address) in IP_PRIVATE: + results.add(nic.address) except ValueError: # pragma: no cover logger.error( - "Could not convert %s to a valid IP object" % addr) - else: - if ip in IP_PRIVATE or not private_only: - results.add(addr) + "Could not convert %s to a valid IP object", + nic.address) + + # TODO: Support the IPv6 address family. + else: + logger.debug("Unsupported address family: %s", nic.family) if not addresses: # pragma: no cover logger.error("No addresses could be found") @@ -198,16 +214,19 @@ def addresses(private_only=True): def interfaces(): """Returns the names of all valid network interface names""" results = set() + net_if_addrs = psutil.net_if_addrs() - for name in netifaces.interfaces(): - # only add network interfaces which have IPv4 - addresses = netifaces.ifaddresses(name) + for name, nics in net_if_addrs.items(): + for nic in nics: + if nic.address is None: + continue - if socket.AF_INET not in addresses: # pragma: no cover - continue + elif nic.family == socket.AF_INET: + results.add(name) - if any(addr.get("addr") for addr in addresses[socket.AF_INET]): - results.add(name) + elif nic.family == socket.AF_INET6: + logger.warning( + "IPv6 not yet supported in interfaces() for %s", name) if not results: # pragma: no cover logger.warning("Failed to find any interfaces") diff --git a/setup.py b/setup.py index b052b791..e66694b7 100644 --- a/setup.py +++ b/setup.py @@ -27,8 +27,7 @@ "pyfarm.core>=0.9.3", # "PyOpenSSL", "service_identity", # required for full SSL support "netaddr", "twisted", "ntplib", "requests!=2.4.0", "treq", - "voluptuous", "jinja2", "psutil>=2.1.0", - "netifaces>=0.10.2", "pyasn1"] + "voluptuous", "jinja2", "psutil>=3.2.2,<4.0.0", "pyasn1"] if "READTHEDOCS" in os.environ: install_requires += ["sphinxcontrib-httpdomain", "sphinx"] diff --git a/tests/test_agent/test_sysinfo.py b/tests/test_agent/test_sysinfo.py index a55e3146..76191aed 100644 --- a/tests/test_agent/test_sysinfo.py +++ b/tests/test_agent/test_sysinfo.py @@ -56,8 +56,8 @@ except ImportError: # pragma: no cover getuid = NotImplemented -import netifaces from mock import Mock, patch +from netaddr import IPAddress from twisted.internet import reactor from twisted.internet.defer import inlineCallbacks, succeed @@ -187,23 +187,52 @@ def test_hostname_trust_dns_mappings(self): "Failed to find a hostname matching %s " "options were %s" % (correct_hostname, reverse_hostnames)) - def test_addresses(self): + def test_addresses_length(self): self.assertGreaterEqual(len(network.addresses(private_only=False)), 1) def test_addresses_type(self): self.assertIsInstance(network.addresses(), tuple) + def test_addresses_ipv4(self): + addresses = network.addresses() + net_if_addrs = psutil.net_if_addrs() + + expected = set() + for name, nics in net_if_addrs.items(): + for nic in nics: + if nic.family == socket.AF_INET and nic.address: + if IPAddress(nic.address) in network.IP_PRIVATE: + expected.add(nic.address) + + self.assertEqual(expected, set(addresses)) + def test_interfaces(self): names = list(network.interfaces()) # We assume all hosts have at least the loopback interface. self.assertGreaterEqual(len(names), 1) - self.assertEqual(isinstance(names, list), True) - self.assertTrue( - all(name in netifaces.interfaces() for name in names)) - addresses = map(netifaces.ifaddresses, names) - self.assertEqual(all(socket.AF_INET in i for i in addresses), True) + for name in names: + for psutil_name, nics in psutil.net_if_addrs().items(): + if name != psutil_name: + continue + for nic in nics: + if (nic.family in (socket.AF_INET, socket.AF_INET6) + and nic.address): + break + else: + self.fail("Failed to locate nic entry for %s" % name) + break + else: + self.fail("Failed to locate nic") + + def test_mac_addresses_does_not_include_loopback(self): + for mac in network.mac_addresses(as_integers=True): + self.assertNotEqual(mac, 0) + + def test_mac_addresses_does_not_include_long_addresses(self): + for mac in network.mac_addresses(as_integers=False): + self.assertLessEqual(len(mac), 17) class TestCPU(TestCase):