Refactoring: transform serializer to visitor, suppress state, isolate logic

cosmo/__main__.py

@@ -9,8 +9,8 @@
 
 from cosmo.clients.netbox import NetboxClient
 from cosmo.log import info
-from cosmo.serializer import RouterSerializer, SwitchSerializer, AbstractRecoverableError, RouterSerializerConfig
-
+from cosmo.serializer import RouterSerializer, SwitchSerializer
+from cosmo.common import AbstractRecoverableError
 
 
 
@@ -57,11 +57,6 @@ def noop(*args, **kwargs):
 
     yaml.emitter.Emitter.process_tag = noop
 
-    for vrf in cosmo_data["vrf_list"]:
-        if len(vrf["export_targets"]) > 1 or len(vrf["import_targets"]) > 1:
-            warnings.warn(f"Currently we only support one import/export target per VRF. {vrf['name']} has {len(vrf['import_targets'])} import targets and {len(vrf['export_targets'])} export targets")
-            continue
-
     for device in cosmo_data["device_list"]:
 
         if 'fqdnSuffix' in cosmo_configuration:
@@ -77,9 +72,7 @@ def noop(*args, **kwargs):
         content = None
         try:
             if device['name'] in cosmo_configuration['devices']['router']:
-
-                router_serializer_cfg = RouterSerializerConfig(cosmo_configuration.get("router_serializer_configuration", {}))
-                serializer = RouterSerializer(router_serializer_cfg, device, cosmo_data['l2vpn_list'], cosmo_data["vrf_list"], cosmo_data["loopbacks"])
+                serializer = RouterSerializer(device, cosmo_data['l2vpn_list'], cosmo_data["loopbacks"])
                 content = serializer.serialize()
             elif device['name'] in cosmo_configuration['devices']['switch']:
                 serializer = SwitchSerializer(device)

cosmo/abstractroutervisitor.py

@@ -0,0 +1,12 @@
+from abc import ABC
+
+from cosmo.types import CosmoLoopbackType
+from cosmo.visitors import AbstractNoopNetboxTypesVisitor
+
+
+class AbstractRouterExporterVisitor(AbstractNoopNetboxTypesVisitor, ABC):
+    _vrf_key = "routing_instances"
+    _interfaces_key = "interfaces"
+    _mgmt_vrf_name = "MGMT-ROUTING-INSTANCE"
+    _l2circuits_key = "l2circuits"
+    _allowed_core_mtus = [9216, 9600, 9230, 9586, 9116]

cosmo/clients/netbox_v4.py

@@ -36,18 +36,26 @@ def _fetch_data(self, kwargs):
         query_template = Template('''
             query {
               interface_list(filters: { tag: "bgp_cpe" }) {
+                __typename
                 id,
                 parent {
+                  __typename
                   id,
                   connected_endpoints {
                     ... on InterfaceType {
+                      __typename
                       name
                       device {
                         primary_ip4 {
+                          __typename
                           address
                         }
                         interfaces {
+                          id
+                          name
+                          __typename
                           ip_addresses {
+                            __typename
                             address
                           }
                         }
@@ -91,20 +99,37 @@ def _fetch_data(self, kwargs):
                 name: {starts_with: "lo"},
                 type: {exact:"loopback"}
               }) {
+                __typename
                 name,
                 child_interfaces {
+                  __typename
                   name,
                   vrf {
+                    __typename
                     id
+                    name
+                    description
+                    rd
+                    export_targets {
+                      __typename
+                      name
+                    }
+                    import_targets {
+                      __typename
+                      name
+                    }
                   },
                   ip_addresses {
+                    __typename
                     address,
                     family {
+                     __typename
                      value,
                     }
                   }
                 }
                 device{
+                  __typename
                   name,
                 }
               }
@@ -127,7 +152,9 @@ def _merge_into(self, data: object, query_data):
             l_ipv6 = next(filter(lambda l: l['family']['value'] == 6, child_interface['ip_addresses']), None)
             loopbacks[device_name] = {
                 "ipv4": l_ipv4['address'] if l_ipv4 else None,
-                "ipv6": l_ipv6['address'] if l_ipv6 else None
+                "ipv6": l_ipv6['address'] if l_ipv6 else None,
+                "__typename": "CosmoLoopbackType",
+                "device": device_name,
             }
 
         return {
@@ -141,20 +168,61 @@ def _fetch_data(self, kwargs):
         query_template = Template('''
          query {
             l2vpn_list (filters: {name: {starts_with: "WAN: "}}) {
+                __typename
                 id
                 name
                 type
                 identifier
                 terminations {
+                  __typename
                   id
                   assigned_object {
                     __typename
                     ... on VLANType {
+                      __typename
                       id
+                      name
+                      interfaces_as_tagged {
+                         id
+                         name
+                         __typename
+                         device {
+                            __typename
+                            id
+                            name
+                         }
+                      }
+                      interfaces_as_untagged {
+                         id
+                         name
+                         __typename
+                         device {
+                            __typename
+                            id
+                            name
+                         }
+                      }
                     }
                     ... on InterfaceType {
+                      __typename
                       id
+                      name
+                      custom_fields
+                      untagged_vlan {
+                        __typename
+                        id
+                        name
+                        vid
+                       }
+                      tagged_vlans {
+                        __typename
+                        id
+                        name
+                        vid
+                      }
                       device {
+                        __typename
+                        id
                         name
                       }
                     }
@@ -173,34 +241,6 @@ def _merge_into(self, data: object, query_data):
         }
 
 
-class VrfDataQuery(ParallelQuery):
-    def _fetch_data(self, kwargs):
-        query_template = Template('''
-            query {
-                vrf_list {
-                    id
-                    name
-                    description
-                    rd
-                    export_targets {
-                      name
-                    }
-                    import_targets {
-                      name
-                    }
-                }
-            }
-        ''')
-
-        return self.client.query(query_template.substitute())['data']
-
-    def _merge_into(self, data: dict, query_data):
-        return {
-            **data,
-            **query_data,
-        }
-
-
 class StaticRouteQuery(ParallelQuery):
 
     def _fetch_data(self, kwargs):
@@ -211,6 +251,8 @@ def _merge_into(self, data: dict, query_data):
         for d in data['device_list']:
             device_static_routes = list(filter(lambda sr: str(sr['device']['id']) == d['id'], query_data))
             d['staticroute_set'] = device_static_routes
+            for e in d['staticroute_set']:
+                e['__typename'] = "CosmoStaticRouteType"
 
         return data
 
@@ -251,24 +293,30 @@ def _fetch_data(self, kwargs):
               device_list(filters: {
                 name: { i_exact: $device },
               }) {
+                __typename
                 id
                 name
                 serial
 
                 device_type {
+                  __typename
                   slug
                 }
                 platform {
+                  __typename
                   manufacturer {
+                    __typename
                     slug
                   }
                   slug
                 }
                 primary_ip4 {
+                  __typename
                   address
                 }
 
                 interfaces {
+                  __typename
                   id
                   name
                   enabled
@@ -277,31 +325,51 @@ def _fetch_data(self, kwargs):
                   mtu
                   description
                   vrf {
+                    __typename
                     id
+                    name
+                    description
+                    rd
+                    export_targets {
+                      __typename
+                      name
+                    }
+                    import_targets {
+                      __typename
+                      name
+                    }
                   }
                   lag {
+                    __typename
                     id
+                    name
                   }
                   ip_addresses {
+                    __typename
                     address
                   }
                   untagged_vlan {
+                    __typename
                     id
                     name
                     vid
                   }
                   tagged_vlans {
+                    __typename
                     id
                     name
                     vid
                   }
                   tags {
+                    __typename
                     name
                     slug
                   }
                   parent {
+                    __typename
                     id
                     mtu
+                    name
                   }
                   custom_fields
                 }
@@ -342,7 +410,6 @@ def get_data(self, device_config):
             )
 
         queries.extend([
-            VrfDataQuery(self.client, device_list=device_list),
             L2VPNDataQuery(self.client, device_list=device_list),
             StaticRouteQuery(self.client, device_list=device_list),
             DeviceMACQuery(self.client, device_list=device_list),

cosmo/common.py

@@ -1,6 +1,30 @@
+import abc
+
+class AbstractRecoverableError(Exception, abc.ABC):
+    pass
+
+
+class DeviceSerializationError(AbstractRecoverableError):
+    pass
+
+
+class InterfaceSerializationError(AbstractRecoverableError):
+    pass
+
+
+# next() can raise StopIteration, so that's why I use this function
+def head(l):
+    return None if not l else l[0]
+
 def deepsort(e):
     if isinstance(e, list):
         return sorted(deepsort(v) for v in e)
     elif isinstance(e, dict):
         return {k: deepsort(v) for k, v in e.items()}
     return e
+
+
+def without_keys(d, keys) -> dict:
+    if type(keys) != list:
+        keys = [keys]
+    return {k: v for k,v in d.items() if k not in keys}

cosmo/cperoutervisitor.py

@@ -0,0 +1,25 @@
+from functools import singledispatchmethod
+
+from cosmo.types import IPAddressType, DeviceType
+from cosmo.visitors import AbstractNoopNetboxTypesVisitor
+
+
+class CpeRouterExporterVisitor(AbstractNoopNetboxTypesVisitor):
+    """
+    This visitor creates a list of networks which are exported from the router
+    via unnumbered bgp. We allow all configured IP networks on a CPE to be
+    exported. By definition the primary IP is our management IP and this IP
+    should not be allowed to be exported via BGP from the router.
+    """
+
+    @singledispatchmethod
+    def accept(self, o):
+        return super().accept(o)
+
+    @accept.register
+    def _(self, o: IPAddressType):
+        primary_ip4 = o.getParent(DeviceType)["primary_ip4"]
+        if primary_ip4 and primary_ip4.getIPAddress() == o.getIPAddress():
+            return # skip
+        ip_interface = o.getIPInterfaceObject()
+        return type(ip_interface), ip_interface.network.with_prefixlen