PUC-740: comparing data from Openstack Networks and Nautobot UCVNIs

Author: haseebsyed12

python/diff-nautobot-understack/README.md

@@ -0,0 +1,9 @@
+
+cd python/diff-nautobot-understack
+python3 -m venv .venv
+source .venv/bin/activate
+poetry lock
+poetry install
+
+export NB_TOKEN=<get_token_from_nautobot_dev>
+poetry run diff-network

python/diff-nautobot-understack/clouds.yaml

@@ -0,0 +1,31 @@
+clouds:
+  uc-dev-infra:
+    auth_type: v3websso
+    identity_provider: sso
+    protocol: openid
+    auth:
+      auth_url: https://keystone.dev.undercloud.rackspace.net/v3
+      project_domain_name: infra
+      project_name: baremetal
+  uc-dev:
+    auth_type: v3websso
+    identity_provider: sso
+    protocol: openid
+    auth:
+      auth_url: https://keystone.dev.undercloud.rackspace.net/v3
+      project_domain_name: Default
+  uc-staging-infra:
+    auth_type: v3websso
+    identity_provider: sso
+    protocol: openid
+    auth:
+      auth_url: https://keystone.staging.undercloud.rackspace.net/v3
+      project_domain_name: infra
+      project_name: baremetal
+  uc-staging:
+    auth_type: v3websso
+    identity_provider: sso
+    protocol: openid
+    auth:
+      auth_url: https://keystone.staging.undercloud.rackspace.net/v3
+      project_domain_name: Default

python/diff-nautobot-understack/diff_nautobot_understack/__init__.py

@@ -0,0 +1,72 @@
+import os
+
+import requests
+import logging
+import inspect
+from urllib.parse import urljoin
+
+NB_URL = os.environ.get("NB_URL", "https://nautobot.dev.undercloud.rackspace.net")
+NB_TOKEN = os.environ.get("NB_TOKEN")
+
+
+class API:
+    CALLER_FRAME = 1
+
+    def __init__(self):
+        self.base_url = NB_URL
+        self.s = requests.Session()
+        self.token = NB_TOKEN
+        self.s.headers.update({"Authorization": f"Token {NB_TOKEN}"})
+
+    def make_api_request(
+        self, url: str, payload: dict | None = None, paginated: bool = False
+    ) -> dict | list:
+        endpoint_url = urljoin(self.base_url, url)
+        caller_function = inspect.stack()[self.CALLER_FRAME].function
+
+        logging.debug(
+            "%(caller_function)s payload: %(payload)s",
+            {"payload": payload, "caller_function": caller_function},
+        )
+
+        if paginated:
+            return self._fetch_paginated_data(endpoint_url, payload, caller_function)
+        else:
+            resp = self.s.get(endpoint_url, timeout=10, json=payload)
+            return self._process_response(resp, caller_function)
+
+    def _fetch_paginated_data(
+        self, endpoint_url: str, payload: dict | None, caller_function: str
+    ) -> list:
+        response_items = []
+        url = endpoint_url
+
+        while url is not None:
+            resp = self.s.get(url, timeout=10, json=payload)
+            resp_data = self._process_response(resp, caller_function)
+
+            response_items.extend(resp_data.get("results", []))
+            url = resp_data.get("next")
+
+        return response_items
+
+    def _process_response(self, resp, caller_function: str) -> dict:
+        if resp.content:
+            resp_data = resp.json()
+        else:
+            resp_data = {"status_code": resp.status_code}
+
+        logging.debug(
+            "%(caller_function)s resp: %(resp)s",
+            {"resp": resp_data, "caller_function": caller_function},
+        )
+
+        self._log_and_raise_for_status(resp)
+        return resp_data
+
+    def _log_and_raise_for_status(self, resp):
+        try:
+            resp.raise_for_status()
+        except Exception as e:
+            logging.error(f"HTTP error occurred: {e}")
+            raise

python/diff-nautobot-understack/diff_nautobot_understack/clients/__init__.py

@@ -0,0 +1,10 @@
+import os
+
+import openstack
+
+OS_CLOUD = os.environ.get("OS_CLOUD", "uc-dev-infra")
+
+
+class API:
+    def __init__(self):
+        self.cloud_connection = openstack.connect(cloud=OS_CLOUD)

python/diff-nautobot-understack/diff_nautobot_understack/clients/nautobot.py

@@ -0,0 +1,28 @@
+from diffsync import Adapter
+from diff_nautobot_understack.clients.openstack import API
+
+from diff_nautobot_understack.network import models
+
+
+class Network(Adapter):
+    network = models.NetworkModel
+
+    top_level = ["network"]
+    type = "OpenstackNetwork"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        openstack_api = API()
+        self.cloud = openstack_api.cloud_connection
+
+    def load(self):
+        for network in self.cloud.network.networks():
+            self.add(
+                self.network(
+                    id=network.id,
+                    name=network.name,
+                    status=network.status.lower(),
+                    provider_physical_network=network.provider_physical_network,
+                    vni_id=network.provider_segmentation_id,
+                )
+            )

python/diff-nautobot-understack/diff_nautobot_understack/clients/openstack.py

@@ -0,0 +1,105 @@
+from diffsync import Adapter
+from pydantic import BaseModel
+from diff_nautobot_understack.clients.nautobot import API
+
+from diff_nautobot_understack.network import models
+
+
+class UcvniDetails(BaseModel):
+    id: str
+    name: str
+    status: str
+    ucvni_id: int
+    ucvni_group: str
+    vlan_group: str
+    vlan_id: int
+
+
+class NautobotError(Exception):
+    message = "Nautobot error"
+
+
+class Network(Adapter):
+    CALLER_FRAME = 1
+    network = models.NetworkModel
+
+    top_level = ["network"]
+    type = "UCVNI"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.api_client = API()
+
+    def load(self):
+        ucvni_data: list[UcvniDetails] = self.ucvni_get()
+        for ucvni_item in ucvni_data:
+            network = self.network(
+                id=ucvni_item.id,
+                name=ucvni_item.name,
+                vni_id=ucvni_item.vlan_id,
+                provider_physical_network=ucvni_item.vlan_group,
+                status=ucvni_item.status,
+            )
+            self.add(network)
+
+    def ucvni_get(
+        self,
+    ) -> list[UcvniDetails]:
+        ucvni_detail_list: list[UcvniDetails] = []
+
+        url = "/api/plugins/undercloud-vni/ucvnis/?include=relationship"
+
+        ucvnis_response = self.api_client.make_api_request(
+            f"{url}/?include=relationships", paginated=True
+        )
+
+        for ucvni_item in ucvnis_response:
+            ucvni_group = self.api_client.make_api_request(
+                url=ucvni_item.get("ucvni_group", {}).get("url")
+            )
+            status = self.api_client.make_api_request(
+                url=ucvni_item.get("status", {}).get("url")
+            )
+            vlan_uuid_objects = (
+                ucvni_item.get("relationships", {})
+                .get("ucvni_vlans", {})
+                .get("destination")
+                .get("objects")
+            )
+            vlan_details = self.get_vlan_details(vlan_uuid_objects)
+            vlan_group, vlan_ids = next(iter(vlan_details.items()))
+            ucvni_details = UcvniDetails(
+                id=ucvni_item.get("id"),
+                name=ucvni_item.get("name"),
+                ucvni_id=ucvni_item.get("ucvni_id"),
+                ucvni_group=ucvni_group.get("name"),
+                status=status.get("name").lower(),
+                vlan_group=vlan_group,
+                vlan_id=int(vlan_ids[0]),
+            )
+            ucvni_detail_list.append(ucvni_details)
+        return ucvni_detail_list
+
+    def get_vlan_details(self, vlan_uuid_objects):
+        vlan_uuids = [vlan_uuid_object["id"] for vlan_uuid_object in vlan_uuid_objects]
+        vlan_details = {}
+
+        vlan_uuids_query_params = "&".join(f"id={value}" for value in vlan_uuids)
+        vlan_url = f"/api/ipam/vlans/?{vlan_uuids_query_params}"
+
+        vlans_response = self.api_client.make_api_request(url=vlan_url, paginated=True)
+
+        for vlan_response in vlans_response:
+            vlan_group_url = vlan_response.get("vlan_group", {}).get("url")
+
+            if vlan_group_url:
+                vlan_group_response = self.api_client.make_api_request(
+                    url=vlan_group_url
+                )
+                vlan_group_name = vlan_group_response.get("name")
+                vlan_id = vlan_response.get("vid")
+
+                if vlan_group_name:
+                    vlan_details.setdefault(vlan_group_name, []).append(vlan_id)
+
+        return vlan_details

python/diff-nautobot-understack/diff_nautobot_understack/network/__init__.py

@@ -0,0 +1,33 @@
+from pprint import pprint
+from diffsync import Diff
+from diffsync.enum import DiffSyncFlags
+from diff_nautobot_understack.network.adapters.openstack_network import (
+    Network as OpenstackNetwork,
+)
+from diff_nautobot_understack.network.adapters.ucvni import Network as UcvniNetwork
+
+
+class MyDiff(Diff):
+    """Custom Diff class to control the order of the site objects."""
+
+    @classmethod
+    def order_children_site(cls, children):
+        """Return the site children ordered in alphabetical order."""
+        keys = sorted(children.keys(), reverse=False)
+        for key in keys:
+            yield children[key]
+
+
+def openstack_network_diff_from_ucvni_network():
+    openstack_network = OpenstackNetwork()
+    openstack_network.load()
+
+    ucvni_network = UcvniNetwork()
+    ucvni_network.load()
+    openstack_network_destination_ucvni_source = openstack_network.diff_from(
+        ucvni_network, flags=DiffSyncFlags.CONTINUE_ON_FAILURE
+    )
+    pprint(" Nautobot ucvnis ⟹ Openstack networks ")
+    summary = openstack_network_destination_ucvni_source.summary()
+    pprint(summary, width=120)
+    pprint(openstack_network_destination_ucvni_source.dict(), width=120)

python/diff-nautobot-understack/diff_nautobot_understack/network/adapters/__init__.py

@@ -0,0 +1,18 @@
+from diffsync import DiffSyncModel
+
+
+class NetworkModel(DiffSyncModel):
+    _modelname = "network"
+    _identifiers = ("id",)
+    _attributes = (
+        "name",
+        "status",
+        "provider_physical_network",
+        "vni_id",
+    )
+
+    id: str
+    name: str
+    status: str
+    provider_physical_network: str
+    vni_id: int

python/diff-nautobot-understack/diff_nautobot_understack/network/adapters/openstack_network.py

@@ -0,0 +1,15 @@
+import openstack
+
+openstack.enable_logging(debug=True)
+
+
+def list_projects(conn):
+    print("List Projects:")
+
+    for project in conn.identity.projects():
+        print(project)
+
+
+if __name__ == "__main__":
+    cloud_connection = openstack.connect(os_cloud="uc-dev-infra")
+    list_projects(cloud_connection)