Merge pull request #624 from rackerlabs/puc-740-diffsync

feat: PUC-740: comparing data from Openstack and Nautobot

Author: nicholaskuechler

python/diff-nautobot-understack/README.md

@@ -0,0 +1,35 @@
+# Nautobot vs OpenStack Comparison Tool
+
+This tool compares data between Nautobot and OpenStack.
+
+## Currently Supported Comparisons
+
+| OpenStack | Nautobot              |
+|-----------|-----------------------|
+| Project   | Tenant                |
+| Network   | UCVNI & Namespace     |
+
+
+---
+
+## Setup Instructions
+
+1. cd python/diff-nautobot-understack
+2. `poetry install`
+   1. poetry will handle the creation of this virtual environment for you. It'll use .venv in the project if you configure it to do so locally on your machine with `poetry config virtualenvs.in-project true`.
+   2. user can create a shell with poetry shell and then when they exit it will clean up auth variables. Or they can run source .venv/bin/activate or poetry run <commands-below>
+
+3. Export environment variables (or add them to a .env file):
+   1. export NAUTOBOT_URL=https://nautobot.url.here
+   2.   users should browse to https://nautobot.url.here/user/api-tokens/ and generate an API token
+   3. export NAUTOBOT_TOKEN= <generated token from above step>
+   4. export OS_CLOUD=uc-dev-infra
+   5. export OS_CLIENT_CONFIG_FILE=./my_clouds.yaml (set this if it is in any other location not [defined here](https://opendev.org/openstack/openstacksdk#getting-started))
+
+
+- Below are some example commands
+```
+    uc-diff --help
+    uc-diff project undercloud -v
+    uc-diff network
+```

python/diff-nautobot-understack/diff_nautobot_understack/__init__.py

@@ -0,0 +1,102 @@
+import os
+import sys
+
+import typer
+from diffsync.diff import Diff
+from rich import print
+from rich.console import Console
+from rich.table import Table
+
+from diff_nautobot_understack.network.main import (
+    openstack_network_diff_from_ucvni_network,
+)
+from diff_nautobot_understack.project.main import (
+    openstack_project_diff_from_nautobot_tenant,
+)
+from diff_nautobot_understack.settings import app_settings as settings
+
+required_env_vars = ["NAUTOBOT_TOKEN", "NAUTOBOT_URL", "OS_CLOUD"]
+
+
+app = typer.Typer(
+    name="diff",
+    add_completion=False,
+    help="compare data between Openstack and Nautobot.",
+)
+diff_outputs = {
+    "project": {"title": "Project Diff", "id_column_name": "Project ID"},
+    "network": {"title": "Network Diff", "id_column_name": "Network ID"},
+}
+
+
+def display_output(
+    diff_result: Diff, diff_output: str, output_format: str | None = None
+):
+    print(diff_result.summary())
+    __output_format = (
+        output_format if output_format is not None else settings.output_format
+    )
+    if __output_format == "table":
+        diff_output_props = diff_outputs.get(diff_output)
+        tabular_output(
+            diff_result.dict().get(diff_output, {}),
+            diff_output_props.get("title"),
+            diff_output_props.get("id_column_name"),
+        )
+    else:
+        print(diff_result.dict())
+
+
+def tabular_output(diffs, title, id_column_name):
+    table = Table(title=title, show_lines=True)
+
+    table.add_column(id_column_name, style="cyan", no_wrap=True)
+    table.add_column("Change Type", style="magenta")
+    table.add_column("Details", style="yellow")
+
+    for diff_id, changes in diffs.items():
+        for change_type, details in changes.items():
+            table.add_row(diff_id, change_type, str(details))
+
+    console = Console()
+    console.print(table)
+
+
+@app.command()
+def project(
+    name: str,
+    debug: bool = typer.Option(False, "--debug", "-v", help="Enable debug mode"),
+    output_format: str = typer.Option(
+        "json", "--format", help="Available formats json, table"
+    ),
+):
+    """Nautobot tenants ⟹ Openstack projects"""
+    settings.debug = debug
+    diff_result = openstack_project_diff_from_nautobot_tenant(os_project=name)
+    display_output(diff_result, "project", output_format)
+
+
+@app.command()
+def network(
+    debug: bool = typer.Option(False, "--debug", "-v", help="Enable debug mode"),
+    output_format: str = typer.Option(
+        "json", "--format", help="Available formats json, table"
+    ),
+):
+    """Nautobot ucvnis ⟹ Openstack networks"""
+    settings.debug = debug
+    diff_result = openstack_network_diff_from_ucvni_network()
+    display_output(diff_result, "network", output_format)
+
+
+def check_env_vars(required_vars):
+    missing_vars = [var for var in required_vars if var not in os.environ]
+
+    if missing_vars:
+        print(f"Error: Missing environment variables: {', '.join(missing_vars)}")
+        sys.exit(1)
+    else:
+        print("All required environment variables are set.")
+
+
+check_env_vars(required_env_vars)

python/diff-nautobot-understack/diff_nautobot_understack/cli.py

@@ -0,0 +1,70 @@
+import inspect
+import logging
+from urllib.parse import urljoin
+
+import requests
+
+from diff_nautobot_understack.settings import app_settings as settings
+
+
+class API:
+    CALLER_FRAME = 1
+
+    def __init__(self):
+        self.base_url = settings.nautobot_url
+        self.s = requests.Session()
+        self.token = settings.nautobot_token
+        self.s.headers.update({"Authorization": f"Token {self.token}"})
+
+    def make_api_request(
+        self, url: str, payload: dict | None = None, paginated: bool = False
+    ) -> dict | list:
+        endpoint_url = urljoin(self.base_url, url)
+        caller_function = inspect.stack()[self.CALLER_FRAME].function
+
+        logging.debug(
+            "%(caller_function)s payload: %(payload)s",
+            {"payload": payload, "caller_function": caller_function},
+        )
+
+        if paginated:
+            return self._fetch_paginated_data(endpoint_url, payload, caller_function)
+        else:
+            resp = self.s.get(endpoint_url, timeout=10, json=payload)
+            return self._process_response(resp, caller_function)
+
+    def _fetch_paginated_data(
+        self, endpoint_url: str, payload: dict | None, caller_function: str
+    ) -> list:
+        response_items = []
+        url = endpoint_url
+
+        while url is not None:
+            resp = self.s.get(url, timeout=10, json=payload)
+            resp_data = self._process_response(resp, caller_function)
+
+            response_items.extend(resp_data.get("results", []))
+            url = resp_data.get("next")
+
+        return response_items
+
+    def _process_response(self, resp, caller_function: str) -> dict:
+        if resp.content:
+            resp_data = resp.json()
+        else:
+            resp_data = {"status_code": resp.status_code}
+
+        logging.debug(
+            "%(caller_function)s resp: %(resp)s",
+            {"resp": resp_data, "caller_function": caller_function},
+        )
+
+        self._log_and_raise_for_status(resp)
+        return resp_data
+
+    def _log_and_raise_for_status(self, resp):
+        try:
+            resp.raise_for_status()
+        except Exception as e:
+            logging.error(f"HTTP error occurred: {e}")
+            raise

python/diff-nautobot-understack/diff_nautobot_understack/clients/__init__.py

@@ -0,0 +1,12 @@
+import openstack
+
+from diff_nautobot_understack.settings import app_settings as settings
+
+
+class API:
+    def __init__(self):
+        cloud_name = settings.os_cloud
+        debug = settings.debug
+
+        openstack.enable_logging(debug=debug)
+        self.cloud_connection = openstack.connect(cloud=cloud_name)

python/diff-nautobot-understack/diff_nautobot_understack/clients/nautobot.py

@@ -0,0 +1,28 @@
+from diffsync import Adapter
+
+from diff_nautobot_understack.clients.openstack import API
+from diff_nautobot_understack.network import models
+
+
+class Network(Adapter):
+    network = models.NetworkModel
+
+    top_level = ["network"]
+    type = "OpenstackNetwork"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        openstack_api = API()
+        self.cloud = openstack_api.cloud_connection
+
+    def load(self):
+        for network in self.cloud.network.networks():
+            self.add(
+                self.network(
+                    id=network.id,
+                    name=network.name,
+                    status=network.status.lower(),
+                    provider_physical_network=network.provider_physical_network,
+                    vni_id=network.provider_segmentation_id,
+                )
+            )

python/diff-nautobot-understack/diff_nautobot_understack/clients/openstack.py

@@ -0,0 +1,105 @@
+from diffsync import Adapter
+from pydantic import BaseModel
+
+from diff_nautobot_understack.clients.nautobot import API
+from diff_nautobot_understack.network import models
+
+
+class UcvniDetails(BaseModel):
+    id: str
+    name: str
+    status: str
+    ucvni_id: int
+    ucvni_group: str
+    vlan_group: str
+    vlan_id: int
+
+
+class NautobotError(Exception):
+    message = "Nautobot error"
+
+
+class Network(Adapter):
+    CALLER_FRAME = 1
+    network = models.NetworkModel
+
+    top_level = ["network"]
+    type = "UCVNI"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.api_client = API()
+
+    def load(self):
+        ucvni_data: list[UcvniDetails] = self.ucvni_get()
+        for ucvni_item in ucvni_data:
+            network = self.network(
+                id=ucvni_item.id,
+                name=ucvni_item.name,
+                vni_id=ucvni_item.vlan_id,
+                provider_physical_network=ucvni_item.vlan_group,
+                status=ucvni_item.status,
+            )
+            self.add(network)
+
+    def ucvni_get(
+        self,
+    ) -> list[UcvniDetails]:
+        ucvni_detail_list: list[UcvniDetails] = []
+
+        url = "/api/plugins/undercloud-vni/ucvnis/?include=relationship"
+
+        ucvnis_response = self.api_client.make_api_request(
+            f"{url}/?include=relationships", paginated=True
+        )
+
+        for ucvni_item in ucvnis_response:
+            ucvni_group = self.api_client.make_api_request(
+                url=ucvni_item.get("ucvni_group", {}).get("url")
+            )
+            status = self.api_client.make_api_request(
+                url=ucvni_item.get("status", {}).get("url")
+            )
+            vlan_uuid_objects = (
+                ucvni_item.get("relationships", {})
+                .get("ucvni_vlans", {})
+                .get("destination")
+                .get("objects")
+            )
+            vlan_details = self.get_vlan_details(vlan_uuid_objects)
+            vlan_group, vlan_ids = next(iter(vlan_details.items()))
+            ucvni_details = UcvniDetails(
+                id=ucvni_item.get("id"),
+                name=ucvni_item.get("name"),
+                ucvni_id=ucvni_item.get("ucvni_id"),
+                ucvni_group=ucvni_group.get("name"),
+                status=status.get("name").lower(),
+                vlan_group=vlan_group,
+                vlan_id=int(vlan_ids[0]),
+            )
+            ucvni_detail_list.append(ucvni_details)
+        return ucvni_detail_list
+
+    def get_vlan_details(self, vlan_uuid_objects):
+        vlan_uuids = [vlan_uuid_object["id"] for vlan_uuid_object in vlan_uuid_objects]
+        vlan_details = {}
+
+        vlan_uuids_query_params = "&".join(f"id={value}" for value in vlan_uuids)
+        vlan_url = f"/api/ipam/vlans/?{vlan_uuids_query_params}"
+
+        vlans_response = self.api_client.make_api_request(url=vlan_url, paginated=True)
+
+        for vlan_response in vlans_response:
+            vlan_group_url = vlan_response.get("vlan_group", {}).get("url")
+
+            if vlan_group_url:
+                vlan_group_response = self.api_client.make_api_request(
+                    url=vlan_group_url
+                )
+                vlan_group_name = vlan_group_response.get("name")
+                vlan_id = vlan_response.get("vid")
+
+                if vlan_group_name:
+                    vlan_details.setdefault(vlan_group_name, []).append(vlan_id)
+
+        return vlan_details

python/diff-nautobot-understack/diff_nautobot_understack/network/__init__.py

@@ -0,0 +1,25 @@
+from diffsync.diff import Diff
+from diffsync.enum import DiffSyncFlags
+from rich import print
+
+from diff_nautobot_understack.network.adapters.openstack_network import (
+    Network as OpenstackNetwork,
+)
+from diff_nautobot_understack.network.adapters.ucvni import Network as UcvniNetwork
+
+
+def openstack_network_diff_from_ucvni_network() -> Diff:
+    openstack_network = OpenstackNetwork()
+    try:
+        openstack_network.load()
+    except Exception:
+        print("Error retrieving networks from Openstack")
+    ucvni_network = UcvniNetwork()
+    try:
+        ucvni_network.load()
+    except Exception:
+        print("Error retrieving ucvnis from Nautobot")
+    ucvni_network_destination_openstack_source = ucvni_network.diff_from(
+        openstack_network, flags=DiffSyncFlags.CONTINUE_ON_FAILURE
+    )
+    return ucvni_network_destination_openstack_source