From e91a45d4210784ee01abd92651f508346a1e09eb Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Tue, 18 Apr 2023 18:23:14 +0200 Subject: [PATCH 01/17] RM: Bump astarte deps astarte_core, astarte_rpc, astarte_data_access Signed-off-by: Arnaldo Cesco --- .../lib/astarte_realm_management/c_system.ex | 6 +++--- .../lib/astarte_realm_management/engine.ex | 3 +-- apps/astarte_realm_management/mix.exs | 8 ++++---- apps/astarte_realm_management/mix.lock | 6 +++--- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/apps/astarte_realm_management/lib/astarte_realm_management/c_system.ex b/apps/astarte_realm_management/lib/astarte_realm_management/c_system.ex index 35f3545dc..a9df74c77 100644 --- a/apps/astarte_realm_management/lib/astarte_realm_management/c_system.ex +++ b/apps/astarte_realm_management/lib/astarte_realm_management/c_system.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2020 Ispirata Srl +# Copyright 2020 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -67,7 +67,7 @@ defmodule CSystem do with {:ok, res} <- Xandra.execute(conn, query, %{}, consistency: :one) do schema_versions = res - |> Stream.map(&Map.fetch!(&1, "schema_version")) + |> Stream.map(&Map.fetch!(&1, :schema_version)) |> Stream.uniq() |> Enum.to_list() @@ -83,7 +83,7 @@ defmodule CSystem do res |> Enum.take(1) |> List.first() - |> Map.fetch!("schema_version") + |> Map.fetch!(:schema_version) {:ok, schema_version} end diff --git a/apps/astarte_realm_management/lib/astarte_realm_management/engine.ex b/apps/astarte_realm_management/lib/astarte_realm_management/engine.ex index d5ed113f5..a9b1d79b8 100644 --- a/apps/astarte_realm_management/lib/astarte_realm_management/engine.ex +++ b/apps/astarte_realm_management/lib/astarte_realm_management/engine.ex @@ -410,8 +410,7 @@ defmodule Astarte.RealmManagement.Engine do end def execute_interface_deletion(client, realm_name, name, major) do - with {:ok, interface_row} <- - Interface.retrieve_interface_row(realm_name, name, major), + with {:ok, interface_row} <- Interface.retrieve_interface_row(realm_name, name, major), {:ok, descriptor} <- InterfaceDescriptor.from_db_result(interface_row), :ok <- Queries.delete_interface_storage(client, descriptor), :ok <- Queries.delete_devices_with_data_on_interface(client, name) do diff --git a/apps/astarte_realm_management/mix.exs b/apps/astarte_realm_management/mix.exs index 7bac64880..91dfe921a 100644 --- a/apps/astarte_realm_management/mix.exs +++ b/apps/astarte_realm_management/mix.exs @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017-2021 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -63,9 +63,9 @@ defmodule Astarte.RealmManagement.Mixfile do defp astarte_required_modules(_) do [ - {:astarte_core, "~> 1.1"}, - {:astarte_data_access, "~> 1.1"}, - {:astarte_rpc, "~> 1.1"} + {:astarte_core, github: "astarte-platform/astarte_core", override: true}, + {:astarte_data_access, github: "astarte-platform/astarte_data_access"}, + {:astarte_rpc, github: "Annopaolo/astarte_rpc", branch: "delete-device"} ] end diff --git a/apps/astarte_realm_management/mix.lock b/apps/astarte_realm_management/mix.lock index 07ebf3b75..fe31e89c3 100644 --- a/apps/astarte_realm_management/mix.lock +++ b/apps/astarte_realm_management/mix.lock @@ -1,9 +1,9 @@ %{ "amqp": {:hex, :amqp, "2.1.2", "eab047abb54f7e30022b81b9534b797e51c6e7756f1b112ec6dcee3c3ac20eac", [:mix], [{:amqp_client, "~> 3.8.0", [hex: :amqp_client, repo: "hexpm", optional: false]}], "hexpm", "535901c611a979221d045839e9e7a661bf33d04590b796c8fa30f487511fde04"}, "amqp_client": {:hex, :amqp_client, "3.8.35", "e81dbec62057155b5aff857ac9ee85a63af2baf6e0fd4e9d02a3aff46a3de836", [:make, :rebar3], [{:rabbit_common, "3.8.35", [hex: :rabbit_common, repo: "hexpm", optional: false]}], "hexpm", "ca8066e8d12530e31a9879789bc44bb2a4877dcd2d4b65e56b3d301b5e727688"}, - "astarte_core": {:hex, :astarte_core, "1.1.0", "de3ec13feba526ac7ffffe34e822507d9b2ef27c5ca9176c8f81fc32f5fb82ed", [:mix], [{:cyanide, "~> 2.0", [hex: :cyanide, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_morph, "~> 0.1.23", [hex: :ecto_morph, repo: "hexpm", optional: false]}, {:elixir_uuid, "~> 1.2", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:protobuf, "~> 0.12", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "4b6175ec088cf6096fcfc0d02d86b67cf305b42750cad145a64c8bf7f1eabd91"}, - "astarte_data_access": {:hex, :astarte_data_access, "1.1.0", "807677199fde1a53bde55a23fa7fddc6d4bef98d231d414b84fe0068a5c0a918", [:mix], [{:astarte_core, "~> 1.1", [hex: :astarte_core, repo: "hexpm", optional: false]}, {:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:cqex, "~> 1.0", [hex: :cqex, repo: "hexpm", optional: false]}, {:skogsra, "~> 2.2", [hex: :skogsra, repo: "hexpm", optional: false]}, {:xandra, "~> 0.11", [hex: :xandra, repo: "hexpm", optional: false]}], "hexpm", "3bbdb2a66d43b35d762805e73cc28a95cce4cb27a5a61cb37dabe5faae326f21"}, - "astarte_rpc": {:hex, :astarte_rpc, "1.1.0", "61cae0468df48c53cef3a279282aa07b2d758427b5a3c2af8a5993c8f86f9cdf", [:mix], [{:amqp, "~> 2.1", [hex: :amqp, repo: "hexpm", optional: false]}, {:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:protobuf, "~> 0.12", [hex: :protobuf, repo: "hexpm", optional: false]}, {:skogsra, "~> 2.2", [hex: :skogsra, repo: "hexpm", optional: false]}], "hexpm", "1f0933cbd4a8ca8d5624b093abcfc68ef4df27cb38849368072a0d65ffc9a597"}, + "astarte_core": {:git, "https://github.com/astarte-platform/astarte_core.git", "4fcb19e67b5afcaeba569d28847a6756f017c3e2", []}, + "astarte_data_access": {:git, "https://github.com/astarte-platform/astarte_data_access.git", "183cea6c9d2fbad22c313935c20915cbb3a90731", []}, + "astarte_rpc": {:git, "https://github.com/Annopaolo/astarte_rpc.git", "381fcab135b0b749a9e37d8467300273b189ddaf", [branch: "delete-device"]}, "castore": {:hex, :castore, "0.1.18", "deb5b9ab02400561b6f5708f3e7660fc35ca2d51bfc6a940d2f513f89c2975fc", [:mix], [], "hexpm", "61bbaf6452b782ef80b33cdb45701afbcf0a918a45ebe7e73f1130d661e66a06"}, "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, From b82562a9148bb04fd9c649c884e2b99036f4abce Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Fri, 14 Apr 2023 18:55:22 +0200 Subject: [PATCH 02/17] RM: add device deletion queries Signed-off-by: Arnaldo Cesco --- .../lib/astarte_realm_management/engine.ex | 8 + .../lib/astarte_realm_management/queries.ex | 581 ++++++++++++++++++ 2 files changed, 589 insertions(+) diff --git a/apps/astarte_realm_management/lib/astarte_realm_management/engine.ex b/apps/astarte_realm_management/lib/astarte_realm_management/engine.ex index a9b1d79b8..93957b0a2 100644 --- a/apps/astarte_realm_management/lib/astarte_realm_management/engine.ex +++ b/apps/astarte_realm_management/lib/astarte_realm_management/engine.ex @@ -31,6 +31,7 @@ defmodule Astarte.RealmManagement.Engine do alias Astarte.Core.Triggers.Trigger alias Astarte.Core.Triggers.Policy alias Astarte.Core.Triggers.PolicyProtobuf.Policy, as: PolicyProto + alias Astarte.Core.Device alias Astarte.DataAccess.Database alias Astarte.DataAccess.Interface alias Astarte.DataAccess.Mappings @@ -993,4 +994,11 @@ defmodule Astarte.RealmManagement.Engine do {:error, :database_connection_error} end end + + def delete_device(realm_name, device_id) do + # TODO check if allow_extended_id + with {:ok, device_id} <- Device.decode_device_id(device_id) do + Queries.insert_device_into_deleted(realm_name, device_id) + end + end end diff --git a/apps/astarte_realm_management/lib/astarte_realm_management/queries.ex b/apps/astarte_realm_management/lib/astarte_realm_management/queries.ex index 066c12412..2196ae059 100644 --- a/apps/astarte_realm_management/lib/astarte_realm_management/queries.ex +++ b/apps/astarte_realm_management/lib/astarte_realm_management/queries.ex @@ -1655,4 +1655,585 @@ defmodule Astarte.RealmManagement.Queries do {:error, :database_error} end end + + def check_device_exists(realm_name, device_id) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_check_device_exists(&1, realm_name, device_id) + ) + end + + defp do_check_device_exists(conn, realm_name, device_id) do + # TODO: validate realm name + statement = """ + SELECT COUNT (*) + FROM #{realm_name}.devices + WHERE device_id = :device_id + """ + + params = %{device_id: device_id} + + with {:ok, prepared} <- Xandra.prepare(conn, statement), + {:ok, [result]} <- + execute_device_exists_query(conn, prepared, params, + consistency: :quorum, + uuid_format: :binary + ) do + {:ok, device_exists_result_to_boolean(result)} + end + end + + defp device_exists_result_to_boolean(%{count: 1}), do: true + defp device_exists_result_to_boolean(_), do: false + + defp execute_device_exists_query(conn, prepared, params, opts) do + case Xandra.execute(conn, prepared, params, opts) do + {:ok, %Xandra.Page{} = page} -> + {:ok, Enum.to_list(page)} + + {:error, %Xandra.ConnectionError{}} -> + _ = + Logger.warn( + "Cannot check if device exists, connection error", + tag: "check_device_exists_connection_error" + ) + + {:error, :database_connection_error} + + {:error, %Xandra.Error{} = error} -> + _ = + Logger.warn( + "Cannot check if device exists, reason #{error.message}", + tag: "check_device_exists_error" + ) + + {:error, error.reason} + end + end + + def insert_device_into_deletion_in_progress(realm_name, device_id) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_insert_device_into_deletion_in_progress(&1, realm_name, device_id) + ) + end + + defp do_insert_device_into_deletion_in_progress(conn, realm_name, device_id) do + # TODO: validate realm name + statement = """ + INSERT INTO #{realm_name}.deletion_in_progress + (device_id, vmq_ack, dup_ack) + VALUES (:device_id, false, false) + """ + + params = %{device_id: device_id} + + with {:ok, prepared} <- Xandra.prepare(conn, statement) do + case Xandra.execute(conn, prepared, params, + consistency: :quorum, + uuid_format: :binary + ) do + {:ok, result} -> + {:ok, result} + + {:error, %Xandra.ConnectionError{}} -> + _ = + Logger.warn( + "Cannot insert device #{inspect(device_id)} into deleted, connection error", + tag: "insert_device_into_deleted_connection_error" + ) + + {:error, :database_connection_error} + + {:error, %Xandra.Error{} = error} -> + _ = + Logger.warn( + "Cannot insert device #{inspect(device_id)} into deleted, reason #{error.message}", + tag: "insert_device_into_deleted_error" + ) + + {:error, error.reason} + end + end + end + + # TODO maybe move to AstarteDataAccess + def retrieve_device_introspection_map!(realm_name, device_id) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_retrieve_introspection_map!(&1, realm_name, device_id) + ) + end + + defp do_retrieve_introspection_map!(conn, realm_name, device_id) do + # TODO: validate realm name + statement = """ + SELECT introspection + FROM #{realm_name}.devices + WHERE device_id=:device_id + """ + + params = %{device_id: device_id} + prepared = Xandra.prepare!(conn, statement) + + [%{introspection: introspection_map}] = + Xandra.execute!(conn, prepared, params, consistency: :quorum, uuid_format: :binary) + |> Enum.to_list() + + # Introspection might be still empty: handle the nil case + introspection_map || %{} + end + + def retrieve_interface_descriptor!( + realm_name, + interface_name, + interface_major + ) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_retrieve_interface_descriptor!( + &1, + realm_name, + interface_name, + interface_major + ) + ) + end + + defp do_retrieve_interface_descriptor!( + conn, + realm_name, + interface_name, + interface_major + ) do + # TODO: validate realm name + statement = """ + SELECT * + FROM #{realm_name}.interfaces + WHERE name=:name AND major_version=:major_version + """ + + params = %{name: interface_name, major_version: interface_major} + + prepared = Xandra.prepare!(conn, statement) + + Xandra.execute!(conn, prepared, params, consistency: :quorum) + |> Enum.to_list() + # If we're looking for a descriptor of an interface of known name and major, we're fairly sure the result is unique + |> hd() + |> Astarte.Core.InterfaceDescriptor.from_db_result!() + end + + def retrieve_individual_datastreams_keys!(realm_name, device_id) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_retrieve_individual_datastreams_keys!(&1, realm_name, device_id) + ) + end + + defp do_retrieve_individual_datastreams_keys!(conn, realm_name, device_id) do + # TODO: validate realm name + statement = """ + SELECT DISTINCT device_id, interface_id, endpoint_id, path + FROM #{realm_name}.individual_datastreams + WHERE device_id=:device_id ALLOW FILTERING + """ + + params = %{device_id: device_id} + + prepared = Xandra.prepare!(conn, statement) + Xandra.execute!(conn, prepared, params, uuid_format: :binary) |> Enum.to_list() + end + + def delete_individual_datastream_values!( + realm_name, + device_id, + interface_id, + endpoint_id, + path + ) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_delete_individual_datastream_values!( + &1, + realm_name, + device_id, + interface_id, + endpoint_id, + path + ) + ) + end + + defp do_delete_individual_datastream_values!( + conn, + realm_name, + device_id, + interface_id, + endpoint_id, + path + ) do + # TODO: validate realm name + statement = """ + DELETE FROM #{realm_name}.individual_datastreams + WHERE device_id=:device_id AND interface_id=:interface_id + AND endpoint_id=:endpoint_id AND path=:path + """ + + params = %{ + device_id: device_id, + interface_id: interface_id, + endpoint_id: endpoint_id, + path: path + } + + prepared = Xandra.prepare!(conn, statement) + + Xandra.execute!(conn, prepared, params, + consistency: :local_quorum, + uuid_format: :binary + ) + end + + def retrieve_individual_properties_keys!(realm_name, device_id) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_retrieve_individual_properties_keys!(&1, realm_name, device_id) + ) + end + + defp do_retrieve_individual_properties_keys!(conn, realm_name, device_id) do + # TODO: validate realm name + statement = """ + SELECT DISTINCT device_id, interface_id + FROM #{realm_name}.individual_properties + WHERE device_id=:device_id ALLOW FILTERING + """ + + params = %{device_id: device_id} + + prepared = Xandra.prepare!(conn, statement) + Xandra.execute!(conn, prepared, params, uuid_format: :binary) |> Enum.to_list() + end + + def delete_individual_properties_values!(realm_name, device_id, interface_id) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_delete_individual_properties_values!(&1, realm_name, device_id, interface_id) + ) + end + + defp do_delete_individual_properties_values!( + conn, + realm_name, + device_id, + interface_id + ) do + # TODO: validate realm name + statement = """ + DELETE FROM #{realm_name}.individual_properties + WHERE device_id=:device_id AND interface_id=:interface_id + """ + + params = %{ + device_id: device_id, + interface_id: interface_id + } + + prepared = Xandra.prepare!(conn, statement) + + Xandra.execute!(conn, prepared, params, + consistency: :local_quorum, + uuid_format: :binary + ) + end + + def retrieve_object_datastream_keys!(realm_name, device_id, table_name) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_retrieve_object_datastream_keys!(&1, realm_name, device_id, table_name) + ) + end + + defp do_retrieve_object_datastream_keys!( + conn, + realm_name, + device_id, + table_name + ) do + # TODO: validate realm name + statement = """ + SELECT DISTINCT device_id, path + FROM #{realm_name}.#{table_name} + WHERE device_id=:device_id ALLOW FILTERING + """ + + params = %{device_id: device_id} + + prepared = Xandra.prepare!(conn, statement) + Xandra.execute!(conn, prepared, params, uuid_format: :binary) |> Enum.to_list() + end + + def delete_object_datastream_values!(realm_name, device_id, path, table_name) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_delete_object_datastream_values!(&1, realm_name, device_id, path, table_name) + ) + end + + defp do_delete_object_datastream_values!( + conn, + realm_name, + device_id, + path, + table_name + ) do + # TODO: validate realm name + statement = """ + DELETE FROM #{realm_name}.#{table_name} + WHERE device_id=:device_id AND path=:path + """ + + params = %{ + device_id: device_id, + path: path + } + + prepared = Xandra.prepare!(conn, statement) + + Xandra.execute!(conn, prepared, params, + consistency: :local_quorum, + uuid_format: :binary + ) + end + + def retrieve_aliases!(realm_name, device_id) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_retrieve_aliases!(&1, realm_name, device_id) + ) + end + + defp do_retrieve_aliases!(conn, realm_name, device_id) do + # TODO: validate realm name + statement = """ + SELECT object_name + FROM #{realm_name}.names + WHERE object_uuid =:device_id ALLOW FILTERING + """ + + params = %{device_id: device_id} + + prepared = Xandra.prepare!(conn, statement) + Xandra.execute!(conn, prepared, params, uuid_format: :binary) |> Enum.to_list() + end + + def delete_alias_values!(realm_name, device_alias) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_delete_alias_values!(&1, realm_name, device_alias) + ) + end + + defp do_delete_alias_values!(conn, realm_name, device_alias) do + # TODO: validate realm name + statement = """ + DELETE FROM #{realm_name}.names + WHERE object_name = :device_alias + """ + + params = %{device_alias: device_alias} + + prepared = Xandra.prepare!(conn, statement) + + Xandra.execute!(conn, prepared, params, + consistency: :local_quorum, + uuid_format: :binary + ) + end + + def retrieve_groups_keys!(realm_name, device_id) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_retrieve_groups_keys!(&1, realm_name, device_id) + ) + end + + defp do_retrieve_groups_keys!(conn, realm_name, device_id) do + # TODO: validate realm name + statement = """ + SELECT group_name, insertion_uuid, device_id + FROM #{realm_name}.grouped_devices + WHERE device_id=:device_id ALLOW FILTERING + """ + + params = %{device_id: device_id} + + prepared = Xandra.prepare!(conn, statement) + + Xandra.execute!(conn, prepared, params, uuid_format: :binary, timeuuid_format: :binary) + |> Enum.to_list() + end + + def delete_group_values!(realm_name, device_id, group_name, insertion_uuid) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_delete_group_values!(&1, realm_name, device_id, group_name, insertion_uuid) + ) + end + + defp do_delete_group_values!( + conn, + realm_name, + device_id, + group_name, + insertion_uuid + ) do + # TODO: validate realm name + statement = """ + DELETE FROM #{realm_name}.grouped_devices + WHERE group_name = :group_name AND insertion_uuid = :insertion_uuid AND device_id = :device_id + """ + + params = %{ + group_name: group_name, + insertion_uuid: insertion_uuid, + device_id: device_id + } + + prepared = Xandra.prepare!(conn, statement) + + Xandra.execute!(conn, prepared, params, + consistency: :local_quorum, + uuid_format: :binary, + timeuuid_format: :binary + ) + end + + def retrieve_kv_store_entries!(realm_name, device_id) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_retrieve_kv_store_entries!(&1, realm_name, device_id) + ) + end + + defp do_retrieve_kv_store_entries!(conn, realm_name, encoded_device_id) do + # TODO: validate realm name + statement = """ + SELECT group, key + FROM #{realm_name}.kv_store + WHERE key=:key ALLOW FILTERING + """ + + params = %{key: encoded_device_id} + + prepared = Xandra.prepare!(conn, statement) + Xandra.execute!(conn, prepared, params, uuid_format: :binary) |> Enum.to_list() + end + + def delete_kv_store_entry!(realm_name, group, key) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_delete_kv_store_entry!(&1, realm_name, group, key) + ) + end + + defp do_delete_kv_store_entry!(conn, realm_name, group, key) do + # TODO: validate realm name + statement = """ + DELETE FROM #{realm_name}.kv_store + WHERE group = :group AND key = :key + """ + + params = %{group: group, key: key} + + prepared = Xandra.prepare!(conn, statement) + + Xandra.execute!(conn, prepared, params, + consistency: :local_quorum, + uuid_format: :binary + ) + end + + def delete_device!(realm_name, device_id) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_delete_device!(&1, realm_name, device_id) + ) + end + + defp do_delete_device!(conn, realm_name, device_id) do + # TODO: validate realm name + statement = """ + DELETE FROM #{realm_name}.devices + WHERE device_id = :device_id + """ + + params = %{device_id: device_id} + + prepared = Xandra.prepare!(conn, statement) + + Xandra.execute!(conn, prepared, params, + consistency: :local_quorum, + uuid_format: :binary + ) + end + + def remove_device_from_deletion_in_progress!(realm_name, device_id) do + Xandra.Cluster.run( + :xandra_device_deletion, + &do_remove_device_from_deletion_in_progress!(&1, realm_name, device_id) + ) + end + + defp do_remove_device_from_deletion_in_progress!(conn, realm_name, device_id) do + # TODO: validate realm name + statement = """ + DELETE FROM #{realm_name}.deletion_in_progress + WHERE device_id = :device_id + """ + + params = %{device_id: device_id} + + prepared = Xandra.prepare!(conn, statement) + + Xandra.execute!(conn, prepared, params, + consistency: :local_quorum, + uuid_format: :binary + ) + end + + def retrieve_realms!() do + statement = """ + SELECT * + FROM astarte.realms + """ + + realms = + Xandra.Cluster.run( + :xandra, + &Xandra.execute!(&1, statement, %{}, consistency: :local_quorum) + ) + + Enum.to_list(realms) + end + + def retrieve_devices_to_delete!(realm_name) do + Xandra.Cluster.run(:xandra_device_deletion, &do_retrieve_devices_to_delete!(&1, realm_name)) + end + + defp do_retrieve_devices_to_delete!(conn, realm_name) do + # TODO: validate realm name + statement = """ + SELECT * + FROM #{realm_name}.deletion_in_progress + """ + + Xandra.execute!(conn, statement, %{}, + consistency: :local_quorum, + uuid_format: :binary + ) + |> Enum.to_list() + |> Enum.filter(fn %{vmq_ack: vmq_ack, dup_ack: dup_ack} -> vmq_ack and dup_ack end) + end end From 595eb2de7b3e0d7a9a8fbd153fc2fa3e2d8c382f Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Thu, 7 Sep 2023 15:48:06 +0200 Subject: [PATCH 03/17] RM: test device removal queries Signed-off-by: Arnaldo Cesco --- .../lib/astarte_realm_management/queries.ex | 4 +- apps/astarte_realm_management/mix.exs | 4 + .../astarte_realm_management/queries_test.exs | 264 ++++++++++++++- .../test/support/database_fixtures.ex | 189 +++++++++++ .../test/support/database_test_helper.exs | 300 ++++++++++++++++++ 5 files changed, 759 insertions(+), 2 deletions(-) create mode 100644 apps/astarte_realm_management/test/support/database_fixtures.ex diff --git a/apps/astarte_realm_management/lib/astarte_realm_management/queries.ex b/apps/astarte_realm_management/lib/astarte_realm_management/queries.ex index 2196ae059..8fa9b1e9c 100644 --- a/apps/astarte_realm_management/lib/astarte_realm_management/queries.ex +++ b/apps/astarte_realm_management/lib/astarte_realm_management/queries.ex @@ -2234,6 +2234,8 @@ defmodule Astarte.RealmManagement.Queries do uuid_format: :binary ) |> Enum.to_list() - |> Enum.filter(fn %{vmq_ack: vmq_ack, dup_ack: dup_ack} -> vmq_ack and dup_ack end) + |> Enum.filter(fn %{vmq_ack: vmq_ack, dup_start_ack: dup_start_ack, dup_end_ack: dup_end_ack} -> + vmq_ack and dup_start_ack and dup_end_ack + end) end end diff --git a/apps/astarte_realm_management/mix.exs b/apps/astarte_realm_management/mix.exs index 91dfe921a..79a6f8fb3 100644 --- a/apps/astarte_realm_management/mix.exs +++ b/apps/astarte_realm_management/mix.exs @@ -24,6 +24,7 @@ defmodule Astarte.RealmManagement.Mixfile do app: :astarte_realm_management, elixir: "~> 1.14", version: "1.2.0-dev", + elixirc_paths: elixirc_paths(Mix.env()), build_embedded: Mix.env() == :prod, start_permanent: Mix.env() == :prod, test_coverage: [tool: ExCoveralls], @@ -45,6 +46,9 @@ defmodule Astarte.RealmManagement.Mixfile do ] end + defp elixirc_paths(:test), do: ["test/support", "lib"] + defp elixirc_paths(_), do: ["lib"] + defp dialyzer_cache_directory(:ci) do "dialyzer_cache" end diff --git a/apps/astarte_realm_management/test/astarte_realm_management/queries_test.exs b/apps/astarte_realm_management/test/astarte_realm_management/queries_test.exs index 9d52c1082..2f8f0396d 100644 --- a/apps/astarte_realm_management/test/astarte_realm_management/queries_test.exs +++ b/apps/astarte_realm_management/test/astarte_realm_management/queries_test.exs @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017,2018 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ defmodule Astarte.RealmManagement.QueriesTest do alias Astarte.RealmManagement.DatabaseTestHelper alias Astarte.RealmManagement.Queries alias Astarte.RealmManagement.Config + alias Astarte.Core.CQLUtils @object_datastream_interface_json """ { @@ -188,6 +189,8 @@ defmodule Astarte.RealmManagement.QueriesTest do SELECT value_timestamp FROM individual_datastreams WHERE device_id=536be249-aaaa-4e02-9583-5a4833cbfe49 AND interface_id=:interface_id AND endpoint_id=:endpoint_id AND path='/test/:ind/v'; """ + @realm_name "autotestrealm" + setup do with {:ok, client} <- DatabaseTestHelper.connect_to_test_database() do DatabaseTestHelper.seed_test_data(client) @@ -577,4 +580,263 @@ defmodule Astarte.RealmManagement.QueriesTest do assert Queries.get_jwt_public_key_pem(client) == {:ok, DatabaseTestHelper.jwt_public_key_pem_fixture()} end + + test "retrieve and delete individual datastreams for a device" do + device_id = :crypto.strong_rand_bytes(16) + interface_name = "com.an.individual.datastream.Interface" + interface_major = 0 + endpoint = "/%{sensorId}/value" + path = "/0/value" + + DatabaseTestHelper.seed_individual_datastream_test_data!( + realm_name: @realm_name, + device_id: device_id, + interface_name: interface_name, + interface_major: interface_major, + endpoint: endpoint, + path: path + ) + + assert [ + %{ + device_id: ^device_id, + interface_id: interface_id, + endpoint_id: endpoint_id, + path: ^path + } + ] = + Queries.retrieve_individual_datastreams_keys!( + @realm_name, + device_id + ) + + assert ^interface_id = CQLUtils.interface_id(interface_name, interface_major) + + assert ^endpoint_id = CQLUtils.endpoint_id(interface_name, interface_major, endpoint) + + assert %Xandra.Void{} = + Queries.delete_individual_datastream_values!( + @realm_name, + device_id, + interface_id, + endpoint_id, + path + ) + + assert [] = + Queries.retrieve_individual_datastreams_keys!( + @realm_name, + device_id + ) + end + + test "retrieve and delete individual properties for a device" do + device_id = :crypto.strong_rand_bytes(16) + interface_name = "com.an.individual.property.Interface" + interface_major = 0 + + DatabaseTestHelper.seed_individual_properties_test_data!( + realm_name: @realm_name, + device_id: device_id, + interface_name: interface_name, + interface_major: interface_major + ) + + assert [ + %{ + device_id: ^device_id, + interface_id: interface_id + } + ] = + Queries.retrieve_individual_properties_keys!( + @realm_name, + device_id + ) + + assert ^interface_id = CQLUtils.interface_id(interface_name, interface_major) + + assert %Xandra.Void{} = + Queries.delete_individual_properties_values!( + @realm_name, + device_id, + interface_id + ) + + assert [] = + Queries.retrieve_individual_properties_keys!( + @realm_name, + device_id + ) + end + + test "retrieve and delete object datastreams for a device" do + interface_name = "com.object.datastream.Interface" + interface_major = 0 + device_id = :crypto.strong_rand_bytes(16) + path = "/0/value" + + table_name = CQLUtils.interface_name_to_table_name(interface_name, interface_major) + DatabaseTestHelper.create_object_datastream_table!(table_name) + + DatabaseTestHelper.seed_object_datastream_test_data!( + realm_name: @realm_name, + device_id: device_id, + interface_name: interface_name, + interface_major: interface_major, + path: path + ) + + assert [ + %{ + device_id: ^device_id, + path: ^path + } + ] = + Queries.retrieve_object_datastream_keys!( + @realm_name, + device_id, + table_name + ) + + assert %Xandra.Void{} = + Queries.delete_object_datastream_values!( + @realm_name, + device_id, + path, + table_name + ) + + assert [] = + Queries.retrieve_object_datastream_keys!( + @realm_name, + device_id, + table_name + ) + end + + test "retrieve device introspection" do + device_id = :crypto.strong_rand_bytes(16) + interface_name = "com.an.object.datastream.Interface" + interface_major = 0 + + DatabaseTestHelper.add_interface_to_introspection!( + realm_name: @realm_name, + device_id: device_id, + interface_name: interface_name, + interface_major: interface_major + ) + + assert %{^interface_name => ^interface_major} = + Queries.retrieve_device_introspection_map!( + @realm_name, + device_id + ) + end + + test "retrieve interface from introspection" do + interface_name = "com.an.object.datastream.Interface" + interface_major = 0 + + DatabaseTestHelper.seed_interfaces_table_object_test_data!( + realm_name: @realm_name, + interface_name: interface_name, + interface_major: interface_major + ) + + assert %Astarte.Core.InterfaceDescriptor{ + name: ^interface_name, + major_version: ^interface_major + } = + Queries.retrieve_interface_descriptor!( + @realm_name, + interface_name, + interface_major + ) + end + + test "retrieve and delete aliases" do + device_id = :crypto.strong_rand_bytes(16) + device_alias = "a boring device alias" + + DatabaseTestHelper.seed_aliases_test_data!( + realm_name: @realm_name, + device_id: device_id, + device_alias: device_alias + ) + + assert [ + %{ + object_name: ^device_alias + } + ] = Queries.retrieve_aliases!(@realm_name, device_id) + + assert %Xandra.Void{} = + Queries.delete_alias_values!( + @realm_name, + device_alias + ) + + assert [] = Queries.retrieve_aliases!(@realm_name, device_id) + end + + test "retrieve and delete groups" do + device_id = :crypto.strong_rand_bytes(16) + {insertion_uuid, _state} = :uuid.get_v1(:uuid.new(self())) + group = "group" + + DatabaseTestHelper.seed_groups_test_data!( + realm_name: @realm_name, + group_name: group, + insertion_uuid: insertion_uuid, + device_id: device_id + ) + + assert [ + %{ + device_id: ^device_id, + insertion_uuid: ^insertion_uuid, + group_name: ^group + } + ] = Queries.retrieve_groups_keys!(@realm_name, device_id) + + assert %Xandra.Void{} = + Queries.delete_group_values!( + @realm_name, + device_id, + group, + insertion_uuid + ) + + assert [] = Queries.retrieve_groups_keys!(@realm_name, device_id) + end + + test "retrieve and delete kv_store entries" do + interface_name = "com.an.individual.datastream.Interface" + group = "devices-with-data-on-interface-#{interface_name}-v0" + + device_id = :crypto.strong_rand_bytes(16) + encoded_device_id = Astarte.Core.Device.encode_device_id(device_id) + + DatabaseTestHelper.seed_kv_store_test_data!( + realm_name: @realm_name, + group: group, + key: encoded_device_id + ) + + assert [ + %{ + group: ^group, + key: ^encoded_device_id + } + ] = Queries.retrieve_kv_store_entries!(@realm_name, encoded_device_id) + + assert %Xandra.Void{} = + Queries.delete_kv_store_entry!( + @realm_name, + group, + encoded_device_id + ) + + assert [] = Queries.retrieve_kv_store_entries!(@realm_name, encoded_device_id) + end end diff --git a/apps/astarte_realm_management/test/support/database_fixtures.ex b/apps/astarte_realm_management/test/support/database_fixtures.ex new file mode 100644 index 000000000..bd72ea794 --- /dev/null +++ b/apps/astarte_realm_management/test/support/database_fixtures.ex @@ -0,0 +1,189 @@ +# +# This file is part of Astarte. +# +# Copyright 2023 SECO Mind Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +defmodule Astarte.RealmManagement.DatabaseFixtures do + alias Astarte.Core.Device + alias Astarte.Core.CQLUtils + + def datastream_values do + [ + realm_name: "realm#{System.unique_integer([:positive])}", + device_id: Device.random_device_id(), + interface_name: "com.datastream.Interface#{System.unique_integer([:positive])}", + interface_major: System.unique_integer([:positive]), + endpoint: "/%{sensorId}/value", + path: "/the_#{System.unique_integer([:positive])}_th/value", + value_timestamp: DateTime.utc_now(), + reception_timestamp: DateTime.utc_now(), + reception_timestamp_submillis: System.unique_integer([:positive]), + value: System.unique_integer([:positive]) + ] + end + + def properties_values do + [ + realm_name: "realm#{System.unique_integer([:positive])}", + device_id: Device.random_device_id(), + interface_name: "com.properties.Interface#{System.unique_integer([:positive])}", + interface_major: System.unique_integer([:positive]), + endpoint: "/%{sensorId}/value", + path: "/the_#{System.unique_integer([:positive])}_th/value", + reception_timestamp: DateTime.utc_now(), + reception_timestamp_submillis: System.unique_integer([:positive]), + value: System.unique_integer([:positive]) + ] + end + + def introspection_values do + [ + realm_name: "realm#{System.unique_integer([:positive])}", + device_id: Device.random_device_id(), + interface_name: "com.Interface#{System.unique_integer([:positive])}", + interface_major: System.unique_integer([:positive]) + ] + end + + def alias_values do + [ + realm_name: "realm#{System.unique_integer([:positive])}", + device_alias: "alias_n_#{System.unique_integer([:positive])}", + device_id: Device.random_device_id() + ] + end + + def group_values do + [ + realm_name: "realm#{System.unique_integer([:positive])}", + device_id: Device.random_device_id(), + group_name: "group_n_#{System.unique_integer([:positive])}", + insertion_uuid: time_uuid() + ] + end + + def kv_store_values do + [ + realm_name: "realm#{System.unique_integer([:positive])}", + group: "group_n_#{System.unique_integer([:positive])}", + key: "key_n_#{System.unique_integer([:positive])}", + value: "bigintAsBlob(#{System.unique_integer([:positive])})" + ] + end + + def devices_values do + [ + realm_name: "realm#{System.unique_integer([:positive])}", + device_id: Device.random_device_id() + ] + end + + def interfaces_object_values do + interface_name = "com.object.datastream.Interface#{System.unique_integer([:positive])}" + interface_major = System.unique_integer([:positive]) + + [ + realm_name: "realm#{System.unique_integer([:positive])}", + interface_name: interface_name, + interface_major: interface_major, + interface_minor: System.unique_integer([:positive]), + # Object aggregated interfaces have always storage type 5 (:one_object_datastream_dbtable) + storage_type: 5, + storage: CQLUtils.interface_name_to_table_name(interface_name, interface_major), + type: Enum.random([1, 2]), + ownership: Enum.random([1, 2]), + # Object aggregated interfaces have always aggregation type 2 (:object) + aggregation: 2, + automaton_transitions: :erlang.term_to_binary(<<>>), + automaton_accepting_states: :erlang.term_to_binary(<<>>), + description: "", + doc: "" + ] + end + + def compute_interface_fixtures(opts, fixtures) do + fixtures = Keyword.merge(fixtures, opts) + interface_id = CQLUtils.interface_id(fixtures[:interface_name], fixtures[:interface_major]) + + endpoint_id = + CQLUtils.endpoint_id( + fixtures[:interface_name], + fixtures[:interface_major], + fixtures[:endpoint] + ) + + # Xandra accepts only maps, not keyword lists + Enum.into(fixtures, %{ + interface_id: interface_id, + endpoint_id: endpoint_id + }) + end + + def compute_alias_fixtures(opts, fixtures) do + fixtures = Keyword.merge(fixtures, opts) + + # Xandra accepts only maps, not keyword lists + %{ + realm_name: fixtures[:realm_name], + object_name: fixtures[:device_alias], + object_uuid: fixtures[:device_id] + } + end + + def compute_interfaces_object_fixtures(opts, fixtures) do + fixtures = Keyword.merge(fixtures, opts) + interface_id = CQLUtils.interface_id(fixtures[:interface_name], fixtures[:interface_major]) + + # Xandra accepts only maps, not keyword lists + %{ + realm_name: fixtures[:realm_name], + name: fixtures[:interface_name], + major_version: fixtures[:interface_major], + minor_version: fixtures[:interface_minor], + interface_id: interface_id, + storage_type: fixtures[:storage_type], + storage: fixtures[:storage], + type: fixtures[:type], + ownership: fixtures[:ownership], + aggregation: fixtures[:aggregation], + automaton_transitions: fixtures[:automaton_transitions], + automaton_accepting_states: fixtures[:automaton_accepting_states], + description: fixtures[:description], + doc: fixtures[:doc] + } + end + + def compute_introspection_fixtures(opts, fixtures) do + fixtures = Keyword.merge(fixtures, opts) + + # Xandra accepts only maps, not keyword lists + %{ + realm_name: fixtures[:realm_name], + device_id: fixtures[:device_id], + introspection: %{fixtures[:interface_name] => fixtures[:interface_major]} + } + end + + def compute_generic_fixtures(opts, fixtures) do + # Xandra accepts only maps, not keyword lists + Keyword.merge(fixtures, opts) |> Enum.into(%{}) + end + + defp time_uuid do + {time_uuid, _state} = :uuid.get_v1(:uuid.new(self())) + time_uuid + end +end diff --git a/apps/astarte_realm_management/test/support/database_test_helper.exs b/apps/astarte_realm_management/test/support/database_test_helper.exs index a9d11bd1a..e98664083 100644 --- a/apps/astarte_realm_management/test/support/database_test_helper.exs +++ b/apps/astarte_realm_management/test/support/database_test_helper.exs @@ -23,6 +23,7 @@ defmodule Astarte.RealmManagement.DatabaseTestHelper do alias CQEx.Client, as: DatabaseClient alias CQEx.Result, as: DatabaseResult alias Astarte.RealmManagement.Config + alias Astarte.RealmManagement.DatabaseFixtures require Logger @jwt_public_key_pem """ @@ -139,6 +140,69 @@ defmodule Astarte.RealmManagement.DatabaseTestHelper do VALUES ('auth', 'jwt_public_key_pem', varcharAsBlob(:pem)); """ + @create_individual_datastreams_table """ + CREATE TABLE IF NOT EXISTS autotestrealm.individual_datastreams ( + device_id uuid, + interface_id uuid, + endpoint_id uuid, + path varchar, + value_timestamp timestamp, + reception_timestamp timestamp, + reception_timestamp_submillis smallint, + + double_value double, + integer_value int, + boolean_value boolean, + longinteger_value bigint, + string_value varchar, + binaryblob_value blob, + datetime_value timestamp, + doublearray_value list, + integerarray_value list, + booleanarray_value list, + longintegerarray_value list, + stringarray_value list, + binaryblobarray_value list, + datetimearray_value list, + + PRIMARY KEY((device_id, interface_id, endpoint_id, path), value_timestamp, reception_timestamp, reception_timestamp_submillis) + ) + """ + + @create_names_table """ + CREATE TABLE IF NOT EXISTS autotestrealm.names ( + object_name varchar, + object_uuid uuid, + PRIMARY KEY ((object_name)) + ) + """ + + @create_grouped_devices_table """ + CREATE TABLE IF NOT EXISTS autotestrealm.grouped_devices ( + group_name varchar, + insertion_uuid timeuuid, + device_id uuid, + PRIMARY KEY ((group_name), insertion_uuid, device_id) + ) + """ + + @create_deleted_devices_table """ + CREATE TABLE IF NOT EXISTS autotestrealm.deletion_in_progress ( + device_id uuid, + vmq_ack boolean, + dup_ack boolean, + PRIMARY KEY ((device_id)) + ) + """ + + @create_devices_table """ + CREATE TABLE IF NOT EXISTS autotestrealm.devices ( + device_id uuid, + introspection map, + PRIMARY KEY ((device_id)) + ) + """ + def seed_datastream_test_data(client, device_id, interface_name, major, endpoint_id, path) do interface_id = CQLUtils.interface_id(interface_name, major) @@ -265,6 +329,11 @@ defmodule Astarte.RealmManagement.DatabaseTestHelper do DatabaseQuery.call!(client, @create_individual_properties_table) DatabaseQuery.call!(client, @create_kv_store_table) DatabaseQuery.call!(client, @create_simple_triggers_table) + DatabaseQuery.call!(client, @create_individual_datastreams_table) + DatabaseQuery.call!(client, @create_names_table) + DatabaseQuery.call!(client, @create_grouped_devices_table) + DatabaseQuery.call!(client, @create_deleted_devices_table) + DatabaseQuery.call!(client, @create_devices_table) :ok end @@ -305,4 +374,235 @@ defmodule Astarte.RealmManagement.DatabaseTestHelper do def jwt_public_key_pem_fixture do @jwt_public_key_pem end + + def seed_individual_datastream_test_data!(opts) do + %{ + realm_name: realm_name, + interface_name: interface_name, + device_id: device_id + } = + params = + DatabaseFixtures.compute_interface_fixtures( + opts, + DatabaseFixtures.datastream_values() + ) + + Xandra.Cluster.run(:xandra, fn conn -> + statement = """ + INSERT INTO #{realm_name}.individual_datastreams + (device_id, interface_id, endpoint_id, path, value_timestamp, reception_timestamp, reception_timestamp_submillis, integer_value) + VALUES (:device_id, :interface_id, :endpoint_id, :path, :value_timestamp, :reception_timestamp, :reception_timestamp_submillis, :value); + """ + + prepared = Xandra.prepare!(conn, statement) + Xandra.execute!(conn, prepared, params) + + kv_store_statement = "INSERT INTO #{realm_name}.kv_store (group, key) VALUES (:group, :key)" + + kv_store_params = %{ + group: "devices-with-data-on-interface-#{interface_name}-v0", + key: Device.encode_device_id(device_id) + } + + kv_store_prepared = Xandra.prepare!(conn, kv_store_statement) + Xandra.execute!(conn, kv_store_prepared, kv_store_params) + end) + + :ok + end + + def seed_individual_properties_test_data!(opts) do + %{ + realm_name: realm_name + } = + params = + DatabaseFixtures.compute_interface_fixtures(opts, DatabaseFixtures.properties_values()) + + Xandra.Cluster.run(:xandra, fn conn -> + statement = """ + INSERT INTO #{realm_name}.individual_properties + (device_id, interface_id, endpoint_id, path, reception_timestamp, reception_timestamp_submillis, integer_value) + VALUES (:device_id, :interface_id, :endpoint_id, :path, :reception_timestamp, :reception_timestamp_submillis, :value) + """ + + prepared = Xandra.prepare!(conn, statement) + Xandra.execute!(conn, prepared, params) + end) + + :ok + end + + def add_interface_to_introspection!(opts) do + %{realm_name: realm_name} = + params = + DatabaseFixtures.compute_introspection_fixtures( + opts, + DatabaseFixtures.introspection_values() + ) + + Xandra.Cluster.run(:xandra, fn conn -> + statement = """ + INSERT INTO #{realm_name}.devices + (device_id, introspection) + VALUES (:device_id, :introspection) + """ + + prepared = Xandra.prepare!(conn, statement) + Xandra.execute!(conn, prepared, params) + end) + + :ok + end + + def seed_interfaces_table_object_test_data!(opts) do + %{realm_name: realm_name} = + params = + DatabaseFixtures.compute_interfaces_object_fixtures( + opts, + DatabaseFixtures.interfaces_object_values() + ) + + statement = """ + INSERT INTO #{realm_name}.interfaces + (name, major_version, minor_version, interface_id, storage_type, storage, type, ownership, aggregation, automaton_transitions, automaton_accepting_states, description, doc) + VALUES (:name, :major_version, :minor_version, :interface_id, :storage_type, :storage, :type, :ownership, :aggregation, :automaton_transitions, :automaton_accepting_states, :description, :doc) + """ + + prepared = Xandra.Cluster.prepare!(:xandra, statement) + Xandra.Cluster.execute!(:xandra, prepared, params, uuid_format: :binary) + end + + def create_object_datastream_table!(table_name) do + Xandra.Cluster.execute(:xandra, "TRUNCATE TABLE autotestrealm.#{table_name}") + + Xandra.Cluster.execute!(:xandra, """ + CREATE TABLE IF NOT EXISTS autotestrealm.#{table_name} ( + device_id uuid, + path varchar, + PRIMARY KEY((device_id, path)) + ) + """) + end + + def seed_object_datastream_test_data!(opts) do + %{ + interface_name: interface_name, + interface_major: interface_major, + realm_name: realm_name + } = + params = + DatabaseFixtures.compute_interface_fixtures( + opts, + DatabaseFixtures.datastream_values() + ) + + interface_table = CQLUtils.interface_name_to_table_name(interface_name, interface_major) + + Xandra.Cluster.run(:xandra, fn conn -> + statement = """ + INSERT INTO #{realm_name}.#{interface_table} (device_id, path) + VALUES (:device_id, :path); + """ + + prepared = Xandra.prepare!(conn, statement) + Xandra.execute!(conn, prepared, params) + end) + + :ok + end + + def seed_aliases_test_data!(opts) do + %{realm_name: realm_name} = + params = DatabaseFixtures.compute_alias_fixtures(opts, DatabaseFixtures.alias_values()) + + Xandra.Cluster.run(:xandra, fn conn -> + statement = """ + INSERT INTO #{realm_name}.names + (object_name, object_uuid) + VALUES (:object_name, :object_uuid) + """ + + prepared = Xandra.prepare!(conn, statement) + Xandra.execute!(conn, prepared, params) + end) + + :ok + end + + def seed_groups_test_data!(opts) do + %{realm_name: realm_name} = + params = DatabaseFixtures.compute_generic_fixtures(opts, DatabaseFixtures.group_values()) + + Xandra.Cluster.run(:xandra, fn conn -> + statement = """ + INSERT INTO #{realm_name}.grouped_devices + (group_name, insertion_uuid, device_id) + VALUES (:group_name, :insertion_uuid, :device_id) + """ + + prepared = Xandra.prepare!(conn, statement) + Xandra.execute!(conn, prepared, params, uuid_format: :binary, timeuuid_format: :binary) + end) + + :ok + end + + def seed_kv_store_test_data!(opts) do + %{realm_name: realm_name} = + params = DatabaseFixtures.compute_generic_fixtures(opts, DatabaseFixtures.kv_store_values()) + + Xandra.Cluster.run(:xandra, fn conn -> + statement = """ + INSERT INTO #{realm_name}.kv_store + (group, key, value) + VALUES (:group, :key, :value) + """ + + prepared = Xandra.prepare!(conn, statement) + Xandra.execute!(conn, prepared, params) + end) + + :ok + end + + def seed_devices_test_data!(opts) do + %{realm_name: realm_name} = + params = DatabaseFixtures.compute_generic_fixtures(opts, DatabaseFixtures.devices_values()) + + Xandra.Cluster.run(:xandra, fn conn -> + statement = """ + INSERT INTO #{realm_name}.devices + (device_id) + VALUES (:device_id) + """ + + prepared = Xandra.prepare!(conn, statement) + Xandra.execute!(conn, prepared, params, uuid_format: :binary) + end) + + :ok + end + + def await_xandra_connected() do + await_cluster_connected(:xandra) + await_cluster_connected(:xandra_device_deletion) + end + + # Taken from https://github.com/lexhide/xandra/blob/main/test/support/test_helper.ex#L5 + defp await_cluster_connected(cluster, tries \\ 10) do + fun = &Xandra.execute!(&1, "SELECT * FROM system.local") + + case Xandra.Cluster.run(cluster, _options = [], fun) do + {:error, %Xandra.ConnectionError{} = error} -> raise error + _other -> :ok + end + rescue + Xandra.ConnectionError -> + if tries > 0 do + Process.sleep(100) + await_cluster_connected(cluster, tries - 1) + else + raise("Xandra cluster #{inspect(cluster)} exceeded maximum number of connection attempts") + end + end end From ff31c87add89178424e32a21b34f0b07378baad1 Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Tue, 18 Apr 2023 18:23:49 +0200 Subject: [PATCH 04/17] RM: Add device deletion logic Device deletion begins when a device_id is put in the `deletion_in_progress` table. The actual deletion of data is started only when it has been acknowledged by both DUP (two times) and the broker (once). The DeviceRemoverSupervisor supervises DeviceRemover tasks. Since deletion is asynchronous, it may happen that the RM service goes down during deletion, or that a temporary filure brings down the deletion service. Therefore, a supervision tree is put in place. Data deletion is started (or resumed, if needed) by the DeviceRemoval.Scheduler and it is carried out by the DeviceRemoval.DeviceRemover Task. The DeviceRemoverSupervisor supervises DeviceRemovers. --- .../lib/astarte_realm_management.ex | 27 +++- .../device_removal/device_remover.ex | 149 ++++++++++++++++++ .../device_removal/scheduler.ex | 83 ++++++++++ .../lib/astarte_realm_management/engine.ex | 59 ++++++- .../lib/astarte_realm_management/queries.ex | 6 +- .../astarte_realm_management/engine_test.exs | 21 +++ 6 files changed, 330 insertions(+), 15 deletions(-) create mode 100644 apps/astarte_realm_management/lib/astarte_realm_management/device_removal/device_remover.ex create mode 100644 apps/astarte_realm_management/lib/astarte_realm_management/device_removal/scheduler.ex diff --git a/apps/astarte_realm_management/lib/astarte_realm_management.ex b/apps/astarte_realm_management/lib/astarte_realm_management.ex index 9e6867d9d..246777bfe 100644 --- a/apps/astarte_realm_management/lib/astarte_realm_management.ex +++ b/apps/astarte_realm_management/lib/astarte_realm_management.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017-2018 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -41,20 +41,33 @@ defmodule Astarte.RealmManagement do DataAccessConfig.validate!() RPCConfig.validate!() - xandra_options = Config.xandra_options!() + xandra_opts = Config.xandra_options!() - data_access_opts = [xandra_options: xandra_options] - - rm_xandra_opts = Keyword.put(xandra_options, :name, :xandra) + data_access_opts = [xandra_options: xandra_opts] children = [ Astarte.RealmManagementWeb.Telemetry, - {Xandra.Cluster, rm_xandra_opts}, + xandra_cluster_child_spec(xandra_opts: xandra_opts, name: :xandra), + xandra_cluster_child_spec(xandra_opts: xandra_opts, name: :xandra_device_deletion), {Astarte.DataAccess, data_access_opts}, - {Astarte.RPC.AMQP.Server, [amqp_queue: Protocol.amqp_queue(), handler: Handler]} + {Astarte.RPC.AMQP.Server, [amqp_queue: Protocol.amqp_queue(), handler: Handler]}, + {Task.Supervisor, name: Astarte.RealmManagement.DeviceRemoverSupervisor}, + Astarte.RealmManagement.DeviceRemoval.Scheduler ] opts = [strategy: :one_for_one, name: Astarte.RealmManagement.Supervisor] Supervisor.start_link(children, opts) end + + def xandra_cluster_child_spec(opts) do + name = Keyword.fetch!(opts, :name) + + xandra_opts = + Keyword.fetch!(opts, :xandra_opts) + |> Keyword.put(:name, name) + # TODO move to string keys + |> Keyword.put(:atom_keys, true) + + Supervisor.child_spec({Xandra.Cluster, xandra_opts}, id: name) + end end diff --git a/apps/astarte_realm_management/lib/astarte_realm_management/device_removal/device_remover.ex b/apps/astarte_realm_management/lib/astarte_realm_management/device_removal/device_remover.ex new file mode 100644 index 000000000..960fa2142 --- /dev/null +++ b/apps/astarte_realm_management/lib/astarte_realm_management/device_removal/device_remover.ex @@ -0,0 +1,149 @@ +# +# This file is part of Astarte. +# +# Copyright 2023 SECO Mind Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +defmodule Astarte.RealmManagement.DeviceRemoval.DeviceRemover do + @moduledoc """ + This module handles data deletion for a device using a Task. + The Task may fail at any time, notably if the database is not + available. + See Astarte.RealmManagement.DeviceRemoval.Scheduler for handling failures. + """ + + use Task + require Logger + alias Astarte.Core.InterfaceDescriptor + alias Astarte.Core.CQLUtils + alias Astarte.RealmManagement.Queries + alias Astarte.Core.Device + + @spec run(%{:device_id => <<_::128>>, :realm_name => binary()}) :: :ok | no_return() + def run(%{realm_name: realm_name, device_id: device_id}) do + encoded_device_id = Device.encode_device_id(device_id) + _ = Logger.info("Starting to remove device #{encoded_device_id}", tag: "device_delete_start") + + Queries.retrieve_individual_datastreams_keys!(realm_name, device_id) + |> Enum.each(&delete_individual_datastreams_from_key!(realm_name, &1)) + + Queries.retrieve_individual_properties_keys!(realm_name, device_id) + |> Enum.each(&delete_individual_properties_from_key!(realm_name, &1)) + + retrieve_object_datastream_keys!(realm_name, device_id) + |> Enum.each(&delete_object_datastreams_from_key!(realm_name, &1)) + + retrieve_aliases_for_device!(realm_name, device_id) + |> Enum.each(&Queries.delete_alias_values!(realm_name, &1)) + + Queries.retrieve_groups_keys!(realm_name, device_id) + |> Enum.each(&delete_group_from_key!(realm_name, &1)) + + Queries.retrieve_kv_store_entries!(realm_name, encoded_device_id) + |> Enum.each(&delete_kv_store_entry!(realm_name, &1)) + + Queries.delete_device!(realm_name, device_id) + Queries.remove_device_from_deletion_in_progress!(realm_name, device_id) + _ = Logger.info("Successfully removed device #{encoded_device_id}", tag: "device_delete_ok") + :ok + end + + defp delete_individual_datastreams_from_key!(realm_name, key) do + %{ + device_id: device_id, + interface_id: interface_id, + endpoint_id: endpoint_id, + path: path + } = key + + Queries.delete_individual_datastream_values!( + realm_name, + device_id, + interface_id, + endpoint_id, + path + ) + end + + defp delete_individual_properties_from_key!(realm_name, key) do + %{ + device_id: device_id, + interface_id: interface_id + } = key + + Queries.delete_individual_properties_values!(realm_name, device_id, interface_id) + end + + defp check_interface_has_object_aggregation!(realm_name, {interface_name, interface_major}) do + case Queries.retrieve_interface_descriptor!(realm_name, interface_name, interface_major) do + %InterfaceDescriptor{aggregation: :object} -> true + _ -> false + end + end + + defp object_interface_to_table_name({interface_name, interface_major}) do + CQLUtils.interface_name_to_table_name(interface_name, interface_major) + end + + defp retrieve_object_datastream_keys!(realm_name, device_id) do + Queries.retrieve_device_introspection_map!(realm_name, device_id) + |> Enum.filter(&check_interface_has_object_aggregation!(realm_name, &1)) + |> Enum.map(&object_interface_to_table_name/1) + |> Enum.flat_map(&retrieve_object_datastream_table_keys!(realm_name, device_id, &1)) + end + + defp retrieve_object_datastream_table_keys!(realm_name, device_id, table_name) do + Queries.retrieve_object_datastream_keys!( + realm_name, + device_id, + table_name + ) + |> Enum.map(&Map.put(&1, :table_name, table_name)) + end + + defp delete_object_datastreams_from_key!(realm_name, key) do + %{ + device_id: device_id, + path: path, + table_name: table_name + } = key + + Queries.delete_object_datastream_values!(realm_name, device_id, path, table_name) + end + + defp retrieve_aliases_for_device!(realm_name, device_id) do + Queries.retrieve_aliases!(realm_name, device_id) + |> Enum.map(fn %{object_name: device_alias} -> device_alias end) + end + + defp delete_group_from_key!(realm_name, key) do + %{ + device_id: device_id, + group_name: group_name, + insertion_uuid: insertion_uuid + } = key + + Queries.delete_group_values!(realm_name, device_id, group_name, insertion_uuid) + end + + defp delete_kv_store_entry!(realm_name, entry) do + %{ + group: group_name, + key: key + } = entry + + Queries.delete_kv_store_entry!(realm_name, group_name, key) + end +end diff --git a/apps/astarte_realm_management/lib/astarte_realm_management/device_removal/scheduler.ex b/apps/astarte_realm_management/lib/astarte_realm_management/device_removal/scheduler.ex new file mode 100644 index 000000000..af483cfca --- /dev/null +++ b/apps/astarte_realm_management/lib/astarte_realm_management/device_removal/scheduler.ex @@ -0,0 +1,83 @@ +# +# This file is part of Astarte. +# +# Copyright 2023 SECO Mind Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +defmodule Astarte.RealmManagement.DeviceRemoval.Scheduler do + @moduledoc """ + This module is used to start Astarte.RealmManagement.DeviceRemoval.DeviceRemover tasks. + Starting a DeviceRemover may happen either at startup, + when interrupted device deletions are resumed, or when + the delete_device/2 function is called. + A DeviceRemover might fail, therefore a Task.Supervisor + handles restarting it. + """ + + use GenServer + + alias Astarte.RealmManagement.Queries + alias Astarte.RealmManagement.DeviceRemoval.DeviceRemover + + require Logger + + # TODO expose this via config + @reconciliation_timeout :timer.minutes(5) + def start_link(_args) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + def init(_init_arg) do + # TODO: find a way to run start_device_deletion!() manually + schedule_device_deletion() + {:ok, %{}} + end + + def handle_info(:delete_devices, state) do + _ = Logger.debug("Reconciling devices to delete") + + start_device_deletion!() + schedule_device_deletion() + {:noreply, state} + end + + defp start_device_deletion!() do + device_to_delete_list = retrieve_devices_to_delete!() + + Enum.each(device_to_delete_list, &start_device_deletion/1) + end + + defp schedule_device_deletion() do + Process.send_after(self(), :delete_devices, @reconciliation_timeout) + end + + defp start_device_deletion(args) do + Task.Supervisor.start_child( + Astarte.RealmManagement.DeviceRemoverSupervisor, + DeviceRemover, + :run, + [args], + restart: :transient + ) + end + + defp retrieve_devices_to_delete!() do + realms = Queries.retrieve_realms!() + + Enum.flat_map(realms, fn %{realm_name: realm_name} -> + devices = Queries.retrieve_devices_to_delete!(realm_name) + Enum.map(devices, &Map.put(&1, :realm_name, realm_name)) + end) + end +end diff --git a/apps/astarte_realm_management/lib/astarte_realm_management/engine.ex b/apps/astarte_realm_management/lib/astarte_realm_management/engine.ex index 93957b0a2..b866814ca 100644 --- a/apps/astarte_realm_management/lib/astarte_realm_management/engine.ex +++ b/apps/astarte_realm_management/lib/astarte_realm_management/engine.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017-2020 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -995,10 +995,59 @@ defmodule Astarte.RealmManagement.Engine do end end + @doc """ + Starts the deletion of a device. Deletion is carried out asynchronously. + The device removal scheduler will take care of eventually deleting the device. + See Astarte.RealmManagement.DeviceRemoval.Scheduler. + Returns `:ok` or `{:error, reason}`. + """ + @spec delete_device(binary(), Device.encoded_device_id()) :: :ok | {:error, any()} def delete_device(realm_name, device_id) do - # TODO check if allow_extended_id - with {:ok, device_id} <- Device.decode_device_id(device_id) do - Queries.insert_device_into_deleted(realm_name, device_id) - end + # TODO check that realm exists, too + with {:ok, decoded_device_id} <- + Astarte.Core.Device.decode_device_id(device_id, allow_extended_id: true), + {:ok, true} <- check_device_exists(realm_name, decoded_device_id), + {:ok, %Xandra.Void{}} <- + insert_device_into_deletion_in_progress(realm_name, decoded_device_id) do + _ = Logger.info("Added device #{device_id} to deletion in progress") + :ok end + end + + defp check_device_exists(realm_name, device_id) do + case Queries.check_device_exists(realm_name, device_id) do + {:ok, true} -> + {:ok, true} + + {:ok, false} -> + _ = + Logger.warn( + "Device #{inspect(device_id)} does not exist", + tag: "device_not_found" + ) + + {:error, :device_not_found} + + {:error, reason} -> + Logger.warn( + "Cannot check if device #{inspect(device_id)} exists, reason #{inspect(reason)}", + tag: "device_exists_fail" + ) + + {:error, reason} + end + end + + defp insert_device_into_deletion_in_progress(realm_name, device_id) do + with {:error, reason} <- + Queries.insert_device_into_deletion_in_progress(realm_name, device_id) do + _ = + Logger.warn( + "Cannot start deletion of device #{inspect(device_id)}, reason #{inspect(reason)}", + tag: "insert_device_into_deleted_fail" + ) + + {:error, reason} + end + end end diff --git a/apps/astarte_realm_management/lib/astarte_realm_management/queries.ex b/apps/astarte_realm_management/lib/astarte_realm_management/queries.ex index 8fa9b1e9c..5d7720352 100644 --- a/apps/astarte_realm_management/lib/astarte_realm_management/queries.ex +++ b/apps/astarte_realm_management/lib/astarte_realm_management/queries.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -1722,8 +1722,8 @@ defmodule Astarte.RealmManagement.Queries do # TODO: validate realm name statement = """ INSERT INTO #{realm_name}.deletion_in_progress - (device_id, vmq_ack, dup_ack) - VALUES (:device_id, false, false) + (device_id, vmq_ack, dup_start_ack, dup_end_ack) + VALUES (:device_id, false, false, false) """ params = %{device_id: device_id} diff --git a/apps/astarte_realm_management/test/astarte_realm_management/engine_test.exs b/apps/astarte_realm_management/test/astarte_realm_management/engine_test.exs index 7b10138a0..fcb9c60dd 100644 --- a/apps/astarte_realm_management/test/astarte_realm_management/engine_test.exs +++ b/apps/astarte_realm_management/test/astarte_realm_management/engine_test.exs @@ -1454,6 +1454,27 @@ defmodule Astarte.RealmManagement.EngineTest do ) == :ok end + test "existing device starts to be deleted" do + device_id = Astarte.Core.Device.random_device_id() + DatabaseTestHelper.seed_devices_test_data!("autotestrealm", device_id) + + Engine.delete_device(@test_realm_name, device_id) + + statement = """ + SELECT * FROM #{@test_realm_name}.deletion_in_progress + """ + + assert [%{device_id: ^device_id}] = + Xandra.Cluster.execute!(:xandra, statement, %{}, uuid_format: :binary) + |> Enum.to_list() + end + + test "missing device is not deleted" do + device_id = Astarte.Core.Device.random_device_id() + + assert {:error, :device_does_not_exist} = Engine.delete_device(@test_realm_name, device_id) + end + defp unpack_source({:ok, source}) when is_binary(source) do interface_obj = Jason.decode!(source) From a5486ef9c4097278e4c77ff34249520e423b3e9e Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Wed, 19 Apr 2023 18:21:27 +0200 Subject: [PATCH 05/17] RM: Test device deletion logic --- .../device_remover_test.exs | 171 ++++++++++++++++++ .../astarte_realm_management/engine_test.exs | 18 +- .../test/support/database_test_helper.exs | 30 ++- 3 files changed, 208 insertions(+), 11 deletions(-) create mode 100644 apps/astarte_realm_management/test/astarte_realm_management/device_remover_test.exs diff --git a/apps/astarte_realm_management/test/astarte_realm_management/device_remover_test.exs b/apps/astarte_realm_management/test/astarte_realm_management/device_remover_test.exs new file mode 100644 index 000000000..912e881f4 --- /dev/null +++ b/apps/astarte_realm_management/test/astarte_realm_management/device_remover_test.exs @@ -0,0 +1,171 @@ +# +# This file is part of Astarte. +# +# Copyright 2023 SECO Mind Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +defmodule Astarte.RealmManagement.DeviceRemoval.DeviceRemoverTest do + use ExUnit.Case + require Logger + + alias Astarte.Core.CQLUtils + alias Astarte.RealmManagement.DeviceRemoval.DeviceRemover + alias Astarte.RealmManagement.DatabaseTestHelper + alias Astarte.RealmManagement.Queries + + @realm_name "autotestrealm" + @device_id <<39, 243, 128, 160, 158, 152, 245, 223, 26, 43, 66, 116, 153, 237, 184, 43>> + @individual_datastream_interface "com.an.individual.datastream.Interface" + @object_datastream_interface "com.an.object.datastream.Interface" + @property_interface "com.a.property.Interface" + @interface_major 0 + @endpoint "/a/%{endpoint}" + @path "/a/path" + + setup_all do + {:ok, client} = DatabaseTestHelper.connect_to_test_database() + DatabaseTestHelper.create_test_keyspace(client) + seed_device_data!() + + on_exit(fn -> + {:ok, client} = DatabaseTestHelper.connect_to_test_database() + DatabaseTestHelper.drop_test_keyspace(client) + end) + end + + test "device data are successfully removed" do + deletion_prepared = + Xandra.Cluster.prepare!( + :xandra, + """ + INSERT INTO #{@realm_name}.deletion_in_progress (device_id, vmq_ack, dup_start_ack, dup_end_ack) + VALUES (:device_id, false, false, false) + """ + ) + + params = %{device_id: @device_id} + + Xandra.Cluster.execute!(:xandra, deletion_prepared, params) + + :ok = DeviceRemover.run(%{realm_name: @realm_name, device_id: @device_id}) + + object_aggregated_interface_table_name = + CQLUtils.interface_name_to_table_name( + @object_datastream_interface, + @interface_major + ) + + assert [] = Queries.retrieve_individual_datastreams_keys!(@realm_name, @device_id) + assert [] = Queries.retrieve_individual_properties_keys!(@realm_name, @device_id) + + assert [] = + Queries.retrieve_object_datastream_keys!( + @realm_name, + @device_id, + object_aggregated_interface_table_name + ) + + assert [] = Queries.retrieve_aliases!(@realm_name, @device_id) + assert [] = Queries.retrieve_groups_keys!(@realm_name, @device_id) + + assert [] = + Queries.retrieve_kv_store_entries!( + @realm_name, + @device_id |> Astarte.Core.Device.encode_device_id() + ) + + devices_prepared = + Xandra.Cluster.prepare!( + :xandra, + "SELECT * FROM #{@realm_name}.devices WHERE device_id = :device_id" + ) + + assert [] = + Xandra.Cluster.execute!(:xandra, devices_prepared, %{device_id: @device_id}, + uuid_format: :binary + ) + |> Enum.to_list() + end + + defp seed_device_data!() do + DatabaseTestHelper.seed_individual_datastream_test_data!( + realm_name: @realm_name, + device_id: @device_id, + interface_name: @individual_datastream_interface, + interface_major: @interface_major, + endpoint: @endpoint, + path: @path + ) + + DatabaseTestHelper.seed_individual_properties_test_data!( + realm_name: @realm_name, + device_id: @device_id, + interface_name: @property_interface, + interface_major: @interface_major, + endpoint: @endpoint, + path: @path + ) + + DatabaseTestHelper.add_interface_to_introspection!( + realm_name: @realm_name, + device_id: @device_id, + interface_name: @object_datastream_interface, + interface_major: @interface_major + ) + + DatabaseTestHelper.create_object_datastream_table!( + CQLUtils.interface_name_to_table_name(@object_datastream_interface, @interface_major) + ) + + DatabaseTestHelper.seed_interfaces_table_object_test_data!( + realm_name: @realm_name, + interface_name: @object_datastream_interface, + interface_major: @interface_major + ) + + DatabaseTestHelper.seed_object_datastream_test_data!( + realm_name: @realm_name, + device_id: @device_id, + interface_name: @object_datastream_interface, + interface_major: @interface_major, + path: @path + ) + + DatabaseTestHelper.seed_aliases_test_data!( + realm_name: @realm_name, + device_id: @device_id + ) + + DatabaseTestHelper.seed_groups_test_data!( + realm_name: @realm_name, + device_id: @device_id + ) + + DatabaseTestHelper.seed_kv_store_test_data!( + realm_name: @realm_name, + group: "devices-by-interface-#{@individual_datastream_interface}-v#{@interface_major}", + device_id: @device_id |> Astarte.Core.Device.encode_device_id() + ) + + DatabaseTestHelper.seed_kv_store_test_data!( + realm_name: @realm_name, + group: + "devices-with-data-on-interface-#{@individual_datastream_interface}-v#{@interface_major}", + device_id: @device_id |> Astarte.Core.Device.encode_device_id() + ) + + DatabaseTestHelper.seed_devices_test_data!(realm_name: @realm_name, device_id: @device_id) + end +end diff --git a/apps/astarte_realm_management/test/astarte_realm_management/engine_test.exs b/apps/astarte_realm_management/test/astarte_realm_management/engine_test.exs index fcb9c60dd..ee0186004 100644 --- a/apps/astarte_realm_management/test/astarte_realm_management/engine_test.exs +++ b/apps/astarte_realm_management/test/astarte_realm_management/engine_test.exs @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017,2018 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ defmodule Astarte.RealmManagement.EngineTest do alias Astarte.RealmManagement.Engine alias Astarte.Core.Triggers.SimpleTriggerConfig alias Astarte.Core.Triggers.SimpleTriggersProtobuf.TaggedSimpleTrigger + alias Astarte.Core.Device @test_interface_a_0 """ { @@ -1454,11 +1455,12 @@ defmodule Astarte.RealmManagement.EngineTest do ) == :ok end - test "existing device starts to be deleted" do - device_id = Astarte.Core.Device.random_device_id() - DatabaseTestHelper.seed_devices_test_data!("autotestrealm", device_id) + test "begin deletion of an existing device" do + device_id = Device.random_device_id() + encoded_device_id = Device.encode_device_id(device_id) + DatabaseTestHelper.seed_devices_test_data!(realm_name: "autotestrealm", device_id: device_id) - Engine.delete_device(@test_realm_name, device_id) + assert :ok = Engine.delete_device(@test_realm_name, encoded_device_id) statement = """ SELECT * FROM #{@test_realm_name}.deletion_in_progress @@ -1469,10 +1471,10 @@ defmodule Astarte.RealmManagement.EngineTest do |> Enum.to_list() end - test "missing device is not deleted" do - device_id = Astarte.Core.Device.random_device_id() + test "do not begin deletion of a missing device" do + missing_device_id = Device.random_device_id() |> Device.encode_device_id() - assert {:error, :device_does_not_exist} = Engine.delete_device(@test_realm_name, device_id) + assert {:error, :device_not_found} = Engine.delete_device(@test_realm_name, missing_device_id) end defp unpack_source({:ok, source}) when is_binary(source) do diff --git a/apps/astarte_realm_management/test/support/database_test_helper.exs b/apps/astarte_realm_management/test/support/database_test_helper.exs index e98664083..b131130c7 100644 --- a/apps/astarte_realm_management/test/support/database_test_helper.exs +++ b/apps/astarte_realm_management/test/support/database_test_helper.exs @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -40,6 +40,25 @@ defmodule Astarte.RealmManagement.DatabaseTestHelper do durable_writes = true; """ + @create_astarte_keyspace """ + CREATE KEYSPACE astarte + WITH + replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND + durable_writes = true; + """ + + @create_astarte_realms_table """ + CREATE TABLE astarte.realms ( + realm_name ascii, + PRIMARY KEY (realm_name) + ); + """ + + @insert_autotestrealm_into_realms """ + INSERT INTO astarte.realms (realm_name) + VALUES ('autotestrealm'); + """ + @create_interfaces_table """ CREATE TABLE autotestrealm.interfaces ( name ascii, @@ -190,7 +209,8 @@ defmodule Astarte.RealmManagement.DatabaseTestHelper do CREATE TABLE IF NOT EXISTS autotestrealm.deletion_in_progress ( device_id uuid, vmq_ack boolean, - dup_ack boolean, + dup_start_ack boolean, + dup_end_ack boolean, PRIMARY KEY ((device_id)) ) """ @@ -324,6 +344,9 @@ defmodule Astarte.RealmManagement.DatabaseTestHelper do def create_test_keyspace(client) do DatabaseQuery.call!(client, @create_autotestrealm) + DatabaseQuery.call!(client, @create_astarte_keyspace) + DatabaseQuery.call!(client, @create_astarte_realms_table) + DatabaseQuery.call!(client, @insert_autotestrealm_into_realms) DatabaseQuery.call!(client, @create_interfaces_table) DatabaseQuery.call!(client, @create_endpoints_table) DatabaseQuery.call!(client, @create_individual_properties_table) @@ -358,7 +381,8 @@ defmodule Astarte.RealmManagement.DatabaseTestHelper do end def drop_test_keyspace(client) do - with {:ok, _result} <- DatabaseQuery.call(client, "DROP KEYSPACE autotestrealm") do + with {:ok, _result} <- DatabaseQuery.call(client, "DROP KEYSPACE autotestrealm"), + {:ok, _result} <- DatabaseQuery.call(client, "DROP KEYSPACE astarte") do :ok else error -> From e52d94f3520b42bb2fb12ea43e5b368bbb80d496 Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Thu, 20 Apr 2023 16:08:29 +0200 Subject: [PATCH 06/17] RM: handle device deletion RPC --- .../astarte_realm_management/rpc/handler.ex | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/apps/astarte_realm_management/lib/astarte_realm_management/rpc/handler.ex b/apps/astarte_realm_management/lib/astarte_realm_management/rpc/handler.ex index 5d357d887..c5c117eab 100644 --- a/apps/astarte_realm_management/lib/astarte_realm_management/rpc/handler.ex +++ b/apps/astarte_realm_management/lib/astarte_realm_management/rpc/handler.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017-2018 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -50,7 +50,8 @@ defmodule Astarte.RealmManagement.RPC.Handler do GetTriggerPoliciesListReply, GetTriggerPolicySource, GetTriggerPolicySourceReply, - DeleteTriggerPolicy + DeleteTriggerPolicy, + DeleteDevice } alias Astarte.Core.Triggers.Trigger @@ -166,6 +167,10 @@ defmodule Astarte.RealmManagement.RPC.Handler do {:ok, Reply.encode(%Reply{error: false, reply: {:generic_ok_reply, %GenericOkReply{}}})} end + def encode_reply(:delete_device, :ok) do + {:ok, Reply.encode(%Reply{error: false, reply: {:generic_ok_reply, %GenericOkReply{}}})} + end + def encode_reply(_call_atom, {:ok, :started}) do msg = %GenericOkReply{ async_operation: true @@ -369,6 +374,21 @@ defmodule Astarte.RealmManagement.RPC.Handler do ) ) + {:delete_device, + %DeleteDevice{ + realm_name: realm_name, + device_id: device_id + }} -> + _ = Logger.metadata(realm: realm_name) + + encode_reply( + :delete_device, + Engine.delete_device( + realm_name, + device_id + ) + ) + invalid_call -> _ = Logger.warn("Received unexpected call: #{inspect(invalid_call)}.") {:error, :unexpected_call} From e8ca36f7fb9b9e8ad01e25cc4ef3e32a583aab44 Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Thu, 22 Jun 2023 16:48:26 +0200 Subject: [PATCH 07/17] RM API: bump relevant deps astarte_core and astarte_rpc Signed-off-by: Arnaldo Cesco --- apps/astarte_realm_management_api/mix.exs | 6 +++--- apps/astarte_realm_management_api/mix.lock | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/astarte_realm_management_api/mix.exs b/apps/astarte_realm_management_api/mix.exs index d893d586d..ca3a4f1e5 100644 --- a/apps/astarte_realm_management_api/mix.exs +++ b/apps/astarte_realm_management_api/mix.exs @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017-2021 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -74,8 +74,8 @@ defmodule Astarte.RealmManagement.API.Mixfile do defp astarte_required_modules(_) do [ - {:astarte_core, "~> 1.1"}, - {:astarte_rpc, "~> 1.1"} + {:astarte_core, github: "astarte-platform/astarte_core"}, + {:astarte_rpc, github: "Annopaolo/astarte_rpc", branch: "delete-device"} ] end diff --git a/apps/astarte_realm_management_api/mix.lock b/apps/astarte_realm_management_api/mix.lock index 71ceb5eb0..5691935b9 100644 --- a/apps/astarte_realm_management_api/mix.lock +++ b/apps/astarte_realm_management_api/mix.lock @@ -1,8 +1,8 @@ %{ "amqp": {:hex, :amqp, "2.1.1", "ad8dec713ba885afffffcb81feb619fe7cfcbcabe9377ab65ab7a110bd4f43a0", [:mix], [{:amqp_client, "~> 3.8.0", [hex: :amqp_client, repo: "hexpm", optional: false]}], "hexpm", "b6d926770e4508e30e3e9e476c57b6c8aeda44f7715663bdc38935620ce5be6f"}, "amqp_client": {:hex, :amqp_client, "3.8.14", "7569517aefb47e0d1c41bca2f4768dc8a2d88487daf7819fecca0d78943f293c", [:make, :rebar3], [{:rabbit_common, "3.8.14", [hex: :rabbit_common, repo: "hexpm", optional: false]}], "hexpm", "e5ba3ac18abbe34a1d990a6bcac25633dc7061ab8f8d101c7dcff97f49f4c523"}, - "astarte_core": {:hex, :astarte_core, "1.1.0", "de3ec13feba526ac7ffffe34e822507d9b2ef27c5ca9176c8f81fc32f5fb82ed", [:mix], [{:cyanide, "~> 2.0", [hex: :cyanide, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_morph, "~> 0.1.23", [hex: :ecto_morph, repo: "hexpm", optional: false]}, {:elixir_uuid, "~> 1.2", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:protobuf, "~> 0.12", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "4b6175ec088cf6096fcfc0d02d86b67cf305b42750cad145a64c8bf7f1eabd91"}, - "astarte_rpc": {:hex, :astarte_rpc, "1.1.0", "61cae0468df48c53cef3a279282aa07b2d758427b5a3c2af8a5993c8f86f9cdf", [:mix], [{:amqp, "~> 2.1", [hex: :amqp, repo: "hexpm", optional: false]}, {:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:protobuf, "~> 0.12", [hex: :protobuf, repo: "hexpm", optional: false]}, {:skogsra, "~> 2.2", [hex: :skogsra, repo: "hexpm", optional: false]}], "hexpm", "1f0933cbd4a8ca8d5624b093abcfc68ef4df27cb38849368072a0d65ffc9a597"}, + "astarte_core": {:git, "https://github.com/astarte-platform/astarte_core.git", "4fcb19e67b5afcaeba569d28847a6756f017c3e2", []}, + "astarte_rpc": {:git, "https://github.com/Annopaolo/astarte_rpc.git", "381fcab135b0b749a9e37d8467300273b189ddaf", [branch: "delete-device"]}, "castore": {:hex, :castore, "0.1.16", "2675f717adc700475345c5512c381ef9273eb5df26bdd3f8c13e2636cf4cc175", [:mix], [], "hexpm", "28ed2c43d83b5c25d35c51bc0abf229ac51359c170cba76171a462ced2e4b651"}, "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, "cors_plug": {:hex, :cors_plug, "2.0.3", "316f806d10316e6d10f09473f19052d20ba0a0ce2a1d910ddf57d663dac402ae", [:mix], [{:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "ee4ae1418e6ce117fc42c2ba3e6cbdca4e95ecd2fe59a05ec6884ca16d469aea"}, From f1dab599a6cba40aa95bb577335ed091550468c0 Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Fri, 28 Apr 2023 18:23:18 +0200 Subject: [PATCH 08/17] RM API: support device deletion Allow to delete a device with an HTTP request using the DELETE method on the `/{realm_name}/devices/{device_id}` endpoint. Signed-off-by: Arnaldo Cesco --- .../devices/devices.ex | 25 ++++++++++++++ .../rpc/realm_management.ex | 16 +++++++-- .../controllers/device_controller.ex | 34 +++++++++++++++++++ .../controllers/fallback_controller.ex | 16 ++++++++- .../router.ex | 4 ++- .../views/error_view.ex | 10 +++++- 6 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 apps/astarte_realm_management_api/lib/astarte_realm_management_api/devices/devices.ex create mode 100644 apps/astarte_realm_management_api/lib/astarte_realm_management_api_web/controllers/device_controller.ex diff --git a/apps/astarte_realm_management_api/lib/astarte_realm_management_api/devices/devices.ex b/apps/astarte_realm_management_api/lib/astarte_realm_management_api/devices/devices.ex new file mode 100644 index 000000000..59227da49 --- /dev/null +++ b/apps/astarte_realm_management_api/lib/astarte_realm_management_api/devices/devices.ex @@ -0,0 +1,25 @@ +# +# This file is part of Astarte. +# +# Copyright 2023 SECO Mind Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +defmodule Astarte.RealmManagement.API.Devices do + alias Astarte.RealmManagement.API.RPC.RealmManagement + + def delete_device(realm_name, device_id) do + RealmManagement.delete_device(realm_name, device_id) + end +end diff --git a/apps/astarte_realm_management_api/lib/astarte_realm_management_api/rpc/realm_management.ex b/apps/astarte_realm_management_api/lib/astarte_realm_management_api/rpc/realm_management.ex index 056ec8c56..8535cba7a 100644 --- a/apps/astarte_realm_management_api/lib/astarte_realm_management_api/rpc/realm_management.ex +++ b/apps/astarte_realm_management_api/lib/astarte_realm_management_api/rpc/realm_management.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017-2018 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -48,7 +48,8 @@ defmodule Astarte.RealmManagement.API.RPC.RealmManagement do GetTriggerPoliciesListReply, GetTriggerPolicySource, GetTriggerPolicySourceReply, - DeleteTriggerPolicy + DeleteTriggerPolicy, + DeleteDevice } alias Astarte.Core.Triggers.SimpleTriggersProtobuf.TaggedSimpleTrigger @@ -253,6 +254,17 @@ defmodule Astarte.RealmManagement.API.RPC.RealmManagement do |> extract_reply() end + def delete_device(realm_name, device_id) do + %DeleteDevice{ + realm_name: realm_name, + device_id: device_id + } + |> encode_call(:delete_device) + |> @rpc_client.rpc_call(@destination) + |> decode_reply() + |> extract_reply() + end + defp encode_call(call, callname) do %Call{call: {callname, call}} |> Call.encode() diff --git a/apps/astarte_realm_management_api/lib/astarte_realm_management_api_web/controllers/device_controller.ex b/apps/astarte_realm_management_api/lib/astarte_realm_management_api_web/controllers/device_controller.ex new file mode 100644 index 000000000..5a65a9779 --- /dev/null +++ b/apps/astarte_realm_management_api/lib/astarte_realm_management_api_web/controllers/device_controller.ex @@ -0,0 +1,34 @@ +# +# This file is part of Astarte. +# +# Copyright 2023 SECO Mind Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +defmodule Astarte.RealmManagement.APIWeb.DeviceController do + use Astarte.RealmManagement.APIWeb, :controller + + alias Astarte.RealmManagement.API.Devices + + action_fallback Astarte.RealmManagement.APIWeb.FallbackController + + plug Astarte.RealmManagement.APIWeb.Plug.LogRealm + plug Astarte.RealmManagement.APIWeb.Plug.AuthorizePath + + def delete(conn, %{"realm_name" => realm_name, "device_id" => device_id}) do + with :ok <- Devices.delete_device(realm_name, device_id) do + send_resp(conn, :no_content, "") + end + end +end diff --git a/apps/astarte_realm_management_api/lib/astarte_realm_management_api_web/controllers/fallback_controller.ex b/apps/astarte_realm_management_api/lib/astarte_realm_management_api_web/controllers/fallback_controller.ex index 761884bfb..2e6a0ebae 100644 --- a/apps/astarte_realm_management_api/lib/astarte_realm_management_api_web/controllers/fallback_controller.ex +++ b/apps/astarte_realm_management_api/lib/astarte_realm_management_api_web/controllers/fallback_controller.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017-2018 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -108,6 +108,20 @@ defmodule Astarte.RealmManagement.APIWeb.FallbackController do |> render(:cannot_delete_currently_used_trigger_policy) end + def call(conn, {:error, :invalid_device_id}) do + conn + |> put_status(:bad_request) + |> put_view(Astarte.RealmManagement.APIWeb.ErrorView) + |> render(:invalid_device_id) + end + + def call(conn, {:error, :device_not_found}) do + conn + |> put_status(:not_found) + |> put_view(Astarte.RealmManagement.APIWeb.ErrorView) + |> render(:device_not_found) + end + # This is called when no JWT token is present def auth_error(conn, {:unauthenticated, _reason}, _opts) do conn diff --git a/apps/astarte_realm_management_api/lib/astarte_realm_management_api_web/router.ex b/apps/astarte_realm_management_api/lib/astarte_realm_management_api_web/router.ex index a7f49ee4b..368e84b72 100644 --- a/apps/astarte_realm_management_api/lib/astarte_realm_management_api_web/router.ex +++ b/apps/astarte_realm_management_api/lib/astarte_realm_management_api_web/router.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -38,5 +38,7 @@ defmodule Astarte.RealmManagement.APIWeb.Router do resources "/:realm_name/triggers", TriggerController, except: [:new, :edit] resources "/:realm_name/policies", TriggerPolicyController, except: [:new, :edit] + + delete "/:realm_name/devices/:device_id", DeviceController, :delete end end diff --git a/apps/astarte_realm_management_api/lib/astarte_realm_management_api_web/views/error_view.ex b/apps/astarte_realm_management_api/lib/astarte_realm_management_api_web/views/error_view.ex index af6160b09..db9741877 100644 --- a/apps/astarte_realm_management_api/lib/astarte_realm_management_api_web/views/error_view.ex +++ b/apps/astarte_realm_management_api/lib/astarte_realm_management_api_web/views/error_view.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -63,6 +63,14 @@ defmodule Astarte.RealmManagement.APIWeb.ErrorView do %{errors: %{detail: "Overlapping endpoints in interface mappings"}} end + def render("invalid_device_id.json", _assigns) do + %{errors: %{detail: "The provided id is not a valid device id"}} + end + + def render("device_not_found.json", _assigns) do + %{errors: %{detail: "Device not found"}} + end + # In case no render clause matches or no # template is found, let's render it as 500 def template_not_found(_template, assigns) do From 40b7a4477ff71a6b4bfa858eb9228ad206a04112 Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Fri, 28 Apr 2023 18:24:10 +0200 Subject: [PATCH 09/17] RM API: test support for device deletion Signed-off-by: Arnaldo Cesco --- .../devices/devices_test.exs | 39 ++++++++++++++ .../controllers/device_controller_test.exs | 54 +++++++++++++++++++ .../support/astarte_realm_management_mock.ex | 21 +++++++- .../astarte_realm_management_mock_db.ex | 28 +++++++++- 4 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 apps/astarte_realm_management_api/test/astarte_realm_management_api/devices/devices_test.exs create mode 100644 apps/astarte_realm_management_api/test/astarte_realm_management_api_web/controllers/device_controller_test.exs diff --git a/apps/astarte_realm_management_api/test/astarte_realm_management_api/devices/devices_test.exs b/apps/astarte_realm_management_api/test/astarte_realm_management_api/devices/devices_test.exs new file mode 100644 index 000000000..2a0df792d --- /dev/null +++ b/apps/astarte_realm_management_api/test/astarte_realm_management_api/devices/devices_test.exs @@ -0,0 +1,39 @@ +# +# This file is part of Astarte. +# +# Copyright 2023 SECO Mind Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +defmodule Astarte.RealmManagement.API.DevicesTest do + use Astarte.RealmManagement.API.DataCase + + alias Astarte.RealmManagement.API.Devices + alias Astarte.RealmManagement.Mock.DB + + @realm "testrealm" + + test "delete device succeeds when the device exists" do + device_id = :crypto.strong_rand_bytes(16) |> Base.url_encode64(padding: false) + DB.create_device(@realm, device_id) + + assert :ok = Devices.delete_device(@realm, device_id) + end + + test "delete device fails when the device does not exists" do + missing_device_id = :crypto.strong_rand_bytes(16) |> Base.url_encode64(padding: false) + + assert {:error, :device_not_found} = Devices.delete_device(@realm, missing_device_id) + end +end diff --git a/apps/astarte_realm_management_api/test/astarte_realm_management_api_web/controllers/device_controller_test.exs b/apps/astarte_realm_management_api/test/astarte_realm_management_api_web/controllers/device_controller_test.exs new file mode 100644 index 000000000..6a6670035 --- /dev/null +++ b/apps/astarte_realm_management_api/test/astarte_realm_management_api_web/controllers/device_controller_test.exs @@ -0,0 +1,54 @@ +# +# This file is part of Astarte. +# +# Copyright 2023 SECO Mind Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +defmodule Astarte.RealmManagement.APIWeb.DeviceControllerTest do + use Astarte.RealmManagement.APIWeb.ConnCase + + alias Astarte.RealmManagement.API.JWTTestHelper + alias Astarte.RealmManagement.Mock + + @realm "testrealm" + @device_id :crypto.strong_rand_bytes(16) |> Base.url_encode64(padding: false) + @other_device_id :crypto.strong_rand_bytes(16) |> Base.url_encode64(padding: false) + + setup %{conn: conn} do + Mock.DB.create_device(@realm, @device_id) + token = JWTTestHelper.gen_jwt_all_access_token() + + conn = + conn + |> put_req_header("accept", "application/json") + |> put_req_header("authorization", "Bearer #{token}") + + {:ok, conn: conn} + end + + describe "delete" do + test "deletes existing device", %{conn: conn} do + delete_conn = delete(conn, device_path(conn, :delete, @realm, @device_id)) + + assert response(delete_conn, 204) + end + + test "renders error on non-existing device", %{conn: conn} do + delete_conn = delete(conn, device_path(conn, :delete, @realm, @other_device_id)) + + assert json_response(delete_conn, 404)["errors"] != %{} + end + end +end diff --git a/apps/astarte_realm_management_api/test/support/astarte_realm_management_mock.ex b/apps/astarte_realm_management_api/test/support/astarte_realm_management_mock.ex index 84b8b7bca..ffd72dd79 100644 --- a/apps/astarte_realm_management_api/test/support/astarte_realm_management_mock.ex +++ b/apps/astarte_realm_management_api/test/support/astarte_realm_management_mock.ex @@ -22,7 +22,8 @@ defmodule Astarte.RealmManagement.Mock do GetTriggerPoliciesListReply, DeleteTriggerPolicy, GetTriggerPolicySource, - GetTriggerPolicySourceReply + GetTriggerPolicySourceReply, + DeleteDevice } alias Astarte.Core.Interface @@ -228,6 +229,24 @@ defmodule Astarte.RealmManagement.Mock do end end + defp execute_rpc( + {:delete_device, + %DeleteDevice{ + realm_name: realm_name, + device_id: device_id + }} + ) do + with :ok <- DB.delete_device(realm_name, device_id) do + %GenericOkReply{} + |> encode_reply(:generic_ok_reply) + |> ok_wrap + else + {:error, reason} -> + generic_error(reason) + |> ok_wrap + end + end + defp generic_ok(async_operation \\ false) do %GenericOkReply{async_operation: async_operation} |> encode_reply(:generic_ok_reply) diff --git a/apps/astarte_realm_management_api/test/support/astarte_realm_management_mock_db.ex b/apps/astarte_realm_management_api/test/support/astarte_realm_management_mock_db.ex index b7d88b850..c3fb656f6 100644 --- a/apps/astarte_realm_management_api/test/support/astarte_realm_management_mock_db.ex +++ b/apps/astarte_realm_management_api/test/support/astarte_realm_management_mock_db.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2021 Ispirata Srl +# Copyright 2021 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,7 +21,9 @@ defmodule Astarte.RealmManagement.Mock.DB do alias Astarte.Core.Triggers.Policy def start_link do - Agent.start_link(fn -> %{interfaces: %{}, trigger_policies: %{}} end, name: __MODULE__) + Agent.start_link(fn -> %{interfaces: %{}, trigger_policies: %{}, devices: %{}} end, + name: __MODULE__ + ) end def drop_interfaces() do @@ -164,4 +166,26 @@ defmodule Astarte.RealmManagement.Mock.DB do nil end end + + def create_device(realm, device_id) do + Agent.update(__MODULE__, fn %{devices: devices} = state -> + %{state | devices: Map.put(devices, {realm, device_id}, {realm, device_id})} + end) + end + + def get_device(realm, device_id) do + Agent.get(__MODULE__, fn %{devices: devices} -> + Map.get(devices, {realm, device_id}) + end) + end + + def delete_device(realm, device_id) do + if get_device(realm, device_id) == nil do + {:error, :device_not_found} + else + Agent.update(__MODULE__, fn %{devices: devices} = state -> + %{state | devices: Map.delete(devices, {realm, device_id})} + end) + end + end end From 6fc43486e7e1a74eaacf322e9abe50afc6cb8293 Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Thu, 22 Jun 2023 16:47:42 +0200 Subject: [PATCH 10/17] HK: bump astarte core deps astarte_core, astarte_rpc, astarte-data_access Signed-off-by: Arnaldo Cesco --- apps/astarte_housekeeping/mix.exs | 6 +++--- apps/astarte_housekeeping/mix.lock | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/astarte_housekeeping/mix.exs b/apps/astarte_housekeeping/mix.exs index 6b19ddc8a..e4aa1ef85 100644 --- a/apps/astarte_housekeeping/mix.exs +++ b/apps/astarte_housekeeping/mix.exs @@ -71,9 +71,9 @@ defmodule Astarte.Housekeeping.Mixfile do defp astarte_required_modules(_) do [ - {:astarte_core, "~> 1.1"}, - {:astarte_data_access, "~> 1.1"}, - {:astarte_rpc, "~> 1.1"} + {:astarte_core, github: "astarte-platform/astarte_core", override: true}, + {:astarte_data_access, github: "astarte-platform/astarte_data_access"}, + {:astarte_rpc, github: "Annopaolo/astarte_rpc", branch: "delete-device"} ] end diff --git a/apps/astarte_housekeeping/mix.lock b/apps/astarte_housekeeping/mix.lock index 8cec4a01b..dc0211815 100644 --- a/apps/astarte_housekeeping/mix.lock +++ b/apps/astarte_housekeeping/mix.lock @@ -1,9 +1,9 @@ %{ "amqp": {:hex, :amqp, "2.1.1", "ad8dec713ba885afffffcb81feb619fe7cfcbcabe9377ab65ab7a110bd4f43a0", [:mix], [{:amqp_client, "~> 3.8.0", [hex: :amqp_client, repo: "hexpm", optional: false]}], "hexpm", "b6d926770e4508e30e3e9e476c57b6c8aeda44f7715663bdc38935620ce5be6f"}, "amqp_client": {:hex, :amqp_client, "3.8.14", "7569517aefb47e0d1c41bca2f4768dc8a2d88487daf7819fecca0d78943f293c", [:make, :rebar3], [{:rabbit_common, "3.8.14", [hex: :rabbit_common, repo: "hexpm", optional: false]}], "hexpm", "e5ba3ac18abbe34a1d990a6bcac25633dc7061ab8f8d101c7dcff97f49f4c523"}, - "astarte_core": {:hex, :astarte_core, "1.1.0", "de3ec13feba526ac7ffffe34e822507d9b2ef27c5ca9176c8f81fc32f5fb82ed", [:mix], [{:cyanide, "~> 2.0", [hex: :cyanide, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_morph, "~> 0.1.23", [hex: :ecto_morph, repo: "hexpm", optional: false]}, {:elixir_uuid, "~> 1.2", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:protobuf, "~> 0.12", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "4b6175ec088cf6096fcfc0d02d86b67cf305b42750cad145a64c8bf7f1eabd91"}, - "astarte_data_access": {:hex, :astarte_data_access, "1.1.0", "807677199fde1a53bde55a23fa7fddc6d4bef98d231d414b84fe0068a5c0a918", [:mix], [{:astarte_core, "~> 1.1", [hex: :astarte_core, repo: "hexpm", optional: false]}, {:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:cqex, "~> 1.0", [hex: :cqex, repo: "hexpm", optional: false]}, {:skogsra, "~> 2.2", [hex: :skogsra, repo: "hexpm", optional: false]}, {:xandra, "~> 0.11", [hex: :xandra, repo: "hexpm", optional: false]}], "hexpm", "3bbdb2a66d43b35d762805e73cc28a95cce4cb27a5a61cb37dabe5faae326f21"}, - "astarte_rpc": {:hex, :astarte_rpc, "1.1.0", "61cae0468df48c53cef3a279282aa07b2d758427b5a3c2af8a5993c8f86f9cdf", [:mix], [{:amqp, "~> 2.1", [hex: :amqp, repo: "hexpm", optional: false]}, {:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:protobuf, "~> 0.12", [hex: :protobuf, repo: "hexpm", optional: false]}, {:skogsra, "~> 2.2", [hex: :skogsra, repo: "hexpm", optional: false]}], "hexpm", "1f0933cbd4a8ca8d5624b093abcfc68ef4df27cb38849368072a0d65ffc9a597"}, + "astarte_core": {:git, "https://github.com/astarte-platform/astarte_core.git", "4fcb19e67b5afcaeba569d28847a6756f017c3e2", []}, + "astarte_data_access": {:git, "https://github.com/astarte-platform/astarte_data_access.git", "183cea6c9d2fbad22c313935c20915cbb3a90731", []}, + "astarte_rpc": {:git, "https://github.com/Annopaolo/astarte_rpc.git", "381fcab135b0b749a9e37d8467300273b189ddaf", [branch: "delete-device"]}, "castore": {:hex, :castore, "0.1.16", "2675f717adc700475345c5512c381ef9273eb5df26bdd3f8c13e2636cf4cc175", [:mix], [], "hexpm", "28ed2c43d83b5c25d35c51bc0abf229ac51359c170cba76171a462ced2e4b651"}, "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, From 06645509737f1e731bf9612bbf342e6ceb94f3db Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Wed, 26 Apr 2023 18:44:49 +0200 Subject: [PATCH 11/17] HK: support device deletion for pre-existing realms Add a migration for creating the `deletion_in_progress` table. Signed-off-by: Arnaldo Cesco --- .../realm/0005_create_deletion_in_progress_table.sql | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 apps/astarte_housekeeping/priv/migrations/realm/0005_create_deletion_in_progress_table.sql diff --git a/apps/astarte_housekeeping/priv/migrations/realm/0005_create_deletion_in_progress_table.sql b/apps/astarte_housekeeping/priv/migrations/realm/0005_create_deletion_in_progress_table.sql new file mode 100644 index 000000000..f823c5549 --- /dev/null +++ b/apps/astarte_housekeeping/priv/migrations/realm/0005_create_deletion_in_progress_table.sql @@ -0,0 +1,7 @@ +CREATE TABLE deletion_in_progress ( + device_id uuid, + vmq_ack boolean, + dup_start_ack boolean, + dup_end_ack boolean, + PRIMARY KEY (device_id) +); From 85ea0090e59426ce15fc1eeaca5e3696120e25d4 Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Wed, 26 Apr 2023 18:46:48 +0200 Subject: [PATCH 12/17] HK: add table to keep track of devices being deleted Track devices currently under deletion in the `deletion_in_progress` table. It also contains fields to track the acknowledgement of the deletion by DUP and the broker. Signed-off-by: Arnaldo Cesco --- .../lib/astarte_housekeeping/queries.ex | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/apps/astarte_housekeeping/lib/astarte_housekeeping/queries.ex b/apps/astarte_housekeeping/lib/astarte_housekeeping/queries.ex index c16272071..4ade04c7f 100644 --- a/apps/astarte_housekeeping/lib/astarte_housekeeping/queries.ex +++ b/apps/astarte_housekeeping/lib/astarte_housekeeping/queries.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017-2018 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -157,6 +157,7 @@ defmodule Astarte.Housekeeping.Queries do :ok <- create_individual_properties_table(realm_conn), :ok <- create_simple_triggers_table(realm_conn), :ok <- create_grouped_devices_table(realm_conn), + :ok <- create_deletion_in_progress_table(realm_conn), :ok <- insert_realm_public_key(realm_conn, public_key_pem), :ok <- insert_realm_astarte_schema_version(realm_conn), :ok <- insert_realm(realm_conn) do @@ -558,6 +559,35 @@ defmodule Astarte.Housekeeping.Queries do end end + defp create_deletion_in_progress_table({conn, realm}) do + query = """ + CREATE TABLE #{realm}.deletion_in_progress ( + device_id uuid, + vmq_ack boolean, + dup_start_ack boolean, + dup_end_ack boolean, + PRIMARY KEY (device_id) + ); + """ + + case CSystem.execute_schema_change(conn, query) do + {:ok, %Xandra.SchemaChange{}} -> + :ok + + {:error, %Xandra.Error{} = err} -> + _ = Logger.warn("Database error: #{Exception.message(err)}.", tag: "database_error") + {:error, :database_error} + + {:error, %Xandra.ConnectionError{} = err} -> + _ = + Logger.warn("Database connection error: #{Exception.message(err)}.", + tag: "database_connection_error" + ) + + {:error, :database_connection_error} + end + end + defp insert_realm_public_key({conn, realm}, public_key_pem) do query = """ INSERT INTO #{realm}.kv_store (group, key, value) From 354d9b86f94f617ceb5a0bccf533c017a8a94d77 Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Fri, 5 May 2023 16:12:28 +0200 Subject: [PATCH 13/17] DUP: bump astarte core deps Bump cyanide, astarte_core, astarte_rpc, astarte-data_access. Contextually, remove some outdated macros. Signed-off-by: Arnaldo Cesco --- apps/astarte_data_updater_plant/mix.exs | 8 ++++---- apps/astarte_data_updater_plant/mix.lock | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/apps/astarte_data_updater_plant/mix.exs b/apps/astarte_data_updater_plant/mix.exs index b31c70508..4303f3364 100644 --- a/apps/astarte_data_updater_plant/mix.exs +++ b/apps/astarte_data_updater_plant/mix.exs @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017-2021 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -63,9 +63,9 @@ defmodule Astarte.DataUpdaterPlant.Mixfile do defp astarte_required_modules(_) do [ - {:astarte_core, "~> 1.1"}, - {:astarte_data_access, "~> 1.1"}, - {:astarte_rpc, "~> 1.1"} + {:astarte_core, github: "astarte-platform/astarte_core", override: true}, + {:astarte_data_access, github: "astarte-platform/astarte_data_access"}, + {:astarte_rpc, github: "Annopaolo/astarte_rpc", branch: "delete-device"} ] end diff --git a/apps/astarte_data_updater_plant/mix.lock b/apps/astarte_data_updater_plant/mix.lock index 8a57293e3..59bead107 100644 --- a/apps/astarte_data_updater_plant/mix.lock +++ b/apps/astarte_data_updater_plant/mix.lock @@ -1,9 +1,9 @@ %{ "amqp": {:hex, :amqp, "2.1.1", "ad8dec713ba885afffffcb81feb619fe7cfcbcabe9377ab65ab7a110bd4f43a0", [:mix], [{:amqp_client, "~> 3.8.0", [hex: :amqp_client, repo: "hexpm", optional: false]}], "hexpm", "b6d926770e4508e30e3e9e476c57b6c8aeda44f7715663bdc38935620ce5be6f"}, "amqp_client": {:hex, :amqp_client, "3.8.14", "7569517aefb47e0d1c41bca2f4768dc8a2d88487daf7819fecca0d78943f293c", [:make, :rebar3], [{:rabbit_common, "3.8.14", [hex: :rabbit_common, repo: "hexpm", optional: false]}], "hexpm", "e5ba3ac18abbe34a1d990a6bcac25633dc7061ab8f8d101c7dcff97f49f4c523"}, - "astarte_core": {:hex, :astarte_core, "1.1.0", "de3ec13feba526ac7ffffe34e822507d9b2ef27c5ca9176c8f81fc32f5fb82ed", [:mix], [{:cyanide, "~> 2.0", [hex: :cyanide, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_morph, "~> 0.1.23", [hex: :ecto_morph, repo: "hexpm", optional: false]}, {:elixir_uuid, "~> 1.2", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:protobuf, "~> 0.12", [hex: :protobuf, repo: "hexpm", optional: false]}], "hexpm", "4b6175ec088cf6096fcfc0d02d86b67cf305b42750cad145a64c8bf7f1eabd91"}, - "astarte_data_access": {:hex, :astarte_data_access, "1.1.0", "807677199fde1a53bde55a23fa7fddc6d4bef98d231d414b84fe0068a5c0a918", [:mix], [{:astarte_core, "~> 1.1", [hex: :astarte_core, repo: "hexpm", optional: false]}, {:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:cqex, "~> 1.0", [hex: :cqex, repo: "hexpm", optional: false]}, {:skogsra, "~> 2.2", [hex: :skogsra, repo: "hexpm", optional: false]}, {:xandra, "~> 0.11", [hex: :xandra, repo: "hexpm", optional: false]}], "hexpm", "3bbdb2a66d43b35d762805e73cc28a95cce4cb27a5a61cb37dabe5faae326f21"}, - "astarte_rpc": {:hex, :astarte_rpc, "1.1.0", "61cae0468df48c53cef3a279282aa07b2d758427b5a3c2af8a5993c8f86f9cdf", [:mix], [{:amqp, "~> 2.1", [hex: :amqp, repo: "hexpm", optional: false]}, {:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:protobuf, "~> 0.12", [hex: :protobuf, repo: "hexpm", optional: false]}, {:skogsra, "~> 2.2", [hex: :skogsra, repo: "hexpm", optional: false]}], "hexpm", "1f0933cbd4a8ca8d5624b093abcfc68ef4df27cb38849368072a0d65ffc9a597"}, + "astarte_core": {:git, "https://github.com/astarte-platform/astarte_core.git", "4fcb19e67b5afcaeba569d28847a6756f017c3e2", []}, + "astarte_data_access": {:git, "https://github.com/astarte-platform/astarte_data_access.git", "183cea6c9d2fbad22c313935c20915cbb3a90731", []}, + "astarte_rpc": {:git, "https://github.com/Annopaolo/astarte_rpc.git", "381fcab135b0b749a9e37d8467300273b189ddaf", [branch: "delete-device"]}, "castore": {:hex, :castore, "0.1.16", "2675f717adc700475345c5512c381ef9273eb5df26bdd3f8c13e2636cf4cc175", [:mix], [], "hexpm", "28ed2c43d83b5c25d35c51bc0abf229ac51359c170cba76171a462ced2e4b651"}, "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, @@ -22,9 +22,9 @@ "ecto_morph": {:hex, :ecto_morph, "0.1.27", "08dda130f23fe0b5893189dab17c0d791438c59fadf8edb996cad670199790d3", [:mix], [{:ecto, ">= 3.0.3", [hex: :ecto, repo: "hexpm", optional: false]}], "hexpm", "1f2152494c68c3458b8ad7ee316e2f2b414f3364d928c3da4cc2013e7eb23ca9"}, "elixir_uuid": {:hex, :elixir_uuid, "1.2.1", "dce506597acb7e6b0daeaff52ff6a9043f5919a4c3315abb4143f0b00378c097", [:mix], [], "hexpm", "f7eba2ea6c3555cea09706492716b0d87397b88946e6380898c2889d68585752"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, - "excoveralls": {:hex, :excoveralls, "0.15.0", "ac941bf85f9f201a9626cc42b2232b251ad8738da993cf406a4290cacf562ea4", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "9631912006b27eca30a2f3c93562bc7ae15980afb014ceb8147dc5cdd8f376f1"}, + "excoveralls": {:hex, :excoveralls, "0.16.1", "0bd42ed05c7d2f4d180331a20113ec537be509da31fed5c8f7047ce59ee5a7c5", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "dae763468e2008cf7075a64cb1249c97cb4bc71e236c5c2b5e5cdf1cfa2bf138"}, "goldrush": {:hex, :goldrush, "0.1.9", "f06e5d5f1277da5c413e84d5a2924174182fb108dabb39d5ec548b27424cd106", [:rebar3], [], "hexpm", "99cb4128cffcb3227581e5d4d803d5413fa643f4eb96523f77d9e6937d994ceb"}, - "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~>2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"}, + "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~> 2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"}, "jsx": {:hex, :jsx, "2.11.0", "08154624050333919b4ac1b789667d5f4db166dc50e190c4d778d1587f102ee0", [:rebar3], [], "hexpm", "eed26a0d04d217f9eecefffb89714452556cf90eb38f290a27a4d45b9988f8c0"}, From f198bbc79a8e0120e2d03a90a0d9f8ca0872e435 Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Wed, 26 Apr 2023 19:09:48 +0200 Subject: [PATCH 14/17] DUP: handle device deletion A DataUpdater process checks periodically if the related device is being deleted. If that is the case, the process updates its `deletion_in_progress` state subfield to `true` and writes to db that deletion has started (`dup_start_ack` field). From now on, all received messages will be acked but ignored, in order to end consuming all inflight messages. The DataUpdater also performs an RPC to the broker to forcefully disconnect the device and prevent it from reconnecting. The last message to be processed by the DataUpdater is by construction received on the internal `"/f"` path (see https://github.com/astarte-platform/astarte_vmq_plugin/pull/75). After that, the DUP deletion end ack is written to the database and the DataUpdater process is terminated. In order to allow deletion of offline (or old) devices, the RemovalScheduler periodically checks the `deletion_in_progress` table and makes the device process start just to perform the deletion check. Signed-off-by: Arnaldo Cesco --- .../consumers_supervisor.ex | 6 +- .../data_updater.ex | 11 +- .../data_updater/deletion_scheduler.ex | 105 ++++++++++++ .../data_updater/impl.ex | 134 +++++++++++++++- .../data_updater/queries.ex | 149 +++++++++++++++++- .../data_updater/server.ex | 14 +- .../data_updater/state.ex | 6 +- .../rpc/vmq_plugin.ex | 14 +- 8 files changed, 425 insertions(+), 14 deletions(-) create mode 100644 apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/deletion_scheduler.ex diff --git a/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/consumers_supervisor.ex b/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/consumers_supervisor.ex index a266b87e2..764f0eb9e 100644 --- a/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/consumers_supervisor.ex +++ b/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/consumers_supervisor.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2019 Ispirata Srl +# Copyright 2019 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,6 +22,7 @@ defmodule Astarte.DataUpdaterPlant.ConsumersSupervisor do alias Astarte.DataUpdaterPlant.AMQPDataConsumer alias Astarte.DataUpdaterPlant.Config + alias Astarte.DataUpdater.DeletionScheduler def start_link(init_arg) do Supervisor.start_link(__MODULE__, init_arg, name: __MODULE__) @@ -34,7 +35,8 @@ defmodule Astarte.DataUpdaterPlant.ConsumersSupervisor do children = [ {Registry, [keys: :unique, name: Registry.AMQPDataConsumer]}, {AMQPDataConsumer.ConnectionManager, amqp_opts: Config.amqp_consumer_options!()}, - AMQPDataConsumer.Supervisor + AMQPDataConsumer.Supervisor, + DeletionScheduler ] opts = [strategy: :rest_for_one, name: __MODULE__] diff --git a/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater.ex b/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater.ex index cf3970a80..f7d866e29 100644 --- a/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater.ex +++ b/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -160,6 +160,15 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater do |> GenServer.call({:dump_state}) end + def start_device_deletion(realm, encoded_device_id, timestamp) do + with :ok <- verify_device_exists(realm, encoded_device_id) do + message_tracker = get_message_tracker(realm, encoded_device_id, offload_start: true) + + get_data_updater_process(realm, encoded_device_id, message_tracker, offload_start: true) + |> GenServer.call({:start_device_deletion, timestamp}) + end + end + def get_data_updater_process(realm, encoded_device_id, message_tracker, opts \\ []) do with {:ok, device_id} <- Device.decode_device_id(encoded_device_id) do case Registry.lookup(Registry.DataUpdater, {realm, device_id}) do diff --git a/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/deletion_scheduler.ex b/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/deletion_scheduler.ex new file mode 100644 index 000000000..82f574c71 --- /dev/null +++ b/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/deletion_scheduler.ex @@ -0,0 +1,105 @@ +# +# This file is part of Astarte. +# +# Copyright 2023 SECO Mind Srl +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +defmodule Astarte.DataUpdater.DeletionScheduler do + @moduledoc """ + This module sends messages to start deletion to a + Astarte.DataUpdater.Server. When a deletion notice + is received, the Server will start the device deletion + procedure, write the dup_start_ack to db, and + synchronously acknowledge it to the Scheduler. + """ + use GenServer + + alias Astarte.DataUpdaterPlant.DataUpdater.Queries + alias Astarte.DataUpdaterPlant.DataUpdater + alias Astarte.DataUpdaterPlant.Config + alias Astarte.Core.Device + + require Logger + + # TODO expose this via config + @reconciliation_timeout :timer.minutes(5) + def start_link(_args) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + def init(_args) do + # TODO: manually start_device_deletion!() when needed + schedule_next_device_deletion() + {:ok, %{}} + end + + def handle_info(:delete_devices, state) do + _ = Logger.debug("Reconciling devices for whom deletion shall begin") + + start_device_deletion!() + schedule_next_device_deletion() + {:noreply, state} + end + + defp start_device_deletion! do + retrieve_devices_to_delete!() + |> Enum.each(fn %{realm_name: realm_name, encoded_device_id: encoded_device_id} -> + timestamp = now_us_x10_timestamp() + # This must be a call, as we want to be sure this was completed + :ok = DataUpdater.start_device_deletion(realm_name, encoded_device_id, timestamp) + end) + end + + defp schedule_next_device_deletion do + Process.send_after(self(), :delete_devices, @reconciliation_timeout) + end + + defp retrieve_devices_to_delete! do + realms = Queries.retrieve_realms!() + + for %{"realm_name" => realm_name} <- realms, + %{"device_id" => device_id} <- + Queries.retrieve_devices_waiting_to_start_deletion!(realm_name), + encoded_device_id = Device.encode_device_id(device_id), + should_handle_data_from_device?(realm_name, encoded_device_id) do + _ = + Logger.debug("Retrieved device to delete", + tag: "device_to_delete", + realm_name: realm_name, + device_id: encoded_device_id + ) + + %{realm_name: realm_name, encoded_device_id: encoded_device_id} + end + end + + defp should_handle_data_from_device?(realm_name, encoded_device_id) do + # TODO extract a function from Astarte.DataUpdaterPlant.AMQPDataConsumer + # This is the same sharding algorithm used in astarte_vmq_plugin + # Make sure they stay in sync + queue_index = + {realm_name, encoded_device_id} + |> :erlang.phash2(Config.data_queue_total_count!()) + + queue_index >= Config.data_queue_range_start!() and + queue_index <= Config.data_queue_range_end!() + end + + # TODO this is copied from astarte_vmq_plugin + defp now_us_x10_timestamp do + DateTime.utc_now() + |> DateTime.to_unix(:microsecond) + |> Kernel.*(10) + end +end diff --git a/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/impl.ex b/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/impl.ex index 10165fd5d..cca21e0e0 100644 --- a/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/impl.ex +++ b/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/impl.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -50,6 +50,7 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.Impl do @interface_lifespan_decimicroseconds 60 * 10 * 1000 * 10000 @device_triggers_lifespan_decimicroseconds 60 * 10 * 1000 * 10000 @groups_lifespan_decimicroseconds 60 * 10 * 1000 * 10000 + @deletion_refresh_lifespan_decimicroseconds 60 * 10 * 1000 * 10000 def init_state(realm, device_id, message_tracker) do MessageTracker.register_data_updater(message_tracker) @@ -74,7 +75,9 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.Impl do last_seen_message: 0, last_device_triggers_refresh: 0, last_groups_refresh: 0, - trigger_id_to_policy_name: %{} + trigger_id_to_policy_name: %{}, + discard_messages: false, + last_deletion_in_progress_refresh: 0 } encoded_device_id = Device.encode_device_id(device_id) @@ -98,6 +101,11 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.Impl do :ok end + def handle_connection(%State{discard_messages: true} = state, _, message_id, _) do + MessageTracker.discard(state.message_tracker, message_id) + state + end + def handle_connection(state, ip_address_string, message_id, timestamp) do {:ok, db_client} = Database.connect(realm: state.realm) @@ -153,6 +161,11 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.Impl do %{new_state | connected: true, last_seen_message: timestamp} end + def handle_heartbeat(%State{discard_messages: true} = state, _, message_id, _) do + MessageTracker.discard(state.message_tracker, message_id) + state + end + # TODO make this private when all heartbeats will be moved to internal def handle_heartbeat(state, message_id, timestamp) do {:ok, db_client} = Database.connect(realm: state.realm) @@ -168,7 +181,14 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.Impl do end def handle_internal(state, "/heartbeat", _payload, message_id, timestamp) do - handle_heartbeat(state, message_id, timestamp) + {:continue, handle_heartbeat(state, message_id, timestamp)} + end + + def handle_internal(%State{discard_messages: true} = state, "/f", _, message_id, _) do + :ok = Queries.ack_end_device_deletion(state.realm, state.device_id) + _ = Logger.info("End device deletion acked.", tag: "device_delete_ack") + MessageTracker.ack_delivery(state.message_tracker, message_id) + {:stop, state} end def handle_internal(state, path, payload, message_id, timestamp) do @@ -200,7 +220,16 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.Impl do timestamp ) - update_stats(new_state, "", nil, path, payload) + {:continue, update_stats(new_state, "", nil, path, payload)} + end + + def start_device_deletion(state, timestamp) do + {:ok, db_client} = Database.connect(realm: state.realm) + + # Device deletion is among time-based actions + new_state = execute_time_based_actions(state, timestamp, db_client) + + {:ok, new_state} end def handle_disconnection(state, message_id, timestamp) do @@ -456,6 +485,11 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.Impl do :ok end + def handle_data(%State{discard_messages: true} = state, _, _, _, message_id, _) do + MessageTracker.discard(state.message_tracker, message_id) + state + end + def handle_data(state, interface, path, payload, message_id, timestamp) do {:ok, db_client} = Database.connect(realm: state.realm) @@ -1147,6 +1181,11 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.Impl do } end + def handle_introspection(%State{discard_messages: true} = state, _, message_id, _) do + MessageTracker.discard(state.message_tracker, message_id) + state + end + def handle_introspection(state, payload, message_id, timestamp) do with {:ok, new_introspection_list} <- PayloadsDecoder.parse_introspection(payload) do process_introspection(state, new_introspection_list, payload, message_id, timestamp) @@ -1409,6 +1448,11 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.Impl do } end + def handle_control(%State{discard_messages: true} = state, _, _, message_id, _) do + MessageTracker.discard(state.message_tracker, message_id) + state + end + def handle_control(state, "/producer/properties", <<0, 0, 0, 0>>, message_id, timestamp) do {:ok, db_client} = Database.connect(realm: state.realm) @@ -1575,6 +1619,16 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.Impl do update_stats(new_state, "", nil, path, payload) end + def handle_install_volatile_trigger( + %State{discard_messages: true} = state, + _, + message_id, + _ + ) do + MessageTracker.ack_delivery(state.message_tracker, message_id) + state + end + def handle_install_volatile_trigger( state, object_id, @@ -1668,6 +1722,11 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.Impl do end end + def handle_delete_volatile_trigger(%State{discard_messages: true} = state, _, message_id, _) do + MessageTracker.discard(state.message_tracker, message_id) + state + end + def handle_delete_volatile_trigger(state, trigger_id) do {new_volatile, maybe_trigger} = Enum.reduce(state.volatile_triggers, {[], nil}, fn item, {acc, found} -> @@ -1836,6 +1895,55 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.Impl do |> reload_groups_on_expiry(timestamp, db_client) |> purge_expired_interfaces(timestamp) |> reload_device_triggers_on_expiry(timestamp, db_client) + |> reload_device_deletion_status_on_expiry(timestamp, db_client) + end + + defp reload_device_deletion_status_on_expiry(state, timestamp, db_client) do + if state.last_deletion_in_progress_refresh + @deletion_refresh_lifespan_decimicroseconds <= + timestamp do + new_state = maybe_start_device_deletion(db_client, state, timestamp) + %State{new_state | last_deletion_in_progress_refresh: timestamp} + else + state + end + end + + defp maybe_start_device_deletion(db_client, state, timestamp) do + if should_start_device_deletion?(state.realm, state.device_id) do + encoded_device_id = Device.encode_device_id(state.device_id) + + :ok = force_device_deletion_from_broker(state.realm, encoded_device_id) + new_state = set_device_disconnected(state, db_client, timestamp) + + _ = + Logger.info("Stop handling data from device in deletion, device_id #{encoded_device_id}") + + # It's ok to repeat that, as we always write ⊤ + Queries.ack_start_device_deletion(state.realm, state.device_id) + + %State{new_state | discard_messages: true} + else + state + end + end + + defp should_start_device_deletion?(realm_name, device_id) do + case Queries.check_device_deletion_in_progress(realm_name, device_id) do + {:ok, true} -> + true + + {:ok, false} -> + false + + {:error, reason} -> + _ = + Logger.warn( + "Cannot check device deletion status for #{inspect(device_id)}, reason #{inspect(reason)}", + tag: "should_start_device_deletion_fail" + ) + + false + end end defp purge_expired_interfaces(state, timestamp) do @@ -2166,6 +2274,24 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.Impl do end end + defp force_device_deletion_from_broker(realm, encoded_device_id) do + _ = Logger.info("Disconnecting device to be deleted, device_id #{encoded_device_id}") + + case VMQPlugin.delete(realm, encoded_device_id) do + # Successfully disconnected + :ok -> + :ok + + # Not found means it was already disconnected, succeed anyway + {:error, :not_found} -> + :ok + + # Some other error, return it + {:error, reason} -> + {:error, reason} + end + end + defp get_on_data_triggers(state, event, interface_id, endpoint_id) do key = {event, interface_id, endpoint_id} diff --git a/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/queries.ex b/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/queries.ex index 6b052573d..eab8ce6ce 100644 --- a/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/queries.ex +++ b/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/queries.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2018 Ispirata Srl +# Copyright 2018 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -943,4 +943,151 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.Queries do {:error, :database_error} end end + + def ack_end_device_deletion(realm_name, device_id) do + Xandra.Cluster.run( + :xandra, + &do_ack_end_device_deletion(&1, realm_name, device_id) + ) + end + + defp do_ack_end_device_deletion(conn, realm_name, device_id) do + statement = """ + UPDATE #{realm_name}.deletion_in_progress + SET dup_end_ack = true + WHERE device_id = :device_id + """ + + with {:ok, prepared} <- Xandra.prepare(conn, statement), + {:ok, %Xandra.Void{}} <- + Xandra.execute(conn, prepared, %{"device_id" => device_id}, uuid_format: :binary) do + :ok + else + {:error, %Xandra.Error{} = error} -> + _ = + Logger.warn( + "Database error while writing device deletion end ack: #{Exception.message(error)}" + ) + + {:error, :database_error} + + {:error, %Xandra.ConnectionError{} = error} -> + _ = + Logger.warn( + "Database connection error while writing device deletion end ack: #{Exception.message(error)}" + ) + + {:error, :database_connection_error} + end + end + + def ack_start_device_deletion(realm_name, device_id) do + Xandra.Cluster.run( + :xandra, + &do_ack_start_device_deletion(&1, realm_name, device_id) + ) + end + + defp do_ack_start_device_deletion(conn, realm_name, device_id) do + statement = """ + UPDATE #{realm_name}.deletion_in_progress + SET dup_start_ack = true + WHERE device_id = :device_id + """ + + with {:ok, prepared} <- Xandra.prepare(conn, statement), + {:ok, %Xandra.Void{}} <- + Xandra.execute(conn, prepared, %{"device_id" => device_id}, uuid_format: :binary) do + :ok + else + {:error, %Xandra.Error{} = error} -> + _ = + Logger.warn( + "Database error while writing device deletion start ack: #{Exception.message(error)}" + ) + + {:error, :database_error} + + {:error, %Xandra.ConnectionError{} = error} -> + _ = + Logger.warn( + "Database connection error while writing device deletion start ack: #{Exception.message(error)}" + ) + + {:error, :database_connection_error} + end + end + + def check_device_deletion_in_progress(realm_name, device_id) do + Xandra.Cluster.run( + :xandra, + &do_check_device_deletion_in_progress(&1, realm_name, device_id) + ) + end + + defp do_check_device_deletion_in_progress(conn, realm_name, device_id) do + statement = """ + SELECT * + FROM #{realm_name}.deletion_in_progress + WHERE device_id = :device_id + """ + + with {:ok, prepared} <- Xandra.prepare(conn, statement), + {:ok, %Xandra.Page{} = page} <- + Xandra.execute(conn, prepared, %{"device_id" => device_id}, uuid_format: :binary) do + result_not_empty? = not Enum.empty?(page) + {:ok, result_not_empty?} + else + {:error, %Xandra.Error{} = error} -> + _ = + Logger.warn( + "Database error while checking device deletion in progress: #{Exception.message(error)}" + ) + + {:error, :database_error} + + {:error, %Xandra.ConnectionError{} = error} -> + _ = + Logger.warn( + "Database connection error while checking device deletion in progress: #{Exception.message(error)}" + ) + + {:error, :database_connection_error} + end + end + + def retrieve_realms! do + statement = """ + SELECT * + FROM astarte.realms + """ + + realms = + Xandra.Cluster.run( + :xandra, + &Xandra.execute!(&1, statement, %{}, consistency: :local_quorum) + ) + + Enum.to_list(realms) + end + + def retrieve_devices_waiting_to_start_deletion!(realm_name) do + Xandra.Cluster.run( + :xandra, + &do_retrieve_devices_waiting_to_start_deletion!(&1, realm_name) + ) + end + + defp do_retrieve_devices_waiting_to_start_deletion!(conn, realm_name) do + statement = """ + SELECT * + FROM #{realm_name}.deletion_in_progress + """ + + Xandra.execute!(conn, statement, %{}, + consistency: :local_quorum, + uuid_format: :binary + ) + |> Enum.to_list() + end end diff --git a/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/server.ex b/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/server.ex index df8e5442b..7d2edc5a4 100644 --- a/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/server.ex +++ b/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/server.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -67,8 +67,11 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.Server do def handle_cast({:handle_internal, payload, path, message_id, timestamp}, state) do if MessageTracker.can_process_message(state.message_tracker, message_id) do - new_state = Impl.handle_internal(state, payload, path, message_id, timestamp) - {:noreply, new_state} + case Impl.handle_internal(state, payload, path, message_id, timestamp) do + {:continue, new_state} -> {:noreply, new_state} + # No more messages from this device, time out now in order to stop this process + {:stop, new_state} -> {:noreply, new_state, 0} + end else {:noreply, state} end @@ -152,6 +155,11 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.Server do {:reply, state, state, timeout} end + def handle_call({:start_device_deletion, timestamp}, _from, state) do + {result, new_state} = Impl.start_device_deletion(state, timestamp) + {:reply, result, new_state} + end + def handle_info({:initialize, realm, device_id, message_tracker}, nil) do timeout = Config.data_updater_deactivation_interval_ms!() diff --git a/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/state.ex b/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/state.ex index fe8952240..7ee2260c0 100644 --- a/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/state.ex +++ b/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/data_updater/state.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017-2018 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -43,6 +43,8 @@ defmodule Astarte.DataUpdaterPlant.DataUpdater.State do :last_device_triggers_refresh, :last_groups_refresh, :datastream_maximum_storage_retention, - :trigger_id_to_policy_name + :trigger_id_to_policy_name, + :discard_messages, + :last_deletion_in_progress_refresh ] end diff --git a/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/rpc/vmq_plugin.ex b/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/rpc/vmq_plugin.ex index 54ed97646..a1fd654cc 100644 --- a/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/rpc/vmq_plugin.ex +++ b/apps/astarte_data_updater_plant/lib/astarte_data_updater_plant/rpc/vmq_plugin.ex @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2018 Ispirata Srl +# Copyright 2018 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,6 +25,7 @@ defmodule Astarte.DataUpdaterPlant.RPC.VMQPlugin do alias Astarte.RPC.Protocol.VMQ.Plugin.{ Call, + Delete, Disconnect, GenericErrorReply, GenericOkReply, @@ -53,6 +54,17 @@ defmodule Astarte.DataUpdaterPlant.RPC.VMQPlugin do end end + def delete(realm_name, device_id) do + %Delete{ + realm_name: realm_name, + device_id: device_id + } + |> encode_call(:delete) + |> @rpc_client.rpc_call(@destination) + |> decode_reply() + |> extract_reply() + end + def disconnect(client_id, discard_state) when is_binary(client_id) and is_boolean(discard_state) do %Disconnect{ From 3caf93ad8c4af81051f528e6babe151fb2f7468c Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Thu, 11 May 2023 17:29:48 +0200 Subject: [PATCH 15/17] DUP: enable mocks using Mox Add the Mox library in order to mock the Astarte RPC client. Signed-off-by: Arnaldo Cesco --- apps/astarte_data_updater_plant/mix.exs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/astarte_data_updater_plant/mix.exs b/apps/astarte_data_updater_plant/mix.exs index 4303f3364..8ab4257ad 100644 --- a/apps/astarte_data_updater_plant/mix.exs +++ b/apps/astarte_data_updater_plant/mix.exs @@ -26,6 +26,7 @@ defmodule Astarte.DataUpdaterPlant.Mixfile do version: "1.2.0-dev", build_embedded: Mix.env() == :prod, start_permanent: Mix.env() == :prod, + elixirc_paths: elixirc_paths(Mix.env()), test_coverage: [tool: ExCoveralls], preferred_cli_env: [ coveralls: :test, @@ -45,6 +46,10 @@ defmodule Astarte.DataUpdaterPlant.Mixfile do ] end + # Compile order is relevant: we make sure support files are available when testing + defp elixirc_paths(:test), do: ["test/support", "lib"] + defp elixirc_paths(_), do: ["lib"] + defp dialyzer_cache_directory(:ci) do "dialyzer_cache" end From 4054ab826d74af17b08dd4c6bcc338b9183db7f9 Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Fri, 5 May 2023 16:19:58 +0200 Subject: [PATCH 16/17] DUP: add tests for supporting device deletion Signed-off-by: Arnaldo Cesco --- .../config/test.exs | 2 + apps/astarte_data_updater_plant/mix.exs | 1 + apps/astarte_data_updater_plant/mix.lock | 2 +- .../data_updater_test.exs | 163 +++++++++++++++++- .../test/support/database_test_helper.exs | 14 +- .../test/support/mocks.ex | 1 + 6 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 apps/astarte_data_updater_plant/test/support/mocks.ex diff --git a/apps/astarte_data_updater_plant/config/test.exs b/apps/astarte_data_updater_plant/config/test.exs index e9bc2fbe4..9114616d7 100644 --- a/apps/astarte_data_updater_plant/config/test.exs +++ b/apps/astarte_data_updater_plant/config/test.exs @@ -8,3 +8,5 @@ config :astarte_data_updater_plant, :amqp_consumer_options, config :logger, :console, format: {PrettyLog.UserFriendlyFormatter, :format}, metadata: [:realm, :device_id, :function] + +config :astarte_data_updater_plant, :rpc_client, MockRPCClient diff --git a/apps/astarte_data_updater_plant/mix.exs b/apps/astarte_data_updater_plant/mix.exs index 8ab4257ad..a14ac28cf 100644 --- a/apps/astarte_data_updater_plant/mix.exs +++ b/apps/astarte_data_updater_plant/mix.exs @@ -80,6 +80,7 @@ defmodule Astarte.DataUpdaterPlant.Mixfile do {:castore, "~> 0.1.0"}, {:cyanide, "~> 2.0"}, {:excoveralls, "~> 0.15", only: :test}, + {:mox, "~> 1.0", only: :test}, {:pretty_log, "~> 0.1"}, {:plug_cowboy, "~> 2.1"}, {:telemetry_metrics_prometheus_core, "~> 0.4"}, diff --git a/apps/astarte_data_updater_plant/mix.lock b/apps/astarte_data_updater_plant/mix.lock index 59bead107..85e52db7d 100644 --- a/apps/astarte_data_updater_plant/mix.lock +++ b/apps/astarte_data_updater_plant/mix.lock @@ -34,7 +34,7 @@ "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mime": {:hex, :mime, "1.5.0", "203ef35ef3389aae6d361918bf3f952fa17a09e8e43b5aa592b93eba05d0fb8d", [:mix], [], "hexpm", "55a94c0f552249fc1a3dd9cd2d3ab9de9d3c89b559c2bd01121f824834f24746"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, - "mox": {:hex, :mox, "0.5.2", "55a0a5ba9ccc671518d068c8dddd20eeb436909ea79d1799e2209df7eaa98b6c", [:mix], [], "hexpm", "df4310628cd628ee181df93f50ddfd07be3e5ecc30232d3b6aadf30bdfe6092b"}, + "mox": {:hex, :mox, "1.0.2", "dc2057289ac478b35760ba74165b4b3f402f68803dd5aecd3bfd19c183815d64", [:mix], [], "hexpm", "f9864921b3aaf763c8741b5b8e6f908f44566f1e427b2630e89e9a73b981fef2"}, "observer_cli": {:hex, :observer_cli, "1.6.1", "d176f967c978ab8b8a29c35c12524f78b7bb36fd4e9b8276dd75c9cb56e07e42", [:mix, :rebar3], [{:recon, "~>2.5.1", [hex: :recon, repo: "hexpm", optional: false]}], "hexpm", "3418e319764b9dff1f469e43cbdffd7fd54ea47cbf765027c557abd146a19fb3"}, "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"}, "plug": {:hex, :plug, "1.11.1", "f2992bac66fdae679453c9e86134a4201f6f43a687d8ff1cd1b2862d53c80259", [:mix], [{:mime, "~> 1.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "23524e4fefbb587c11f0833b3910bfb414bf2e2534d61928e920f54e3a1b881f"}, diff --git a/apps/astarte_data_updater_plant/test/astarte_data_updater_plant/data_updater_test.exs b/apps/astarte_data_updater_plant/test/astarte_data_updater_plant/data_updater_test.exs index dc2a92c00..188305c90 100644 --- a/apps/astarte_data_updater_plant/test/astarte_data_updater_plant/data_updater_test.exs +++ b/apps/astarte_data_updater_plant/test/astarte_data_updater_plant/data_updater_test.exs @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017,2018 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,6 +18,8 @@ defmodule Astarte.DataUpdaterPlant.DataUpdaterTest do use ExUnit.Case + import Mox + alias Astarte.Core.Device alias Astarte.Core.Triggers.SimpleEvents.DeviceConnectedEvent alias Astarte.Core.Triggers.SimpleEvents.IncomingDataEvent @@ -40,6 +42,19 @@ defmodule Astarte.DataUpdaterPlant.DataUpdaterTest do alias Astarte.Core.CQLUtils alias CQEx.Query, as: DatabaseQuery alias CQEx.Result, as: DatabaseResult + alias Astarte.RPC.Protocol.VMQ.Plugin, as: Protocol + + alias Astarte.RPC.Protocol.VMQ.Plugin.{ + Call, + Delete, + GenericOkReply, + Disconnect, + Reply + } + + @vmq_plugin_destination Protocol.amqp_queue() + @encoded_generic_ok_reply %Reply{reply: {:generic_ok_reply, %GenericOkReply{}}} + |> Reply.encode() setup_all do {:ok, _client} = Astarte.DataUpdaterPlant.DatabaseTestHelper.create_test_keyspace() @@ -1652,6 +1667,152 @@ defmodule Astarte.DataUpdaterPlant.DataUpdaterTest do DataUpdater.dump_state(realm, encoded_device_id) end + setup [:set_mox_from_context, :verify_on_exit!] + + test "device deletion is acked and related DataUpdater process stops" do + AMQPTestHelper.clean_queue() + + realm = "autotestrealm" + + encoded_device_id = + :crypto.strong_rand_bytes(16) + |> Base.url_encode64(padding: false) + + {:ok, device_id} = Device.decode_device_id(encoded_device_id) + + # Register the device with some fake data + total_received_messages = 42 + total_received_bytes = 4242 + + insert_opts = [ + total_received_msgs: total_received_messages, + total_received_bytes: total_received_bytes + ] + + DatabaseTestHelper.insert_device(device_id, insert_opts) + + # Set device deletion to in progress + deletion_in_progress_statement = """ + INSERT INTO #{realm}.deletion_in_progress (device_id) + VALUES (:device_id) + """ + + Xandra.Cluster.run(:xandra, fn conn -> + prepared = Xandra.prepare!(conn, deletion_in_progress_statement) + + %Xandra.Void{} = + Xandra.execute!(conn, prepared, %{"device_id" => device_id}, uuid_format: :binary) + end) + + # We expect that sooner or later the device will be disconnected + MockRPCClient + |> expect(:rpc_call, fn serialized_call, @vmq_plugin_destination -> + assert %Call{call: {:delete, %Delete{} = delete_call}} = Call.decode(serialized_call) + + assert %Delete{ + realm_name: realm, + device_id: encoded_device_id + } = delete_call + + {:ok, @encoded_generic_ok_reply} + end) + + timestamp_us_x_10 = make_timestamp("2017-10-09T15:00:32+00:00") + timestamp_ms = div(timestamp_us_x_10, 10_000) + + DataUpdater.start_device_deletion(realm, encoded_device_id, timestamp_ms) + + # Check DUP start ack in deleted_devices table + dup_start_ack_statement = """ + SELECT dup_start_ack + FROM #{realm}.deletion_in_progress + WHERE device_id = :device_id + """ + + dup_start_ack_result = + Xandra.Cluster.run(:xandra, fn conn -> + prepared = Xandra.prepare!(conn, dup_start_ack_statement) + + %Xandra.Page{} = + page = + Xandra.execute!(conn, prepared, %{"device_id" => device_id}, uuid_format: :binary) + + Enum.to_list(page) + end) + + assert [%{"dup_start_ack" => true}] = dup_start_ack_result + + # Check that no data is being handled + DataUpdater.handle_data( + realm, + encoded_device_id, + "this.interface.does.not.Exist", + "/don/t/care", + :dontcare, + gen_tracking_id(), + make_timestamp("2017-10-09T14:30:15+00:00") + ) + + received_data_statement = """ + SELECT total_received_msgs, total_received_bytes + FROM #{realm}.devices WHERE device_id=:device_id; + """ + + received_data_result = + Xandra.Cluster.run(:xandra, fn conn -> + prepared = Xandra.prepare!(conn, received_data_statement) + + %Xandra.Page{} = + page = + Xandra.execute!(conn, prepared, %{"device_id" => device_id}, uuid_format: :binary) + + Enum.to_list(page) + end) + + assert [ + %{ + "total_received_msgs" => ^total_received_messages, + "total_received_bytes" => ^total_received_bytes + } + ] = received_data_result + + # Now process the device's last message + DataUpdater.handle_internal( + realm, + encoded_device_id, + "/f", + :dontcare, + gen_tracking_id(), + timestamp_us_x_10 + ) + + # Let the process handle device's last message + Process.sleep(100) + + # Check DUP end ack in deleted_devices table + dup_end_ack_statement = """ + SELECT dup_end_ack + FROM #{realm}.deletion_in_progress + WHERE device_id = :device_id + """ + + dup_end_ack_result = + Xandra.Cluster.run(:xandra, fn conn -> + prepared = Xandra.prepare!(conn, dup_end_ack_statement) + + %Xandra.Page{} = + page = + Xandra.execute!(conn, prepared, %{"device_id" => device_id}, uuid_format: :binary) + + Enum.to_list(page) + end) + + assert [%{"dup_end_ack" => true}] = dup_end_ack_result + + # Finally, check that the related DataUpdater process exists no more + assert [] = Registry.lookup(Registry.DataUpdater, {realm, device_id}) + end + defp retrieve_endpoint_id(client, interface_name, interface_major, path) do query = DatabaseQuery.new() diff --git a/apps/astarte_data_updater_plant/test/support/database_test_helper.exs b/apps/astarte_data_updater_plant/test/support/database_test_helper.exs index 5effd76b8..d73b4741e 100644 --- a/apps/astarte_data_updater_plant/test/support/database_test_helper.exs +++ b/apps/astarte_data_updater_plant/test/support/database_test_helper.exs @@ -1,7 +1,7 @@ # # This file is part of Astarte. # -# Copyright 2017 Ispirata Srl +# Copyright 2017 - 2023 SECO Mind Srl # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -354,6 +354,16 @@ defmodule Astarte.DataUpdaterPlant.DatabaseTestHelper do VALUES (:object_id, :object_type, :parent_trigger_id, :simple_trigger_id, :trigger_data, :trigger_target); """ + @create_deletion_in_progress_table """ + CREATE TABLE autotestrealm.deletion_in_progress ( + device_id uuid, + dup_start_ack boolean, + dup_end_ack boolean, + + PRIMARY KEY (device_id) + ); + """ + def create_test_keyspace do {:ok, client} = DatabaseClient.new(List.first(Config.cqex_nodes!())) @@ -757,6 +767,8 @@ defmodule Astarte.DataUpdaterPlant.DatabaseTestHelper do DatabaseQuery.call!(client, query) + DatabaseQuery.call!(client, @create_deletion_in_progress_table) + {:ok, client} %{msg: msg} -> diff --git a/apps/astarte_data_updater_plant/test/support/mocks.ex b/apps/astarte_data_updater_plant/test/support/mocks.ex new file mode 100644 index 000000000..4586b2420 --- /dev/null +++ b/apps/astarte_data_updater_plant/test/support/mocks.ex @@ -0,0 +1 @@ +Mox.defmock(MockRPCClient, for: Astarte.RPC.Client) From a47a2a883176531e1d74b98b873e377c30d30d91 Mon Sep 17 00:00:00 2001 From: Arnaldo Cesco Date: Fri, 8 Sep 2023 16:45:11 +0200 Subject: [PATCH 17/17] Allow VerneMQ to work in docker compose setup Now the Astarte VerneMQ Plugin needs access to the astarte database. Add the relevant env var in the docker-compose.yml file to allow it to connect. Signed-off-by: Arnaldo Cesco --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 76abbdd39..13225cd10 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -248,6 +248,7 @@ services: DOCKER_VERNEMQ_ASTARTE_VMQ_PLUGIN__AMQP__PASSWORD: "guest" DOCKER_VERNEMQ_ASTARTE_VMQ_PLUGIN__AMQP__HOST: "rabbitmq" DOCKER_VERNEMQ_USER_appengine: "appengine" + DOCKER_VERNEMQ_ASTARTE_VMQ_PLUGIN__CASSANDRA__NODES: "scylla:9042" CFSSL_URL: "http://cfssl:8080" volumes: - vernemq-data:/opt/vernemq/data