From fba8db35b643c5f741572c581a731c39e37064be Mon Sep 17 00:00:00 2001 From: Ben McIlwain Date: Tue, 16 Sep 2025 17:58:18 -0400 Subject: [PATCH] Add a batch action to remove all contacts from domains This implements the first part of Minimum Data Set phase 3, wherein we delete all contact data. This action is necessary to leave a permanent record on the domain (in the form of a domain history entry) documenting when the contacts were removed by the administrative user. Then, after this has finished removing all contact assocations, we can simply empty out or drop the Contact/ContactHistory tables and associated join tables. --- .../batch/RemoveAllDomainContactsAction.java | 232 ++++++++++++++++++ .../registry/module/RequestComponent.java | 3 + .../backend/BackendRequestComponent.java | 3 + .../registry/batch/domain_remove_contacts.xml | 22 ++ .../RemoveAllDomainContactsActionTest.java | 116 +++++++++ .../module/backend/backend_routing.txt | 1 + .../google/registry/module/routing.txt | 1 + 7 files changed, 378 insertions(+) create mode 100644 core/src/main/java/google/registry/batch/RemoveAllDomainContactsAction.java create mode 100644 core/src/main/resources/google/registry/batch/domain_remove_contacts.xml create mode 100644 core/src/test/java/google/registry/batch/RemoveAllDomainContactsActionTest.java diff --git a/core/src/main/java/google/registry/batch/RemoveAllDomainContactsAction.java b/core/src/main/java/google/registry/batch/RemoveAllDomainContactsAction.java new file mode 100644 index 00000000000..7adc1861372 --- /dev/null +++ b/core/src/main/java/google/registry/batch/RemoveAllDomainContactsAction.java @@ -0,0 +1,232 @@ +// Copyright 2025 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.batch; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.net.MediaType.PLAIN_TEXT_UTF_8; +import static google.registry.flows.FlowUtils.marshalWithLenientRetry; +import static google.registry.model.common.FeatureFlag.FeatureName.MINIMUM_DATASET_CONTACTS_PROHIBITED; +import static google.registry.persistence.transaction.TransactionManagerFactory.tm; +import static google.registry.util.DateTimeUtils.END_OF_TIME; +import static google.registry.util.ResourceUtils.readResourceUtf8; +import static jakarta.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR; +import static jakarta.servlet.http.HttpServletResponse.SC_NO_CONTENT; +import static jakarta.servlet.http.HttpServletResponse.SC_OK; +import static java.nio.charset.StandardCharsets.US_ASCII; + +import com.google.common.base.Ascii; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.flogger.FluentLogger; +import google.registry.config.RegistryConfig.Config; +import google.registry.flows.EppController; +import google.registry.flows.EppRequestSource; +import google.registry.flows.PasswordOnlyTransportCredentials; +import google.registry.flows.StatelessRequestSessionMetadata; +import google.registry.model.common.FeatureFlag; +import google.registry.model.contact.Contact; +import google.registry.model.domain.DesignatedContact; +import google.registry.model.domain.Domain; +import google.registry.model.eppcommon.ProtocolDefinition; +import google.registry.model.eppoutput.EppOutput; +import google.registry.persistence.VKey; +import google.registry.request.Action; +import google.registry.request.Action.GaeService; +import google.registry.request.Response; +import google.registry.request.auth.Auth; +import google.registry.request.lock.LockHandler; +import jakarta.inject.Inject; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.logging.Level; +import javax.annotation.Nullable; +import org.joda.time.Duration; + +/** + * An action that removes all contacts from all active (non-deleted) domains. + * + *

This implements part 1 of phase 3 of the Minimum Dataset migration, wherein we remove all uses + * of contact objects in preparation for later removing all contact data from the system. + * + *

This runs as a singly threaded, resumable action that loads batches of domains still + * containing contacts, and runs a superuser domain update on each one to remove the contacts, + * leaving behind a record recording that update. + */ +@Action( + service = GaeService.BACKEND, + path = RemoveAllDomainContactsAction.PATH, + method = Action.Method.POST, + auth = Auth.AUTH_ADMIN) +public class RemoveAllDomainContactsAction implements Runnable { + + public static final String PATH = "/_dr/task/removeAllDomainContacts"; + private static final String LOCK_NAME = "Remove all domain contacts"; + private static final String CONTACT_FMT = "%s"; + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + private final EppController eppController; + private final String registryAdminClientId; + private final LockHandler lockHandler; + private final Response response; + private final String updateDomainXml; + private int successes = 0; + private int failures = 0; + + private static final int BATCH_SIZE = 10000; + + @Inject + RemoveAllDomainContactsAction( + EppController eppController, + @Config("registryAdminClientId") String registryAdminClientId, + LockHandler lockHandler, + Response response) { + this.eppController = eppController; + this.registryAdminClientId = registryAdminClientId; + this.lockHandler = lockHandler; + this.response = response; + this.updateDomainXml = + readResourceUtf8(RemoveAllDomainContactsAction.class, "domain_remove_contacts.xml"); + } + + @Override + public void run() { + checkState( + tm().transact(() -> FeatureFlag.isActiveNow(MINIMUM_DATASET_CONTACTS_PROHIBITED)), + "Minimum dataset migration must be completed prior to running this action"); + response.setContentType(PLAIN_TEXT_UTF_8); + + Callable runner = + () -> { + try { + runLocked(); + response.setStatus(SC_OK); + } catch (Exception e) { + logger.atSevere().withCause(e).log("Errored out during execution."); + response.setStatus(SC_INTERNAL_SERVER_ERROR); + response.setPayload(String.format("Errored out with cause: %s", e)); + } + return null; + }; + + if (!lockHandler.executeWithLocks(runner, null, Duration.standardHours(1), LOCK_NAME)) { + // Send a 200-series status code to prevent this conflicting action from retrying. + response.setStatus(SC_NO_CONTENT); + response.setPayload("Could not acquire lock; already running?"); + } + } + + private void runLocked() { + logger.atInfo().log("Removing contacts on all active domains."); + + List domainRepoIdsBatch; + do { + domainRepoIdsBatch = + tm().>transact( + () -> + tm().getEntityManager() + .createQuery( + """ + SELECT repoId FROM Domain WHERE deletionTime = :end_of_time AND NOT ( + adminContact IS NULL AND billingContact IS NULL + AND registrantContact IS NULL AND techContact IS NULL) + """) + .setParameter("end_of_time", END_OF_TIME) + .setMaxResults(BATCH_SIZE) + .getResultList()); + + domainRepoIdsBatch.forEach(this::runDomainUpdateFlow); + } while (!domainRepoIdsBatch.isEmpty()); + String msg = + String.format( + "Finished; %d domains were successfully updated and %d errored out.", + successes, failures); + logger.at(failures == 0 ? Level.INFO : Level.WARNING).log(msg); + response.setPayload(msg); + } + + private void runDomainUpdateFlow(String repoId) { + // Create a new transaction that the flow's execution will be enlisted in that loads the domain + // transactionally. This way we can ensure that nothing else has modified the domain in question + // in the intervening period since the query above found it. + boolean success = tm().transact(() -> runDomainUpdateFlowInner(repoId)); + if (success) { + successes++; + } else { + failures++; + } + } + + /** + * Runs the actual domain update flow and returns whether the contact removals were successful. + */ + private boolean runDomainUpdateFlowInner(String repoId) { + Domain domain = tm().loadByKey(VKey.create(Domain.class, repoId)); + if (!domain.getDeletionTime().equals(END_OF_TIME)) { + // Domain has been deleted since the action began running; nothing further to be + // done here. + logger.atInfo().log("Nothing to process for deleted domain '%s'.", domain.getDomainName()); + return false; + } + logger.atInfo().log("Attempting to remove contacts on domain '%s'.", domain.getDomainName()); + + StringBuilder sb = new StringBuilder(); + ImmutableMap, Contact> contacts = + tm().loadByKeys( + domain.getContacts().stream() + .map(DesignatedContact::getContactKey) + .collect(ImmutableSet.toImmutableSet())); + + // Collect all the (non-registrant) contacts referenced by the domain and compile an EPP XML + // string that removes each one. + for (DesignatedContact designatedContact : domain.getContacts()) { + @Nullable Contact contact = contacts.get(designatedContact.getContactKey()); + if (contact == null) { + logger.atWarning().log( + "Domain '%s' referenced contact with repo ID '%s' that couldn't be" + " loaded.", + domain.getDomainName(), designatedContact.getContactKey().getKey()); + continue; + } + sb.append( + String.format( + CONTACT_FMT, + Ascii.toLowerCase(designatedContact.getType().name()), + contact.getContactId())) + .append("\n"); + } + + String compiledXml = + updateDomainXml + .replace("%DOMAIN%", domain.getDomainName()) + .replace("%CONTACTS%", sb.toString()); + EppOutput output = + eppController.handleEppCommand( + new StatelessRequestSessionMetadata( + registryAdminClientId, ProtocolDefinition.getVisibleServiceExtensionUris()), + new PasswordOnlyTransportCredentials(), + EppRequestSource.BACKEND, + false, + true, + compiledXml.getBytes(US_ASCII)); + if (output.isSuccess()) { + logger.atInfo().log( + "Successfully removed contacts from domain '%s'.", domain.getDomainName()); + } else { + logger.atWarning().log( + "Failed removing contacts from domain '%s' with error %s.", + domain.getDomainName(), new String(marshalWithLenientRetry(output), US_ASCII)); + } + return output.isSuccess(); + } +} diff --git a/core/src/main/java/google/registry/module/RequestComponent.java b/core/src/main/java/google/registry/module/RequestComponent.java index 9fca8fdf948..7b1a6223bfd 100644 --- a/core/src/main/java/google/registry/module/RequestComponent.java +++ b/core/src/main/java/google/registry/module/RequestComponent.java @@ -23,6 +23,7 @@ import google.registry.batch.DeleteProberDataAction; import google.registry.batch.ExpandBillingRecurrencesAction; import google.registry.batch.RelockDomainAction; +import google.registry.batch.RemoveAllDomainContactsAction; import google.registry.batch.ResaveAllEppResourcesPipelineAction; import google.registry.batch.ResaveEntityAction; import google.registry.batch.SendExpiringCertificateNotificationEmailAction; @@ -270,6 +271,8 @@ interface RequestComponent { ReadinessProbeActionFrontend readinessProbeActionFrontend(); + RemoveAllDomainContactsAction removeAllDomainContactsAction(); + RdapAutnumAction rdapAutnumAction(); RdapDomainAction rdapDomainAction(); diff --git a/core/src/main/java/google/registry/module/backend/BackendRequestComponent.java b/core/src/main/java/google/registry/module/backend/BackendRequestComponent.java index f0ba49c6c3f..621fcdf15f8 100644 --- a/core/src/main/java/google/registry/module/backend/BackendRequestComponent.java +++ b/core/src/main/java/google/registry/module/backend/BackendRequestComponent.java @@ -23,6 +23,7 @@ import google.registry.batch.DeleteProberDataAction; import google.registry.batch.ExpandBillingRecurrencesAction; import google.registry.batch.RelockDomainAction; +import google.registry.batch.RemoveAllDomainContactsAction; import google.registry.batch.ResaveAllEppResourcesPipelineAction; import google.registry.batch.ResaveEntityAction; import google.registry.batch.SendExpiringCertificateNotificationEmailAction; @@ -153,6 +154,8 @@ public interface BackendRequestComponent { RelockDomainAction relockDomainAction(); + RemoveAllDomainContactsAction removeAllDomainContactsAction(); + ResaveAllEppResourcesPipelineAction resaveAllEppResourcesPipelineAction(); ResaveEntityAction resaveEntityAction(); diff --git a/core/src/main/resources/google/registry/batch/domain_remove_contacts.xml b/core/src/main/resources/google/registry/batch/domain_remove_contacts.xml new file mode 100644 index 00000000000..a10e528c7fc --- /dev/null +++ b/core/src/main/resources/google/registry/batch/domain_remove_contacts.xml @@ -0,0 +1,22 @@ + + + + + %DOMAIN% + + %CONTACTS% + + + + + + + + + Registry minimum data set phase 3: Removed all contacts from domain. + false + + + ABC-12345 + + diff --git a/core/src/test/java/google/registry/batch/RemoveAllDomainContactsActionTest.java b/core/src/test/java/google/registry/batch/RemoveAllDomainContactsActionTest.java new file mode 100644 index 00000000000..6f7cbbaa703 --- /dev/null +++ b/core/src/test/java/google/registry/batch/RemoveAllDomainContactsActionTest.java @@ -0,0 +1,116 @@ +// Copyright 2025 The Nomulus Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package google.registry.batch; + +import static com.google.common.truth.Truth.assertThat; +import static google.registry.model.common.FeatureFlag.FeatureName.MINIMUM_DATASET_CONTACTS_PROHIBITED; +import static google.registry.model.common.FeatureFlag.FeatureStatus.ACTIVE; +import static google.registry.testing.DatabaseHelper.createTld; +import static google.registry.testing.DatabaseHelper.loadByEntity; +import static google.registry.testing.DatabaseHelper.newDomain; +import static google.registry.testing.DatabaseHelper.persistActiveContact; +import static google.registry.testing.DatabaseHelper.persistResource; +import static google.registry.util.DateTimeUtils.START_OF_TIME; + +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedMap; +import google.registry.flows.DaggerEppTestComponent; +import google.registry.flows.EppController; +import google.registry.flows.EppTestComponent.FakesAndMocksModule; +import google.registry.model.common.FeatureFlag; +import google.registry.model.contact.Contact; +import google.registry.model.domain.Domain; +import google.registry.persistence.transaction.JpaTestExtensions; +import google.registry.persistence.transaction.JpaTestExtensions.JpaIntegrationTestExtension; +import google.registry.testing.FakeClock; +import google.registry.testing.FakeLockHandler; +import google.registry.testing.FakeResponse; +import java.util.Optional; +import org.joda.time.DateTime; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** Unit tests for {@link RemoveAllDomainContactsAction}. */ +class RemoveAllDomainContactsActionTest { + + @RegisterExtension + final JpaIntegrationTestExtension jpa = + new JpaTestExtensions.Builder().buildIntegrationTestExtension(); + + private final FakeResponse response = new FakeResponse(); + private RemoveAllDomainContactsAction action; + + @BeforeEach + void beforeEach() { + createTld("tld"); + persistResource( + new FeatureFlag.Builder() + .setFeatureName(MINIMUM_DATASET_CONTACTS_PROHIBITED) + .setStatusMap(ImmutableSortedMap.of(START_OF_TIME, ACTIVE)) + .build()); + EppController eppController = + DaggerEppTestComponent.builder() + .fakesAndMocksModule(FakesAndMocksModule.create(new FakeClock())) + .build() + .startRequest() + .eppController(); + action = + new RemoveAllDomainContactsAction( + eppController, "NewRegistrar", new FakeLockHandler(true), response); + } + + @Test + void test_removesAllContactsFromMultipleDomains_andDoesntModifyDomainThatHasNoContacts() { + Contact c1 = persistActiveContact("contact12345"); + Domain d1 = persistResource(newDomain("foo.tld", c1)); + assertThat(d1.getAllContacts()).hasSize(3); + Contact c2 = persistActiveContact("contact23456"); + Domain d2 = persistResource(newDomain("bar.tld", c2)); + assertThat(d2.getAllContacts()).hasSize(3); + Domain d3 = + persistResource( + newDomain("baz.tld") + .asBuilder() + .setRegistrant(Optional.empty()) + .setContacts(ImmutableSet.of()) + .build()); + assertThat(d3.getAllContacts()).isEmpty(); + DateTime lastUpdate = d3.getUpdateTimestamp().getTimestamp(); + + action.run(); + assertThat(loadByEntity(d1).getAllContacts()).isEmpty(); + assertThat(loadByEntity(d2).getAllContacts()).isEmpty(); + assertThat(loadByEntity(d3).getUpdateTimestamp().getTimestamp()).isEqualTo(lastUpdate); + } + + @Test + void test_removesContacts_onDomainsThatOnlyPartiallyHaveContacts() { + Contact c1 = persistActiveContact("contact12345"); + Domain d1 = + persistResource( + newDomain("foo.tld", c1).asBuilder().setContacts(ImmutableSet.of()).build()); + assertThat(d1.getAllContacts()).hasSize(1); + Contact c2 = persistActiveContact("contact23456"); + Domain d2 = + persistResource( + newDomain("bar.tld", c2).asBuilder().setRegistrant(Optional.empty()).build()); + assertThat(d2.getAllContacts()).hasSize(2); + + action.run(); + assertThat(loadByEntity(d1).getAllContacts()).isEmpty(); + assertThat(loadByEntity(d2).getAllContacts()).isEmpty(); + } +} diff --git a/core/src/test/resources/google/registry/module/backend/backend_routing.txt b/core/src/test/resources/google/registry/module/backend/backend_routing.txt index 4c7e40f365f..6de8ea5b4da 100644 --- a/core/src/test/resources/google/registry/module/backend/backend_routing.txt +++ b/core/src/test/resources/google/registry/module/backend/backend_routing.txt @@ -26,6 +26,7 @@ BACKEND /_dr/task/rdeUpload RdeUploadAction BACKEND /_dr/task/readDnsRefreshRequests ReadDnsRefreshRequestsAction POST y APP ADMIN BACKEND /_dr/task/refreshDnsOnHostRename RefreshDnsOnHostRenameAction POST n APP ADMIN BACKEND /_dr/task/relockDomain RelockDomainAction POST y APP ADMIN +BACKEND /_dr/task/removeAllDomainContacts RemoveAllDomainContactsAction POST n APP ADMIN BACKEND /_dr/task/resaveAllEppResourcesPipeline ResaveAllEppResourcesPipelineAction GET n APP ADMIN BACKEND /_dr/task/resaveEntity ResaveEntityAction POST n APP ADMIN BACKEND /_dr/task/sendExpiringCertificateNotificationEmail SendExpiringCertificateNotificationEmailAction GET n APP ADMIN diff --git a/core/src/test/resources/google/registry/module/routing.txt b/core/src/test/resources/google/registry/module/routing.txt index 5250a322c1d..cf13db90900 100644 --- a/core/src/test/resources/google/registry/module/routing.txt +++ b/core/src/test/resources/google/registry/module/routing.txt @@ -44,6 +44,7 @@ BACKEND /_dr/task/readDnsRefreshRequests ReadDnsRefreshReques BACKEND /_dr/task/refreshDnsForAllDomains RefreshDnsForAllDomainsAction GET n APP ADMIN BACKEND /_dr/task/refreshDnsOnHostRename RefreshDnsOnHostRenameAction POST n APP ADMIN BACKEND /_dr/task/relockDomain RelockDomainAction POST y APP ADMIN +BACKEND /_dr/task/removeAllDomainContacts RemoveAllDomainContactsAction POST n APP ADMIN BACKEND /_dr/task/resaveAllEppResourcesPipeline ResaveAllEppResourcesPipelineAction GET n APP ADMIN BACKEND /_dr/task/resaveEntity ResaveEntityAction POST n APP ADMIN BACKEND /_dr/task/sendExpiringCertificateNotificationEmail SendExpiringCertificateNotificationEmailAction GET n APP ADMIN