Update: Rework orphan deletion for scale (PaloAltoNetworks#83)

Our current orphan deletion logic does not scale due to its not in approach. This rework swaps the logic to an in approach by doing the following:

- On namespace deletion, a deletion record is created that will stick around for 1 day (mongo expiration time)
- When an orphan job starts, it will pull all existing deletion records and delete any related objects that the service manages that are 
    - In the namespace in the deletion record AND
    - Have a create time that is less than the deletion time of the record, or no create time at all

Fixes: https://redlock.atlassian.net/browse/CNS-7191

cmd/a3s/main.go

@@ -93,6 +93,14 @@ func main() {
 		zap.L().Fatal("Unable to create exp expiration index for oidccache", zap.Error(err))
 	}
 
+	if err := manipmongo.EnsureIndex(m, api.NamespaceDeletionRecordIdentity, mgo.Index{
+		Key:         []string{"deletetime"},
+		ExpireAfter: 24 * time.Hour,
+		Name:        "index_expiration_deletetime",
+	}); err != nil {
+		zap.L().Fatal("Unable to create expiration index for namesapce deletion records", zap.Error(err))
+	}
+
 	if err := createRootNamespaceIfNeeded(m); err != nil {
 		zap.L().Fatal("Unable to handle root namespace", zap.Error(err))
 	}
@@ -345,6 +353,7 @@ func main() {
 	bahamut.RegisterProcessorOrDie(server, processors.NewPermissionsProcessor(retriever), api.PermissionsIdentity)
 	bahamut.RegisterProcessorOrDie(server, processors.NewAuthzProcessor(pauthz, jwks, cfg.JWT.JWTIssuer, cfg.JWT.JWTAudience), api.AuthzIdentity)
 	bahamut.RegisterProcessorOrDie(server, processors.NewNamespacesProcessor(m, pubsub), api.NamespaceIdentity)
+	bahamut.RegisterProcessorOrDie(server, processors.NewNamespaceDeletionRecordsProcessor(m), api.NamespaceDeletionRecordIdentity)
 	bahamut.RegisterProcessorOrDie(server, processors.NewAuthorizationProcessor(m, pubsub, retriever, cfg.JWT.JWTIssuer), api.AuthorizationIdentity)
 	bahamut.RegisterProcessorOrDie(server, processors.NewImportProcessor(bmanipMaker, pauthz), api.ImportIdentity)
 
@@ -676,11 +685,17 @@ func makeNamespaceCleaner(ctx context.Context, m manipulate.Manipulator) notific
 		ns := msg.Data.(string)
 
 		for _, i := range api.Manager().AllIdentities() {
+
+			if i.IsEqual(api.NamespaceDeletionRecordIdentity) {
+				continue
+			}
+
 			mctx := manipulate.NewContext(
 				ctx,
 				manipulate.ContextOptionNamespace(ns),
 				manipulate.ContextOptionRecursive(true),
 			)
+
 			if err := m.DeleteMany(mctx, i); err != nil {
 				zap.L().Error("Unable to clean namespace", zap.String("ns", ns), zap.Error(err))
 			}

internal/processors/namespacedeletionrecords.go

@@ -0,0 +1,25 @@
+package processors
+
+import (
+	"go.aporeto.io/a3s/pkgs/api"
+	"go.aporeto.io/a3s/pkgs/crud"
+	"go.aporeto.io/bahamut"
+	"go.aporeto.io/manipulate"
+)
+
+// A NamespaceDeletionRecordsProcessor is a bahamut processor for NamespaceDeletionRecord.
+type NamespaceDeletionRecordsProcessor struct {
+	manipulator manipulate.Manipulator
+}
+
+// NewNamespaceDeletionRecordsProcessor returns a new NamespaceDeletionRecordsProcessor.
+func NewNamespaceDeletionRecordsProcessor(manipulator manipulate.Manipulator) *NamespaceDeletionRecordsProcessor {
+	return &NamespaceDeletionRecordsProcessor{
+		manipulator: manipulator,
+	}
+}
+
+// ProcessRetrieveMany handles the retrieve many requests for NamespaceDeletionRecord.
+func (p *NamespaceDeletionRecordsProcessor) ProcessRetrieveMany(bctx bahamut.Context) error {
+	return crud.RetrieveMany(bctx, p.manipulator, &api.NamespaceDeletionRecordsList{})
+}

internal/processors/namespaces.go

@@ -3,6 +3,7 @@ package processors
 import (
 	"net/http"
 	"strings"
+	"time"
 
 	"go.aporeto.io/a3s/pkgs/api"
 	"go.aporeto.io/a3s/pkgs/crud"
@@ -11,6 +12,7 @@ import (
 	"go.aporeto.io/bahamut"
 	"go.aporeto.io/elemental"
 	"go.aporeto.io/manipulate"
+	"go.uber.org/zap"
 )
 
 // A NamespacesProcessor is a bahamut processor for Namespaces.
@@ -71,7 +73,21 @@ func (p *NamespacesProcessor) ProcessUpdate(bctx bahamut.Context) error {
 // ProcessDelete handles the delete requests for Namespaces.
 func (p *NamespacesProcessor) ProcessDelete(bctx bahamut.Context) error {
 	return crud.Delete(bctx, p.manipulator, api.NewNamespace(),
-		crud.OptionPostWriteHook(p.makeNotify(bctx.Request().Operation)),
+		crud.OptionPostWriteHook(func(obj elemental.Identifiable) {
+
+			ndr := api.NewNamespaceDeletionRecord()
+			ndr.Namespace = obj.(*api.Namespace).Name
+			ndr.DeleteTime = time.Now()
+
+			if err := p.manipulator.Create(manipulate.NewContext(bctx.Context()), ndr); err != nil {
+				zap.L().Error("Unable to create namespace deletion record",
+					zap.String("namespace", ndr.Namespace),
+					zap.Error(err),
+				)
+			}
+
+			p.makeNotify(bctx.Request().Operation)(obj)
+		}),
 	)
 }
 

pkgs/api/doc/documentation.md

@@ -1556,6 +1556,41 @@ Type: `string`
 
 The namespace of the object.
 
+### NamespaceDeletionRecord
+
+A namespace deletion record holds the namespace that was deleted and the date it
+was deleted.
+
+#### Relations
+
+##### `GET /namespacedeletionrecords`
+
+Retrieves the list of namespace deletion records.
+
+Parameters:
+
+- `q` (`string`): This is an example.
+
+#### Attributes
+
+##### `ID` [`identifier`,`autogenerated`,`read_only`]
+
+Type: `string`
+
+ID is the identifier of the object.
+
+##### `deleteTime`
+
+Type: `time`
+
+Deletion date of the object.
+
+##### `namespace`
+
+Type: `string`
+
+Namespace that got deleted.
+
 ## core/import
 
 ### Import