diff --git a/epoxy-adapter/src/main/java/com/airbnb/epoxy/EpoxyVisibilityTracker.java b/epoxy-adapter/src/main/java/com/airbnb/epoxy/EpoxyVisibilityTracker.java
index 4c180d5d7d..08ed3275e2 100644
--- a/epoxy-adapter/src/main/java/com/airbnb/epoxy/EpoxyVisibilityTracker.java
+++ b/epoxy-adapter/src/main/java/com/airbnb/epoxy/EpoxyVisibilityTracker.java
@@ -298,7 +298,10 @@ private void processChildRecyclerViewAttached(@NonNull RecyclerView childRecycle
   }
 
   private void processChildRecyclerViewDetached(@NonNull RecyclerView childRecyclerView) {
-    nestedTrackers.remove(childRecyclerView);
+    EpoxyVisibilityTracker tracker = nestedTrackers.remove(childRecyclerView);
+    if (tracker != null) {
+      tracker.detach(childRecyclerView);
+    }
   }
 
   /**