Add thought chunk dedup to eliminate codex doubling

Codex-acp sends cumulative re-deliveries of thought text after
incremental tokens, causing doubled output in the buffer.
agent-shell--thought-chunk-delta detects prefix/suffix overlap and
emits only the genuinely new tail. Accumulator state resets at each
new thought group.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: timvisher-dd

agent-shell-streaming.el

@@ -352,6 +352,35 @@ back to content-text extraction."
          :expanded agent-shell-tool-use-expand-by-default))
       (agent-shell--tool-call-clear-output state .toolCallId))))
 
+;;; Thought chunk dedup
+
+(defun agent-shell--thought-chunk-delta (accumulated chunk)
+  "Return the portion of CHUNK not already present in ACCUMULATED.
+When an agent re-delivers the full accumulated thought text (e.g.
+codex-acp sending a cumulative summary after incremental tokens),
+only the genuinely new tail is returned.
+
+Three cases are handled:
+  ;; Cumulative from start (prefix match)
+  (agent-shell--thought-chunk-delta \"AB\" \"ABCD\")  => \"CD\"
+
+  ;; Already present (suffix match, e.g. leading whitespace trimmed)
+  (agent-shell--thought-chunk-delta \"\\n\\nABCD\" \"ABCD\")  => \"\"
+
+  ;; Incremental token (no overlap)
+  (agent-shell--thought-chunk-delta \"AB\" \"CD\")  => \"CD\""
+  (cond
+   ((or (null accumulated) (string-empty-p accumulated))
+    chunk)
+   ;; Chunk starts with all accumulated text (cumulative from start).
+   ((string-prefix-p accumulated chunk)
+    (substring chunk (length accumulated)))
+   ;; Chunk is already fully contained as a suffix of accumulated
+   ;; (e.g. re-delivery omits leading whitespace tokens).
+   ((string-suffix-p chunk accumulated)
+    "")
+   (t chunk)))
+
 ;;; Cancellation
 
 (defun agent-shell--mark-tool-calls-cancelled (state)

agent-shell.el

@@ -588,6 +588,7 @@ OUTGOING-REQUEST-DECORATOR (passed through to `acp-make-client')."
                              (cons :modes nil)))
         (cons :last-entry-type nil)
         (cons :chunked-group-count 0)
+        (cons :thought-accumulated nil)
         (cons :request-count 0)
         (cons :tool-calls nil)
         (cons :available-commands nil)
@@ -1181,30 +1182,34 @@ COMMAND, when present, may be a shell command string or an argv vector."
                (map-put! state :last-entry-type "tool_call"))
               ((equal (map-elt update 'sessionUpdate) "agent_thought_chunk")
                (let-alist update
-                 ;; (message "agent_thought_chunk: last-type=%s, will-append=%s"
-                 ;;          (map-elt state :last-entry-type)
-                 ;;          (equal (map-elt state :last-entry-type) "agent_thought_chunk"))
-                 (unless (equal (map-elt state :last-entry-type)
-                                "agent_thought_chunk")
-                   (map-put! state :chunked-group-count (1+ (map-elt state :chunked-group-count)))
-                   (agent-shell--append-transcript
-                    :text (format "## Agent's Thoughts (%s)\n\n" (format-time-string "%F %T"))
-                    :file-path agent-shell--transcript-file))
-                 (agent-shell--append-transcript
-                  :text .content.text
-                  :file-path agent-shell--transcript-file)
-                 (agent-shell--update-fragment
-                  :state state
-                  :block-id (format "%s-agent_thought_chunk"
-                                    (map-elt state :chunked-group-count))
-                  :label-left  (concat
-                                agent-shell-thought-process-icon
-                                " "
-                                (propertize "Thought process" 'font-lock-face font-lock-doc-markup-face))
-                  :body .content.text
-                  :append (equal (map-elt state :last-entry-type)
-                                 "agent_thought_chunk")
-                  :expanded agent-shell-thought-process-expand-by-default))
+                 (let ((new-group (not (equal (map-elt state :last-entry-type)
+                                              "agent_thought_chunk"))))
+                   (when new-group
+                     (map-put! state :chunked-group-count (1+ (map-elt state :chunked-group-count)))
+                     (map-put! state :thought-accumulated nil)
+                     (agent-shell--append-transcript
+                      :text (format "## Agent's Thoughts (%s)\n\n" (format-time-string "%F %T"))
+                      :file-path agent-shell--transcript-file))
+                   (let ((delta (agent-shell--thought-chunk-delta
+                                 (map-elt state :thought-accumulated)
+                                 .content.text)))
+                     (map-put! state :thought-accumulated
+                               (concat (or (map-elt state :thought-accumulated) "") delta))
+                     (when (and delta (not (string-empty-p delta)))
+                       (agent-shell--append-transcript
+                        :text delta
+                        :file-path agent-shell--transcript-file)
+                       (agent-shell--update-fragment
+                        :state state
+                        :block-id (format "%s-agent_thought_chunk"
+                                          (map-elt state :chunked-group-count))
+                        :label-left  (concat
+                                      agent-shell-thought-process-icon
+                                      " "
+                                      (propertize "Thought process" 'font-lock-face font-lock-doc-markup-face))
+                        :body delta
+                        :append (not new-group)
+                        :expanded agent-shell-thought-process-expand-by-default)))))
                (map-put! state :last-entry-type "agent_thought_chunk"))
               ((equal (map-elt update 'sessionUpdate) "agent_message_chunk")
                (unless (equal (map-elt state :last-entry-type) "agent_message_chunk")

tests/agent-shell-streaming-tests.el

@@ -223,5 +223,74 @@ c)))
   (kill-buffer buffer)))))
 
 
+;;; Thought chunk dedup tests
+
+(ert-deftest agent-shell--thought-chunk-delta-incremental-test ()
+  "Incremental tokens with no prefix overlap pass through unchanged."
+  (should (equal (agent-shell--thought-chunk-delta "AB" "CD") "CD"))
+  (should (equal (agent-shell--thought-chunk-delta nil "hello") "hello"))
+  (should (equal (agent-shell--thought-chunk-delta "" "hello") "hello")))
+
+(ert-deftest agent-shell--thought-chunk-delta-cumulative-test ()
+  "Cumulative re-delivery returns only the new tail."
+  (should (equal (agent-shell--thought-chunk-delta "AB" "ABCD") "CD"))
+  (should (equal (agent-shell--thought-chunk-delta "hello " "hello world") "world")))
+
+(ert-deftest agent-shell--thought-chunk-delta-exact-duplicate-test ()
+  "Exact duplicate returns empty string."
+  (should (equal (agent-shell--thought-chunk-delta "ABCD" "ABCD") "")))
+
+(ert-deftest agent-shell--thought-chunk-delta-suffix-test ()
+  "Chunk already present as suffix of accumulated returns empty string.
+This handles the case where leading whitespace tokens were streamed
+incrementally but the re-delivery omits them."
+  (should (equal (agent-shell--thought-chunk-delta "\n\nABCD" "ABCD") ""))
+  (should (equal (agent-shell--thought-chunk-delta "\n\n**bold**" "**bold**") "")))
+
+(ert-deftest agent-shell--thought-chunk-no-duplication-test ()
+  "Thought chunks must not produce duplicate output in the buffer.
+Replays the codex doubling pattern: incremental tokens followed by
+a cumulative re-delivery of the complete thought text."
+  (let* ((buffer (get-buffer-create " *agent-shell-thought-dedup*"))
+         (agent-shell--state (agent-shell--make-state :buffer buffer))
+         (agent-shell--transcript-file nil)
+         (thought-text "**Checking beads**\n\nLooking for .beads directory."))
+    (map-put! agent-shell--state :client 'test-client)
+    (map-put! agent-shell--state :request-count 1)
+    (with-current-buffer buffer
+      (erase-buffer)
+      (agent-shell-mode))
+    (unwind-protect
+        (with-current-buffer buffer
+          ;; Send incremental tokens
+          (dolist (token (list "\n\n" "**Checking" " beads**" "\n\n"
+                               "Looking" " for" " .beads" " directory."))
+            (agent-shell--on-notification
+             :state agent-shell--state
+             :notification `((method . "session/update")
+                             (params . ((update
+                                         . ((sessionUpdate . "agent_thought_chunk")
+                                            (content (type . "text")
+                                                     (text . ,token)))))))))
+          ;; Cumulative re-delivery of the complete text
+          (agent-shell--on-notification
+           :state agent-shell--state
+           :notification `((method . "session/update")
+                           (params . ((update
+                                       . ((sessionUpdate . "agent_thought_chunk")
+                                          (content (type . "text")
+                                                   (text . ,thought-text))))))))
+          (let* ((buf-text (buffer-substring-no-properties (point-min) (point-max)))
+                 (count (let ((c 0) (s 0))
+                          (while (string-match "Checking beads" buf-text s)
+                            (setq c (1+ c) s (match-end 0)))
+                          c)))
+            ;; Content must be present
+            (should (string-match-p "Checking beads" buf-text))
+            ;; Must appear exactly once (no duplication)
+            (should (= count 1))))
+      (when (buffer-live-p buffer)
+        (kill-buffer buffer)))))
+
 (provide 'agent-shell-streaming-tests)
 ;;; agent-shell-streaming-tests.el ends here