Provide detailed Exception message when token count exceeds max

Closes #4835

Signed-off-by: John Blum <[email protected]>

Author: jxblum

spring-ai-commons/src/main/java/org/springframework/ai/embedding/TokenCountBatchingStrategy.java

@@ -51,6 +51,7 @@
  * @author Laura Trotta
  * @author Jihoon Kim
  * @author Yanming Zhou
+ * @author John Blum
  * @since 1.0.0
  */
 public class TokenCountBatchingStrategy implements BatchingStrategy {
@@ -148,8 +149,9 @@ public List<List<Document>> batch(List<Document> documents) {
 			int tokenCount = this.tokenCountEstimator
 				.estimate(document.getFormattedContent(this.contentFormatter, this.metadataMode));
 			if (tokenCount > this.maxInputTokenCount) {
-				throw new IllegalArgumentException(
-						"Tokens in a single document exceeds the maximum number of allowed input tokens");
+				String message = "Tokens [%d] in document with ID [%s] exceeds maximum number of allowed input tokens [%d]"
+					.formatted(tokenCount, document.getId(), this.maxInputTokenCount);
+				throw new IllegalArgumentException(message);
 			}
 			documentTokens.put(document, tokenCount);
 		}

spring-ai-commons/src/test/java/org/springframework/ai/embedding/TokenCountBatchingStrategyTests.java

@@ -22,17 +22,30 @@
 
 import org.junit.jupiter.api.Test;
 
+import org.springframework.ai.document.ContentFormatter;
 import org.springframework.ai.document.Document;
+import org.springframework.ai.document.MetadataMode;
+import org.springframework.ai.tokenizer.TokenCountEstimator;
 import org.springframework.core.io.DefaultResourceLoader;
 import org.springframework.core.io.Resource;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoMoreInteractions;
 
 /**
- * Basic unit test for {@link TokenCountBatchingStrategy}.
+ * Basic unit tests for {@link TokenCountBatchingStrategy}.
  *
  * @author Soby Chacko
+ * @author John Blum
  */
 public class TokenCountBatchingStrategyTests {
 
@@ -54,4 +67,28 @@ void batchEmbeddingWithLargeDocumentExceedsMaxTokenSize() throws IOException {
 			.isInstanceOf(IllegalArgumentException.class);
 	}
 
+	@Test
+	void documentTokenCountExceedsConfiguredMaxTokenCount() {
+
+		Document mockDocument = mock(Document.class);
+		ContentFormatter mockContentFormatter = mock(ContentFormatter.class);
+		TokenCountEstimator mockTokenCountEstimator = mock(TokenCountEstimator.class);
+
+		doReturn("123abc").when(mockDocument).getId();
+		doReturn(10).when(mockTokenCountEstimator).estimate(anyString());
+		doReturn("test").when(mockDocument).getFormattedContent(any(), any());
+
+		TokenCountBatchingStrategy batchingStrategy = new TokenCountBatchingStrategy(mockTokenCountEstimator, 9, 0.0d,
+				mockContentFormatter, MetadataMode.EMBED);
+
+		assertThatIllegalArgumentException().isThrownBy(() -> batchingStrategy.batch(List.of(mockDocument)))
+			.withMessage("Tokens [10] in document with ID [123abc] exceeds maximum number of allowed input tokens [9]")
+			.withNoCause();
+
+		verify(mockDocument, times(1)).getId();
+		verify(mockDocument, times(1)).getFormattedContent(eq(mockContentFormatter), eq(MetadataMode.EMBED));
+		verify(mockTokenCountEstimator, times(1)).estimate(eq("test"));
+		verifyNoMoreInteractions(mockDocument, mockTokenCountEstimator);
+	}
+
 }