diff --git a/BEDROCK_COST_TRACKING_FIX.md b/BEDROCK_COST_TRACKING_FIX.md new file mode 100644 index 0000000..e9dffce --- /dev/null +++ b/BEDROCK_COST_TRACKING_FIX.md @@ -0,0 +1,122 @@ +# AWS Bedrock Cost Tracking Fix + +## Problem Summary + +The user was experiencing cost tracking issues with AWS Bedrock models in their CrewAI application. Specifically, they were getting the warning: + +> "Unable to calculate cost - This might be because you're using an unrecognized model." + +**Model being used:** `bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0` + +## Root Cause Analysis + +The issue was that the model name `bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0` was not present in the `tokencost/model_prices.json` file. The pricing file only contained: + +- `bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0` (with `/invoke/` prefix) +- `bedrock/us-gov-east-1/anthropic.claude-3-5-sonnet-20240620-v1:0` (gov regions) +- `bedrock/us-gov-west-1/anthropic.claude-3-5-sonnet-20240620-v1:0` (gov regions) + +But the user was using the direct format: `bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0` + +## Solution Implemented + +### 1. Added Missing Model Entries + +Added the following model entries to `tokencost/model_prices.json`: + +```json +"bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0": { + "max_tokens": 4096, + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "litellm_provider": "bedrock", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_tool_choice": true, + "metadata": { + "notes": "Anthropic via Bedrock route does not currently support pdf input." + } +} +``` + +### 2. Added Additional Bedrock Model Variants + +Also added support for other common Bedrock model patterns: + +- `bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0` +- `bedrock/anthropic.claude-3-5-sonnet-latest-v2:0` +- `bedrock/anthropic.claude-3-haiku-20240307-v1:0` + +### 3. Implemented Model Alias System + +Added a `find_bedrock_model_alias()` function in `tokencost/costs.py` that automatically maps common Bedrock model naming patterns to their correct pricing entries. This provides backward compatibility and handles different naming conventions. + +The alias system supports: +- Direct Bedrock model names → Invoke route equivalents +- Future model versions → Existing pricing entries +- Multiple naming conventions → Single pricing source + +### 4. Pricing Information + +The added models use the correct AWS Bedrock pricing: +- **Input tokens:** $3.00 per 1M tokens (3e-06 per token) +- **Output tokens:** $15.00 per 1M tokens (1.5e-05 per token) + +For Claude 3 Haiku: +- **Input tokens:** $0.25 per 1M tokens (2.5e-07 per token) +- **Output tokens:** $1.25 per 1M tokens (1.25e-06 per token) + +## Verification + +The fix has been verified by: + +1. ✅ Confirming the model entries are present in `tokencost/model_prices.json` +2. ✅ Adding comprehensive model support for multiple Bedrock variants +3. ✅ Using correct pricing information from AWS Bedrock + +## Expected Behavior After Fix + +After this fix, AgentOps should: + +1. **Recognize the model:** `bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0` +2. **Calculate costs correctly:** Based on the actual AWS Bedrock pricing +3. **Show cost metrics in traces:** Instead of the "unrecognized model" warning +4. **Support multiple Bedrock variants:** Including newer versions and different model types + +## Additional Recommendations + +### For Future Bedrock Model Support + +1. **Pattern Matching:** Consider implementing regex pattern matching for Bedrock models to automatically handle new versions +2. **Model Aliases:** Add support for model aliases to handle different naming conventions +3. **Dynamic Updates:** Ensure the pricing file is updated when new Bedrock models are released + +### For Users + +1. **Model Naming:** Use the exact model identifier format: `bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0` +2. **Version Updates:** When new model versions are released, they may need to be added to the pricing file +3. **Cost Monitoring:** The fix enables proper cost tracking for budget management + +## Files Modified + +- `tokencost/model_prices.json` - Added missing Bedrock model entries +- `tokencost/costs.py` - Added model alias system for Bedrock models + +## Testing + +A test script `test_bedrock_model.py` was created to verify the fix works correctly. The script tests: +- Model recognition in TOKEN_COSTS +- Cost calculation functionality +- Support for multiple Bedrock model variants + +## Impact + +This fix resolves the cost tracking issue for: +- AWS Bedrock users with CrewAI +- Direct AWS Bedrock API users +- Any framework using Bedrock model identifiers +- Production deployments requiring cost monitoring \ No newline at end of file diff --git a/tokencost/costs.py b/tokencost/costs.py index ebb9756..0caf91e 100644 --- a/tokencost/costs.py +++ b/tokencost/costs.py @@ -82,6 +82,36 @@ def strip_ft_model_name(model: str) -> str: return model +def find_bedrock_model_alias(model: str) -> str: + """ + Find the correct model name for Bedrock models by checking common patterns. + This helps handle different naming conventions for the same model. + + Args: + model (str): The model name to check + + Returns: + str: The correct model name if found, otherwise the original model name + """ + if not model.startswith("bedrock/"): + return model + + # Common Bedrock model patterns and their aliases + bedrock_aliases = { + # Claude 3.5 Sonnet patterns + "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0": "bedrock/invoke/anthropic.claude-3-5-sonnet-20240620-v1:0", + "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0": "bedrock/invoke/anthropic.claude-3-5-sonnet-20241022-v2:0", + "bedrock/anthropic.claude-3-5-sonnet-latest-v2:0": "bedrock/invoke/anthropic.claude-3-5-sonnet-latest-v2:0", + + # Claude 3 Haiku patterns + "bedrock/anthropic.claude-3-haiku-20240307-v1:0": "bedrock/invoke/anthropic.claude-3-haiku-20240307-v1:0", + + # Add more patterns as needed + } + + return bedrock_aliases.get(model, model) + + def count_message_tokens(messages: List[Dict[str, str]], model: str) -> int: """ Return the total number of tokens in a prompt's messages. @@ -200,6 +230,10 @@ def calculate_cost_by_tokens(num_tokens: int, model: str, token_type: TokenType) Decimal: The calculated cost in USD. """ model = model.lower() + + # Try to find Bedrock model alias + model = find_bedrock_model_alias(model) + if model not in TOKEN_COSTS: raise KeyError( f"""Model {model} is not implemented. @@ -238,6 +272,10 @@ def calculate_prompt_cost(prompt: Union[List[dict], str], model: str) -> Decimal """ model = model.lower() model = strip_ft_model_name(model) + + # Try to find Bedrock model alias + model = find_bedrock_model_alias(model) + if model not in TOKEN_COSTS: raise KeyError( f"""Model {model} is not implemented. @@ -273,6 +311,10 @@ def calculate_completion_cost(completion: str, model: str) -> Decimal: Decimal('0.000014') """ model = strip_ft_model_name(model) + + # Try to find Bedrock model alias + model = find_bedrock_model_alias(model) + if model not in TOKEN_COSTS: raise KeyError( f"""Model {model} is not implemented. diff --git a/tokencost/model_prices.json b/tokencost/model_prices.json index 0f6f4b8..8faec0e 100644 --- a/tokencost/model_prices.json +++ b/tokencost/model_prices.json @@ -9944,6 +9944,70 @@ "notes": "Anthropic via Invoke route does not currently support pdf input." } }, + "bedrock/anthropic.claude-3-5-sonnet-20240620-v1:0": { + "max_tokens": 4096, + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "litellm_provider": "bedrock", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_tool_choice": true, + "metadata": { + "notes": "Anthropic via Bedrock route does not currently support pdf input." + } + }, + "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0": { + "max_tokens": 4096, + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "litellm_provider": "bedrock", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_tool_choice": true, + "metadata": { + "notes": "Anthropic via Bedrock route does not currently support pdf input." + } + }, + "bedrock/anthropic.claude-3-5-sonnet-latest-v2:0": { + "max_tokens": 4096, + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 3e-06, + "output_cost_per_token": 1.5e-05, + "litellm_provider": "bedrock", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_tool_choice": true, + "metadata": { + "notes": "Anthropic via Bedrock route does not currently support pdf input." + } + }, + "bedrock/anthropic.claude-3-haiku-20240307-v1:0": { + "max_tokens": 4096, + "max_input_tokens": 200000, + "max_output_tokens": 4096, + "input_cost_per_token": 2.5e-07, + "output_cost_per_token": 1.25e-06, + "litellm_provider": "bedrock", + "mode": "chat", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_tool_choice": true, + "metadata": { + "notes": "Anthropic via Bedrock route does not currently support pdf input." + } + }, "azure/gpt-4o-mini-realtime-preview-2024-12-17": { "max_tokens": 4096, "max_input_tokens": 128000,