diff --git a/instrumentation-loongsuite/README.md b/instrumentation-loongsuite/README.md index 03d08ed6b..e7c11e6f3 100644 --- a/instrumentation-loongsuite/README.md +++ b/instrumentation-loongsuite/README.md @@ -1,9 +1,10 @@ | Instrumentation | Supported Packages | Metrics support | Semconv status | | --------------- | ------------------ | --------------- | -------------- | -| [loongsuite-instrumentation-agentscope](./loongsuite-instrumentation-agentscope) | agentscope >= 0.1.5.dev0 | No | development -| [loongsuite-instrumentation-agno](./loongsuite-instrumentation-agno) | agno >= 1.5.0 | No | development +| [loongsuite-instrumentation-agentscope](./loongsuite-instrumentation-agentscope) | agentscope >= 1.0.0 | No | development +| [loongsuite-instrumentation-agno](./loongsuite-instrumentation-agno) | agno | No | development +| [loongsuite-instrumentation-dashscope](./loongsuite-instrumentation-dashscope) | dashscope >= 1.0.0 | No | development | [loongsuite-instrumentation-dify](./loongsuite-instrumentation-dify) | dify | No | development -| [loongsuite-instrumentation-langchain](./loongsuite-instrumentation-langchain) | langchain_core >= 0.1.0 | Yes | development -| [loongsuite-instrumentation-mcp](./loongsuite-instrumentation-mcp) | mcp>=1.3.0 | Yes | development +| [loongsuite-instrumentation-langchain](./loongsuite-instrumentation-langchain) | langchain_core >= 0.1.0 | No | development +| [loongsuite-instrumentation-mcp](./loongsuite-instrumentation-mcp) | mcp >= 1.3.0, <= 1.13.1 | No | development | [loongsuite-instrumentation-mem0](./loongsuite-instrumentation-mem0) | mem0ai >= 1.0.0 | No | development \ No newline at end of file diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/pyproject.toml index d372b764f..bea19dbd2 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/pyproject.toml +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/pyproject.toml @@ -25,8 +25,8 @@ classifiers = [ ] dependencies = [ "opentelemetry-api ~= 1.37", - "opentelemetry-instrumentation ~= 0.58b0", - "opentelemetry-semantic-conventions ~= 0.58b0", + "opentelemetry-instrumentation >= 0.58b0", + "opentelemetry-semantic-conventions >= 0.58b0", "opentelemetry-util-genai", "wrapt", ] diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/__init__.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/__init__.py index 309f802a1..17ef57f15 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/__init__.py +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/__init__.py @@ -96,6 +96,10 @@ def _setup_tracing_patch(self, wrapped, instance, args, kwargs): """Replace setup_tracing with no-op to use OTEL instead.""" pass + def _check_tracing_enabled_patch(self, wrapped, instance, args, kwargs): + """Return False to disable tracing in native AgentScope library.""" + return False + def _instrument(self, **kwargs: Any) -> None: """Enable AgentScope instrumentation.""" tracer_provider = kwargs.get("tracer_provider") @@ -199,6 +203,18 @@ def wrap_formatter_with_tracer(wrapped, instance, args, kwargs): logger.debug("Patched setup_tracing") except Exception as e: logger.warning(f"Failed to patch setup_tracing: {e}") + + # Patch _check_tracing_enabled to return False + # We always want to disable tracing in native AgentScope library + try: + wrap_function_wrapper( + module="agentscope.tracing._trace", + name="_check_tracing_enabled", + wrapper=self._check_tracing_enabled_patch, + ) + logger.debug("Patched _check_tracing_enabled") + except Exception as e: + logger.warning(f"Failed to patch _check_tracing_enabled: {e}") def _uninstrument(self, **kwargs: Any) -> None: """Disable AgentScope instrumentation.""" @@ -269,3 +285,11 @@ def _uninstrument(self, **kwargs: Any) -> None: logger.debug("Uninstrumented setup_tracing") except Exception as e: logger.warning(f"Failed to uninstrument setup_tracing: {e}") + + try: + import agentscope.tracing # noqa: PLC0415 + + unwrap(agentscope.tracing, "_check_tracing_enabled") + logger.debug("Uninstrumented _check_tracing_enabled") + except Exception as e: + logger.warning(f"Failed to uninstrument _check_tracing_enabled: {e}") \ No newline at end of file diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/utils.py b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/utils.py index 10782e110..3225de680 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/utils.py +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agentscope/src/opentelemetry/instrumentation/agentscope/utils.py @@ -432,14 +432,7 @@ def convert_agentscope_messages_to_genai_format( if isinstance(msg, Msg): msg_dict = _format_msg_to_parts(msg) elif isinstance(msg, dict): - if provider_name: - try: - converted = get_message_converter(provider_name)([msg]) - msg_dict = converted[0] if converted else msg - except Exception: - msg_dict = msg - else: - msg_dict = msg + msg_dict = msg else: continue diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-agno/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-agno/pyproject.toml index c3d8fc16e..16bcf2fb7 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-agno/pyproject.toml +++ b/instrumentation-loongsuite/loongsuite-instrumentation-agno/pyproject.toml @@ -26,8 +26,8 @@ classifiers = [ ] dependencies = [ "opentelemetry-api ~= 1.37", - "opentelemetry-instrumentation ~= 0.58b0", - "opentelemetry-semantic-conventions ~= 0.58b0", + "opentelemetry-instrumentation >= 0.58b0", + "opentelemetry-semantic-conventions >= 0.58b0", ] [project.optional-dependencies] diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-dashscope/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-dashscope/pyproject.toml index 99f8e06fb..a22cacb8f 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-dashscope/pyproject.toml +++ b/instrumentation-loongsuite/loongsuite-instrumentation-dashscope/pyproject.toml @@ -26,9 +26,9 @@ classifiers = [ ] dependencies = [ "opentelemetry-api ~= 1.37", - "opentelemetry-instrumentation ~= 0.58b0", - "opentelemetry-semantic-conventions ~= 0.58b0", - "opentelemetry-util-genai ~= 0.2b0", + "opentelemetry-instrumentation >= 0.58b0", + "opentelemetry-semantic-conventions >= 0.58b0", + "opentelemetry-util-genai > 0.2b0", ] [project.optional-dependencies] diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-dify/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-dify/pyproject.toml index b498abac2..5d492609d 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-dify/pyproject.toml +++ b/instrumentation-loongsuite/loongsuite-instrumentation-dify/pyproject.toml @@ -26,8 +26,8 @@ classifiers = [ ] dependencies = [ "opentelemetry-api ~= 1.37", - "opentelemetry-instrumentation ~= 0.58b0", - "opentelemetry-semantic-conventions ~= 0.58b0", + "opentelemetry-instrumentation >= 0.58b0", + "opentelemetry-semantic-conventions >= 0.58b0", ] [project.optional-dependencies] diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-langchain/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-langchain/pyproject.toml index ea1dafe2d..2853b613f 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-langchain/pyproject.toml +++ b/instrumentation-loongsuite/loongsuite-instrumentation-langchain/pyproject.toml @@ -26,8 +26,8 @@ classifiers = [ ] dependencies = [ "opentelemetry-api ~= 1.37", - "opentelemetry-instrumentation ~= 0.58b0", - "opentelemetry-semantic-conventions ~= 0.58b0", + "opentelemetry-instrumentation >= 0.58b0", + "opentelemetry-semantic-conventions >= 0.58b0", ] [project.optional-dependencies] diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-mcp/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-mcp/pyproject.toml index e1e548b4d..e106d2074 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-mcp/pyproject.toml +++ b/instrumentation-loongsuite/loongsuite-instrumentation-mcp/pyproject.toml @@ -8,7 +8,7 @@ dynamic = ["version"] description = "OpenTelemetry MCP (Model Context Protocol) instrumentation" readme = "README.md" license = "Apache-2.0" -requires-python = ">=3.10, <=3.13" +requires-python = ">=3.10" authors = [ { name = "LoongSuite Python Agent Authors"}, ] @@ -31,7 +31,7 @@ dependencies = [ [project.optional-dependencies] instruments = [ - "mcp >= 1.3.0, <= 1.13.1", + "mcp >= 1.3.0, <= 1.25.0", ] test = [ "opentelemetry-sdk", diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-mcp/src/opentelemetry/instrumentation/mcp/__init__.py b/instrumentation-loongsuite/loongsuite-instrumentation-mcp/src/opentelemetry/instrumentation/mcp/__init__.py index 0f589b6ba..2213faff5 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-mcp/src/opentelemetry/instrumentation/mcp/__init__.py +++ b/instrumentation-loongsuite/loongsuite-instrumentation-mcp/src/opentelemetry/instrumentation/mcp/__init__.py @@ -19,6 +19,7 @@ ) from opentelemetry.instrumentation.mcp.utils import ( _get_logger, + _get_streamable_http_client_name, _is_version_supported, _is_ws_installed, ) @@ -53,6 +54,8 @@ for method_name, rpc_name in RPC_NAME_MAPPING.items() ] +_streamable_http_client_name = _get_streamable_http_client_name() + class MCPInstrumentor(BaseInstrumentor): """ @@ -99,7 +102,7 @@ def _instrument(self, **kwargs: Any) -> None: ) wrap_function_wrapper( module="mcp.client.streamable_http", - name="streamablehttp_client", + name=_streamable_http_client_name, wrapper=streamable_http_client_wrapper(), ) wrap_function_wrapper( @@ -140,10 +143,10 @@ def _uninstrument(self, **kwargs: Any) -> None: try: import mcp.client.streamable_http # noqa: PLC0415 - unwrap(mcp.client.streamable_http, "streamablehttp_client") + unwrap(mcp.client.streamable_http, _streamable_http_client_name) except Exception: logger.warning( - "Fail to uninstrument streamablehttp_client", exc_info=True + "Fail to uninstrument streamable_http_client", exc_info=True ) try: diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-mcp/src/opentelemetry/instrumentation/mcp/utils.py b/instrumentation-loongsuite/loongsuite-instrumentation-mcp/src/opentelemetry/instrumentation/mcp/utils.py index 42655cdc3..23c33e243 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-mcp/src/opentelemetry/instrumentation/mcp/utils.py +++ b/instrumentation-loongsuite/loongsuite-instrumentation-mcp/src/opentelemetry/instrumentation/mcp/utils.py @@ -18,10 +18,17 @@ _has_mcp_types = False MIN_SUPPORTED_VERSION = (1, 3, 0) -MAX_SUPPORTED_VERSION = (1, 13, 1) +MAX_SUPPORTED_VERSION = (1, 25, 0) MCP_PACKAGE_NAME = "mcp" DEFAULT_MAX_ATTRIBUTE_LENGTH = 1024 * 1024 +# Version thresholds for API changes +# v1.24.0: streamable_http_client was added (streamablehttp_client deprecated) +STREAMABLE_HTTP_CLIENT_NEW_NAME_VERSION = (1, 24, 0) +# Streamable HTTP client function names +STREAMABLE_HTTP_CLIENT_NEW_NAME = "streamable_http_client" +STREAMABLE_HTTP_CLIENT_OLD_NAME = "streamablehttp_client" + _max_attributes_length = None @@ -77,6 +84,18 @@ def _is_version_supported() -> bool: ) +def _get_streamable_http_client_name() -> str: + """ + Get the correct streamable HTTP client function name based on MCP version. + - v1.24.0+: uses `streamable_http_client` (new name) + - v1.3.0 - v1.23.x: uses `streamablehttp_client` (old name) + """ + current_version = _get_mcp_version() + if current_version >= STREAMABLE_HTTP_CLIENT_NEW_NAME_VERSION: + return STREAMABLE_HTTP_CLIENT_NEW_NAME + return STREAMABLE_HTTP_CLIENT_OLD_NAME + + def _is_capture_content_enabled() -> bool: capture_content = environ.get( MCPEnvironmentVariables.CAPTURE_INPUT_ENABLED, "true" diff --git a/instrumentation-loongsuite/loongsuite-instrumentation-mem0/pyproject.toml b/instrumentation-loongsuite/loongsuite-instrumentation-mem0/pyproject.toml index 0e4a685c7..ff3c03bbe 100644 --- a/instrumentation-loongsuite/loongsuite-instrumentation-mem0/pyproject.toml +++ b/instrumentation-loongsuite/loongsuite-instrumentation-mem0/pyproject.toml @@ -26,9 +26,9 @@ classifiers = [ dependencies = [ "wrapt >=1.17.3", "opentelemetry-api ~=1.37", - "opentelemetry-instrumentation ~=0.58b0", - "opentelemetry-semantic-conventions ~=0.58b0", - "opentelemetry-util-genai ~= 0.2b0", + "opentelemetry-instrumentation >=0.58b0", + "opentelemetry-semantic-conventions >=0.58b0", + "opentelemetry-util-genai >= 0.2b0", ] [project.optional-dependencies] diff --git a/loongsuite-distro/BOOTSTRAP_REVIEW.md b/loongsuite-distro/BOOTSTRAP_REVIEW.md new file mode 100644 index 000000000..2ebff4fb6 --- /dev/null +++ b/loongsuite-distro/BOOTSTRAP_REVIEW.md @@ -0,0 +1,272 @@ +# bootstrap.py 流程梳理与优化建议 + +## 一、基本流程梳理 + +### 1. 安装流程 (`install_from_tar`) + +``` +main() + └─> install_from_tar() + ├─> resolve_tar_path() # 解析 tar 路径，可能需要下载 + ├─> extract_tar() # 解压 tar 文件，获取所有 .whl 文件 + ├─> filter_packages() # 过滤包（核心逻辑） + │ ├─> get_package_name_from_whl() # 从 whl 文件名提取包名 + │ ├─> _is_instrumentation_in_bootstrap_gen() # 检查是否为 instrumentation + │ ├─> check_python_version_compatibility() # 检查 Python 版本兼容性 + │ ├─> check_dependency_compatibility() # 检查依赖版本兼容性 + │ └─> get_target_libraries_from_bootstrap_gen() + _is_library_installed() # 自动检测 + └─> install_packages() # 使用 pip 安装 +``` + +### 2. 卸载流程 (`uninstall_loongsuite_packages`) + +``` +main() + └─> uninstall_loongsuite_packages() + ├─> get_installed_loongsuite_packages() # 获取已安装的包列表 + └─> uninstall_packages() # 使用 pip 卸载 +``` + +### 3. 核心辅助函数 + +- **包名处理**： + - `get_package_name_from_whl()`: 从 whl 文件名提取包名 + - `get_installed_package_version()`: 获取已安装包的版本（处理下划线/连字符变体） + +- **元数据提取**： + - `get_metadata_from_whl()`: 从 whl 文件提取 METADATA + - `get_python_requirement_from_whl()`: 提取 Python 版本要求 + +- **兼容性检查**： + - `check_python_version_compatibility()`: 检查 Python 版本 + - `check_dependency_compatibility()`: 检查依赖版本 + +- **bootstrap_gen 查询**： + - `_is_instrumentation_in_bootstrap_gen()`: 检查是否为 instrumentation + - `get_target_libraries_from_bootstrap_gen()`: 获取目标库列表 + +- **库检测**： + - `_is_library_installed()`: 检查库是否已安装 + +## 二、发现的问题和优化建议 + +### 1. 🔴 包名规范化逻辑重复 + +**问题**： +- `get_installed_package_version()` 中三次尝试（原始名、下划线→连字符、连字符→下划线） +- `_is_library_installed()` 中也有类似逻辑 +- 多个地方都有 `normalized_name = package_name.replace("_", "-")` 的重复 + +**建议**： +```python +def normalize_package_name(package_name: str) -> str: + """统一规范化包名：将下划线转换为连字符""" + return package_name.replace("_", "-") + +def get_package_name_variants(package_name: str) -> List[str]: + """获取包名的所有可能变体（用于查找）""" + normalized = normalize_package_name(package_name) + variants = [package_name] + if normalized != package_name: + variants.append(normalized) + # 如果需要，也可以添加反向变体 + return variants +``` + +### 2. 🔴 从 requirement 字符串提取包名的逻辑重复 + +**问题**： +在 `_is_instrumentation_in_bootstrap_gen()` 和 `get_target_libraries_from_bootstrap_gen()` 中都有： +```python +default_pkg_name = ( + default_instr.split("==")[0] + .split(">=")[0] + .split("<=")[0] + .split("~=")[0] + .split("!=")[0] + .strip() +) +``` + +**建议**： +```python +def extract_package_name_from_requirement(req_str: str) -> str: + """从 requirement 字符串中提取包名""" + try: + return Requirement(req_str).name + except Exception: + # Fallback: 手动解析 + for op in ["==", ">=", "<=", "~=", "!=", ">", "<"]: + if op in req_str: + return req_str.split(op)[0].strip() + return req_str.strip() +``` + +### 3. 🟡 get_installed_package_version 中的重复代码 + +**问题**： +三个几乎相同的 try-except 块，只是包名不同。 + +**建议**： +```python +def get_installed_package_version(package_name: str) -> Optional[str]: + """获取已安装包的版本""" + variants = get_package_name_variants(package_name) + + for variant in variants: + version = _try_get_version(variant) + if version: + return version + return None + +def _try_get_version(package_name: str) -> Optional[str]: + """尝试获取单个包名变体的版本""" + cmd = [sys.executable, "-m", "pip", "show", package_name] + try: + result = subprocess.run( + cmd, capture_output=True, text=True, check=True, timeout=5 + ) + for line in result.stdout.splitlines(): + if line.startswith("Version:"): + return line.split(":", 1)[1].strip() + except (subprocess.CalledProcessError, subprocess.TimeoutExpired): + pass + return None +``` + +### 4. 🟡 filter_packages 函数过长 + +**问题**： +`filter_packages()` 函数有 130+ 行，包含太多逻辑，可读性差。 + +**建议**： +拆分为多个小函数： +```python +def filter_packages(...): + """主函数，协调各个过滤步骤""" + base_packages = [] + instrumentation_packages = [] + + for whl_file in whl_files: + package_name = get_package_name_from_whl(whl_file) + + if _should_skip_package(package_name, whl_file, blacklist, whitelist, + skip_version_check, auto_detect): + continue + + if package_name in BASE_DEPENDENCIES: + base_packages.append(whl_file) + else: + if _should_install_instrumentation(package_name, whl_file, auto_detect): + instrumentation_packages.append(whl_file) + + return base_packages, instrumentation_packages + +def _should_skip_package(...) -> bool: + """检查是否应该跳过该包""" + # 黑名单/白名单检查 + # Python 版本检查 + # 依赖版本检查 + pass + +def _should_install_instrumentation(...) -> bool: + """检查是否应该安装该 instrumentation""" + # auto-detect 逻辑 + pass +``` + +### 5. 🟡 包名匹配逻辑重复 + +**问题**： +多处都有 `normalized_name == package_name or default_pkg_name == package_name` 这样的匹配。 + +**建议**： +```python +def package_names_match(name1: str, name2: str) -> bool: + """检查两个包名是否匹配（考虑规范化）""" + normalized1 = normalize_package_name(name1) + normalized2 = normalize_package_name(name2) + return (normalized1 == normalized2 or + name1 == name2 or + normalized1 == name2 or + name1 == normalized2) +``` + +### 6. 🟢 常量提取 + +**问题**： +`EXCLUDED_PACKAGES` 在函数内部定义，应该移到模块级别。 + +**建议**： +```python +# 在模块级别定义 +UNINSTALL_EXCLUDED_PACKAGES = { + "loongsuite-distro", + "opentelemetry-api", + "opentelemetry-sdk", + "opentelemetry-instrumentation", +} +``` + +### 7. 🟢 错误处理改进 + +**问题**： +多处使用 `except Exception: pass`，可能隐藏重要错误。 + +**建议**： +更具体地捕获异常，至少记录警告： +```python +except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + logger.debug(f"Failed to get version for {package_name}: {e}") + return None +except Exception as e: + logger.warning(f"Unexpected error getting version for {package_name}: {e}") + return None +``` + +### 8. 🟢 使用 packaging 库解析 requirement + +**问题**： +手动解析 requirement 字符串（split("==")[0]...）不够健壮。 + +**建议**： +统一使用 `packaging.requirements.Requirement` 解析（已经在用，但有些地方还在手动解析）。 + +### 9. 🟡 模块化建议 + +**建议**将代码拆分为多个模块**： + +``` +loongsuite/distro/ + ├── bootstrap.py # 主入口和 CLI + ├── package_utils.py # 包名处理、版本获取等工具函数 + ├── metadata.py # whl 元数据提取 + ├── compatibility.py # 兼容性检查 + └── bootstrap_gen.py # bootstrap_gen 查询（已存在） +``` + +## 三、优先级建议 + +### 高优先级（立即优化） +1. ✅ 提取包名规范化函数（减少重复，提高一致性） +2. ✅ 提取 requirement 解析函数（多处使用，容易出错） +3. ✅ 简化 `get_installed_package_version()`（消除重复代码） + +### 中优先级（后续优化） +4. ⚠️ 拆分 `filter_packages()` 函数（提高可读性） +5. ⚠️ 提取包名匹配函数（统一匹配逻辑） +6. ⚠️ 改进错误处理（更好的调试体验） + +### 低优先级（可选） +7. 💡 模块化拆分（如果文件继续增长） +8. 💡 使用更专业的 metadata 解析库（如果遇到解析问题） + +## 四、总结 + +当前代码功能完整，但存在以下主要问题： +1. **代码重复**：包名规范化、requirement 解析等逻辑在多处重复 +2. **函数过长**：`filter_packages()` 函数包含太多逻辑 +3. **错误处理**：过于宽泛的异常捕获可能隐藏问题 + +建议优先解决代码重复问题，这将提高代码的可维护性和一致性。 + diff --git a/loongsuite-distro/README.rst b/loongsuite-distro/README.rst new file mode 100644 index 000000000..142d2d6fb --- /dev/null +++ b/loongsuite-distro/README.rst @@ -0,0 +1,77 @@ +LoongSuite Distro +================= + +LoongSuite Python Agent's Distro package, providing LoongSuite-specific configuration and tools. + +Installation +------------ + +:: + + pip install loongsuite-distro + +Optional dependencies: + +:: + + # Install with baggage processor support + pip install loongsuite-distro[baggage] + + # Install with OTLP exporter support + pip install loongsuite-distro[otlp] + + # Install with both + pip install loongsuite-distro[baggage,otlp] + +Features +-------- + +1. **LoongSuite Distro**: Provides LoongSuite-specific OpenTelemetry configuration +2. **LoongSuite Bootstrap**: Install all LoongSuite components from tar package +3. **Baggage Processor**: Optional baggage span processor with prefix matching and stripping support + +Usage +----- + +### Configure LoongSuite Distro + +Specify using LoongSuite Distro via environment variable:: + + export OTEL_PYTHON_DISTRO=loongsuite + +### Use LoongSuite Bootstrap + +Install all components from tar package:: + + loongsuite-bootstrap -t loongsuite-python-agent-1.0.0.tar.gz + +Install from GitHub Releases:: + + loongsuite-bootstrap -v 1.0.0 + +Install latest version:: + + loongsuite-bootstrap --latest + +### Configure Baggage Processor + +The baggage processor is automatically loaded if configured via environment variables. +First, install the optional dependency:: + + pip install loongsuite-distro[baggage] + +Then configure via environment variables:: + + export LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES="traffic.,app." + export LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES="traffic." + +The processor will only be loaded if ``LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES`` is set. + +For more usage, please refer to `LOONGSUITE_BOOTSTRAP_README.md`. + +References +---------- + +* `LoongSuite Python Agent `_ + + diff --git a/loongsuite-distro/pyproject.toml b/loongsuite-distro/pyproject.toml new file mode 100644 index 000000000..0cdc6e522 --- /dev/null +++ b/loongsuite-distro/pyproject.toml @@ -0,0 +1,68 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "loongsuite-distro" +dynamic = ["version"] +description = "LoongSuite Python Agent Distro" +readme = "README.rst" +license = "Apache-2.0" +requires-python = ">=3.9" +authors = [ + { name = "LoongSuite Python Agent Authors", email = "qp467389@alibaba-inc.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Typing :: Typed", +] +dependencies = [ + "opentelemetry-api ~= 1.12", + "opentelemetry-sdk ~= 1.13", + "opentelemetry-instrumentation >= 0.58b0", +] + +[project.optional-dependencies] +otlp = [ + "opentelemetry-exporter-otlp ~= 1.40", +] +baggage = [ + "loongsuite-processor-baggage", +] + +[project.entry-points.opentelemetry_configurator] +loongsuite = "loongsuite.distro:LoongSuiteConfigurator" + +[project.entry-points.opentelemetry_distro] +loongsuite = "loongsuite.distro:LoongSuiteDistro" + +[project.scripts] +loongsuite-bootstrap = "loongsuite.distro.bootstrap:main" +loongsuite-instrument = "opentelemetry.instrumentation.auto_instrumentation:run" + +[project.urls] +Homepage = "https://github.com/alibaba/loongsuite-python-agent" +Repository = "https://github.com/alibaba/loongsuite-python-agent" + +[tool.hatch.version] +path = "src/loongsuite/distro/version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/loongsuite"] + + diff --git a/loongsuite-distro/src/loongsuite/__init__.py b/loongsuite-distro/src/loongsuite/__init__.py new file mode 100644 index 000000000..476fb73aa --- /dev/null +++ b/loongsuite-distro/src/loongsuite/__init__.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + diff --git a/loongsuite-distro/src/loongsuite/distro/__init__.py b/loongsuite-distro/src/loongsuite/distro/__init__.py new file mode 100644 index 000000000..375f38909 --- /dev/null +++ b/loongsuite-distro/src/loongsuite/distro/__init__.py @@ -0,0 +1,180 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +from typing import TYPE_CHECKING, Any, Optional, Set, cast + +from opentelemetry import trace +from opentelemetry.environment_variables import ( + OTEL_LOGS_EXPORTER, + OTEL_METRICS_EXPORTER, + OTEL_TRACES_EXPORTER, +) +from opentelemetry.instrumentation.distro import BaseDistro +from opentelemetry.sdk._configuration import _OTelSDKConfigurator +from opentelemetry.sdk.environment_variables import OTEL_EXPORTER_OTLP_PROTOCOL +from opentelemetry.sdk.trace import TracerProvider + +if TYPE_CHECKING: + from opentelemetry.sdk.trace import SpanProcessor + +logger = logging.getLogger(__name__) + +# Environment variable names for baggage processor configuration +_LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES = ( + "LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES" +) +_LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES = ( + "LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES" +) + + +class LoongSuiteConfigurator(_OTelSDKConfigurator): + """ + LoongSuite configurator, inherits from OpenTelemetry SDK configurator + + Automatically adds LoongSuiteBaggageSpanProcessor if configured via environment variables. + Only loads the processor if LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES is set. + """ + + def _configure(self, **kwargs: Any) -> None: + # Call parent method to complete base initialization + super()._configure(**kwargs) # type: ignore[misc] + + # Get tracer provider + tracer_provider = trace.get_tracer_provider() + + if isinstance(tracer_provider, TracerProvider): + # Get additional processors + additional_processors = self._get_additional_span_processors( + **kwargs + ) + + # Add additional processors + for processor in additional_processors: + tracer_provider.add_span_processor(processor) + + def _get_additional_span_processors( + self, **kwargs: Any + ) -> list["SpanProcessor"]: + """ + Return additional span processors to add to trace provider + + Subclasses can override this method to provide custom processors. + + Supports configuration via environment variables for baggage processor: + - LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES: Comma-separated list of prefixes for matching baggage keys + - LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES: Comma-separated list of prefixes to strip from baggage keys + + The baggage processor is only loaded if LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES is set. + + Args: + **kwargs: Arguments passed to _configure + + Returns: + List of span processors to add + """ + processors: list["SpanProcessor"] = [] + + # Check if baggage allowed prefixes is configured + allowed_prefixes_str = os.getenv( + _LOONGSUITE_PROCESSOR_BAGGAGE_ALLOWED_PREFIXES + ) + + if allowed_prefixes_str: + # Try to load loongsuite-processor-baggage + try: + # Dynamic import to avoid type checker errors + from loongsuite.processor.baggage import ( # noqa: PLC0415 + LoongSuiteBaggageSpanProcessor, + ) + + # Parse allowed prefixes + allowed_prefixes = self._parse_prefixes(allowed_prefixes_str) + + # Parse strip prefixes + strip_prefixes_str = os.getenv( + _LOONGSUITE_PROCESSOR_BAGGAGE_STRIP_PREFIXES + ) + strip_prefixes = ( + self._parse_prefixes(strip_prefixes_str) + if strip_prefixes_str + else None + ) + + # Create processor + # LoongSuiteBaggageSpanProcessor inherits from SpanProcessor + processor_instance = LoongSuiteBaggageSpanProcessor( # type: ignore[misc] + allowed_prefixes=allowed_prefixes + if allowed_prefixes + else None, + strip_prefixes=strip_prefixes if strip_prefixes else None, + ) + # Type cast since LoongSuiteBaggageSpanProcessor inherits from SpanProcessor + processor = cast("SpanProcessor", processor_instance) + processors.append(processor) + + logger.info( + "Loaded LoongSuiteBaggageSpanProcessor with allowed_prefixes=%s, strip_prefixes=%s", + allowed_prefixes, + strip_prefixes, + ) + except ImportError as e: + logger.warning( + "Failed to import loongsuite.processor.baggage: %s. " + "Baggage processor will not be loaded. " + "Please install loongsuite-processor-baggage package.", + e, + ) + + return processors + + @staticmethod + def _parse_prefixes(prefixes_str: str) -> Optional[Set[str]]: + """ + Parse comma-separated prefix string + + Args: + prefixes_str: Comma-separated prefix string, e.g., "traffic.,app." + + Returns: + Set of prefixes, or None if input is empty + """ + if not prefixes_str or not prefixes_str.strip(): + return None + + # Split and strip whitespace + prefixes = { + prefix.strip() + for prefix in prefixes_str.split(",") + if prefix.strip() + } + + return prefixes if prefixes else None + + +class LoongSuiteDistro(BaseDistro): + """ + LoongSuite Distro configures default OpenTelemetry settings. + + This is the Distro provided by LoongSuite, which configures default exporters and protocols. + """ + + # pylint: disable=no-self-use + def _configure(self, **kwargs: Any) -> None: + os.environ.setdefault(OTEL_TRACES_EXPORTER, "otlp") + os.environ.setdefault(OTEL_METRICS_EXPORTER, "otlp") + os.environ.setdefault(OTEL_LOGS_EXPORTER, "otlp") + os.environ.setdefault(OTEL_EXPORTER_OTLP_PROTOCOL, "grpc") diff --git a/loongsuite-distro/src/loongsuite/distro/bootstrap.py b/loongsuite-distro/src/loongsuite/distro/bootstrap.py new file mode 100644 index 000000000..1470e1c0b --- /dev/null +++ b/loongsuite-distro/src/loongsuite/distro/bootstrap.py @@ -0,0 +1,1177 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +LoongSuite Bootstrap Tool + +Install all components of loongsuite Python Agent from tar.gz package. +Supports blacklist/whitelist to control which instrumentations to install. +""" + +import argparse +import json as json_lib +import logging +import shutil +import subprocess +import sys +import tarfile +import tempfile +import urllib.request +import zipfile +from pathlib import Path +from typing import Any, List, Optional, Set, Tuple, Union + +from loongsuite.distro.bootstrap_gen import ( + default_instrumentations as gen_default_instrumentations, +) +from loongsuite.distro.bootstrap_gen import libraries as gen_libraries +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet + +logger = logging.getLogger(__name__) + +# Base dependency packages (must be installed) +BASE_DEPENDENCIES = { + "opentelemetry-api", + "opentelemetry-sdk", + "opentelemetry-instrumentation", + "opentelemetry-util-genai", + "opentelemetry-semantic-conventions", +} + +# Packages to exclude from uninstallation +UNINSTALL_EXCLUDED_PACKAGES = { + "loongsuite-distro", + "opentelemetry-api", + "opentelemetry-sdk", + "opentelemetry-instrumentation", +} + + +def normalize_package_name(package_name: str) -> str: + """ + Normalize package name by converting underscores to hyphens. + + Package names in PyPI use hyphens, but wheel filenames may use underscores. + This function ensures consistent package name format. + + Args: + package_name: Package name (may contain underscores or hyphens) + + Returns: + Normalized package name with hyphens + """ + return package_name.replace("_", "-") + + +def get_package_name_variants(package_name: str) -> List[str]: + """ + Get all possible variants of a package name for lookup. + + This is useful when checking if a package is installed, as package names + may be stored with either underscores or hyphens. + + Args: + package_name: Package name + + Returns: + List of package name variants to try + """ + variants = [package_name] + normalized = normalize_package_name(package_name) + if normalized != package_name: + variants.append(normalized) + # Also try reverse (hyphens to underscores) for completeness + reverse = package_name.replace("-", "_") + if reverse != package_name and reverse not in variants: + variants.append(reverse) + return variants + + +def extract_package_name_from_requirement(req_str: str) -> str: + """ + Extract package name from a requirement string. + + Examples: + "redis >= 2.6" -> "redis" + "opentelemetry-instrumentation==0.60b0" -> "opentelemetry-instrumentation" + "package-name~=1.0" -> "package-name" + + Args: + req_str: Requirement string + + Returns: + Package name extracted from requirement + """ + try: + return Requirement(req_str).name + except Exception: + # Fallback: manual parsing if Requirement parsing fails + for op in ["==", ">=", "<=", "~=", "!=", ">", "<"]: + if op in req_str: + return req_str.split(op)[0].strip() + return req_str.strip() + + +def package_names_match(name1: str, name2: str) -> bool: + """ + Check if two package names match (considering normalization). + + Args: + name1: First package name + name2: Second package name + + Returns: + True if names match (after normalization), False otherwise + """ + normalized1 = normalize_package_name(name1) + normalized2 = normalize_package_name(name2) + return ( + normalized1 == normalized2 + or name1 == name2 + or normalized1 == name2 + or name1 == normalized2 + ) + + +def load_list_file(file_path: Path) -> Set[str]: + """Load list from file (one package name per line)""" + if not file_path.exists(): + return set() + + packages = set() + with open(file_path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + packages.add(line) + + return packages + + +def get_package_name_from_whl(whl_path: Path) -> str: + """ + Extract package name from whl filename + + Wheel filename format: {package_name}-{version}-{python_tag}-{abi_tag}-{platform_tag}.whl + Example: loongsuite_instrumentation_mem0-0.1.0-py3-none-any.whl + + Returns normalized package name with hyphens (e.g., "loongsuite-instrumentation-mem0") + """ + name = whl_path.stem # Remove .whl extension + parts = name.split("-") + + if len(parts) < 2: + # If no hyphens, return as-is (shouldn't happen for valid wheels) + return name.replace("_", "-") + + package_parts = [] + for part in parts: + # Check if this part looks like a version number + # Version numbers typically: + # - Start with a digit + # - Contain dots (e.g., "0.1.0", "1.2.3") + # - Or are build tags like "dev", "b0", etc. + # - Or are Python/ABI/platform tags + + # Check for version-like patterns: starts with digit and contains dot, or is a known tag + is_version_like = ( + ( + part and part[0].isdigit() and "." in part + ) # e.g., "0.1.0", "1.2.3" + or part in ("dev", "b0", "b1", "rc0", "rc1") # Build tags + or part.startswith("py") # Python tags: "py3", "py2", "py" + or part in ("none", "any") # ABI/platform tags + ) + + if is_version_like: + break + + package_parts.append(part) + + if not package_parts: + # Fallback: if we couldn't extract, use first part + result = parts[0] if parts else name + else: + # Join with hyphens + result = "-".join(package_parts) + + # Normalize: convert underscores to hyphens for package name consistency + # (wheel filenames may use underscores, but package names use hyphens) + result = result.replace("_", "-") + return result + + +def get_metadata_from_whl(whl_path: Path) -> Optional[dict[str, Any]]: + """ + Extract metadata from whl file + + Args: + whl_path: Path to whl file + + Returns: + Dictionary with metadata fields, or None if not found + """ + try: + with zipfile.ZipFile(whl_path, "r") as whl_zip: + # Look for METADATA file in the wheel + metadata_path = None + for name in whl_zip.namelist(): + if name.endswith("/METADATA") or name == "METADATA": + metadata_path = name + break + + if not metadata_path: + return None + + metadata = {} + current_field = None + # Read METADATA file + with whl_zip.open(metadata_path) as metadata_file: + for line in metadata_file: + line_str = line.decode("utf-8").strip() + if not line_str: + current_field = None + continue + + # Check for continuation line + if line_str.startswith(" ") or line_str.startswith("\t"): + if current_field and current_field in metadata: + if isinstance(metadata[current_field], list): + if metadata[current_field]: + metadata[current_field][-1] += ( + " " + line_str.strip() + ) + else: + metadata[current_field] += ( + " " + line_str.strip() + ) + continue + + # Parse field name and value + if ":" in line_str: + field_name, field_value = line_str.split(":", 1) + field_name = field_name.strip() + field_value = field_value.strip() + current_field = field_name + + if field_name == "Requires-Python": + metadata["requires_python"] = field_value + elif field_name == "Requires-Dist": + if "requires_dist" not in metadata: + metadata["requires_dist"] = [] + metadata["requires_dist"].append(field_value) + elif field_name == "Provides-Extra": + if "provides_extra" not in metadata: + metadata["provides_extra"] = [] + metadata["provides_extra"].append(field_value) + + return metadata if metadata else None + except Exception: + pass + + return None + + +def get_python_requirement_from_whl(whl_path: Path) -> Optional[str]: + """ + Extract Python version requirement from whl file metadata + + Args: + whl_path: Path to whl file + + Returns: + Python version requirement string (e.g., ">=3.10, <=3.13") or None if not found + """ + metadata = get_metadata_from_whl(whl_path) + return metadata.get("requires_python") if metadata else None + + +def _try_get_package_version(package_name: str) -> Optional[str]: + """ + Try to get version of a package using pip show. + + Args: + package_name: Package name to check + + Returns: + Version string if found, None otherwise + """ + cmd = [sys.executable, "-m", "pip", "show", package_name] + try: + result = subprocess.run( + cmd, capture_output=True, text=True, check=True, timeout=5 + ) + for line in result.stdout.splitlines(): + if line.startswith("Version:"): + return line.split(":", 1)[1].strip() + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + logger.debug(f"Failed to get version for {package_name}: {e}") + except Exception as e: + logger.warning(f"Unexpected error getting version for {package_name}: {e}") + return None + + +def get_installed_package_version(package_name: str) -> Optional[str]: + """ + Get installed version of a package. + + Tries multiple name variants (with underscores/hyphens) to handle + different naming conventions. + + Args: + package_name: Package name (may contain hyphens or underscores) + + Returns: + Installed version string, or None if not installed + """ + variants = get_package_name_variants(package_name) + for variant in variants: + version = _try_get_package_version(variant) + if version: + return version + return None + + +def _is_library_installed(req_str: str) -> bool: + """ + Check if a library is installed and version satisfies requirement. + + Similar to opentelemetry-bootstrap's _is_installed function. + + Args: + req_str: Requirement string (e.g., "redis >= 2.6") + + Returns: + True if library is installed and version satisfies requirement, False otherwise + """ + try: + req = Requirement(req_str) + package_name = req.name + + # get_installed_package_version already tries multiple variants + dist_version = get_installed_package_version(package_name) + + if dist_version is None: + return False + + # Check if installed version satisfies requirement + return req.specifier.contains(dist_version) + except Exception as e: + logger.debug(f"Failed to check if library is installed for {req_str}: {e}") + return False + + +def _is_instrumentation_in_bootstrap_gen(package_name: str) -> bool: + """ + Check if a package is an instrumentation listed in bootstrap_gen.py. + + Args: + package_name: Package name to check + + Returns: + True if the package is in bootstrap_gen.py (either in libraries or default_instrumentations) + """ + if not package_name: + return False + + # Check default instrumentations + for default_instr in gen_default_instrumentations: + if isinstance(default_instr, str): + default_pkg_name = extract_package_name_from_requirement(default_instr) + if package_names_match(default_pkg_name, package_name): + return True + + # Check libraries mapping + for lib_mapping in gen_libraries: + instrumentation = lib_mapping.get("instrumentation", "") + if isinstance(instrumentation, str): + instr_pkg_name = extract_package_name_from_requirement(instrumentation) + if package_names_match(instr_pkg_name, package_name): + return True + + return False + + +def get_target_libraries_from_bootstrap_gen( + package_name: str, +) -> Tuple[List[str], bool]: + """ + Get target library requirements from bootstrap_gen.py. + + This function uses the pre-generated bootstrap_gen.py file to get + target library information, similar to opentelemetry-bootstrap. + + Args: + package_name: Name of the instrumentation package (e.g., "opentelemetry-instrumentation-redis") + May contain hyphens or underscores, will be normalized + + Returns: + Tuple of (target_libraries list, is_default_instrumentation bool) + target_libraries contains library requirement strings (e.g., ["redis >= 2.6"]) + is_default_instrumentation is True if this is a default instrumentation + """ + if not package_name: + return [], False + + # Check if it's a default instrumentation + for default_instr in gen_default_instrumentations: + if isinstance(default_instr, str): + default_pkg_name = extract_package_name_from_requirement(default_instr) + if package_names_match(default_pkg_name, package_name): + return [], True + + # Look up in libraries mapping + target_libraries = [] + for lib_mapping in gen_libraries: + instrumentation = lib_mapping.get("instrumentation", "") + if isinstance(instrumentation, str): + instr_pkg_name = extract_package_name_from_requirement(instrumentation) + if package_names_match(instr_pkg_name, package_name): + target_lib = lib_mapping.get("library", "") + if target_lib and isinstance(target_lib, str): + target_libraries.append(target_lib) + + return target_libraries, False + + +def check_dependency_compatibility( + whl_path: Path, skip_version_check: bool = False +) -> Tuple[bool, Optional[str]]: + """ + Check if package dependencies are compatible with installed packages + + Args: + whl_path: Path to whl file + skip_version_check: If True, skip version compatibility check + + Returns: + (is_compatible, conflict_message) + is_compatible: True if compatible, False otherwise + conflict_message: Description of conflict if incompatible, None otherwise + """ + if skip_version_check: + return True, None + + metadata = get_metadata_from_whl(whl_path) + if not metadata or "requires_dist" not in metadata: + return True, None + + # Key packages to check compatibility + key_packages = { + "opentelemetry-instrumentation", + "opentelemetry-semantic-conventions", + } + + conflicts = [] + for req_str in metadata.get("requires_dist", []): + try: + req = Requirement(req_str) + if req.name.lower() in key_packages: + installed_version = get_installed_package_version(req.name) + if installed_version: + # Check if installed version satisfies requirement + if not req.specifier.contains(installed_version): + conflicts.append( + f"{req.name} {installed_version} does not satisfy {req_str}" + ) + except Exception: + # If parsing fails, assume compatible to avoid false positives + continue + + if conflicts: + conflict_msg = "; ".join(conflicts) + return False, conflict_msg + + return True, None + + +def check_python_version_compatibility( + whl_path: Path, current_version: Tuple[int, int] +) -> Tuple[bool, Optional[str]]: + """ + Check if current Python version is compatible with whl file requirements + + Args: + whl_path: Path to whl file + current_version: Current Python version as (major, minor) tuple + + Returns: + (is_compatible, requirement_string) + is_compatible: True if compatible, False otherwise + requirement_string: Python requirement string if found, None otherwise + """ + requirement_str = get_python_requirement_from_whl(whl_path) + + if not requirement_str: + # If no requirement found, assume compatible + return True, None + + try: + # Parse the requirement string + spec = SpecifierSet(requirement_str) + # Convert current version to string format + current_version_str = f"{current_version[0]}.{current_version[1]}" + # Check if current version satisfies the requirement + is_compatible = spec.contains(current_version_str) + return is_compatible, requirement_str + except Exception: + # If parsing fails, assume compatible to avoid false positives + return True, requirement_str + + +def download_file(url: str, dest: Path) -> Path: + """Download file to specified path""" + logger.info(f"Downloading file: {url}") + urllib.request.urlretrieve(url, dest) + logger.info(f"Download completed: {dest}") + return dest + + +def extract_tar(tar_path: Path, extract_dir: Path) -> List[Path]: + """Extract tar.gz file, return all whl file paths""" + logger.info(f"Extracting tar file: {tar_path} -> {extract_dir}") + + whl_files = [] + with tarfile.open(tar_path, "r:gz") as tar: + tar.extractall(extract_dir) + + # Find all whl files + for member in tar.getmembers(): + if member.name.endswith(".whl"): + whl_path = extract_dir / member.name + if whl_path.exists(): + whl_files.append(whl_path) + + logger.info(f"Extraction completed, found {len(whl_files)} whl files") + return sorted(whl_files) + + +def filter_packages( + whl_files: List[Path], + blacklist: Optional[Set[str]] = None, + whitelist: Optional[Set[str]] = None, + skip_version_check: bool = False, + auto_detect: bool = False, +) -> Tuple[List[Path], List[Path]]: + """ + Filter packages based on blacklist/whitelist, Python version compatibility, + dependency version compatibility, and optionally auto-detect installed libraries + + Args: + whl_files: List of whl file paths + blacklist: blacklist (do not install these packages) + whitelist: whitelist (only install these packages if specified) + skip_version_check: If True, skip dependency version compatibility check + auto_detect: If True, only install instrumentation packages if their target libraries are installed + + Returns: + (base dependency packages list, instrumentation packages list) + """ + base_packages = [] + instrumentation_packages = [] + + blacklist = blacklist or set() + whitelist = whitelist or set() + + # Get current Python version + current_version = (sys.version_info.major, sys.version_info.minor) + current_version_str = f"{current_version[0]}.{current_version[1]}" + + logger.info(f"Scanning {len(whl_files)} packages for installation...") + if auto_detect: + logger.info( + "Auto-detect mode enabled: will only install instrumentations for detected libraries" + ) + + for whl_file in whl_files: + package_name = get_package_name_from_whl(whl_file) + + # Check blacklist + if blacklist and package_name in blacklist: + logger.info(f"Skipping {package_name} (blacklist)") + continue + + # Check whitelist + if whitelist and package_name not in whitelist: + logger.info(f"Skipping {package_name} (not in whitelist)") + continue + + # Check Python version compatibility (only for instrumentations in bootstrap_gen.py) + # Base dependencies and utility packages are installed without Python version check + is_instrumentation = _is_instrumentation_in_bootstrap_gen(package_name) + if is_instrumentation: + is_compatible, requirement_str = ( + check_python_version_compatibility(whl_file, current_version) + ) + if not is_compatible: + logger.info( + f"Skipping {package_name} (Python version incompatible: requires {requirement_str}, current: {current_version_str})" + ) + continue + + # Check dependency version compatibility (only for base dependencies) + # Instrumentation packages will be checked by pip during installation + if package_name in BASE_DEPENDENCIES: + is_dep_compatible, conflict_msg = check_dependency_compatibility( + whl_file, skip_version_check + ) + if not is_dep_compatible: + logger.warning( + f"Skipping {package_name} (dependency version incompatible: {conflict_msg})" + ) + continue + + # Classify: base dependencies vs instrumentation + if package_name in BASE_DEPENDENCIES: + base_packages.append(whl_file) + else: + # For instrumentation packages, check if auto-detect is enabled + if auto_detect: + target_libraries, is_default = ( + get_target_libraries_from_bootstrap_gen(package_name) + ) + + # Default instrumentations are always installed (like opentelemetry-bootstrap) + if is_default: + logger.info( + f"Will install {package_name} (default instrumentation)" + ) + instrumentation_packages.append(whl_file) + elif target_libraries: + # Check if any target library is installed + library_installed = False + installed_libs = [] + not_installed_libs = [] + for lib_req in target_libraries: + if _is_library_installed(lib_req): + library_installed = True + try: + req = Requirement(lib_req) + installed_libs.append(req.name) + except Exception: + installed_libs.append(lib_req) + else: + try: + req = Requirement(lib_req) + not_installed_libs.append(req.name) + except Exception: + not_installed_libs.append(lib_req) + + if library_installed: + logger.info( + f"Will install {package_name} (detected libraries: {', '.join(installed_libs)})" + ) + instrumentation_packages.append(whl_file) + else: + logger.info( + f"Skipping {package_name} (required libraries not installed: {', '.join(not_installed_libs)})" + ) + continue + else: + # No mapping found in bootstrap_gen.py, skip it + logger.info( + f"Skipping {package_name} (no target libraries mapping in bootstrap_gen.py)" + ) + continue + else: + # Auto-detect disabled, install all instrumentation packages + logger.info(f"Will install {package_name}") + instrumentation_packages.append(whl_file) + + return base_packages, instrumentation_packages + + +def install_packages( + whl_files: List[Path], find_links_dir: Path, upgrade: bool = False +): + """Install packages using pip""" + if not whl_files: + logger.warning("No packages to install") + return + + cmd = [ + sys.executable, + "-m", + "pip", + "install", + "--find-links", + str(find_links_dir), + ] + + if upgrade: + cmd.append("--upgrade") + + # Add all whl files + cmd.extend([str(whl) for whl in whl_files]) + + logger.info(f"Executing install command: {' '.join(cmd)}") + try: + subprocess.run(cmd, check=True) + logger.info("Installation completed") + except subprocess.CalledProcessError as e: + logger.error(f"Installation failed: {e}") + raise + + +def get_installed_loongsuite_packages() -> List[str]: + """ + Get list of installed loongsuite and opentelemetry packages to uninstall + + Excludes: + - loongsuite-distro + - opentelemetry-api + - opentelemetry-sdk + - opentelemetry-instrumentation + + Returns: + List of installed package names to uninstall + """ + cmd = [sys.executable, "-m", "pip", "list", "--format=json"] + try: + result = subprocess.run( + cmd, capture_output=True, text=True, check=True + ) + installed_packages = json_lib.loads(result.stdout) + + # Filter packages to uninstall + packages_to_uninstall = [] + for pkg in installed_packages: + name = pkg.get("name", "") + name_lower = name.lower() + + # Skip excluded packages + if name_lower in UNINSTALL_EXCLUDED_PACKAGES: + continue + + # Include loongsuite-* packages (except loongsuite-distro) + if name_lower.startswith("loongsuite-"): + packages_to_uninstall.append(name) + # Include opentelemetry-* packages (except opentelemetry-api and opentelemetry-sdk) + elif name_lower.startswith("opentelemetry-"): + packages_to_uninstall.append(name) + + return packages_to_uninstall + except subprocess.CalledProcessError as e: + logger.error(f"Failed to get installed packages: {e}") + raise + except json_lib.JSONDecodeError as e: + logger.error(f"Failed to parse pip list output: {e}") + raise + + +def uninstall_packages(package_names: List[str], yes: bool = False): + """Uninstall packages using pip""" + if not package_names: + logger.warning("No packages to uninstall") + return + + cmd = [ + sys.executable, + "-m", + "pip", + "uninstall", + ] + + if yes: + cmd.append("-y") + + # Add all package names + cmd.extend(package_names) + + logger.info(f"Executing uninstall command: {' '.join(cmd)}") + try: + subprocess.run(cmd, check=True) + logger.info("Uninstallation completed") + except subprocess.CalledProcessError as e: + logger.error(f"Uninstallation failed: {e}") + raise + + +def resolve_tar_path( + tar_path: Union[Path, str], +) -> Tuple[Path, Optional[Path]]: + """ + Resolve tar path, downloading from URI if necessary + + Args: + tar_path: tar file path or URI (can be Path or str) + + Returns: + (local_tar_path, temp_dir_to_cleanup) + local_tar_path: Path to local tar file + temp_dir_to_cleanup: Path to temporary directory to clean up (None if not downloaded) + """ + tar_path_str = str(tar_path) + if tar_path_str.startswith(("http://", "https://")): + # Download from URI + temp_dir = Path(tempfile.mkdtemp(prefix="loongsuite-download-")) + temp_tar = temp_dir / "loongsuite.tar.gz" + download_file(tar_path_str, temp_tar) + return temp_tar, temp_dir + else: + tar_path = Path(tar_path) + if not tar_path.exists(): + raise FileNotFoundError(f"Tar file does not exist: {tar_path}") + return tar_path, None + + +def get_package_names_from_tar( + tar_path: Path, + blacklist: Optional[Set[str]] = None, + whitelist: Optional[Set[str]] = None, +) -> List[str]: + """ + Extract package names from tar file + + Args: + tar_path: Path to tar file + blacklist: blacklist (do not include these packages) + whitelist: whitelist (only include these packages if specified) + + Returns: + List of package names + """ + temp_dir = Path(tempfile.mkdtemp(prefix="loongsuite-")) + try: + whl_files = extract_tar(tar_path, temp_dir) + if not whl_files: + raise ValueError("No whl files found in tar file") + + base_packages, instrumentation_packages = filter_packages( + whl_files, blacklist, whitelist, auto_detect=False + ) + + # Get package names + package_names = [] + for whl in base_packages + instrumentation_packages: + package_name = get_package_name_from_whl(whl) + package_names.append(package_name) + + return package_names + finally: + shutil.rmtree(temp_dir, ignore_errors=True) + + +def install_from_tar( + tar_path: Union[Path, str], + blacklist: Optional[Set[str]] = None, + whitelist: Optional[Set[str]] = None, + upgrade: bool = False, + keep_temp: bool = False, + skip_version_check: bool = False, + auto_detect: bool = False, +): + """ + Install loongsuite packages from tar package + + Args: + tar_path: tar file path or URI (can be Path or str) + blacklist: blacklist (do not install these packages) + whitelist: whitelist (only install these packages if specified) + upgrade: whether to upgrade already installed packages + keep_temp: whether to keep temporary directory + skip_version_check: If True, skip dependency version compatibility check + auto_detect: If True, only install instrumentation packages if their target libraries are installed + """ + # Resolve tar path (download from URI if necessary) + local_tar_path, temp_tar_dir = resolve_tar_path(tar_path) + + # Create temporary directory for extraction + temp_dir = Path(tempfile.mkdtemp(prefix="loongsuite-")) + + try: + logger.info("Extracting packages from tar file...") + # Extract tar file + whl_files = extract_tar(local_tar_path, temp_dir) + + if not whl_files: + raise ValueError("No whl files found in tar file") + + logger.info(f"Found {len(whl_files)} packages in tar file") + + # Filter packages + logger.info("Filtering packages...") + base_packages, instrumentation_packages = filter_packages( + whl_files, blacklist, whitelist, skip_version_check, auto_detect + ) + + # Ensure base dependencies must be installed + if not base_packages: + logger.warning( + "Warning: No base dependency packages found, this may cause installation to fail" + ) + + # Merge all packages to install + all_packages = base_packages + instrumentation_packages + + if not all_packages: + logger.warning("No packages to install after filtering") + return + + logger.info( + f"Will install {len(base_packages)} base dependency packages" + ) + logger.info( + f"Will install {len(instrumentation_packages)} instrumentation packages" + ) + + if instrumentation_packages: + logger.info("Instrumentation packages to install:") + for pkg in instrumentation_packages: + pkg_name = get_package_name_from_whl(pkg) + logger.info(f" - {pkg_name}") + + # Install + logger.info("Installing packages...") + install_packages(all_packages, temp_dir, upgrade) + logger.info("Installation completed successfully!") + + finally: + if not keep_temp: + shutil.rmtree(temp_dir, ignore_errors=True) + if temp_tar_dir and temp_tar_dir.exists(): + shutil.rmtree(temp_tar_dir, ignore_errors=True) + else: + logger.info(f"Temporary directory kept at: {temp_dir}") + if temp_tar_dir: + logger.info(f"Downloaded tar file kept at: {local_tar_path}") + + +def uninstall_loongsuite_packages( + blacklist: Optional[Set[str]] = None, + whitelist: Optional[Set[str]] = None, + yes: bool = False, +): + """ + Uninstall installed loongsuite packages + + Args: + blacklist: blacklist (do not uninstall these packages) + whitelist: whitelist (only uninstall these packages if specified) + yes: automatically confirm uninstallation + """ + # Get installed loongsuite packages + installed_packages = get_installed_loongsuite_packages() + + if not installed_packages: + logger.warning("No loongsuite packages found installed") + return + + # Apply blacklist/whitelist filters + blacklist = blacklist or set() + whitelist = whitelist or set() + + package_names = [] + for pkg in installed_packages: + # Check blacklist + if blacklist and pkg in blacklist: + logger.debug(f"Skipping package (blacklist): {pkg}") + continue + + # Check whitelist + if whitelist and pkg not in whitelist: + logger.debug(f"Skipping package (not in whitelist): {pkg}") + continue + + package_names.append(pkg) + + if not package_names: + logger.warning("No packages to uninstall after filtering") + return + + logger.info(f"Will uninstall {len(package_names)} packages:") + for name in package_names: + logger.info(f" - {name}") + + # Uninstall + uninstall_packages(package_names, yes) + + +def get_latest_release_url( + repo: str = "alibaba/loongsuite-python-agent", +) -> str: + """Get latest release tar.gz URL from GitHub API""" + api_url = f"https://api.github.com/repos/{repo}/releases/latest" + logger.info(f"Fetching latest release: {api_url}") + + try: + with urllib.request.urlopen(api_url) as response: + data = json_lib.loads(response.read()) + for asset in data.get("assets", []): + if asset["name"].endswith(".tar.gz"): + return asset["browser_download_url"] + + # If no asset found, try to build URL from tag + tag = data.get("tag_name", "").lstrip("v") + return f"https://github.com/{repo}/releases/download/{data.get('tag_name')}/loongsuite-python-agent-{tag}.tar.gz" + except Exception as e: + logger.error(f"Failed to fetch latest release: {e}") + raise + + +def main(): + parser = argparse.ArgumentParser( + description=""" + LoongSuite Bootstrap - Install/Uninstall loongsuite Python Agent from tar package + + This tool installs or uninstalls all loongsuite components from tar.gz file. + Supports blacklist/whitelist to control which instrumentations to install/uninstall. + """ + ) + + parser.add_argument( + "-a", + "--action", + choices=["install", "uninstall"], + required=True, + help="action type: install to install packages, uninstall to uninstall packages", + ) + + # Common arguments + parser.add_argument( + "--blacklist", + type=Path, + help="blacklist file path (one package name per line, do not install/uninstall these packages)", + ) + parser.add_argument( + "--whitelist", + type=Path, + help="whitelist file path (one package name per line, only install/uninstall these packages)", + ) + + # Install-specific arguments + install_group = parser.add_argument_group("install options") + install_group.add_argument( + "-t", + "--tar", + type=str, + help="tar package path or URI (required for install action, supports http:// and https://)", + ) + install_group.add_argument( + "-v", + "--version", + type=str, + help="version number, download from GitHub Releases (e.g., 1.0.0) (for install action)", + ) + install_group.add_argument( + "--latest", + action="store_true", + help="install latest version (from GitHub Releases) (for install action)", + ) + install_group.add_argument( + "--upgrade", + action="store_true", + help="upgrade already installed packages (for install action)", + ) + install_group.add_argument( + "--keep-temp", + action="store_true", + help="keep temporary directory (for debugging)", + ) + install_group.add_argument( + "--force", + action="store_true", + help="force installation even if dependency versions are incompatible", + ) + install_group.add_argument( + "--auto-detect", + action="store_true", + help="only install instrumentation packages if their target libraries are installed (similar to opentelemetry-bootstrap)", + ) + install_group.add_argument( + "--verbose", + action="store_true", + help="enable verbose debug logging", + ) + + # Uninstall-specific arguments + uninstall_group = parser.add_argument_group("uninstall options") + uninstall_group.add_argument( + "-y", + "--yes", + action="store_true", + help="automatically confirm uninstallation (for uninstall action)", + ) + + args = parser.parse_args() + + # Configure logging level + if args.verbose or ( + hasattr(args, "action") + and args.action == "install" + and hasattr(args, "auto_detect") + and args.auto_detect + ): + logging.basicConfig( + level=logging.DEBUG, + format="%(levelname)s: %(message)s", + force=True, + ) + logger.setLevel(logging.DEBUG) + else: + logging.basicConfig( + level=logging.INFO, format="%(levelname)s: %(message)s", force=True + ) + logger.setLevel(logging.INFO) + + # Load blacklist/whitelist + blacklist = load_list_file(args.blacklist) if args.blacklist else None + whitelist = load_list_file(args.whitelist) if args.whitelist else None + + if blacklist: + logger.info(f"Blacklist: {len(blacklist)} packages") + if whitelist: + logger.info(f"Whitelist: {len(whitelist)} packages") + + if args.action == "install": + # Determine tar file path + tar_path = None + if args.tar: + tar_path = args.tar + elif args.version: + tar_path = f"https://github.com/alibaba/loongsuite-python-agent/releases/download/v{args.version}/loongsuite-python-agent-{args.version}.tar.gz" + elif args.latest: + tar_path = get_latest_release_url() + else: + parser.error( + "For install action, must specify one of --tar, --version, or --latest" + ) + + # Install + install_from_tar( + tar_path, + blacklist=blacklist, + whitelist=whitelist, + upgrade=args.upgrade, + keep_temp=args.keep_temp, + skip_version_check=args.force, + auto_detect=args.auto_detect, + ) + + elif args.action == "uninstall": + # Uninstall installed loongsuite packages + uninstall_loongsuite_packages( + blacklist=blacklist, + whitelist=whitelist, + yes=args.yes, + ) + + +if __name__ == "__main__": + logging.basicConfig( + level=logging.INFO, + format="%(levelname)s: %(message)s", + ) + main() diff --git a/loongsuite-distro/src/loongsuite/distro/bootstrap_gen.py b/loongsuite-distro/src/loongsuite/distro/bootstrap_gen.py new file mode 100644 index 000000000..19e31b0eb --- /dev/null +++ b/loongsuite-distro/src/loongsuite/distro/bootstrap_gen.py @@ -0,0 +1,86 @@ + +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT. THIS FILE WAS AUTOGENERATED FROM INSTRUMENTATION PACKAGES. +# RUN `python scripts/generate_loongsuite_bootstrap.py` TO REGENERATE. + +libraries = [ + {"library": "openai >= 1.26.0", "instrumentation": "opentelemetry-instrumentation-openai-v2"}, + {"library": "google-cloud-aiplatform >= 1.64", "instrumentation": "opentelemetry-instrumentation-vertexai>=2.0b0"}, + {"library": "aio_pika >= 7.2.0, < 10.0.0", "instrumentation": "opentelemetry-instrumentation-aio-pika==0.61b0.dev"}, + {"library": "aiohttp ~= 3.0", "instrumentation": "opentelemetry-instrumentation-aiohttp-client==0.61b0.dev"}, + {"library": "aiohttp ~= 3.0", "instrumentation": "opentelemetry-instrumentation-aiohttp-server==0.61b0.dev"}, + {"library": "aiokafka >= 0.8, < 1.0", "instrumentation": "opentelemetry-instrumentation-aiokafka==0.61b0.dev"}, + {"library": "aiopg >= 0.13.0, < 2.0.0", "instrumentation": "opentelemetry-instrumentation-aiopg==0.61b0.dev"}, + {"library": "asgiref ~= 3.0", "instrumentation": "opentelemetry-instrumentation-asgi==0.61b0.dev"}, + {"library": "asyncclick ~= 8.0", "instrumentation": "opentelemetry-instrumentation-asyncclick==0.61b0.dev"}, + {"library": "asyncpg >= 0.12.0", "instrumentation": "opentelemetry-instrumentation-asyncpg==0.61b0.dev"}, + {"library": "boto~=2.0", "instrumentation": "opentelemetry-instrumentation-boto==0.61b0.dev"}, + {"library": "boto3 ~= 1.0", "instrumentation": "opentelemetry-instrumentation-boto3sqs==0.61b0.dev"}, + {"library": "botocore ~= 1.0", "instrumentation": "opentelemetry-instrumentation-botocore==0.61b0.dev"}, + {"library": "cassandra-driver ~= 3.25", "instrumentation": "opentelemetry-instrumentation-cassandra==0.61b0.dev"}, + {"library": "scylla-driver ~= 3.25", "instrumentation": "opentelemetry-instrumentation-cassandra==0.61b0.dev"}, + {"library": "celery >= 4.0, < 6.0", "instrumentation": "opentelemetry-instrumentation-celery==0.61b0.dev"}, + {"library": "click >= 8.1.3, < 9.0.0", "instrumentation": "opentelemetry-instrumentation-click==0.61b0.dev"}, + {"library": "confluent-kafka >= 1.8.2, <= 2.11.0", "instrumentation": "opentelemetry-instrumentation-confluent-kafka==0.61b0.dev"}, + {"library": "django >= 1.10", "instrumentation": "opentelemetry-instrumentation-django==0.61b0.dev"}, + {"library": "elasticsearch >= 6.0", "instrumentation": "opentelemetry-instrumentation-elasticsearch==0.61b0.dev"}, + {"library": "falcon >= 1.4.1, < 5.0.0", "instrumentation": "opentelemetry-instrumentation-falcon==0.61b0.dev"}, + {"library": "fastapi ~= 0.92", "instrumentation": "opentelemetry-instrumentation-fastapi==0.61b0.dev"}, + {"library": "flask >= 1.0", "instrumentation": "opentelemetry-instrumentation-flask==0.61b0.dev"}, + {"library": "grpcio >= 1.42.0", "instrumentation": "opentelemetry-instrumentation-grpc==0.61b0.dev"}, + {"library": "httpx >= 0.18.0", "instrumentation": "opentelemetry-instrumentation-httpx==0.61b0.dev"}, + {"library": "jinja2 >= 2.7, < 4.0", "instrumentation": "opentelemetry-instrumentation-jinja2==0.61b0.dev"}, + {"library": "kafka-python >= 2.0, < 3.0", "instrumentation": "opentelemetry-instrumentation-kafka-python==0.61b0.dev"}, + {"library": "kafka-python-ng >= 2.0, < 3.0", "instrumentation": "opentelemetry-instrumentation-kafka-python==0.61b0.dev"}, + {"library": "mysql-connector-python >= 8.0, < 10.0", "instrumentation": "opentelemetry-instrumentation-mysql==0.61b0.dev"}, + {"library": "mysqlclient < 3", "instrumentation": "opentelemetry-instrumentation-mysqlclient==0.61b0.dev"}, + {"library": "pika >= 0.12.0", "instrumentation": "opentelemetry-instrumentation-pika==0.61b0.dev"}, + {"library": "psycopg >= 3.1.0", "instrumentation": "opentelemetry-instrumentation-psycopg==0.61b0.dev"}, + {"library": "psycopg2 >= 2.7.3.1", "instrumentation": "opentelemetry-instrumentation-psycopg2==0.61b0.dev"}, + {"library": "psycopg2-binary >= 2.7.3.1", "instrumentation": "opentelemetry-instrumentation-psycopg2==0.61b0.dev"}, + {"library": "pymemcache >= 1.3.5, < 5", "instrumentation": "opentelemetry-instrumentation-pymemcache==0.61b0.dev"}, + {"library": "pymongo >= 3.1, < 5.0", "instrumentation": "opentelemetry-instrumentation-pymongo==0.61b0.dev"}, + {"library": "pymssql >= 2.1.5, < 3", "instrumentation": "opentelemetry-instrumentation-pymssql==0.61b0.dev"}, + {"library": "PyMySQL < 2", "instrumentation": "opentelemetry-instrumentation-pymysql==0.61b0.dev"}, + {"library": "pyramid >= 1.7", "instrumentation": "opentelemetry-instrumentation-pyramid==0.61b0.dev"}, + {"library": "redis >= 2.6", "instrumentation": "opentelemetry-instrumentation-redis==0.61b0.dev"}, + {"library": "remoulade >= 0.50", "instrumentation": "opentelemetry-instrumentation-remoulade==0.61b0.dev"}, + {"library": "requests ~= 2.0", "instrumentation": "opentelemetry-instrumentation-requests==0.61b0.dev"}, + {"library": "sqlalchemy >= 1.0.0, < 2.1.0", "instrumentation": "opentelemetry-instrumentation-sqlalchemy==0.61b0.dev"}, + {"library": "starlette >= 0.13", "instrumentation": "opentelemetry-instrumentation-starlette==0.61b0.dev"}, + {"library": "psutil >= 5", "instrumentation": "opentelemetry-instrumentation-system-metrics==0.61b0.dev"}, + {"library": "tornado >= 5.1.1", "instrumentation": "opentelemetry-instrumentation-tornado==0.61b0.dev"}, + {"library": "tortoise-orm >= 0.17.0", "instrumentation": "opentelemetry-instrumentation-tortoiseorm==0.61b0.dev"}, + {"library": "pydantic >= 1.10.2", "instrumentation": "opentelemetry-instrumentation-tortoiseorm==0.61b0.dev"}, + {"library": "urllib3 >= 1.0.0, < 3.0.0", "instrumentation": "opentelemetry-instrumentation-urllib3==0.61b0.dev"}, + {"library": "agentscope >= 1.0.0", "instrumentation": "loongsuite-instrumentation-agentscope==1.0.0"}, + {"library": "agno", "instrumentation": "loongsuite-instrumentation-agno==0.1b0.dev"}, + {"library": "dashscope >= 1.0.0", "instrumentation": "loongsuite-instrumentation-dashscope==0.1.0.dev0"}, + {"library": "langchain_core >= 0.1.0", "instrumentation": "loongsuite-instrumentation-langchain==1.0.0"}, + {"library": "mcp >= 1.3.0, <= 1.13.1", "instrumentation": "loongsuite-instrumentation-mcp==0.1.0"}, + {"library": "mem0ai >= 1.0.0", "instrumentation": "loongsuite-instrumentation-mem0==0.1.0"}, +] + +default_instrumentations = [ + "opentelemetry-instrumentation-asyncio==0.61b0.dev", + "opentelemetry-instrumentation-dbapi==0.61b0.dev", + "opentelemetry-instrumentation-logging==0.61b0.dev", + "opentelemetry-instrumentation-sqlite3==0.61b0.dev", + "opentelemetry-instrumentation-threading==0.61b0.dev", + "opentelemetry-instrumentation-urllib==0.61b0.dev", + "opentelemetry-instrumentation-wsgi==0.61b0.dev", + "loongsuite-instrumentation-dify==1.1.0", +] diff --git a/loongsuite-distro/src/loongsuite/distro/py.typed b/loongsuite-distro/src/loongsuite/distro/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/loongsuite-distro/src/loongsuite/distro/version.py b/loongsuite-distro/src/loongsuite/distro/version.py new file mode 100644 index 000000000..4effd145c --- /dev/null +++ b/loongsuite-distro/src/loongsuite/distro/version.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.1.0.dev" diff --git a/pkg-requirements.txt b/pkg-requirements.txt new file mode 100644 index 000000000..67491c71d --- /dev/null +++ b/pkg-requirements.txt @@ -0,0 +1,3 @@ +build>=1.0.0 +setuptools>=65.0.0 +wheel>=0.40.0 diff --git a/processor/loongsuite-processor-baggage/CHANGELOG.md b/processor/loongsuite-processor-baggage/CHANGELOG.md new file mode 100644 index 000000000..8b67f34c5 --- /dev/null +++ b/processor/loongsuite-processor-baggage/CHANGELOG.md @@ -0,0 +1,18 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## Unreleased + +## Version 0.1.0 + +### Added + +- Initial release of LoongSuite Baggage Span Processor +- Support for prefix matching to filter baggage keys +- Support for prefix stripping to remove prefixes from baggage keys before adding to span attributes +- Integration with LoongSuite Configurator via environment variables + diff --git a/processor/loongsuite-processor-baggage/LICENSE b/processor/loongsuite-processor-baggage/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/processor/loongsuite-processor-baggage/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/processor/loongsuite-processor-baggage/README.rst b/processor/loongsuite-processor-baggage/README.rst new file mode 100644 index 000000000..054fae8fe --- /dev/null +++ b/processor/loongsuite-processor-baggage/README.rst @@ -0,0 +1,81 @@ +LoongSuite Baggage Span Processor +================================== + +The LoongSuite Baggage Span Processor reads entries stored in Baggage +from the parent context and adds the baggage entries' keys and values +to the span as attributes on span start. + +This processor supports: +- Prefix matching: Only process baggage keys that match specified prefixes +- Prefix stripping: Remove specified prefixes from baggage keys before adding to attributes + +Installation +------------ + +:: + + pip install loongsuite-processor-baggage + +Usage +----- + +Add the span processor when configuring the tracer provider. + +Example 1: Match specific prefixes and strip one of them + +:: + + from loongsuite.processor.baggage import LoongSuiteBaggageSpanProcessor + from opentelemetry.sdk.trace import TracerProvider + + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes={"traffic.", "app."}, + strip_prefixes={"traffic."} + ) + ) + + # baggage: traffic.hello_key = "value" + # Result: attributes will have hello_key = "value" (traffic. prefix stripped) + + # baggage: app.user_id = "123" + # Result: attributes will have app.user_id = "123" (app. prefix not stripped) + +Example 2: Allow all prefixes but strip specific ones + +:: + + from loongsuite.processor.baggage import LoongSuiteBaggageSpanProcessor + from opentelemetry.sdk.trace import TracerProvider + + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes=None, # Allow all + strip_prefixes={"traffic.", "app."} + ) + ) + +Example 3: Only match specific prefixes without stripping + +:: + + from loongsuite.processor.baggage import LoongSuiteBaggageSpanProcessor + from opentelemetry.sdk.trace import TracerProvider + + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes={"loongsuite."}, + strip_prefixes=None # No stripping + ) + ) + +⚠ Warning ⚠️ + +Do not put sensitive information in Baggage. + +To repeat: a consequence of adding data to Baggage is that the keys and +values will appear in all outgoing HTTP headers from the application. + diff --git a/processor/loongsuite-processor-baggage/pyproject.toml b/processor/loongsuite-processor-baggage/pyproject.toml new file mode 100644 index 000000000..7f10eb7a9 --- /dev/null +++ b/processor/loongsuite-processor-baggage/pyproject.toml @@ -0,0 +1,46 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "loongsuite-processor-baggage" +dynamic = ["version"] +description = "LoongSuite Baggage Span Processor with prefix matching and stripping" +readme = "README.rst" +license = "Apache-2.0" +requires-python = ">=3.9" +authors = [ + { name = "LoongSuite Python Agent Authors", email = "zmh405877@alibaba-inc.com" }, +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", +] +dependencies = [ + "opentelemetry-api ~= 1.5", + "opentelemetry-sdk ~= 1.5", +] + +[project.urls] +Homepage = "https://github.com/alibaba/loongsuite-python-agent" +Repository = "https://github.com/alibaba/loongsuite-python-agent" + +[tool.hatch.version] +path = "src/loongsuite/processor/baggage/version.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/tests", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/loongsuite"] diff --git a/processor/loongsuite-processor-baggage/src/loongsuite/__init__.py b/processor/loongsuite-processor-baggage/src/loongsuite/__init__.py new file mode 100644 index 000000000..f87ce79b7 --- /dev/null +++ b/processor/loongsuite-processor-baggage/src/loongsuite/__init__.py @@ -0,0 +1,14 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/processor/loongsuite-processor-baggage/src/loongsuite/processor/__init__.py b/processor/loongsuite-processor-baggage/src/loongsuite/processor/__init__.py new file mode 100644 index 000000000..f87ce79b7 --- /dev/null +++ b/processor/loongsuite-processor-baggage/src/loongsuite/processor/__init__.py @@ -0,0 +1,14 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/__init__.py b/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/__init__.py new file mode 100644 index 000000000..83844521d --- /dev/null +++ b/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/__init__.py @@ -0,0 +1,18 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .processor import LoongSuiteBaggageSpanProcessor +from .version import __version__ + +__all__ = ["LoongSuiteBaggageSpanProcessor", "__version__"] diff --git a/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/processor.py b/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/processor.py new file mode 100644 index 000000000..6221bb941 --- /dev/null +++ b/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/processor.py @@ -0,0 +1,129 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Set + +from opentelemetry.baggage import get_all as get_all_baggage +from opentelemetry.context import Context +from opentelemetry.sdk.trace import SpanProcessor +from opentelemetry.trace import Span + + +class LoongSuiteBaggageSpanProcessor(SpanProcessor): + """ + LoongSuite Baggage Span Processor + + Reads Baggage entries from the parent context and adds matching baggage + key-value pairs to span attributes based on configured prefix matching rules. + + Supported features: + 1. Prefix matching: Only process baggage keys that match specified prefixes + 2. Prefix stripping: Remove specified prefixes before writing to attributes + + Example: + # Configure matching prefixes: "traffic.", "app." + # Configure stripping prefix: "traffic." + # baggage: traffic.hello_key = "value" + # Result: attributes will have hello_key = "value" (prefix stripped) + + # baggage: app.user_id = "123" + # Result: attributes will have app.user_id = "123" (app. prefix not stripped) + + ⚠ Warning ⚠️ + + Do not put sensitive information in Baggage. + + To repeat: a consequence of adding data to Baggage is that the keys and + values will appear in all outgoing HTTP headers from the application. + """ + + def __init__( + self, + allowed_prefixes: Optional[Set[str]] = None, + strip_prefixes: Optional[Set[str]] = None, + ) -> None: + """ + Initialize LoongSuite Baggage Span Processor + + Args: + allowed_prefixes: Set of allowed baggage key prefixes. If None or empty, + all baggage keys are allowed. If specified, only keys + matching these prefixes will be processed. + strip_prefixes: Set of prefixes to strip. If a baggage key matches these + prefixes, they will be removed before writing to attributes. + """ + self._allowed_prefixes = allowed_prefixes or set() + self._strip_prefixes = strip_prefixes or set() + + # If allowed_prefixes is empty, allow all prefixes + self._allow_all = len(self._allowed_prefixes) == 0 + + def _should_process_key(self, key: str) -> bool: + """ + Determine whether this baggage key should be processed + + Args: + key: baggage key + + Returns: + True if the key should be processed, False otherwise + """ + if self._allow_all: + return True + + # Check if key matches any of the allowed prefixes + for prefix in self._allowed_prefixes: + if key.startswith(prefix): + return True + + return False + + def _strip_prefix(self, key: str) -> str: + """ + Strip matching prefix from key + + Args: + key: original baggage key + + Returns: + key with prefix stripped + """ + for prefix in self._strip_prefixes: + if key.startswith(prefix): + return key[len(prefix) :] + return key + + def on_start( + self, span: "Span", parent_context: Optional[Context] = None + ) -> None: + """ + Called when a span starts, adds matching baggage entries to span attributes + + Args: + span: span to add attributes to + parent_context: parent context used to retrieve baggage + """ + baggage = get_all_baggage(parent_context) + + for key, value in baggage.items(): + # Check if this key should be processed + if not self._should_process_key(key): + continue + + # Strip prefix if needed + attribute_key = self._strip_prefix(key) + + # Add to span attributes + # Baggage values are strings, which are valid AttributeValue + span.set_attribute(attribute_key, value) # type: ignore[arg-type] diff --git a/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/version.py b/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/version.py new file mode 100644 index 000000000..5fd301e2e --- /dev/null +++ b/processor/loongsuite-processor-baggage/src/loongsuite/processor/baggage/version.py @@ -0,0 +1,15 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.1.0" diff --git a/processor/loongsuite-processor-baggage/test-requirements.txt b/processor/loongsuite-processor-baggage/test-requirements.txt new file mode 100644 index 000000000..2fe69045f --- /dev/null +++ b/processor/loongsuite-processor-baggage/test-requirements.txt @@ -0,0 +1,5 @@ +pytest>=7.0.0 +pytest-cov>=4.0.0 +opentelemetry-api>=1.5.0 +opentelemetry-sdk>=1.5.0 + diff --git a/processor/loongsuite-processor-baggage/tests/__init__.py b/processor/loongsuite-processor-baggage/tests/__init__.py new file mode 100644 index 000000000..f87ce79b7 --- /dev/null +++ b/processor/loongsuite-processor-baggage/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + diff --git a/processor/loongsuite-processor-baggage/tests/test_baggage_processor.py b/processor/loongsuite-processor-baggage/tests/test_baggage_processor.py new file mode 100644 index 000000000..11225a504 --- /dev/null +++ b/processor/loongsuite-processor-baggage/tests/test_baggage_processor.py @@ -0,0 +1,171 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from loongsuite.processor.baggage import LoongSuiteBaggageSpanProcessor + +from opentelemetry.baggage import set_baggage +from opentelemetry.context import attach, detach +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SpanProcessor + + +class LoongSuiteBaggageSpanProcessorTest(unittest.TestCase): + def test_check_the_baggage_processor(self): + self.assertIsInstance(LoongSuiteBaggageSpanProcessor(), SpanProcessor) + + def test_allow_all_prefixes(self): + """Test allowing all prefixes""" + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor(allowed_prefixes=None) + ) + + tracer = tracer_provider.get_tracer("my-tracer") + ctx = set_baggage("any_key", "any_value") + + with tracer.start_as_current_span(name="test", context=ctx) as span: + self.assertEqual(span._attributes["any_key"], "any_value") + + def test_prefix_matching(self): + """Test prefix matching functionality""" + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes={"traffic.", "app."} + ) + ) + + tracer = tracer_provider.get_tracer("my-tracer") + ctx = set_baggage("traffic.hello", "world") + ctx = set_baggage("app.user_id", "123", context=ctx) + ctx = set_baggage("other.key", "value", context=ctx) + + with tracer.start_as_current_span(name="test", context=ctx) as span: + # Matching prefixes should be added + self.assertEqual(span._attributes["traffic.hello"], "world") + self.assertEqual(span._attributes["app.user_id"], "123") + # Non-matching prefixes should not be added + self.assertNotIn("other.key", span._attributes) + + def test_prefix_stripping(self): + """Test prefix stripping functionality""" + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes={"traffic.", "app."}, + strip_prefixes={"traffic."}, + ) + ) + + tracer = tracer_provider.get_tracer("my-tracer") + ctx = set_baggage("traffic.hello_key", "value") + ctx = set_baggage("app.user_id", "123", context=ctx) + + with tracer.start_as_current_span(name="test", context=ctx) as span: + # traffic. prefix should be stripped + self.assertEqual(span._attributes["hello_key"], "value") + self.assertNotIn("traffic.hello_key", span._attributes) + # app. prefix should not be stripped (not in strip_prefixes) + self.assertEqual(span._attributes["app.user_id"], "123") + + def test_multiple_strip_prefixes(self): + """Test multiple strip prefixes""" + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes=None, strip_prefixes={"traffic.", "app."} + ) + ) + + tracer = tracer_provider.get_tracer("my-tracer") + ctx = set_baggage("traffic.key1", "value1") + ctx = set_baggage("app.key2", "value2", context=ctx) + ctx = set_baggage("other.key3", "value3", context=ctx) + + with tracer.start_as_current_span(name="test", context=ctx) as span: + self.assertEqual(span._attributes["key1"], "value1") + self.assertEqual(span._attributes["key2"], "value2") + self.assertEqual(span._attributes["other.key3"], "value3") + + def test_nested_spans(self): + """Test nested spans""" + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes={"traffic."}, strip_prefixes={"traffic."} + ) + ) + + tracer = tracer_provider.get_tracer("my-tracer") + ctx = set_baggage("traffic.queen", "bee") + + with tracer.start_as_current_span( + name="parent", context=ctx + ) as parent_span: + self.assertEqual(parent_span._attributes["queen"], "bee") + + with tracer.start_as_current_span( + name="child", context=ctx + ) as child_span: + self.assertEqual(child_span._attributes["queen"], "bee") + + def test_context_token(self): + """Test using context token""" + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes={"traffic."}, strip_prefixes={"traffic."} + ) + ) + + tracer = tracer_provider.get_tracer("my-tracer") + token = attach(set_baggage("traffic.bumble", "bee")) + + try: + with tracer.start_as_current_span("parent") as span: + self.assertEqual(span._attributes["bumble"], "bee") + + token2 = attach(set_baggage("traffic.moar", "bee")) + try: + with tracer.start_as_current_span("child") as child_span: + self.assertEqual( + child_span._attributes["bumble"], "bee" + ) + self.assertEqual(child_span._attributes["moar"], "bee") + finally: + detach(token2) + finally: + detach(token) + + def test_empty_prefixes(self): + """Test empty prefix sets""" + tracer_provider = TracerProvider() + tracer_provider.add_span_processor( + LoongSuiteBaggageSpanProcessor( + allowed_prefixes=set(), # Empty set, should allow all + strip_prefixes=set(), + ) + ) + + tracer = tracer_provider.get_tracer("my-tracer") + ctx = set_baggage("any_key", "any_value") + + with tracer.start_as_current_span(name="test", context=ctx) as span: + self.assertEqual(span._attributes["any_key"], "any_value") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/build_loongsuite_package.py b/scripts/build_loongsuite_package.py new file mode 100755 index 000000000..c31875105 --- /dev/null +++ b/scripts/build_loongsuite_package.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +""" +Build script: Package all required whl files into tar.gz + +This script will: +1. Build all packages under instrumentation/ +2. Build all packages under instrumentation-genai/ +3. Build all packages under instrumentation-loongsuite/ +4. Build util/opentelemetry-util-genai/ +5. Build processor/loongsuite-processor-baggage/ +6. Skip duplicate packages according to config file +7. Package all whl files into tar.gz + +Note: loongsuite-distro is not included as it is published separately to PyPI. +""" + +import argparse +import json +import logging +import subprocess +import sys +import tarfile +from pathlib import Path +from typing import List, Set + +logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s") +logger = logging.getLogger(__name__) + + +def load_skip_config(config_path: Path) -> Set[str]: + """Load package names to skip from config file""" + if not config_path.exists(): + logger.warning( + f"Config file {config_path} does not exist, using default config" + ) + return set() + + with open(config_path, "r", encoding="utf-8") as f: + config = json.load(f) + + skip_packages = set(config.get("skip_packages", [])) + logger.info( + f"Loaded {len(skip_packages)} packages to skip from config file: {skip_packages}" + ) + return skip_packages + + +def get_package_name_from_whl(whl_path: Path) -> str: + """Extract package name from whl filename""" + # Format: package_name-version-py3-none-any.whl + # Or: package_name-version-cp39-cp39-linux_x86_64.whl + name = whl_path.stem # Remove .whl + # Find the part before the first number (version number) after the first - + parts = name.split("-") + if len(parts) >= 2: + # Package name is all parts before version number, joined with - + # Example: opentelemetry-instrumentation-langchain-2.0.0b0-py3-none-any + # Package name is: opentelemetry-instrumentation-langchain + # Need to find the position of version number (first part that looks like a version) + package_parts = [] + for part in parts: + # Version numbers usually contain digits and dots, or contain b0, dev, etc. + if any(c.isdigit() for c in part) or part in ( + "dev", + "b0", + "b1", + "rc0", + "rc1", + ): + break + package_parts.append(part) + return "-".join(package_parts) + return name + + +def build_package( + package_dir: Path, dist_dir: Path, existing_whl_files: Set[Path] +) -> List[Path]: + """Build whl file for a single package""" + pyproject_toml = package_dir / "pyproject.toml" + if not pyproject_toml.exists(): + logger.debug(f"Skipping {package_dir}, no pyproject.toml") + return [] + + logger.info(f"Building package: {package_dir}") + try: + # Record whl files before build + before_whl_files = set(dist_dir.glob("*.whl")) + + result = subprocess.run( + [ + sys.executable, + "-m", + "build", + "--wheel", + "--outdir", + str(dist_dir), + ], + cwd=package_dir, + check=True, + capture_output=True, + text=True, + ) + + # Find newly generated whl files (exist after build but not before) + after_whl_files = set(dist_dir.glob("*.whl")) + new_whl_files = [ + f for f in after_whl_files - before_whl_files if f.suffix == ".whl" + ] + + if not new_whl_files: + logger.warning( + f"No new whl files found after building {package_dir}" + ) + if result.stdout: + logger.debug(f"stdout: {result.stdout}") + if result.stderr: + logger.debug(f"stderr: {result.stderr}") + + return sorted(new_whl_files) + except subprocess.CalledProcessError as e: + logger.error(f"Failed to build {package_dir}: {e}") + if e.stdout: + logger.error(f"stdout: {e.stdout}") + if e.stderr: + logger.error(f"stderr: {e.stderr}") + return [] + + +def collect_packages( + base_dir: Path, + dist_dir: Path, + skip_packages: Set[str], +) -> List[Path]: + """Collect all packages that need to be built""" + all_whl_files = [] + existing_whl_files = set(dist_dir.glob("*.whl")) + + # 1. Build packages under instrumentation/ + instrumentation_dir = base_dir / "instrumentation" + if instrumentation_dir.exists(): + logger.info("Building packages under instrumentation/...") + for package_dir in sorted(instrumentation_dir.iterdir()): + if ( + package_dir.is_dir() + and (package_dir / "pyproject.toml").exists() + ): + whl_files = build_package( + package_dir, dist_dir, existing_whl_files + ) + all_whl_files.extend(whl_files) + existing_whl_files.update(whl_files) + + # 2. Build packages under instrumentation-genai/ + instrumentation_genai_dir = base_dir / "instrumentation-genai" + if instrumentation_genai_dir.exists(): + logger.info("Building packages under instrumentation-genai/...") + for package_dir in sorted(instrumentation_genai_dir.iterdir()): + if ( + package_dir.is_dir() + and (package_dir / "pyproject.toml").exists() + ): + whl_files = build_package( + package_dir, dist_dir, existing_whl_files + ) + all_whl_files.extend(whl_files) + existing_whl_files.update(whl_files) + + # 3. Build packages under instrumentation-loongsuite/ + instrumentation_loongsuite_dir = base_dir / "instrumentation-loongsuite" + if instrumentation_loongsuite_dir.exists(): + logger.info("Building packages under instrumentation-loongsuite/...") + for package_dir in sorted(instrumentation_loongsuite_dir.iterdir()): + if ( + package_dir.is_dir() + and (package_dir / "pyproject.toml").exists() + ): + whl_files = build_package( + package_dir, dist_dir, existing_whl_files + ) + all_whl_files.extend(whl_files) + existing_whl_files.update(whl_files) + + # 4. Build util/opentelemetry-util-genai/ + util_genai_dir = base_dir / "util" / "opentelemetry-util-genai" + if ( + util_genai_dir.exists() + and (util_genai_dir / "pyproject.toml").exists() + ): + logger.info("Building util/opentelemetry-util-genai/...") + whl_files = build_package(util_genai_dir, dist_dir, existing_whl_files) + all_whl_files.extend(whl_files) + existing_whl_files.update(whl_files) + + # 5. Build processor/loongsuite-processor-baggage/ + processor_baggage_dir = ( + base_dir / "processor" / "loongsuite-processor-baggage" + ) + if ( + processor_baggage_dir.exists() + and (processor_baggage_dir / "pyproject.toml").exists() + ): + logger.info("Building processor/loongsuite-processor-baggage/...") + whl_files = build_package( + processor_baggage_dir, dist_dir, existing_whl_files + ) + all_whl_files.extend(whl_files) + existing_whl_files.update(whl_files) + + # 6. Filter out packages that need to be skipped + filtered_whl_files = [] + skipped_count = 0 + seen_packages = {} # Used to detect duplicate packages + + for whl_file in all_whl_files: + package_name = get_package_name_from_whl(whl_file) + + # Check if in skip list + if package_name in skip_packages: + logger.info( + f"Skipping package: {package_name} (according to config file)" + ) + skipped_count += 1 + # Delete skipped whl file + whl_file.unlink() + continue + + # Check for duplicate packages (same package may have multiple whl files, e.g., different platforms) + if package_name in seen_packages: + # Keep the newest file + existing_file = seen_packages[package_name] + if whl_file.stat().st_mtime > existing_file.stat().st_mtime: + logger.debug( + f"Replacing duplicate package {package_name}: {existing_file.name} -> {whl_file.name}" + ) + existing_file.unlink() + seen_packages[package_name] = whl_file + filtered_whl_files.remove(existing_file) + filtered_whl_files.append(whl_file) + else: + logger.debug( + f"Skipping older version {package_name}: {whl_file.name}" + ) + whl_file.unlink() + else: + seen_packages[package_name] = whl_file + filtered_whl_files.append(whl_file) + + logger.info(f"Built {len(all_whl_files)} whl files in total") + logger.info(f"Skipped {skipped_count} packages") + logger.info(f"Final package contains {len(filtered_whl_files)} whl files") + + return filtered_whl_files + + +def create_tar_archive(whl_files: List[Path], output_path: Path): + """Package all whl files into tar.gz""" + logger.info(f"Creating tar archive: {output_path}") + + with tarfile.open(output_path, "w:gz") as tar: + for whl_file in sorted(whl_files): + # Only save filename, not path + tar.add(whl_file, arcname=whl_file.name) + logger.debug(f"Added to archive: {whl_file.name}") + + logger.info( + f"Successfully created archive: {output_path} ({output_path.stat().st_size / 1024 / 1024:.2f} MB)" + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Build loongsuite Python Agent release package" + ) + parser.add_argument( + "--base-dir", + type=Path, + default=Path(__file__).parent.parent, + help="Project root directory (default: script's parent directory)", + ) + parser.add_argument( + "--dist-dir", + type=Path, + default=None, + help="Build output directory (default: base-dir/dist)", + ) + parser.add_argument( + "--config", + type=Path, + default=Path(__file__).parent / "loongsuite-build-config.json", + help="Config file path (default: scripts/loongsuite-build-config.json)", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Output tar.gz file path (default: dist/loongsuite-python-agent-.tar.gz)", + ) + parser.add_argument( + "--version", + type=str, + default="dev", + help="Version number (for output filename)", + ) + + args = parser.parse_args() + + base_dir = args.base_dir.resolve() + dist_dir = args.dist_dir or (base_dir / "dist") + dist_dir.mkdir(parents=True, exist_ok=True) + + # Clean old whl files + logger.info(f"Cleaning old build files: {dist_dir}") + for old_file in dist_dir.glob("*.whl"): + old_file.unlink() + + # Load skip config + skip_packages = load_skip_config(args.config) + + # Collect and build all packages + whl_files = collect_packages(base_dir, dist_dir, skip_packages) + + if not whl_files: + logger.error("No whl files found, build failed") + sys.exit(1) + + # Create tar archive + output_path = args.output or ( + dist_dir / f"loongsuite-python-agent-{args.version}.tar.gz" + ) + create_tar_archive(whl_files, output_path) + + logger.info("Build completed!") + logger.info(f"Output file: {output_path}") + + +if __name__ == "__main__": + main() diff --git a/scripts/generate_loongsuite_bootstrap.py b/scripts/generate_loongsuite_bootstrap.py new file mode 100644 index 000000000..98848958f --- /dev/null +++ b/scripts/generate_loongsuite_bootstrap.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 + +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ast +import logging +import subprocess +from pathlib import Path + +import tomli +from generate_instrumentation_bootstrap import ( + independent_packages, + packages_to_exclude, +) +from otel_packaging import get_instrumentation_packages as get_upstream_packages + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("loongsuite_bootstrap_generator") + +# Get root path +scripts_path = Path(__file__).parent +root_path = scripts_path.parent + +_template = """ +{header} + +# DO NOT EDIT. THIS FILE WAS AUTOGENERATED FROM INSTRUMENTATION PACKAGES. +# RUN `python scripts/generate_loongsuite_bootstrap.py` TO REGENERATE. + +{source} +""" + +_source_tmpl = """ +libraries = [] +default_instrumentations = [] +""" + +gen_path = root_path / "loongsuite-distro" / "src" / "loongsuite" / "distro" / "bootstrap_gen.py" + + +def get_instrumentation_packages(): + """Get all instrumentation packages from various directories""" + packages = [] + + logger.info("Scanning instrumentation packages...") + + # Get packages from upstream directories (instrumentation, instrumentation-genai) + # using otel_packaging + logger.info("Processing upstream packages (instrumentation, instrumentation-genai)...") + for pkg in get_upstream_packages(independent_packages=independent_packages): + if pkg["name"] not in packages_to_exclude: + packages.append(pkg) + + # Scan instrumentation-loongsuite directory (reuse same logic as otel_packaging) + loongsuite_dir = root_path / "instrumentation-loongsuite" + if loongsuite_dir.exists(): + logger.info("Processing loongsuite packages (instrumentation-loongsuite)...") + pkg_dirs = sorted([d for d in loongsuite_dir.iterdir() if d.is_dir()]) + for pkg_dir in pkg_dirs: + pyproject_toml = pkg_dir / "pyproject.toml" + if not pyproject_toml.exists(): + continue + + try: + # Get version using hatch command (same as otel_packaging) + # Suppress hatch's verbose output + version = subprocess.check_output( + "hatch version", + shell=True, + cwd=pkg_dir, + universal_newlines=True, + ).strip() + + # Read pyproject.toml + with open(pyproject_toml, "rb") as f: + pyproject = tomli.load(f) + + pkg_name = pyproject["project"]["name"] + + # Skip if this package is in the exclusion list + if pkg_name in packages_to_exclude: + continue + + # Get optional dependencies + optional_deps = pyproject["project"]["optional-dependencies"] + instruments = optional_deps.get("instruments", []) + instruments_any = optional_deps.get("instruments-any", []) + + # Handle independent packages + if pkg_name in independent_packages: + specifier = independent_packages[pkg_name] + requirement = f"{pkg_name}{specifier}" if specifier else f"{pkg_name}=={version}" + else: + requirement = f"{pkg_name}=={version}" + + packages.append({ + "name": pkg_name, + "version": version, + "instruments": instruments, + "instruments-any": instruments_any, + "requirement": requirement, + }) + except subprocess.CalledProcessError as e: + logger.warning(f"Could not get hatch version from {pkg_dir.name}: {e}") + continue + except Exception as e: + logger.warning(f"Failed to process {pkg_dir.name}: {e}") + continue + + return packages + + +def main(): + # Read license header + header_path = scripts_path / "license_header.txt" + if header_path.exists(): + with open(header_path, "r", encoding="utf-8") as f: + header = f.read() + else: + header = "# Copyright The OpenTelemetry Authors\n# Licensed under the Apache License, Version 2.0\n" + + # Get all packages + packages = get_instrumentation_packages() + logger.info(f"Found {len(packages)} instrumentation packages") + + # Build AST nodes + default_instrumentations = ast.List(elts=[]) + libraries = ast.List(elts=[]) + + logger.info("Building bootstrap configuration...") + for pkg in packages: + # If no instruments and no instruments-any, it's a default instrumentation + if not pkg["instruments"] and not pkg["instruments-any"]: + default_instrumentations.elts.append( + ast.Constant(value=pkg["requirement"]) + ) + else: + # Add instruments (all must be installed) + for target_pkg in pkg["instruments"]: + libraries.elts.append( + ast.Dict( + keys=[ + ast.Constant(value="library"), + ast.Constant(value="instrumentation"), + ], + values=[ + ast.Constant(value=target_pkg), + ast.Constant(value=pkg["requirement"]), + ], + ) + ) + + # Add instruments-any (at least one must be installed) + for target_pkg in pkg["instruments-any"]: + libraries.elts.append( + ast.Dict( + keys=[ + ast.Constant(value="library"), + ast.Constant(value="instrumentation"), + ], + values=[ + ast.Constant(value=target_pkg), + ast.Constant(value=pkg["requirement"]), + ], + ) + ) + + # Generate source code manually (avoiding astor dependency) + logger.info("Generating source code...") + # Build libraries list string + libraries_lines = ["libraries = ["] + for lib_mapping in libraries.elts: + if isinstance(lib_mapping, ast.Dict): + # Extract keys and values + lib_key = lib_mapping.keys[0] + instr_key = lib_mapping.keys[1] + lib_val_node = lib_mapping.values[0] + instr_val_node = lib_mapping.values[1] + + # Get string values + lib_key_str = lib_key.value if isinstance(lib_key, ast.Constant) else (lib_key.s if hasattr(lib_key, 's') else "library") + instr_key_str = instr_key.value if isinstance(instr_key, ast.Constant) else (instr_key.s if hasattr(instr_key, 's') else "instrumentation") + lib_val_str = lib_val_node.value if isinstance(lib_val_node, ast.Constant) else (lib_val_node.s if hasattr(lib_val_node, 's') else "") + instr_val_str = instr_val_node.value if isinstance(instr_val_node, ast.Constant) else (instr_val_node.s if hasattr(instr_val_node, 's') else "") + + # Escape quotes in values + lib_val_str = lib_val_str.replace('"', '\\"') + instr_val_str = instr_val_str.replace('"', '\\"') + + libraries_lines.append(f' {{"{lib_key_str}": "{lib_val_str}", "{instr_key_str}": "{instr_val_str}"}},') + libraries_lines.append("]") + + # Build default_instrumentations list string + default_lines = ["default_instrumentations = ["] + for default_instr in default_instrumentations.elts: + if isinstance(default_instr, ast.Constant): + instr_val = default_instr.value.replace('"', '\\"') + default_lines.append(f' "{instr_val}",') + default_lines.append("]") + + # Combine source + source = "\n".join(libraries_lines) + "\n\n" + "\n".join(default_lines) + + # Format with header + formatted_source = _template.format(header=header, source=source) + + # Write to file + gen_path.parent.mkdir(parents=True, exist_ok=True) + with open(gen_path, "w", encoding="utf-8") as f: + f.write(formatted_source) + + logger.info("generated %s", gen_path) + logger.info(" - %d default instrumentations", len(default_instrumentations.elts)) + logger.info(" - %d library mappings", len(libraries.elts)) + + +if __name__ == "__main__": + main() + diff --git a/scripts/generate_loongsuite_readme.py b/scripts/generate_loongsuite_readme.py new file mode 100644 index 000000000..f89cdeae8 --- /dev/null +++ b/scripts/generate_loongsuite_readme.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import logging +import os +from pathlib import Path + +import tomli + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("loongsuite_readme_generator") + +_prefix = "loongsuite-instrumentation-" + +header = """ +| Instrumentation | Supported Packages | Metrics support | Semconv status | +| --------------- | ------------------ | --------------- | -------------- |""" + + +def main(base_instrumentation_path): + table = [header] + for instrumentation in sorted(os.listdir(base_instrumentation_path)): + instrumentation_path = os.path.join( + base_instrumentation_path, instrumentation + ) + if not os.path.isdir( + instrumentation_path + ) or not instrumentation.startswith(_prefix): + continue + + pyproject_toml = Path(instrumentation_path) / "pyproject.toml" + if not pyproject_toml.exists(): + continue + + try: + with open(pyproject_toml, "rb") as f: + pyproject = tomli.load(f) + + project = pyproject.get("project", {}) + optional_deps = project.get("optional-dependencies", {}) + instruments = optional_deps.get("instruments", []) + instruments_any = optional_deps.get("instruments-any", []) + + # Extract package name from instrumentation directory name + # e.g., "loongsuite-instrumentation-agentscope" -> "agentscope" + name = instrumentation.replace(_prefix, "") + + instruments_all = () + if not instruments and not instruments_any: + instruments_all = (name,) + else: + instruments_all = tuple(instruments + instruments_any) + + # Try to get metrics support and semconv status from pyproject.toml + # These might not be present, so use defaults + supports_metrics = project.get("supports_metrics", False) + semconv_status = project.get("semconv_status", "development") + + metric_column = "Yes" if supports_metrics else "No" + + table.append( + f"| [{instrumentation}](./{instrumentation}) | {','.join(instruments_all)} | {metric_column} | {semconv_status}" + ) + except Exception as e: + logger.warning(f"Failed to process {instrumentation}: {e}") + continue + + readme_path = os.path.join(base_instrumentation_path, "README.md") + with open(readme_path, "w", encoding="utf-8") as fh: + fh.write("\n".join(table)) + logger.info(f"Generated {readme_path}") + + +if __name__ == "__main__": + root_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + instrumentation_path = os.path.join(root_path, "instrumentation-loongsuite") + if os.path.exists(instrumentation_path): + main(instrumentation_path) + else: + logger.warning(f"Instrumentation path does not exist: {instrumentation_path}") + diff --git a/scripts/loongsuite-build-config.json b/scripts/loongsuite-build-config.json new file mode 100644 index 000000000..4b567e738 --- /dev/null +++ b/scripts/loongsuite-build-config.json @@ -0,0 +1,11 @@ +{ + "skip_packages": [ + "opentelemetry-instrumentation-langchain" + ], + "description": "Build configuration file, defines packages to skip (to avoid duplicates)", + "notes": [ + "When there are packages with the same name in instrumentation-genai and instrumentation-loongsuite,", + "prefer the version in instrumentation-loongsuite and skip the version in instrumentation-genai" + ] +} + diff --git a/tox-loongsuite.ini b/tox-loongsuite.ini index bafb6b6e7..96f5dc0f1 100644 --- a/tox-loongsuite.ini +++ b/tox-loongsuite.ini @@ -40,6 +40,13 @@ envlist = py3{10,11,12,13}-test-loongsuite-instrumentation-mem0-{oldest,latest} lint-loongsuite-instrumentation-mem0 + ; loongsuite-processor-baggage + py3{9,10,11,12,13}-test-loongsuite-processor-baggage + lint-loongsuite-processor-baggage + + ; generate tasks + generate-loongsuite + [testenv] test_deps = opentelemetry-api@{env:CORE_REPO}\#egg=opentelemetry-api&subdirectory=opentelemetry-api @@ -77,6 +84,9 @@ deps = loongsuite-mem0-latest: {[testenv]test_deps} loongsuite-mem0-latest: -r {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-mem0/test-requirements-latest.txt + + loongsuite-processor-baggage: {[testenv]test_deps} + loongsuite-processor-baggage: -r {toxinidir}/processor/loongsuite-processor-baggage/test-requirements.txt ; FIXME: add coverage testing allowlist_externals = @@ -117,9 +127,25 @@ commands = test-loongsuite-instrumentation-mem0: pytest {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-mem0/tests {posargs} lint-loongsuite-instrumentation-mem0: python -m ruff check {toxinidir}/instrumentation-loongsuite/loongsuite-instrumentation-mem0 + test-loongsuite-processor-baggage: pytest {toxinidir}/processor/loongsuite-processor-baggage/tests {posargs} + lint-loongsuite-processor-baggage: python -m ruff check {toxinidir}/processor/loongsuite-processor-baggage + ; TODO: add coverage commands ; coverage: {toxinidir}/scripts/coverage.sh +[testenv:generate-loongsuite] +deps = + -r {toxinidir}/gen-requirements.txt + +allowlist_externals = + {toxinidir}/scripts/generate_loongsuite_bootstrap.py + {toxinidir}/scripts/generate_loongsuite_readme.py + pytest + +commands = + {toxinidir}/scripts/generate_loongsuite_bootstrap.py + {toxinidir}/scripts/generate_loongsuite_readme.py + [testenv:docs] deps = -c {toxinidir}/dev-requirements.txt diff --git a/util/opentelemetry-util-genai/pyproject.toml b/util/opentelemetry-util-genai/pyproject.toml index 407820396..e4a02b132 100644 --- a/util/opentelemetry-util-genai/pyproject.toml +++ b/util/opentelemetry-util-genai/pyproject.toml @@ -25,9 +25,9 @@ classifiers = [ "Programming Language :: Python :: 3.13", ] dependencies = [ - "opentelemetry-instrumentation ~= 0.58b0", - "opentelemetry-semantic-conventions ~= 0.58b0", - "opentelemetry-api>=1.31.0", + "opentelemetry-instrumentation >= 0.58b0", + "opentelemetry-semantic-conventions >= 0.58b0", + "opentelemetry-api >= 1.31.0", ] [project.entry-points.opentelemetry_genai_completion_hook] diff --git a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py index dc47508d6..77661704f 100644 --- a/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py +++ b/util/opentelemetry-util-genai/src/opentelemetry/util/genai/utils.py @@ -95,13 +95,19 @@ def default(self, o: Any) -> Any: gen_ai_json_dump = partial( - json.dump, separators=(",", ":"), cls=_GenAiJsonEncoder + json.dump, + separators=(",", ":"), + cls=_GenAiJsonEncoder, + ensure_ascii=False, # LoongSuite Extension ) """Should be used by GenAI instrumentations when serializing objects that may contain bytes, datetimes, etc. for GenAI observability.""" gen_ai_json_dumps = partial( - json.dumps, separators=(",", ":"), cls=_GenAiJsonEncoder + json.dumps, + separators=(",", ":"), + cls=_GenAiJsonEncoder, + ensure_ascii=False, # LoongSuite Extension ) """Should be used by GenAI instrumentations when serializing objects that may contain bytes, datetimes, etc. for GenAI observability."""