From a8993791ab7cc64c1780bbfd897631485b252848 Mon Sep 17 00:00:00 2001 From: ghostbird03 Date: Sun, 10 May 2026 20:15:27 +0800 Subject: [PATCH 1/4] feat: migrate dependency install robustness from langbot-app-plugin-sdk#54 Migrate PR #54 to add pre-check, retry, and verification for plugin dependency installation. - errors.py: add DependencyInstallError, DependencyVerificationError - pkgmgr.py: add verify_dependencies, install_with_retry, precheck_dependencies - mgr.py: wire precheck -> retry -> verify into install flow, track failed deps, raise on failure after all retries exhausted --- src/langbot_plugin/entities/io/errors.py | 27 +++++ src/langbot_plugin/runtime/helper/pkgmgr.py | 109 ++++++++++++++++++++ src/langbot_plugin/runtime/plugin/mgr.py | 51 +++++++-- 3 files changed, 178 insertions(+), 9 deletions(-) diff --git a/src/langbot_plugin/entities/io/errors.py b/src/langbot_plugin/entities/io/errors.py index 32974c92..77827faa 100644 --- a/src/langbot_plugin/entities/io/errors.py +++ b/src/langbot_plugin/entities/io/errors.py @@ -21,6 +21,33 @@ def __str__(self): return self.message +class DependencyInstallError(Exception): + """Dependency installation failed.""" + + def __init__(self, package: str, returncode: int, stderr: str): + self.package = package + self.returncode = returncode + self.stderr = stderr + super().__init__(f"Failed to install {package}: {stderr}") + + def __str__(self): + return f"Failed to install {self.package}: {self.stderr}" + + +class DependencyVerificationError(Exception): + """Dependency verification failed.""" + + def __init__(self, missing: list[str], version_mismatch: list[str] = None): + self.missing = missing + self.version_mismatch = version_mismatch or [] + super().__init__( + f"Missing dependencies: {missing}, Version mismatch: {self.version_mismatch}" + ) + + def __str__(self): + return f"Missing dependencies: {self.missing}, Version mismatch: {self.version_mismatch}" + + class ActionCallError(Exception): """The action call failed.""" diff --git a/src/langbot_plugin/runtime/helper/pkgmgr.py b/src/langbot_plugin/runtime/helper/pkgmgr.py index c918db3c..7db21342 100644 --- a/src/langbot_plugin/runtime/helper/pkgmgr.py +++ b/src/langbot_plugin/runtime/helper/pkgmgr.py @@ -1,4 +1,5 @@ import asyncio +import importlib.util import os import re import subprocess @@ -139,3 +140,111 @@ def _parse_downloaded_bytes(output: str) -> int: else: total += int(val) return total + + +async def verify_dependencies(deps: list[str]) -> list[str]: + """Verify that installed dependencies are actually importable. + + Args: + deps: List of dependency specs, e.g. ['pkg==1.0.0', 'pkg>=2.0'] + + Returns: + List of dependencies that failed import verification. + """ + missing = [] + for dep in deps: + pkg_name = ( + dep.split("==")[0] + .split(">=")[0] + .split("<=")[0] + .split("<")[0] + .split(">")[0] + .split("[")[0] + .strip() + ) + pkg_name_normalized = pkg_name.replace("-", "_") + if importlib.util.find_spec(pkg_name_normalized) is None: + if importlib.util.find_spec(pkg_name) is None: + missing.append(dep) + return missing + + +async def install_with_retry( + package: str, + max_retries: int = 3, + retry_delay: float = 1.0, + extra_params: list | None = None, +) -> tuple[int, int, str]: + """Install a package with retry on failure. + + Args: + package: Package name to install. + max_retries: Maximum number of retry attempts. + retry_delay: Delay between retries in seconds. + extra_params: Additional pip parameters. + + Returns: + (returncode, downloaded_bytes, error_message) + """ + last_error = "" + for attempt in range(max_retries): + returncode, downloaded_bytes, _ = await install_single_async( + package, extra_params + ) + if returncode == 0: + return returncode, downloaded_bytes, "" + + last_error = ( + f"Attempt {attempt + 1}/{max_retries} failed with code {returncode}" + ) + + if attempt < max_retries - 1: + await asyncio.sleep(retry_delay) + + return returncode, 0, last_error + + +async def precheck_dependencies(requirements_file: str) -> dict: + """Pre-check dependency status before installation. + + Args: + requirements_file: Path to requirements.txt. + + Returns: + { + 'deps': list[str], + 'already_installed': list[str], + 'to_install': list[str], + 'conflicts': list[str], + } + """ + deps = parse_requirements(requirements_file) + already_installed = [] + to_install = [] + + for dep in deps: + pkg_name = ( + dep.split("==")[0] + .split(">=")[0] + .split("<=")[0] + .split("<")[0] + .split(">")[0] + .split("[")[0] + .strip() + ) + pkg_name_normalized = pkg_name.replace("-", "_") + + if ( + importlib.util.find_spec(pkg_name_normalized) is not None + or importlib.util.find_spec(pkg_name) is not None + ): + already_installed.append(dep) + else: + to_install.append(dep) + + return { + "deps": deps, + "already_installed": already_installed, + "to_install": to_install, + "conflicts": [], + } diff --git a/src/langbot_plugin/runtime/plugin/mgr.py b/src/langbot_plugin/runtime/plugin/mgr.py index ed874847..f5ed05f0 100644 --- a/src/langbot_plugin/runtime/plugin/mgr.py +++ b/src/langbot_plugin/runtime/plugin/mgr.py @@ -310,10 +310,22 @@ async def install_plugin( yield {"current_action": "installing dependencies"} requirements_file = os.path.join(plugin_path, "requirements.txt") if os.path.exists(requirements_file): - deps = pkgmgr_helper.parse_requirements(requirements_file) + precheck_result = await pkgmgr_helper.precheck_dependencies( + requirements_file + ) + deps = precheck_result["deps"] + to_install = precheck_result["to_install"] + already_installed = precheck_result["already_installed"] + + logger.info( + f"Dependency precheck: {len(already_installed)} already installed, " + f"{len(to_install)} to install" + ) + total_deps = len(deps) total_downloaded = 0 start_time = time.time() + failed_deps = [] for i, dep in enumerate(deps): elapsed = time.time() - start_time @@ -326,34 +338,55 @@ async def install_plugin( "current_dep": dep, "deps_downloaded_size": total_downloaded, "deps_speed": total_downloaded / elapsed if elapsed > 0 else 0, + "already_installed": len(already_installed), + "to_install": len(to_install), }, } - returncode, downloaded_bytes, output = ( - await pkgmgr_helper.install_single_async(dep) + returncode, downloaded_bytes, error_msg = ( + await pkgmgr_helper.install_with_retry(dep, max_retries=3) ) total_downloaded += downloaded_bytes if returncode != 0: - error_message = f"Failed to install dependency: {dep}" - if output: - error_message = f"{error_message}\n{output.strip()}" - logger.error(error_message) - raise RuntimeError(error_message) + logger.error( + f"Failed to install dependency after retries: {dep}, " + f"error: {error_msg}" + ) + failed_deps.append(dep) + + missing_deps = await pkgmgr_helper.verify_dependencies(deps) + if missing_deps: + logger.warning( + f"Dependency verification failed, missing: {missing_deps}" + ) + for dep in missing_deps: + if dep not in failed_deps: + failed_deps.append(dep) elapsed = time.time() - start_time yield { "current_action": "installing dependencies", "metadata": { "deps_total": total_deps, - "deps_installed": total_deps, + "deps_installed": total_deps - len(failed_deps), "deps_remaining": 0, + "deps_failed": len(failed_deps), + "failed_deps": failed_deps, "current_dep": "", "deps_downloaded_size": total_downloaded, "deps_speed": total_downloaded / elapsed if elapsed > 0 else 0, }, } + if failed_deps: + error_message = ( + f"Plugin {plugin_author}/{plugin_name} has {len(failed_deps)} " + f"failed dependencies: {failed_deps}" + ) + logger.error(error_message) + raise RuntimeError(error_message) + # initialize plugin settings yield {"current_action": "initializing plugin settings"} await self.context.control_handler.call_action( From bf77a672fd9440d28963a198b38b656e5f96b973 Mon Sep 17 00:00:00 2001 From: ghostbird03 Date: Sun, 10 May 2026 20:16:34 +0800 Subject: [PATCH 2/4] fix: use importlib.metadata to resolve pip package name vs module name mismatch verify_dependencies and precheck_dependencies used find_spec with the normalized pip package name, which fails for packages whose pip name differs from their Python import name (e.g. yiri-mirai -> mirai, PyYAML -> yaml, Pillow -> PIL). Fix: check distribution metadata via importlib.metadata.distribution() first (authoritative pip install check), then resolve actual import names via packages_distributions() reverse mapping as fallback. --- src/langbot_plugin/runtime/helper/pkgmgr.py | 113 +++++++++++++------- 1 file changed, 77 insertions(+), 36 deletions(-) diff --git a/src/langbot_plugin/runtime/helper/pkgmgr.py b/src/langbot_plugin/runtime/helper/pkgmgr.py index 7db21342..88f63312 100644 --- a/src/langbot_plugin/runtime/helper/pkgmgr.py +++ b/src/langbot_plugin/runtime/helper/pkgmgr.py @@ -1,4 +1,5 @@ import asyncio +import importlib.metadata import importlib.util import os import re @@ -142,30 +143,87 @@ def _parse_downloaded_bytes(output: str) -> int: return total -async def verify_dependencies(deps: list[str]) -> list[str]: - """Verify that installed dependencies are actually importable. +_dist_to_packages: dict[str, set[str]] | None = None - Args: - deps: List of dependency specs, e.g. ['pkg==1.0.0', 'pkg>=2.0'] + +def _extract_package_name(dep_spec: str) -> str: + """Extract the pip package name from a dependency spec string.""" + for sep in ("==", ">=", "<=", "<", ">", "["): + dep_spec = dep_spec.split(sep)[0] + return dep_spec.strip() + + +def _is_distribution_installed(pkg_name: str) -> bool: + """Check whether a pip distribution is installed via its metadata. + + This is the authoritative check, regardless of whether the pip + package name matches the actual Python module name. + """ + candidates = {pkg_name, pkg_name.replace("-", "_"), pkg_name.replace("_", "-")} + for name in candidates: + try: + importlib.metadata.distribution(name) + return True + except importlib.metadata.PackageNotFoundError: + continue + return False + + +def _resolve_import_names(pkg_name: str) -> set[str]: + """Resolve top-level Python import names for a pip distribution. + + Many pip package names differ from their importable module name. + E.g. yiri-mirai -> mirai, PyYAML -> yaml, Pillow -> PIL. + Uses importlib.metadata.packages_distributions() reverse mapping. + """ + global _dist_to_packages + + if _dist_to_packages is None: + _dist_to_packages = {} + try: + for top_pkg, dist_names in importlib.metadata.packages_distributions().items(): + for dist_name in dist_names: + key = dist_name.lower().replace("_", "-") + _dist_to_packages.setdefault(key, set()).add(top_pkg) + except Exception: + pass + + key = pkg_name.lower().replace("_", "-") + names = _dist_to_packages.get(key, set()).copy() + names.add(pkg_name.replace("-", "_")) + return names + + +def _check_dependency_installed(dep_spec: str) -> bool: + """Check whether a dependency is installed and importable. + + Resolution order: + 1. importlib.metadata.distribution() - authoritative pip install check + 2. packages_distributions() reverse mapping - find actual import names + 3. fallback: find_spec on the normalized package name + """ + pkg_name = _extract_package_name(dep_spec) + + if _is_distribution_installed(pkg_name): + return True + + for import_name in _resolve_import_names(pkg_name): + if importlib.util.find_spec(import_name) is not None: + return True + + return False + + +async def verify_dependencies(deps: list[str]) -> list[str]: + """Verify installed dependencies are actually importable. Returns: - List of dependencies that failed import verification. + List of dependency specs that failed verification. """ missing = [] for dep in deps: - pkg_name = ( - dep.split("==")[0] - .split(">=")[0] - .split("<=")[0] - .split("<")[0] - .split(">")[0] - .split("[")[0] - .strip() - ) - pkg_name_normalized = pkg_name.replace("-", "_") - if importlib.util.find_spec(pkg_name_normalized) is None: - if importlib.util.find_spec(pkg_name) is None: - missing.append(dep) + if not _check_dependency_installed(dep): + missing.append(dep) return missing @@ -207,9 +265,6 @@ async def install_with_retry( async def precheck_dependencies(requirements_file: str) -> dict: """Pre-check dependency status before installation. - Args: - requirements_file: Path to requirements.txt. - Returns: { 'deps': list[str], @@ -223,21 +278,7 @@ async def precheck_dependencies(requirements_file: str) -> dict: to_install = [] for dep in deps: - pkg_name = ( - dep.split("==")[0] - .split(">=")[0] - .split("<=")[0] - .split("<")[0] - .split(">")[0] - .split("[")[0] - .strip() - ) - pkg_name_normalized = pkg_name.replace("-", "_") - - if ( - importlib.util.find_spec(pkg_name_normalized) is not None - or importlib.util.find_spec(pkg_name) is not None - ): + if _check_dependency_installed(dep): already_installed.append(dep) else: to_install.append(dep) From b12349f42de38f014732a098062246a82aa3edbf Mon Sep 17 00:00:00 2001 From: ghostbird03 Date: Sun, 10 May 2026 21:01:06 +0800 Subject: [PATCH 3/4] fix: use packaging.Requirement for package name extraction, skip already-installed deps in install loop - Replace fragile chained split in _extract_package_name with packaging.requirements.Requirement for proper PEP 508 parsing - Pre-checked already-installed dependencies are skipped in the install loop instead of being re-installed - install_with_retry now captures pip stderr output on failure for meaningful error messages --- src/langbot_plugin/runtime/helper/pkgmgr.py | 23 ++++++++++++++++----- src/langbot_plugin/runtime/plugin/mgr.py | 12 ++++++----- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/langbot_plugin/runtime/helper/pkgmgr.py b/src/langbot_plugin/runtime/helper/pkgmgr.py index 88f63312..a22ff512 100644 --- a/src/langbot_plugin/runtime/helper/pkgmgr.py +++ b/src/langbot_plugin/runtime/helper/pkgmgr.py @@ -6,6 +6,7 @@ import subprocess import sys +from packaging.requirements import Requirement from pip._internal import main as pipmain @@ -147,10 +148,20 @@ def _parse_downloaded_bytes(output: str) -> int: def _extract_package_name(dep_spec: str) -> str: - """Extract the pip package name from a dependency spec string.""" - for sep in ("==", ">=", "<=", "<", ">", "["): - dep_spec = dep_spec.split(sep)[0] - return dep_spec.strip() + """Extract the pip package name from a dependency spec string. + + Uses ``packaging.requirements.Requirement`` for proper PEP 508 parsing, + handling all version operators, extras, environment markers, and URL + references (e.g. ``yiri-mirai>=1.0`` → ``yiri-mirai``). + + Falls back to a simple operator split for truly malformed specs. + """ + try: + return Requirement(dep_spec).name + except Exception: + for sep in ("==", ">=", "<=", "!=", "~=", "===", "<", ">", "["): + dep_spec = dep_spec.split(sep)[0] + return dep_spec.strip() def _is_distribution_installed(pkg_name: str) -> bool: @@ -246,7 +257,7 @@ async def install_with_retry( """ last_error = "" for attempt in range(max_retries): - returncode, downloaded_bytes, _ = await install_single_async( + returncode, downloaded_bytes, output = await install_single_async( package, extra_params ) if returncode == 0: @@ -255,6 +266,8 @@ async def install_with_retry( last_error = ( f"Attempt {attempt + 1}/{max_retries} failed with code {returncode}" ) + if output.strip(): + last_error += f"\n{output.strip()}" if attempt < max_retries - 1: await asyncio.sleep(retry_delay) diff --git a/src/langbot_plugin/runtime/plugin/mgr.py b/src/langbot_plugin/runtime/plugin/mgr.py index f5ed05f0..387ee78d 100644 --- a/src/langbot_plugin/runtime/plugin/mgr.py +++ b/src/langbot_plugin/runtime/plugin/mgr.py @@ -326,20 +326,22 @@ async def install_plugin( total_downloaded = 0 start_time = time.time() failed_deps = [] + already_installed_count = len(already_installed) + to_install_count = len(to_install) - for i, dep in enumerate(deps): + for i, dep in enumerate(to_install): elapsed = time.time() - start_time yield { "current_action": "installing dependencies", "metadata": { "deps_total": total_deps, - "deps_installed": i, - "deps_remaining": total_deps - i, + "deps_installed": already_installed_count + i, + "deps_remaining": to_install_count - i, "current_dep": dep, "deps_downloaded_size": total_downloaded, "deps_speed": total_downloaded / elapsed if elapsed > 0 else 0, - "already_installed": len(already_installed), - "to_install": len(to_install), + "already_installed": already_installed_count, + "to_install": to_install_count, }, } From aa7307da4d8733316b6aba28a8b0b317f9a5573a Mon Sep 17 00:00:00 2001 From: ghostbird03 Date: Sat, 16 May 2026 16:39:12 +0800 Subject: [PATCH 4/4] fix(pkgmgr): check version constraints and markers in dependency precheck/verify MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes addressing review feedback on #55: - _check_dependency_installed() now evaluates PEP 508 environment markers (e.g. `foo; python_version < "3.9"` is correctly skipped when the marker does not apply). - Version constraints are verified via specifier.contains(): a package like `requests>=2.32` is only considered satisfied if the installed version actually meets the requirement. - URL requirements (e.g. `foo @ https://...`) return False in precheck so pip decides whether reinstall is needed, but verify_dependencies() accepts distribution existence after pip has run. - Removed _extract_package_name() dead code — Requirement() is now called directly; malformed specs return False to let pip handle them. - Added packaging>=24.0 to pyproject.toml as an explicit runtime dependency. --- pyproject.toml | 1 + src/langbot_plugin/runtime/helper/pkgmgr.py | 80 ++++++++++++++------- 2 files changed, 55 insertions(+), 26 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bd00778d..234627e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "dotenv>=0.9.9", "httpx>=0.28.1", "jinja2>=3.1.6", + "packaging>=24.0", "pip>=25.2", "pydantic>=2.11.5", "pydantic-settings>=2.10.1", diff --git a/src/langbot_plugin/runtime/helper/pkgmgr.py b/src/langbot_plugin/runtime/helper/pkgmgr.py index a22ff512..a01ad2dc 100644 --- a/src/langbot_plugin/runtime/helper/pkgmgr.py +++ b/src/langbot_plugin/runtime/helper/pkgmgr.py @@ -147,23 +147,6 @@ def _parse_downloaded_bytes(output: str) -> int: _dist_to_packages: dict[str, set[str]] | None = None -def _extract_package_name(dep_spec: str) -> str: - """Extract the pip package name from a dependency spec string. - - Uses ``packaging.requirements.Requirement`` for proper PEP 508 parsing, - handling all version operators, extras, environment markers, and URL - references (e.g. ``yiri-mirai>=1.0`` → ``yiri-mirai``). - - Falls back to a simple operator split for truly malformed specs. - """ - try: - return Requirement(dep_spec).name - except Exception: - for sep in ("==", ">=", "<=", "!=", "~=", "===", "<", ">", "["): - dep_spec = dep_spec.split(sep)[0] - return dep_spec.strip() - - def _is_distribution_installed(pkg_name: str) -> bool: """Check whether a pip distribution is installed via its metadata. @@ -206,18 +189,53 @@ def _resolve_import_names(pkg_name: str) -> set[str]: def _check_dependency_installed(dep_spec: str) -> bool: - """Check whether a dependency is installed and importable. + """Check whether a dependency requirement is fully satisfied. + + Returns True only when: + 1. Environment markers do not apply → treat as satisfied (skip) + 2. Distribution is installed AND version satisfies the specifier + 3. Package is importable (fallback for name-mismatch cases, + e.g. yiri-mirai → mirai, PyYAML → yaml) - Resolution order: - 1. importlib.metadata.distribution() - authoritative pip install check - 2. packages_distributions() reverse mapping - find actual import names - 3. fallback: find_spec on the normalized package name + Returns False when the installed version does not meet the specifier + or the package is not installed at all. """ - pkg_name = _extract_package_name(dep_spec) + try: + req = Requirement(dep_spec) + except Exception: + # parse_requirements() already filters comments, empty lines, and + # option lines (-r / --index-url). If Requirement() still fails, the + # input is genuinely malformed and no heuristic name extraction will + # be reliable. Return False so pip gets a chance to install it. + return False - if _is_distribution_installed(pkg_name): + if req.marker and not req.marker.evaluate(): return True + pkg_name = req.name + + if _is_distribution_installed(pkg_name): + if req.url: + # URL requirements (e.g. ``foo @ https://...``) cannot be + # reliably verified from Python — the installed distribution + # may have come from a different source or version. Reading + # pip's internal direct_url.json (PEP 610) is fragile and + # couples us to pip implementation details. Returning True + # would risk silently keeping a mismatched version. + # + # Instead, return False so pip decides. If the URL points to + # an already-satisfied version, pip will output "Requirement + # already satisfied" and exit without downloading — the + # overhead is a single subprocess spawn. + return False + if not req.specifier: + return True + try: + installed_version = importlib.metadata.version(pkg_name) + return installed_version in req.specifier + except importlib.metadata.PackageNotFoundError: + return False + for import_name in _resolve_import_names(pkg_name): if importlib.util.find_spec(import_name) is not None: return True @@ -233,8 +251,18 @@ async def verify_dependencies(deps: list[str]) -> list[str]: """ missing = [] for dep in deps: - if not _check_dependency_installed(dep): - missing.append(dep) + if _check_dependency_installed(dep): + continue + # _check_dependency_installed always returns False for URL + # requirements (preferring pip to decide). After pip has run, + # distribution existence is sufficient verification. + try: + req = Requirement(dep) + if req.url and _is_distribution_installed(req.name): + continue + except Exception: + pass + missing.append(dep) return missing