From daf7bd5b2b11f7061b842e25285e9b1eac3409bb Mon Sep 17 00:00:00 2001 From: traoreera Date: Tue, 12 May 2026 15:27:51 +0000 Subject: [PATCH] ajoue d'un service a xcore (xworker) pour facilite l'utilisation de celeris dans les application --- docgen/aggregator.py | 66 --- docgen/composer.py | 158 ------ docgen/config.yaml | 29 -- docgen/hasher.py | 5 - docgen/pipeline.py | 154 ------ docgen/scanner.py | 84 ---- docgen/summarizer.py | 113 ----- integration.yaml | 15 +- tests/unit/services/test_container.py | 4 +- xcore/cli/__init__.py | 1 - xcore/cli/main.py | 208 -------- xcore/cli/marketplace_cmd.py | 269 ---------- xcore/cli/plugin_cmd.py | 698 -------------------------- xcore/cli/sandbox_cmd.py | 340 ------------- xcore/cli/validate_cmd.py | 80 --- xcore/configurations/loader.py | 11 + xcore/configurations/sections.py | 59 +++ xcore/sdk/__init__.py | 4 + xcore/services/container.py | 25 +- xcore/services/database/migrations.py | 105 +++- xcore/services/xworker | 1 + 21 files changed, 200 insertions(+), 2229 deletions(-) delete mode 100644 docgen/aggregator.py delete mode 100644 docgen/composer.py delete mode 100644 docgen/config.yaml delete mode 100644 docgen/hasher.py delete mode 100644 docgen/pipeline.py delete mode 100644 docgen/scanner.py delete mode 100644 docgen/summarizer.py delete mode 100644 xcore/cli/__init__.py delete mode 100644 xcore/cli/main.py delete mode 100644 xcore/cli/marketplace_cmd.py delete mode 100644 xcore/cli/plugin_cmd.py delete mode 100644 xcore/cli/sandbox_cmd.py delete mode 100644 xcore/cli/validate_cmd.py create mode 160000 xcore/services/xworker diff --git a/docgen/aggregator.py b/docgen/aggregator.py deleted file mode 100644 index 8a525d0e..00000000 --- a/docgen/aggregator.py +++ /dev/null @@ -1,66 +0,0 @@ -from collections import defaultdict - -from summarizer import call_model, summarize_chunk - - -def summarize_by_file(chunks, cfg): - """ - Groupe les chunks par fichier (path) et génère un résumé par chunk. - Retourne un dict : path -> liste de résumés partiels + métadonnées du fichier. - """ - file_map = defaultdict(lambda: {"summaries": [], "module": None, "filename": None}) - - for chunk in chunks: - path = chunk["path"] - summary = summarize_chunk(path, chunk["content"], cfg) - file_map[path]["summaries"].append(summary) - file_map[path]["module"] = chunk["module"] - file_map[path]["filename"] = chunk["filename"] - - return file_map - - -def aggregate_file_summaries(file_map, cfg): - """ - Fusionne les résumés partiels de chaque fichier en un résumé unique. - Retourne un dict : path -> { summary, module, filename } - """ - if not file_map: - raise ValueError("No file summaries to aggregate.") - - aggregated = {} - - for path, data in file_map.items(): - summaries = data["summaries"] - combined = "\n".join(summaries) - - if len(summaries) == 1: - # Pas besoin de fusion si un seul chunk - final_summary = summaries[0] - else: - final_prompt = f"""You are writing technical documentation for a software project. - -The following are partial summaries of different chunks of the same source file: -`{path}`. -Merge them into a single, unified technical summary as if the file had been read in full. - -Rules: -- Write in plain, direct prose. No bullet lists. -- Eliminate any redundancy between the partial summaries. -- Keep only information that is accurate and relevant to understanding this file's role. -- Maximum length: 150 words. -- Do not mention that this is a merge of multiple summaries. - ---- PARTIAL SUMMARIES --- - -{combined} -""" - final_summary = call_model(final_prompt, cfg) - - aggregated[path] = { - "summary": final_summary, - "module": data["module"], - "filename": data["filename"], - } - - return aggregated diff --git a/docgen/composer.py b/docgen/composer.py deleted file mode 100644 index 025c07c6..00000000 --- a/docgen/composer.py +++ /dev/null @@ -1,158 +0,0 @@ -from summarizer import call_model - - -def compose_file_doc(path, filename, summary, cfg): - """ - Génère la documentation Markdown pour un seul fichier source. - """ - prompt = f""" -You are a senior software architect writing official developer documentation. - -Write a clean, well-structured Markdown documentation page for the -following source file. This page will be published in the project's -`docs/` directory. - ---- STYLE GUIDELINES --- -- Write in clear, flowing prose. Use bullet lists only for discrete - enumerable items (e.g. parameters, config keys). -- Use a confident, technical tone. The reader is a developer joining - the project. -- Use Markdown headings (## and ###), backticks for code references - (`ClassName`, `function_name()`), and short paragraphs. -- Do not pad sections. If a section has little to say, keep it brief. -- Never repeat the filename in the body — it will appear as the page title. - ---- REQUIRED SECTIONS --- - -## Overview -What this file does and why it exists. 2–3 sentences max. - -## Responsibilities -The specific tasks this file handles within the system. Write in prose, -not a bullet list. - -## Key Components -For each important class or function, write a short paragraph: what it does, -its signature (if relevant), and any notable behavior or edge cases. - -## Dependencies -What this file imports and relies on — internal modules and external libraries. -Briefly explain why each dependency is needed. - -## How It Fits In -How this file connects to the rest of the codebase: who calls it, what it -produces, and where its output goes. - ---- FILE SUMMARY --- -File: {path} - -{summary} -""" - return call_model(prompt, cfg) - - -def compose_module_index(module_name, file_docs, child_modules, cfg): - """ - Génère un README.md pour un module. - Gère les deux cas : - - module feuille : a des fichiers directs, pas de sous-modules - - module intermédiaire : a des sous-modules (et éventuellement des - fichiers directs) - - module_name : ex "linting" ou "core/utils" - file_docs : dict filename → { summary, doc_path } (fichiers directs) - child_modules : dict child_module_path → description courte (sous-modules - directs) - """ - # Section fichiers directs - if file_docs: - files_section = "### Files\n" + "\n".join( - f"- [`{fname}.py`](./{fname}.md) — {data['summary'][:120].rstrip()}" - for fname, data in file_docs.items() - ) - else: - files_section = "" - - # Section sous-modules directs - # Le lien doit être relatif : depuis docs/core/README.md → ./utils/README.md - if child_modules: - # Extraire uniquement le dernier segment pour le lien relatif - sub_section = "### Sub-modules\n" + "\n".join( - f"- [`{child.split('/')[-1]}/`](./{child.split('/')[-1]}/README.md) " - f"— {desc}" - for child, desc in child_modules.items() - ) - else: - sub_section = "" - - content_block = "\n\n".join(filter(None, [files_section, sub_section])) - - prompt = f"""You are writing the index page for a Python module's -documentation. - -Module: `{module_name}` - -Write a concise README.md in Markdown for this module. -It will serve as the entry point when a developer opens the -`docs/{module_name}/` folder. - ---- STYLE GUIDELINES --- -- Start with 2–4 sentences of prose describing what this module is - responsible for. Be specific. -- Then reproduce the "Files" and "Sub-modules" sections below exactly as - provided — do not rewrite them, just insert them after your prose overview. - They already contain the correct Markdown links. -- Use backticks for module and file names. -- Do not add extra sections. - ---- CONTENT TO INSERT AFTER YOUR OVERVIEW --- -{content_block} -""" - return call_model(prompt, cfg) - - -def compose_root_index(modules_summary, root_files, cfg): - """ - Génère le README.md racine de docs/. - modules_summary : dict module_name → description courte (modules de - premier niveau) - root_files : dict filename → { summary } (fichiers à la racine du - repo, hors module) - """ - modules_list = "\n".join( - f"- [`{mod}/`](./{mod}/README.md) — {desc}" - for mod, desc in modules_summary.items() - ) - - root_files_list = ( - "\n".join( - f"- [`{fname}.py`](./{fname}.md) — {data['summary'][:120].rstrip()}" - for fname, data in root_files.items() - ) - if root_files - else "" - ) - - root_files_section = ( - f"### Root-level files\n{root_files_list}" if root_files_list else "" - ) - - prompt = f"""You are writing the top-level documentation index for a -software project. - -Write a root README.md for the `docs/` folder. This is the first page a -developer reads. - ---- STYLE GUIDELINES --- -- Start with a short, specific project overview (3–4 sentences). No filler. -- Then reproduce the modules list and root files section below exactly as - provided — do not rewrite the links. -- Close with one short paragraph explaining how the documentation is - organized and how to navigate it. - ---- MODULES --- -{modules_list} - -{root_files_section} -""" - return call_model(prompt, cfg) diff --git a/docgen/config.yaml b/docgen/config.yaml deleted file mode 100644 index 6929dc00..00000000 --- a/docgen/config.yaml +++ /dev/null @@ -1,29 +0,0 @@ -project: - name: XCore - repo_path: ./xcore - docs_dir: IAdocs # remplace output_file — la doc est maintenant un dossier - -cache: - enabled: true - path: docgen/cache/summaries.json - -model: - base_url: http://localhost:11434/v1 - api_key: lm-studio - model_name: aleshribar3/deepseek-r1-tool-calling:1.5b - temperature: 0.1 - -generation: - max_file_chunk_chars: 8000 - include_extensions: [".py", ".yaml", ".json", ".md", '.cpp'] - exclude_dirs: - [ - ".git", - "__pycache__", - ".venv", - "docs", - "backgroundtask", - ".vscode", - "docgen", - ] - exclude_files: [ "*.md", "*.pyc", "*.ini", "*.html", "*.json"] # fichiers ignores completement diff --git a/docgen/hasher.py b/docgen/hasher.py deleted file mode 100644 index 0aa35d05..00000000 --- a/docgen/hasher.py +++ /dev/null @@ -1,5 +0,0 @@ -import hashlib - - -def compute_hash(content: str) -> str: - return hashlib.sha256(content.encode()).hexdigest() diff --git a/docgen/pipeline.py b/docgen/pipeline.py deleted file mode 100644 index 2659fcff..00000000 --- a/docgen/pipeline.py +++ /dev/null @@ -1,154 +0,0 @@ -import os -from collections import defaultdict - -from aggregator import aggregate_file_summaries, summarize_by_file -from composer import compose_file_doc, compose_module_index, compose_root_index -from scanner import scan_repo -from summarizer import load_config - - -def write_file(path, content): - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "w", encoding="utf-8") as f: - f.write(content) - print(f"[WRITTEN] {path}") - - -def get_all_ancestor_modules(module): - """ - Retourne tous les niveaux intermédiaires d'un module. - Exemple : "core/utils/parsers" → ["core", "core/utils", "core/utils/parsers"] - """ - parts = module.split("/") - return ["/".join(parts[: i + 1]) for i in range(len(parts))] - - -def main(): - cfg = load_config() - docs_root = cfg["project"]["docs_dir"] # ex: "docs" - - # 1. Scanner le repo - chunks = scan_repo(cfg) - if not chunks: - print( - "No files found to process. Check repo_path and " - "include_extensions in config." - ) - return - - # 2. Résumer chaque fichier (avec cache) - file_map = summarize_by_file(chunks, cfg) - file_summaries = aggregate_file_summaries(file_map, cfg) - - # 3. Générer une doc .md par fichier source - # Structure : docs//.md - # - # On construit aussi deux index : - # - modules_files : module_path → { filename → { summary, doc_path } } - # - modules_children : module_path → set de sous-modules directs - # (pour les README des niveaux intermédiaires) - - modules_files = defaultdict(dict) # module -> fichiers directs - modules_children = defaultdict(set) # module -> sous-modules directs - root_files = {} # fichiers à la racine du repo - - for path, data in file_summaries.items(): - module = data["module"] # ex: "core/utils" | None - filename = data["filename"] # ex: "scanner" - summary = data["summary"] - - # Calcul du chemin de sortie - if module: - doc_path = os.path.join(docs_root, module, f"{filename}.md") - else: - doc_path = os.path.join(docs_root, f"{filename}.md") - - # Génération de la doc du fichier - print(f"[COMPOSING] {doc_path}") - doc_content = compose_file_doc(path, filename, summary, cfg) - write_file(doc_path, doc_content) - - if module: - # Enregistrer le fichier dans son module direct - modules_files[module][filename] = { - "summary": summary, - "doc_path": doc_path, - } - # Enregistrer tous les liens parent → enfant pour les niveaux intermédiaires - # ex: "core/utils" → enregistre core→core/utils et core/utils (feuille) - ancestors = get_all_ancestor_modules(module) - for i in range(len(ancestors) - 1): - parent = ancestors[i] - child = ancestors[i + 1] - modules_children[parent].add(child) - else: - root_files[filename] = { - "summary": summary, - "doc_path": doc_path, - } - - # 4. Générer un README.md pour chaque module (tous niveaux confondus) - # On traite du plus profond au plus haut pour que les descriptions - # des sous-modules soient disponibles quand on génère les parents. - # - # all_modules = union de tous les modules qui ont des fichiers directs - # + tous les modules intermédiaires (qui n'ont que des enfants) - all_modules = set(modules_files.keys()) | set(modules_children.keys()) - - # Trier du plus profond (plus de "/" dans le chemin) au plus haut - sorted_modules = sorted(all_modules, key=lambda m: m.count("/"), reverse=True) - - # Stocker une description courte par module (pour les index parents) - module_descriptions = {} - - for module in sorted_modules: - index_path = os.path.join(docs_root, module, "README.md") - print(f"[MODULE INDEX] {index_path}") - - # Fichiers directs dans ce module - direct_files = modules_files.get(module, {}) - - # Sous-modules directs (enfants immédiats seulement) - direct_children = sorted(modules_children.get(module, set())) - child_descriptions = { - child: module_descriptions.get(child, "") for child in direct_children - } - - index_content = compose_module_index( - module_name=module, - file_docs=direct_files, - child_modules=child_descriptions, - cfg=cfg, - ) - write_file(index_path, index_content) - - # Résumé court pour le niveau parent - if direct_files: - first_summary = next(iter(direct_files.values()))["summary"] - elif child_descriptions: - first_summary = next(iter(child_descriptions.values())) - else: - first_summary = "" - module_descriptions[module] = first_summary[:120].rstrip() + "..." - - # 5. Générer le README.md racine de docs/ - # Liste uniquement les modules de premier niveau (sans "/" dans le chemin) - top_level_modules = { - mod: module_descriptions[mod] for mod in sorted(all_modules) if "/" not in mod - } - - root_index_path = os.path.join(docs_root, "README.md") - print(f"[ROOT INDEX] {root_index_path}") - root_index = compose_root_index( - modules_summary=top_level_modules, - root_files=root_files, - cfg=cfg, - ) - write_file(root_index_path, root_index) - - print("\nDocumentation generated successfully.") - print(f"Output: {docs_root}/") - - -if __name__ == "__main__": - main() diff --git a/docgen/scanner.py b/docgen/scanner.py deleted file mode 100644 index 96cb659c..00000000 --- a/docgen/scanner.py +++ /dev/null @@ -1,84 +0,0 @@ -import os - -import yaml - - -def load_config(): - with open("docgen/config.yaml") as f: - return yaml.safe_load(f) - - -def split_content(content, max_chars): - return [content[i : i + max_chars] for i in range(0, len(content), max_chars)] - - -def is_excluded(root, exclude_dirs): - parts = root.replace("\\", "/").split("/") - return any(part in exclude_dirs for part in parts) - - -def get_module_name(path, repo_root): - """ - Retourne le chemin relatif du dossier contenant le fichier, depuis la racine du repo. - Si le fichier est directement à la racine, retourne None. - Exemples : - /repo/linting/scanner.py -> "linting" - /repo/core/utils/helpers.py -> "core/utils" - /repo/pipeline.py -> None - """ - rel = os.path.relpath(path, repo_root) - parts = rel.replace("\\", "/").split("/") - if len(parts) == 1: - return None - return "/".join(parts[:-1]) - - -def scan_repo(cfg=None): - if cfg is None: - from summarizer import load_config - - cfg = load_config() - - repo = cfg["project"]["repo_path"] - max_chars = cfg["generation"]["max_file_chunk_chars"] - exts = tuple(cfg["generation"]["include_extensions"]) - exclude = set(cfg["generation"]["exclude_dirs"]) - - # Fichiers a ignorer completement (configurable, defaut : __init__.py) - ignore_files = set(cfg["generation"].get("exclude_files", ["__init__.py"])) - - chunks = [] - - for root, dirs, files in os.walk(repo): - dirs[:] = [d for d in dirs if d not in exclude] - - if is_excluded(root, exclude): - continue - - for file in files: - if file in ignore_files: - print(f"[SKIP] {os.path.join(root, file)} (excluded file)") - continue - - if file.endswith(exts): - path = os.path.join(root, file) - module = get_module_name(path, repo) - - try: - with open(path, "r", errors="ignore") as f: - content = f.read() - except OSError as e: - print(f"[SKIP] Cannot read {path}: {e}") - continue - - for part in split_content(content, max_chars): - chunks.append( - { - "path": path, - "module": module, - "filename": os.path.splitext(file)[0], - "content": part, - } - ) - - return chunks diff --git a/docgen/summarizer.py b/docgen/summarizer.py deleted file mode 100644 index b9a1bfc3..00000000 --- a/docgen/summarizer.py +++ /dev/null @@ -1,113 +0,0 @@ -import json -import os - -import requests -import yaml -from hasher import compute_hash - - -def load_config(): - with open("docgen/config.yaml") as f: - return yaml.safe_load(f) - - -_cache = None -_cache_dirty = False - - -def load_cache(cfg): - global _cache - if _cache is not None: - return _cache - path = cfg["cache"]["path"] - if not os.path.exists(path): - _cache = {} - else: - with open(path) as f: - _cache = json.load(f) - return _cache - - -def save_cache(cfg, cache): - path = cfg["cache"]["path"] - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, "w") as f: - json.dump(cache, f, indent=2) - - -def call_model(prompt, cfg=None): - if cfg is None: - cfg = load_config() - url = cfg["model"]["base_url"] + "/chat/completions" - - payload = { - "model": cfg["model"]["model_name"], - "temperature": cfg["model"]["temperature"], - "messages": [ - {"role": "system", "content": "You are a senior backend architect."}, - {"role": "user", "content": prompt}, - ], - } - - headers = {"Authorization": f"Bearer {cfg['model']['api_key']}"} - - try: - r = requests.post(url, json=payload, headers=headers, timeout=1000000) - r.raise_for_status() - data = r.json() - return data["choices"][0]["message"]["content"] - except requests.exceptions.RequestException as e: - raise RuntimeError(f"Model request failed: {e}") - except (KeyError, IndexError): - raise RuntimeError(f"Unexpected model response format: {r.text[:200]}") - - -def summarize_chunk(path, content, cfg=None): - if cfg is None: - cfg = load_config() - - file_hash = compute_hash(content) - cache_key = f"{path}:{file_hash}" - - if cfg["cache"]["enabled"]: - cache = load_cache(cfg) - if cache_key in cache: - print(f"[CACHE HIT] {path}") - return cache[cache_key] - - print(f"[PROCESSING] {path}") - - prompt = f"""You are documenting a software project for other developers. -Analyze the following source file chunk and produce a concise technical -summary. - -Write in clear, professional English. Use plain prose — avoid filler phrases -like "this file contains" or "it is responsible for". -Go straight to the point. Be specific: name the actual classes, functions, -and modules involved. - -Cover these aspects in order: -1. **Purpose** — What problem does this file solve? What is its role in the - system? -2. **Key components** — List the main classes and functions with a one-line - description of each. -3. **Dependencies** — What does this file import or rely on (internal - modules, external libraries)? -4. **Interactions** — How does this file connect to the rest of the - codebase (called by, calls into, data it produces or consumes)? - -Keep the summary under 200 words. Do not repeat the file path in your answer. - ---- FILE: {path} --- - -{content} -""" - - summary = call_model(prompt, cfg) - - if cfg["cache"]["enabled"]: - cache = load_cache(cfg) - cache[cache_key] = summary - save_cache(cfg, cache) - - return summary diff --git a/integration.yaml b/integration.yaml index 9984ae73..0c93b48d 100644 --- a/integration.yaml +++ b/integration.yaml @@ -72,7 +72,20 @@ services: # func: myapp.tasks.monitoring:snapshot_metrics # trigger: interval # minutes: 5 - + extensions: + email: + module: extensions.email.service:EmailService + config: + smtp_host: smtp.gmail.com + smtp_port: 587 + smtp_user: ${SMTP_USER} + smtp_password: ${SMTP_PASSWORD} + from_address: noreply@example.com + from_name: "Mon Application" + use_tls: true + timeout: 10 + max_retries: 3 + queue_size: 100 # ── Observabilité ───────────────────────────────────────────── observability: diff --git a/tests/unit/services/test_container.py b/tests/unit/services/test_container.py index 4be0dadc..43b66bf9 100644 --- a/tests/unit/services/test_container.py +++ b/tests/unit/services/test_container.py @@ -16,6 +16,7 @@ ExtensionServiceProvider, SchedulerServiceProvider, ServiceContainer, + XWorkerServiceProvider, ) @@ -76,10 +77,11 @@ def container(self, mock_config): def test_default_providers_present(self): """Test default providers are present.""" - assert len(ServiceContainer.DEFAULT_PROVIDERS) == 4 + assert len(ServiceContainer.DEFAULT_PROVIDERS) == 5 assert DatabaseServiceProvider in ServiceContainer.DEFAULT_PROVIDERS assert CacheServiceProvider in ServiceContainer.DEFAULT_PROVIDERS assert SchedulerServiceProvider in ServiceContainer.DEFAULT_PROVIDERS + assert XWorkerServiceProvider in ServiceContainer.DEFAULT_PROVIDERS assert ExtensionServiceProvider in ServiceContainer.DEFAULT_PROVIDERS def test_get_nonexistent_service(self, container): diff --git a/xcore/cli/__init__.py b/xcore/cli/__init__.py deleted file mode 100644 index 6e00087a..00000000 --- a/xcore/cli/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""CLI xcore — commandes de gestion des plugins.""" diff --git a/xcore/cli/main.py b/xcore/cli/main.py deleted file mode 100644 index 89007cdd..00000000 --- a/xcore/cli/main.py +++ /dev/null @@ -1,208 +0,0 @@ -""" -main.py — Point d'entrée du CLI xcore. - -Commandes: - xcore plugin list - xcore plugin load - xcore plugin reload - xcore plugin sign --key - xcore plugin verify --key - xcore plugin validate - xcore plugin health - xcore plugin install [--source zip|git] [--url ] - xcore plugin remove - xcore plugin info - - xcore sandbox run - xcore sandbox limits - xcore sandbox network - xcore sandbox fs - - xcore marketplace list - xcore marketplace trending - xcore marketplace search - xcore marketplace show - xcore marketplace rate --score <1-5> - - xcore services status - xcore health -""" - -from __future__ import annotations - -import asyncio - - -def main() -> None: - import argparse - - parser = argparse.ArgumentParser( - prog="xcore", - description="xcore v2 — gestion du framework plugin-first", - ) - parser.add_argument("--config", default=None, help="Chemin vers xcore.yaml") - parser.add_argument("--version", action="store_true") - - subparsers = parser.add_subparsers(dest="command") - - # ── plugin ──────────────────────────────────────────────── - plugin_p = subparsers.add_parser("plugin", help="Gestion des plugins") - plugin_sub = plugin_p.add_subparsers(dest="subcommand") - - plugin_sub.add_parser("list", help="Liste les plugins installés") - plugin_sub.add_parser("health", help="Health check de tous les plugins") - - load_p = plugin_sub.add_parser( - "load", help="Charge un plugin sur le serveur en cours" - ) - load_p.add_argument("name") - load_p.add_argument( - "--host", default=None, help="Host du serveur (défaut: 127.0.0.1)" - ) - load_p.add_argument( - "--port", type=int, default=None, help="Port du serveur (défaut: 8000)" - ) - load_p.add_argument( - "--path", - default=None, - help="Préfixe du router ex: /app ou /plugin (défaut: plugin_prefix config)", - ) - load_p.add_argument( - "--key", default=None, help="API key (défaut: secret_key config)" - ) - - reload_p = plugin_sub.add_parser( - "reload", help="Recharge un plugin sur le serveur en cours" - ) - reload_p.add_argument("name") - reload_p.add_argument( - "--host", default=None, help="Host du serveur (défaut: 127.0.0.1)" - ) - reload_p.add_argument( - "--port", type=int, default=None, help="Port du serveur (défaut: 8000)" - ) - reload_p.add_argument( - "--path", - default=None, - help="Préfixe du router ex: /app ou /plugin (défaut: plugin_prefix config)", - ) - reload_p.add_argument( - "--key", default=None, help="API key (défaut: secret_key config)" - ) - - install_p = plugin_sub.add_parser("install", help="Installe un plugin") - install_p.add_argument("name", help="Nom du plugin ou URL") - install_p.add_argument( - "--source", - choices=["zip", "git", "marketplace"], - default="marketplace", - help="Source d'installation (défaut: marketplace)", - ) - install_p.add_argument("--url", default=None, help="URL directe (zip ou git)") - - remove_p = plugin_sub.add_parser("remove", help="Supprime un plugin") - remove_p.add_argument("name") - - info_p = plugin_sub.add_parser("info", help="Affiche les métadonnées d'un plugin") - info_p.add_argument("name") - - sign_p = plugin_sub.add_parser("sign", help="Signe un plugin Trusted") - sign_p.add_argument("path") - sign_p.add_argument("--key", default=None) - - verify_p = plugin_sub.add_parser("verify", help="Vérifie la signature d'un plugin") - verify_p.add_argument("path") - verify_p.add_argument("--key", default=None) - - validate_p = plugin_sub.add_parser( - "validate", help="Valide le manifeste d'un plugin" - ) - validate_p.add_argument("path") - - # ── sandbox ─────────────────────────────────────────────── - sandbox_p = subparsers.add_parser("sandbox", help="Gestion du sandbox runtime") - sandbox_sub = sandbox_p.add_subparsers(dest="subcommand") - - sb_run = sandbox_sub.add_parser("run", help="Lance un plugin en mode sandbox isolé") - sb_run.add_argument("name", help="Nom du plugin") - - sb_limits = sandbox_sub.add_parser( - "limits", help="Affiche les limites ressources d'un plugin" - ) - sb_limits.add_argument("name") - - sb_network = sandbox_sub.add_parser( - "network", help="Affiche la politique réseau d'un plugin" - ) - sb_network.add_argument("name") - - sb_fs = sandbox_sub.add_parser( - "fs", help="Affiche la politique filesystem d'un plugin" - ) - sb_fs.add_argument("name") - - # ── marketplace ─────────────────────────────────────────── - mkt_p = subparsers.add_parser("marketplace", help="Catalogue de plugins") - mkt_sub = mkt_p.add_subparsers(dest="subcommand") - - mkt_sub.add_parser("list", help="Liste tous les plugins du marketplace") - mkt_sub.add_parser("trending", help="Plugins populaires") - - mkt_search = mkt_sub.add_parser("search", help="Recherche un plugin") - mkt_search.add_argument("query") - - mkt_show = mkt_sub.add_parser("show", help="Détails d'un plugin") - mkt_show.add_argument("name") - - mkt_rate = mkt_sub.add_parser("rate", help="Note un plugin") - mkt_rate.add_argument("name") - mkt_rate.add_argument("--score", type=int, choices=range(1, 6), required=True) - - # ── services ────────────────────────────────────────────── - svc_p = subparsers.add_parser("services", help="État des services") - svc_sub = svc_p.add_subparsers(dest="subcommand") - svc_status_p = svc_sub.add_parser("status") - svc_status_p.add_argument("--json", action="store_true", help="Output as JSON") - - health_p = subparsers.add_parser("health", help="Health check global") - health_p.add_argument("--json", action="store_true", help="Output as JSON") - - args = parser.parse_args() - - if args.version: - from xcore import __version__ - - print(f"xcore v{__version__}") - return - - if args.command == "plugin": - from .plugin_cmd import handle_plugin - - asyncio.run(handle_plugin(args)) - - elif args.command == "sandbox": - from .sandbox_cmd import handle_sandbox - - asyncio.run(handle_sandbox(args)) - - elif args.command == "marketplace": - from .marketplace_cmd import handle_marketplace - - asyncio.run(handle_marketplace(args)) - - elif args.command == "services": - from .plugin_cmd import handle_services - - asyncio.run(handle_services(args)) - - elif args.command == "health": - from .plugin_cmd import handle_health - - asyncio.run(handle_health(args)) - - else: - parser.print_help() - - -if __name__ == "__main__": - main() diff --git a/xcore/cli/marketplace_cmd.py b/xcore/cli/marketplace_cmd.py deleted file mode 100644 index 7f05ea49..00000000 --- a/xcore/cli/marketplace_cmd.py +++ /dev/null @@ -1,269 +0,0 @@ -""" -marketplace_cmd.py — Handlers des commandes `xcore marketplace *`. - -xcore marketplace list → liste tous les plugins -xcore marketplace trending → plugins populaires -xcore marketplace search → recherche -xcore marketplace show → détails complets -xcore marketplace rate --score <1-5> -""" - -from __future__ import annotations - -import sys -from typing import Any - -from rich.console import Console -from rich.markup import escape -from rich.panel import Panel -from rich.table import Table - -console = Console() -error_console = Console(stderr=True) - - -def _load_config(args): - from xcore.configurations.loader import ConfigLoader - - return ConfigLoader.load(getattr(args, "config", None)) - - -def _get_client(args): - from xcore.marketplace import MarketplaceClient - - cfg = _load_config(args) - return MarketplaceClient(cfg), cfg - - -async def handle_marketplace(args) -> None: - sub = getattr(args, "subcommand", None) - dispatch = { - "list": _mkt_list, - "trending": _mkt_trending, - "search": _mkt_search, - "show": _mkt_show, - "rate": _mkt_rate, - } - handler = dispatch.get(sub) - if handler: - await handler(args) - else: - print("Usage : xcore marketplace ") - - -# ── list ────────────────────────────────────────────────────── - - -async def _mkt_list(args) -> None: - client, _ = _get_client(args) - with console.status("[bold green]🔍 Récupération du catalogue...[/]"): - try: - plugins = await client.list_plugins() - except Exception as e: - error_console.print( - f"[bold red]❌ Erreur marketplace :[/] {escape(str(e))}" - ) - sys.exit(1) - - if not plugins: - console.print("[yellow]Aucun plugin disponible.[/]") - return - - table = Table(title=f"Catalogue Marketplace ({len(plugins)} plugins)") - table.add_column( - "Nom", style="cyan", no_wrap=True, max_width=32, overflow="ellipsis" - ) - table.add_column("Version", style="magenta", max_width=16, overflow="ellipsis") - table.add_column("Auteur", style="green", max_width=24, overflow="ellipsis") - table.add_column("Note") - table.add_column("Description", style="white") - - for p in plugins: - table.add_row( - escape(p.get("name", "?")), - f"v{escape(str(p.get('version', '?')))}", - escape(p.get("author", "?")), - _stars(p.get("rating", 0)), - escape(p.get("description", "")[:50]), - ) - - console.print(table) - - -# ── trending ────────────────────────────────────────────────── - - -async def _mkt_trending(args) -> None: - client, _ = _get_client(args) - with console.status("[bold orange1]🔥 Récupération des plugins populaires...[/]"): - try: - plugins = await client.trending() - except Exception as e: - error_console.print( - f"[bold red]❌ Erreur marketplace :[/] {escape(str(e))}" - ) - sys.exit(1) - - if not plugins: - console.print("[yellow]Aucun plugin trending.[/]") - return - - table = Table(title="🔥 Plugins Populaires") - table.add_column("#", justify="right", style="dim") - table.add_column("Nom", style="cyan", no_wrap=True) - table.add_column("Version", style="magenta") - table.add_column("Note") - table.add_column("Téléchargements", justify="right", style="green") - table.add_column("Description", style="white") - - for i, p in enumerate(plugins, 1): - table.add_row( - str(i), - escape(p.get("name", "?")), - f"v{escape(str(p.get('version', '?')))}", - _stars(p.get("rating", 0)), - f"{p.get('downloads', 0):,}", - escape(p.get("description", "")[:50]), - ) - - console.print(table) - - -# ── search ──────────────────────────────────────────────────── - - -async def _mkt_search(args) -> None: - client, _ = _get_client(args) - query = args.query - with console.status(f"[bold green]🔍 Recherche : [white]{escape(query)}[/]...[/]"): - try: - results = await client.search(query) - except Exception as e: - error_console.print( - f"[bold red]❌ Erreur marketplace :[/] {escape(str(e))}" - ) - sys.exit(1) - - if not results: - console.print(f"[yellow]Aucun résultat pour '{escape(query)}'.[/]") - return - - table = Table(title=f"🔍 {len(results)} résultat(s) pour '{escape(query)}'") - table.add_column("Nom", style="cyan", no_wrap=True) - table.add_column("Version", style="magenta") - table.add_column("Auteur", style="green") - table.add_column("Note") - table.add_column("Description", style="white") - - for p in results: - table.add_row( - escape(p.get("name", "?")), - f"v{escape(str(p.get('version', '?')))}", - escape(p.get("author", "?")), - _stars(p.get("rating", 0)), - escape(p.get("description", "")[:50]), - ) - - console.print(table) - - -# ── show ────────────────────────────────────────────────────── - - -async def _mkt_show(args) -> None: - client, cfg = _get_client(args) - name = args.name - with console.status( - f"[bold green]🔍 Récupération de [white]{escape(name)}[/]...[/]" - ): - try: - plugin = await client.get_plugin(name) - versions = await client.get_versions(name) - except Exception as e: - error_console.print( - f"[bold red]❌ Erreur marketplace :[/] {escape(str(e))}" - ) - sys.exit(1) - - if not plugin: - error_console.print(f"[bold red]❌ Plugin '{escape(name)}' introuvable.[/]") - sys.exit(1) - - info = [ - f"[bold cyan]Auteur :[/][magenta] " - f"{escape(str(plugin.get('author', '?')))}[/]", - f"[bold cyan]Description :[/] " - f"{escape(str(plugin.get('description', '?')))}", - f"[bold cyan]Mode :[/][yellow] " - f"{escape(str(plugin.get('execution_mode', 'legacy')))}[/]", - f"[bold cyan]Licence :[/][green] " - f"{escape(str(plugin.get('license', '?')))}[/]", - f"[bold cyan]Note :[/] {_stars(plugin.get('rating', 0))} " - f"({plugin.get('rating_count', 0)} votes)", - f"[bold cyan]Téléch. :[/][bold] {plugin.get('downloads', 0):,}[/]", - f"[bold cyan]Dépôt :[/][blue] " - f"{escape(str(plugin.get('repository', '?')))}[/]", - ] - if plugin.get("requires"): - info.append( - f"[bold cyan]Dépendances :[/] {escape(', '.join(plugin['requires']))}" - ) - - content = "\n".join(info) - if versions: - content += "\n\n[bold white]Versions disponibles :[/]\n" - for v in versions[:5]: - tag = " [bold green]← latest[/]" if v.get("latest") else "" - ver_str = escape(str(v.get("version", "?"))) - rel_at = escape(str(v.get("released_at", "?"))) - content += f" {ver_str:12} {rel_at}{tag}\n" - - content += ( - f"\n[italic grey70]Pour installer :[/]\n" - f" [bold]xcore plugin install {escape(name)}[/]" - ) - title = ( - f"[bold green]📦 {escape(plugin.get('name', name))} " - f"v{escape(str(plugin.get('version', '?')))}[/]" - ) - console.print(Panel(content, title=title, expand=False, border_style="cyan")) - - -# ── rate ────────────────────────────────────────────────────── - - -async def _mkt_rate(args) -> None: - client, _ = _get_client(args) - name = args.name - score = args.score - - console.print(f"⭐ Notation de '{name}' : {score}/5") - try: - result = await client.rate_plugin(name, score) - new_rating = result.get("new_rating", "?") - total = result.get("rating_count", "?") - console.print( - f"✅ Note enregistrée. Nouvelle moyenne : {new_rating}/5 ({total} votes)" - ) - except Exception as e: - error_console.print(f"[bold red]❌ Erreur :[/] {escape(str(e))}") - sys.exit(1) - - -# ── helpers ─────────────────────────────────────────────────── - - -def _stars(rating: Any) -> str: - """Convertit une note 0-5 en étoiles ASCII avec markup Rich.""" - try: - rating = max(0.0, min(5.0, float(rating or 0))) - except (ValueError, TypeError): - rating = 0.0 - - full = int(rating) - half = 1 if (rating - full) >= 0.5 else 0 - empty = 5 - full - half - - active = "[bold yellow]" + ("★" * full + "½" * half) + "[/]" - dimmed = "[dim]" + ("☆" * empty) + "[/]" - return active + dimmed diff --git a/xcore/cli/plugin_cmd.py b/xcore/cli/plugin_cmd.py deleted file mode 100644 index 9fae3467..00000000 --- a/xcore/cli/plugin_cmd.py +++ /dev/null @@ -1,698 +0,0 @@ -""" -plugin_cmd.py — Handlers des commandes `xcore plugin *`. -""" - -from __future__ import annotations - -import json -import re -import shutil -import sys -from pathlib import Path -from urllib.parse import urlparse - -from rich.console import Console, Group -from rich.markup import escape -from rich.panel import Panel -from rich.prompt import Confirm -from rich.table import Table - -console = Console() - -# Pattern pour valider le nom d'un plugin : alphanumérique, tirets et -# underscores uniquement. Cela empêche les tentatives de traversal (..) -# ou de chemins absolus. -PLUGIN_NAME_PATTERN = re.compile(r"^[a-zA-Z0-9_-]+$") - - -def _validate_plugin_name(name: str) -> None: - if not name or not PLUGIN_NAME_PATTERN.match(name): - print(f"❌ Nom de plugin invalide : {name!r}") - print(" Le nom doit contenir uniquement des lettres, chiffres, '-' et '_'.") - sys.exit(1) - - -def _ensure_safe_scheme(url: str) -> None: - """Vérifie que l'URL utilise un protocole sécurisé (http/https).""" - scheme = urlparse(url).scheme - if scheme not in ("http", "https"): - console.print( - f"[bold red]❌ Sécurité :[/] Protocole '{scheme}' non supporté pour {url}" - ) - console.print(" Seuls 'http' et 'https' sont autorisés.") - sys.exit(1) - - -def _load_config(args): - from xcore.configurations.loader import ConfigLoader - - return ConfigLoader.load(getattr(args, "config", None)) - - -async def handle_plugin(args) -> None: - sub = getattr(args, "subcommand", None) - dispatch = { - "list": _plugin_list, - "health": _plugin_health, - "load": _plugin_load, - "reload": _plugin_reload, - "install": _plugin_install, - "remove": _plugin_remove, - "info": _plugin_info, - "sign": _plugin_sign, - "verify": _plugin_verify, - "validate": _plugin_validate, - } - handler = dispatch.get(sub) - if handler: - await handler(args) - else: - usage = ( - "Usage : xcore plugin " - ) - print(usage) - - -# ── list ────────────────────────────────────────────────────── - - -async def _plugin_list(args) -> None: - cfg = _load_config(args) - plugin_dir = Path(cfg.plugins.directory) - if not plugin_dir.exists(): - console.print(f"[bold red]❌ Dossier plugins introuvable :[/] {plugin_dir}") - return - plugins = sorted( - d.name - for d in plugin_dir.iterdir() - if d.is_dir() and not d.name.startswith("_") - ) - if not plugins: - console.print("[yellow]Aucun plugin trouvé.[/]") - return - - table = Table(title=f"Plugins dans {plugin_dir} ({len(plugins)})") - table.add_column("Nom", style="cyan", no_wrap=True) - table.add_column("Version", style="magenta") - table.add_column("Mode", style="green") - table.add_column("Description", style="white") - - for p in plugins: - manifest_path = plugin_dir / p / "plugin.yaml" - if manifest_path.exists(): - try: - import yaml - - with open(manifest_path) as f: - m = yaml.safe_load(f) or {} - version = m.get("version", "?") - mode = m.get("execution_mode", "legacy") - desc = m.get("description", "") - table.add_row(p, f"v{version}", mode, desc) - except Exception: - table.add_row(p, "[red]?[/]", "[red]?[/]", "[red]Erreur de lecture[/]") - else: - table.add_row( - p, - "[grey70]?[/]", - "[grey70]?[/]", - "[italic grey70]Manifeste manquant[/]", - ) - - console.print(table) - - -# ── health ──────────────────────────────────────────────────── - - -async def _plugin_health(args) -> None: - cfg = _load_config(args) - plugin_dir = Path(cfg.plugins.directory) - if not plugin_dir.exists(): - console.print(f"[bold red]❌ Dossier plugins introuvable :[/] {plugin_dir}") - return - - plugins = sorted( - d for d in plugin_dir.iterdir() if d.is_dir() and not d.name.startswith("_") - ) - if not plugins: - console.print("[yellow]Aucun plugin trouvé.[/]") - return - - table = Table(title="Health Check des Plugins") - table.add_column("Plugin", style="cyan", no_wrap=True) - table.add_column("Mode", justify="center") - table.add_column("Sig", justify="center") - table.add_column("AST", justify="center") - table.add_column("Manifest", justify="center") - table.add_column("Status", style="dim") - - from xcore.kernel.security.signature import is_signed - from xcore.kernel.security.validation import ASTScanner, ManifestValidator - - ok_count = 0 - err_count = 0 - - with console.status("[bold green]Analyzing plugin health...") as status: - for plugin_dir_entry in plugins: - name = plugin_dir_entry.name - status.update(f"[bold green]Analyzing {escape(name)}...") - try: - validator = ManifestValidator() - manifest, _, _ = validator.load_and_validate(plugin_dir_entry) - - # Signature - signed = "✅" if is_signed(manifest) else "⚠️ " - - # AST scan - scanner = ASTScanner() - result = scanner.scan( - plugin_dir_entry, - whitelist=manifest.allowed_imports, - entry_point=manifest.entry_point, - ) - ast_ok = "✅" if result.passed else "❌" - - mode = manifest.execution_mode.value - table.add_row(name, mode, signed, ast_ok, "✅", "[green]OK[/]") - ok_count += 1 - - except Exception as e: - table.add_row( - name, - "[red]?[/]", - "[red]?[/]", - "[red]?[/]", - "❌", - f"[red]Error: {e}[/]", - ) - err_count += 1 - - console.print(table) - console.print( - f"\n[bold]Result: [green]{ok_count} OK[/], [red]{err_count} Error(s)[/][/]" - ) - - -# ── install ─────────────────────────────────────────────────── - - -async def _plugin_install(args) -> None: - cfg = _load_config(args) - source = getattr(args, "source", "marketplace") - url = getattr(args, "url", None) - name = args.name - _validate_plugin_name(name) - - plugin_dir = Path(cfg.plugins.directory) - plugin_dir.mkdir(parents=True, exist_ok=True) - dest = plugin_dir / name - - if dest.exists(): - console.print( - f"[bold red]❌ Erreur :[/] Plugin '{name}' déjà installé dans {dest}" - ) - console.print( - f" Pour mettre à jour : xcore plugin remove {name} && " - f"xcore plugin install {name}" - ) - sys.exit(1) - - if source == "git" or (url and url.endswith(".git")): - await _install_from_git(name, url or name, dest) - - elif source == "zip" or (url and (url.endswith(".zip") or url.startswith("http"))): - if not url: - console.print("[bold red]❌ Erreur :[/] --url requis pour --source zip") - sys.exit(1) - await _install_from_zip(name, url, dest) - - else: - # marketplace - from xcore.marketplace import MarketplaceClient - - client = MarketplaceClient(cfg) - await _install_from_marketplace(client, name, dest, cfg) - - # ── Signature automatique post-install ──────────────────── - await _auto_sign(dest, cfg) - - print(f"✅ Plugin '{name}' installé dans {dest}") - - -async def _install_from_git(name: str, url: str, dest: Path) -> None: - import asyncio - - print(f"📦 Clonage git : {url}") - proc = await asyncio.create_subprocess_exec( - "git", - "clone", - "--depth=1", - url, - str(dest), - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - _, stderr = await proc.communicate() - if proc.returncode != 0: - console.print(f"[bold red]❌ git clone échoué :[/] {stderr.decode().strip()}") - sys.exit(1) - - -async def _install_from_zip(name: str, url: str, dest: Path) -> None: - import asyncio - import io - import urllib.request - import zipfile - - _ensure_safe_scheme(url) - print(f"📦 Téléchargement : {url}") - try: - loop = asyncio.get_event_loop() - data = await loop.run_in_executor( - None, lambda: urllib.request.urlopen(url).read() - ) - with zipfile.ZipFile(io.BytesIO(data)) as zf: - # Détecte un sous-dossier racine dans le zip - members = zf.namelist() - prefix = members[0].split("/")[0] + "/" if "/" in members[0] else "" - - dest_resolved = dest.resolve() - dest_resolved.mkdir(parents=True, exist_ok=True) - - for member in members: - stripped = ( - member[len(prefix) :] - if prefix and member.startswith(prefix) - else member - ) - if not stripped: - continue - - # Protection Zip Slip: ensure target is within dest - target = (dest_resolved / stripped).resolve() - if not target.is_relative_to(dest_resolved): - print(f"⚠️ Tentative de Zip Slip ignorée : {member}") - continue - - if member.endswith("/"): - target.mkdir(parents=True, exist_ok=True) - else: - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(zf.read(member)) - except Exception as e: - console.print(f"[bold red]❌ Téléchargement échoué :[/] {e}") - sys.exit(1) - - -async def _install_from_marketplace(client, name: str, dest: Path, cfg) -> None: - console.print(f"🔍 Recherche '{name}' sur le marketplace...") - plugin = await client.get_plugin(name) - if not plugin: - console.print( - f"[bold red]❌ Erreur :[/] Plugin '{name}' introuvable sur le marketplace." - ) - sys.exit(1) - - download_url = plugin.get("download_url") - source_type = plugin.get("source_type", "zip") - - print( - f"📦 Plugin trouvé : v{plugin.get('version', '?')} — " - f"{plugin.get('description', '')}" - ) - - if source_type == "git": - await _install_from_git(name, download_url, dest) - else: - await _install_from_zip(name, download_url, dest) - - -async def _auto_sign(plugin_dir: Path, cfg) -> None: - """ - Signe automatiquement le plugin après installation - avec la secret_key définie dans la config plugins. - - - Valide d'abord le manifeste (skip silencieux si invalide). - - Écrit plugin.sig dans le dossier du plugin. - - Affiche un avertissement si la clé est la valeur par défaut. - """ - from xcore.kernel.security.signature import sign_plugin - from xcore.kernel.security.validation import ManifestValidator - - try: - manifest, _, _ = ManifestValidator().load_and_validate(plugin_dir) - except Exception as e: - print(f"⚠️ Signature auto ignorée (manifeste invalide) : {e}") - return - - secret_key = cfg.plugins.secret_key - - if secret_key in (b"change-me-in-production", b"change-me"): - print( - "⚠️ Signature avec la clé par défaut — " - "définissez plugins.secret_key dans xcore.yaml pour la production." - ) - - try: - sig_path = sign_plugin(manifest, secret_key) - print(f"🔑 Signé automatiquement → {sig_path.name}") - except Exception as e: - print(f"⚠️ Signature auto échouée : {e}") - - -# ── remove ──────────────────────────────────────────────────── - - -async def _plugin_remove(args) -> None: - cfg = _load_config(args) - name = args.name - _validate_plugin_name(name) - plugin_dir = Path(cfg.plugins.directory) / name - - if not plugin_dir.exists(): - console.print( - f"[bold red]❌ Erreur :[/] Plugin '{name}' introuvable dans {plugin_dir}" - ) - sys.exit(1) - - confirm = Confirm.ask(f"[bold red]⚠️ Supprimer '{name}' ?[/]", default=False) - if not confirm: - console.print("Annulé.") - return - - shutil.rmtree(plugin_dir) - print(f"✅ Plugin '{name}' supprimé.") - - -# ── info ────────────────────────────────────────────────────── - - -async def _plugin_info(args) -> None: - cfg = _load_config(args) - name = args.name - _validate_plugin_name(name) - plugin_dir = Path(cfg.plugins.directory) / name - - if not plugin_dir.exists(): - console.print(f"[bold red]❌ Erreur :[/] Plugin '{name}' introuvable.") - sys.exit(1) - - from xcore.kernel.security.signature import is_signed - from xcore.kernel.security.validation import ManifestValidator - - try: - validator = ManifestValidator() - manifest, _, _ = validator.load_and_validate(plugin_dir) - except Exception as e: - console.print(f"[bold red]❌ Manifeste invalide :[/] {e}") - sys.exit(1) - - # Construction du contenu du panel - info = [ - f"[bold cyan]Auteur :[/][magenta] {escape(str(manifest.author))}[/]", - f"[bold cyan]Description :[/] " f"{escape(str(manifest.description))}", - f"[bold cyan]Mode :[/][yellow] " - f"{escape(str(manifest.execution_mode.value))}[/]", - f"[bold cyan]Framework :[/][green] " - f"{escape(str(manifest.framework_version))}[/]", - f"[bold cyan]Entry point :[/][blue] " f"{escape(str(manifest.entry_point))}[/]", - f"[bold cyan]Signé :[/] " - f"{'✅ oui' if is_signed(manifest) else '⚠️ non'}", - ] - - if manifest.requires: - deps = ", ".join( - d.name if hasattr(d, "name") else str(d) for d in manifest.requires - ) - info.append(f"[bold cyan]Dépendances :[/] {escape(deps)}") - - if manifest.allowed_imports: - imports = ", ".join(map(str, manifest.allowed_imports)) - info.append(f"[bold cyan]Imports OK :[/] [dim]{escape(imports)}[/]") - - # Ressources - info.append("\n[bold white]Ressources :[/]") - r = manifest.resources - info.append(f" [cyan]timeout :[/][magenta] {r.timeout_seconds}s[/]") - info.append(f" [cyan]mémoire max :[/][magenta] {r.max_memory_mb}MB[/]") - info.append(f" [cyan]disque max :[/][magenta] {r.max_disk_mb}MB[/]") - info.append( - f" [cyan]rate limit :[/][magenta] {r.rate_limit.calls} appels " - f"/ {r.rate_limit.period_seconds}s[/]" - ) - - # Permissions - if manifest.permissions: - info.append(f"\n[bold white]Permissions ({len(manifest.permissions)}) :[/]") - for p in manifest.permissions: - effect = p.get("effect", "allow") - symbol = "✅" if effect == "allow" else "❌" - res = escape(str(p.get("resource", "*"))) - acts = escape(str(p.get("actions", ["*"]))) - info.append(f" {symbol} {res} → {acts}") - - content = Group(*info) - title = f"[bold green]🔌 {escape(manifest.name)} v{escape(manifest.version)}[/]" - console.print(Panel(content, title=title, expand=False, border_style="cyan")) - - -# ── sign / verify / validate ────────────────────────────────── - - -async def _plugin_validate(args) -> None: - from xcore.kernel.security.validation import ManifestValidator - - path = Path(args.path) - if not path.exists(): - console.print(f"[bold red]❌ Erreur :[/] Dossier introuvable : {path}") - sys.exit(1) - try: - v = ManifestValidator() - manifest, _, _ = v.load_and_validate(path) - console.print( - f"✅ Manifeste valide : {manifest.name} v{manifest.version} " - f"[{manifest.execution_mode.value}]" - ) - except Exception as e: - console.print(f"[bold red]❌ Manifeste invalide :[/] {e}") - sys.exit(1) - - -async def _plugin_sign(args) -> None: - from xcore.kernel.security.signature import sign_plugin - from xcore.kernel.security.validation import ManifestValidator - - path = Path(args.path) - key = (args.key or "change-me").encode() - manifest, _, _ = ManifestValidator().load_and_validate(path) - sig = sign_plugin(manifest, key) - print(f"✅ Signé : {sig}") - - -async def _plugin_verify(args) -> None: - from xcore.kernel.security.signature import SignatureError, verify_plugin - from xcore.kernel.security.validation import ManifestValidator - - path = Path(args.path) - key = (args.key or "change-me").encode() - manifest = ManifestValidator().load_and_validate(path) - try: - verify_plugin(manifest, key) - console.print(f"✅ Signature valide : {manifest.name}") - except SignatureError as e: - console.print(f"[bold red]❌ Erreur de signature :[/] {e}") - sys.exit(1) - - -async def _plugin_load(args) -> None: - await _ipc_call(args, action="load", method="POST") - - -async def _plugin_reload(args) -> None: - await _ipc_call(args, action="reload", method="POST") - - -async def _ipc_call(args, action: str, method: str = "POST") -> None: - """ - Appelle l'API HTTP du serveur xcore en cours d'exécution. - - URL construite depuis la config : - POST {app.host}:{app.port}{plugin_prefix}/ipc/{name}/{action} - - L'API key est lue depuis : - 1. --key en argument CLI - 2. cfg.app.secret_key (config) - - Si le serveur ne répond pas, affiche un message clair. - """ - import asyncio - import json - import urllib.request - from urllib.error import HTTPError, URLError - - cfg = _load_config(args) - name = args.name - _validate_plugin_name(name) - - # Construction de l'URL - # Priorité : --host/--port/--path CLI > config > défauts - host = getattr(args, "host", None) or getattr(cfg.app, "host", "127.0.0.1") - port = getattr(args, "port", None) or getattr(cfg.app, "port", 8000) - - # --path : l'utilisateur donne /app ou /plugin, on complète avec /ipc/{name}/{action} - cli_path = getattr(args, "path", None) - base_path = cli_path or cfg.app.plugin_prefix - base_path = "/" + base_path.strip("/") # normalise les slashes - - url = f"http://{host}:{port}{base_path}/ipc/{name}/{action}" - - # Clé API - cli_key = getattr(args, "key", None) - if cli_key: - api_key = cli_key - else: - sk = cfg.app.secret_key - api_key = sk.decode("utf-8") if isinstance(sk, bytes) else sk - - print(f"🔄 {method} {url}") - - def _do_request(): - _ensure_safe_scheme(url) - req = urllib.request.Request( - url, - data=b"{}", - method=method, - headers={ - "Content-Type": "application/json", - "X-Plugin-Key": api_key, - }, - ) - with urllib.request.urlopen(req, timeout=10) as resp: - return json.loads(resp.read().decode("utf-8")) - - loop = asyncio.get_event_loop() - try: - result = await loop.run_in_executor(None, _do_request) - status = result.get("status", "?") - msg = result.get("msg", "") - if status == "ok": - console.print(f"✅ {msg or f'Plugin {name!r} : {action} OK'}") - else: - console.print(f"[bold red]❌ Erreur :[/] {result}") - sys.exit(1) - - except HTTPError as e: - body = e.read().decode("utf-8", errors="replace") - try: - detail = json.loads(body).get("detail", body) - except Exception: - detail = body - console.print(f"[bold red]❌ HTTP {e.code} :[/] {detail}") - sys.exit(1) - - except URLError: - console.print( - f"[bold red]❌ Erreur :[/] Impossible de joindre le serveur " - f"xcore sur {host}:{port}.\n" - f" Vérifiez que le serveur est démarré.", - ) - sys.exit(1) - - except Exception as e: - console.print(f"[bold red]❌ Erreur inattendue :[/] {e}") - sys.exit(1) - - -# ── services / health globaux ───────────────────────────────── - - -async def handle_services(args) -> None: - sub = getattr(args, "subcommand", None) - if sub == "status": - cfg = _load_config(args) - from xcore.services import ServiceContainer - - is_json = getattr(args, "json", False) - - if is_json: - container = ServiceContainer(cfg.services) - await container.init() - status = container.status() - await container.shutdown() - print(json.dumps(status, indent=2, default=str)) - return - - with console.status("[bold green]Récupération de l'état des services...[/]"): - container = ServiceContainer(cfg.services) - await container.init() - status = container.status() - await container.shutdown() - - table = Table(title="État des Services Système") - table.add_column("Service", style="cyan") - table.add_column("Status", justify="center") - table.add_column("Détails", style="dim") - - for svc_id, info in status["services"].items(): - st = info.get("status", "unknown") - color = "green" if st == "ready" else "red" - if st in ("initializing", "uninitialized"): - color = "yellow" - elif st == "degraded": - color = "orange1" - - # On retire les clés déjà affichées dans les colonnes - details = {k: v for k, v in info.items() if k not in ("name", "status")} - details_str = ", ".join(f"{k}={v}" for k, v in details.items()) - - table.add_row( - info.get("name", svc_id), - f"[{color}]{st}[/]", - escape(details_str), - ) - - console.print(table) - if status.get("registered_keys"): - keys = ", ".join(status["registered_keys"]) - console.print(f"\n[bold cyan]Clés enregistrées :[/] [dim]{escape(keys)}[/]") - - -async def handle_health(args) -> None: - cfg = _load_config(args) - from xcore.services import ServiceContainer - - is_json = getattr(args, "json", False) - - if is_json: - container = ServiceContainer(cfg.services) - await container.init() - health = await container.health() - await container.shutdown() - print(json.dumps(health, indent=2, default=str)) - return - - with console.status("[bold green]Exécution du Health Check global...[/]"): - container = ServiceContainer(cfg.services) - await container.init() - health = await container.health() - await container.shutdown() - - table = Table(box=None, show_header=False, padding=(0, 2)) - for svc, info in health["services"].items(): - sym = "✅" if info["ok"] else "❌" - msg = info.get("msg", "") - table.add_row(sym, f"[bold]{svc}[/]", f"[dim]{escape(msg)}[/]") - - status_str = "[bold green]OK[/]" if health["ok"] else "[bold red]DÉGRADÉ[/]" - title = f"Health Check : {status_str}" - - console.print( - Panel( - table, - title=title, - expand=False, - border_style="green" if health["ok"] else "red", - ) - ) diff --git a/xcore/cli/sandbox_cmd.py b/xcore/cli/sandbox_cmd.py deleted file mode 100644 index cfe7f04a..00000000 --- a/xcore/cli/sandbox_cmd.py +++ /dev/null @@ -1,340 +0,0 @@ -""" -sandbox_cmd.py — Handlers des commandes `xcore sandbox *`. - -xcore sandbox run → Lance un plugin en mode sandboxed isolé -xcore sandbox limits → Affiche les limites ressources déclarées -xcore sandbox network → Affiche la politique réseau -xcore sandbox fs → Affiche la politique filesystem -""" - -from __future__ import annotations - -import sys -from pathlib import Path - -from rich.console import Console, Group -from rich.markup import escape -from rich.panel import Panel -from rich.prompt import Confirm -from rich.table import Table - -console = Console() -error_console = Console(stderr=True) - - -def _load_config(args): - from xcore.configurations.loader import ConfigLoader - - return ConfigLoader.load(getattr(args, "config", None)) - - -def _load_manifest(plugin_dir: Path): - from xcore.kernel.security.validation import ManifestValidator - - return ManifestValidator().load_and_validate(plugin_dir) - - -async def handle_sandbox(args) -> None: - sub = getattr(args, "subcommand", None) - dispatch = { - "run": _sandbox_run, - "limits": _sandbox_limits, - "network": _sandbox_network, - "fs": _sandbox_fs, - } - handler = dispatch.get(sub) - if handler: - await handler(args) - else: - print("Usage : xcore sandbox ") - - -# ── run ─────────────────────────────────────────────────────── - - -async def _sandbox_run(args) -> None: - """ - Lance un plugin en mode sandbox isolé (subprocess) et attend un appel ping - pour confirmer qu'il est opérationnel. - """ - cfg = _load_config(args) - name = args.name - plugin_dir = Path(cfg.plugins.directory) / name - - if not plugin_dir.is_dir(): - print(f"❌ Plugin '{name}' introuvable : {plugin_dir}", file=sys.stderr) - sys.exit(1) - - try: - manifest = _load_manifest(plugin_dir) - except Exception as e: - print(f"❌ Manifeste invalide : {e}", file=sys.stderr) - sys.exit(1) - - from xcore.kernel.sandbox.process_manager import ( - SandboxConfig, - SandboxProcessManager, - ) - - info = [ - f"[bold cyan]Mémoire max :[/][magenta] " - f"{manifest.resources.max_memory_mb}MB[/]", - f"[bold cyan]Timeout :[/][magenta] " - f"{manifest.resources.timeout_seconds}s[/]", - ] - title = f"[bold green]🚀 Lancement sandbox : {escape(name)}[/]" - console.print(Panel(Group(*info), title=title, expand=False, border_style="cyan")) - - config = SandboxConfig( - timeout=manifest.resources.timeout_seconds, - max_restarts=manifest.runtime.retry.max_attempts, - startup_timeout=5.0, - ) - mgr = SandboxProcessManager(manifest, config) - - try: - with console.status( - f"[bold green]Démarrage de la sandbox {escape(name)}...[/]" - ): - await mgr.start() - status = mgr.status() - - # Ping de confirmation - resp = await mgr._channel.call("ping", {}) - - if resp.success: - console.print("✅ [bold green]Sandbox opérationnelle[/]") - else: - msg = escape(str(resp.data)) - console.print(f"⚠️ [yellow]Sandbox démarrée mais ping échoué : {msg}[/]") - - table = Table(box=None, show_header=False, padding=(0, 2)) - table.add_row("[bold cyan]PID :[/]", f"[magenta]{status['pid']}[/]") - table.add_row("[bold cyan]État :[/]", f"[yellow]{status['state']}[/]") - table.add_row( - "[bold cyan]Disque:[/]", - f"[magenta]{status['disk']['used_mb']}MB / " - f"{status['disk']['max_mb']}MB[/]", - ) - console.print(table) - - except Exception as e: - error_console.print( - f"[bold red]❌ Échec démarrage sandbox :[/] {escape(str(e))}" - ) - sys.exit(1) - finally: - await mgr.stop() - console.print(f"[bold red]🛑 Sandbox {escape(name)} arrêtée.[/]") - - -# ── limits ──────────────────────────────────────────────────── - - -async def _sandbox_limits(args) -> None: - """Affiche les limites ressources déclarées dans le manifeste.""" - cfg = _load_config(args) - name = args.name - plugin_dir = Path(cfg.plugins.directory) / name - - if not plugin_dir.is_dir(): - error_console.print( - f"[bold red]❌ Erreur :[/] Plugin '{escape(name)}' introuvable." - ) - sys.exit(1) - - try: - manifest = _load_manifest(plugin_dir) - except Exception as e: - error_console.print(f"[bold red]❌ Manifeste invalide :[/] {escape(str(e))}") - sys.exit(1) - - r = manifest.resources - rt = manifest.runtime - - info = [ - f"[bold cyan]Mémoire max :[/][magenta] " f"{r.max_memory_mb} MB[/]", - f"[bold cyan]Disque max :[/][magenta] " f"{r.max_disk_mb} MB[/]", - f"[bold cyan]Timeout appel :[/][magenta] " f"{r.timeout_seconds} s[/]", - f"[bold cyan]Rate limit :[/][magenta] " - f"{r.rate_limit.calls} appels / {r.rate_limit.period_seconds}s[/]", - "\n[bold white]Runtime :[/]", - f" [cyan]Health check :[/][yellow] " - f"{'activé' if rt.health_check.enabled else 'désactivé'}[/]", - ] - - if rt.health_check.enabled: - info.append(f" [dim]intervalle : {rt.health_check.interval_seconds}s[/]") - info.append(f" [dim]timeout : {rt.health_check.timeout_seconds}s[/]") - - info.append( - f" [cyan]Retry :[/][yellow] {rt.retry.max_attempts} tentative(s)[/]" - ) - if rt.retry.max_attempts > 1: - info.append(f" [dim]backoff : {rt.retry.backoff_seconds}s[/]") - - # Vérifie le disque actuel si le plugin est installé - data_dir = plugin_dir / "data" - if data_dir.exists(): - from xcore.kernel.sandbox.isolation import DiskWatcher - - watcher = DiskWatcher(data_dir, r.max_disk_mb) - stats = watcher.stats() - symbol = "✅" if stats["ok"] else "❌" - info.append("\n[bold white]État actuel :[/]") - info.append( - f" [cyan]Disque utilisé :[/][magenta] " - f"{stats['used_mb']}MB / {stats['max_mb']}MB " - f"({stats['used_pct']}%) {symbol}[/]" - ) - - content = Group(*info) - title = f"[bold green]🛡️ Limites ressources : {escape(name)}[/]" - console.print(Panel(content, title=title, expand=False, border_style="cyan")) - - -# ── network ─────────────────────────────────────────────────── - - -async def _sandbox_network(args) -> None: - """ - Affiche la politique réseau du plugin. - Note : le blocage réseau actif nécessite un OS supportant les namespaces - (Linux uniquement). Cette commande affiche l'état déclaré et détecte - si le plugin tente des imports réseau via l'AST scan. - """ - cfg = _load_config(args) - name = args.name - plugin_dir = Path(cfg.plugins.directory) / name - - if not plugin_dir.is_dir(): - error_console.print( - f"[bold red]❌ Erreur :[/] Plugin '{escape(name)}' introuvable." - ) - sys.exit(1) - - try: - manifest = _load_manifest(plugin_dir) - except Exception as e: - error_console.print(f"[bold red]❌ Manifeste invalide :[/] {escape(str(e))}") - sys.exit(1) - - info = [] - NETWORK_IMPORTS = { - "socket", - "ssl", - "http", - "urllib", - "httpx", - "requests", - "aiohttp", - "websockets", - } - - from xcore.kernel.security.validation import ASTScanner - - scanner = ASTScanner() - result = scanner.scan( - plugin_dir, - whitelist=manifest.allowed_imports, - entry_point=manifest.entry_point, - ) - - # Vérifie si des imports réseau sont dans la whitelist du plugin - allowed_network = [i for i in manifest.allowed_imports if i in NETWORK_IMPORTS] - blocked_by_ast = [ - e for e in result.errors if any(net in e for net in NETWORK_IMPORTS) - ] - - if allowed_network: - info.append( - f"⚠️ [yellow]Imports réseau autorisés (whitelist) :[/] {escape(str(allowed_network))}" - ) - else: - info.append("✅ [green]Aucun import réseau autorisé[/]") - - if blocked_by_ast: - info.append( - "\n❌ [bold red]Imports réseau détectés et bloqués par AST scan :[/]" - ) - for b in blocked_by_ast: - info.append(f" [red]→ {escape(str(b))}[/]") - else: - info.append("✅ [green]Aucun import réseau détecté dans le code[/]") - - info.append("\n[dim]Note : isolation réseau OS (namespaces) — Linux uniquement.[/]") - info.append("[dim]Pour une isolation complète, utilisez un conteneur Docker.[/]") - - content = Group(*info) - title = f"[bold green]🌐 Politique réseau : {escape(name)}[/]" - console.print(Panel(content, title=title, expand=False, border_style="cyan")) - - -# ── fs ──────────────────────────────────────────────────────── - - -async def _sandbox_fs(args) -> None: - """Affiche et valide la politique filesystem du plugin.""" - cfg = _load_config(args) - name = args.name - plugin_dir = Path(cfg.plugins.directory) / name - - if not plugin_dir.is_dir(): - error_console.print( - f"[bold red]❌ Erreur :[/] Plugin '{escape(name)}' introuvable." - ) - sys.exit(1) - - try: - manifest = _load_manifest(plugin_dir) - except Exception as e: - error_console.print(f"[bold red]❌ Manifeste invalide :[/] {escape(str(e))}") - sys.exit(1) - - fs = manifest.filesystem - info = [] - - table = Table(box=None, padding=(0, 2), show_header=False) - table.add_column("Path", style="cyan") - table.add_column("Status", justify="right") - - info.append("[bold white]Chemins autorisés :[/]") - if not fs.allowed_paths: - info.append(" [dim]Aucun chemin autorisé explicitement[/]") - else: - for p in fs.allowed_paths: - abs_path = plugin_dir / p - status = ( - "[green]✅ existe[/]" - if abs_path.exists() - else "[yellow]⚠️ manquant[/]" - ) - table.add_row(escape(str(p)), status) - info.append(table) - - if fs.denied_paths: - info.append("\n[bold white]Chemins bloqués :[/]") - denied_table = Table(box=None, padding=(0, 2), show_header=False) - for p in fs.denied_paths: - denied_table.add_row(escape(str(p)), "[red]❌ bloqué[/]") - info.append(denied_table) - - info.append("\n[italic dim]Comportement : fail-closed[/]") - info.append("[dim]Tout chemin hors de 'allowed' est refusé.[/]") - - content = Group(*info) - title = f"[bold green]📁 Politique filesystem : {escape(name)}[/]" - console.print(Panel(content, title=title, expand=False, border_style="cyan")) - - # Vérifie si le dossier data/ existe, le créer si nécessaire - data_dir = plugin_dir / "data" - if not data_dir.exists(): - console.print("") - rel_path = escape(str(data_dir.relative_to(plugin_dir.parent))) - create = Confirm.ask( - f"[yellow]⚠️ Le dossier {rel_path} n'existe pas. Créer ?[/]", - default=False, - ) - if create: - data_dir.mkdir(parents=True) - console.print(f"✅ [green]{data_dir} créé.[/]") diff --git a/xcore/cli/validate_cmd.py b/xcore/cli/validate_cmd.py deleted file mode 100644 index 462ddebd..00000000 --- a/xcore/cli/validate_cmd.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -validate_cmd.py — Validation complète d'un plugin (manifeste + AST + signature). - -Usage: - xcore plugin validate ./plugins/my_plugin -""" - -from __future__ import annotations - -import sys -from pathlib import Path - - -def validate_full( - plugin_path: str, secret_key: bytes = b"", strict: bool = False -) -> bool: - """ - Validation complète d'un plugin : - 1. Lecture manifeste - 2. Validation schéma - 3. Scan AST - 4. Vérification signature (si trusted + secret_key fourni) - - Retourne True si tout est valide. - """ - from xcore.kernel.api.contract import ExecutionMode - from xcore.kernel.security.validation import ASTScanner, ManifestValidator - - path = Path(plugin_path).resolve() - ok = True - - print(f"\n{'='*50}") - print(f" Validation : {path.name}") - print(f"{'='*50}") - - # 1. Manifeste - try: - validator = ManifestValidator() - manifest = validator.load_and_validate(path) - console.print( - f"✅ Manifeste : {manifest.name} v{manifest.version} " - f"[{manifest.execution_mode.value}]" - ) - except Exception as e: - print(f"❌ Manifeste : {e}", file=sys.stderr) - return False - - # 2. AST scan - scanner = ASTScanner() - result = scanner.scan( - path, whitelist=manifest.allowed_imports, entry_point=manifest.entry_point - ) - if result.passed: - print(f"✅ Scan AST : {len(result.scanned)} fichier(s) analysé(s)") - else: - print("❌ Scan AST :") - for err in result.errors: - print(f" {err}", file=sys.stderr) - ok = False - for w in result.warnings: - print(f"⚠️ AST warning : {w}") - - # 3. Signature (Trusted seulement) - if manifest.execution_mode == ExecutionMode.TRUSTED and secret_key: - from xcore.kernel.security.signature import SignatureError, verify_plugin - - try: - verify_plugin(manifest, secret_key) - print("✅ Signature : valide") - except SignatureError as e: - if strict: - print(f"❌ Signature : {e}", file=sys.stderr) - ok = False - else: - print(f"⚠️ Signature : {e}") - - print(f"\n{'='*50}") - print(f" Résultat : {'✅ VALIDE' if ok else '❌ INVALIDE'}") - print(f"{'='*50}\n") - return ok diff --git a/xcore/configurations/loader.py b/xcore/configurations/loader.py index 70f8ef4f..4e68d8c2 100644 --- a/xcore/configurations/loader.py +++ b/xcore/configurations/loader.py @@ -27,6 +27,7 @@ SecurityConfig, ServicesConfig, TracingConfig, + WorkerConfig, XcoreConfig, ) @@ -204,11 +205,21 @@ def _parse_services(cls, d: dict) -> ServicesConfig: timezone=s.get("timezone", "UTC"), jobs=s.get("jobs", []), ) + xworker_raw = d.get("xworker") + celery_raw = d.get("celery") + + if xworker_raw is None and celery_raw is not None: + xworker_raw = celery_raw + + xworker = WorkerConfig.from_dict(xworker_raw or {}) + celery = WorkerConfig.from_dict(celery_raw or {}) if celery_raw else None return ServicesConfig( databases=dbs, cache=cache, scheduler=scheduler, extensions=d.get("extensions", {}), + xworker=xworker, + celery=celery, ) @staticmethod diff --git a/xcore/configurations/sections.py b/xcore/configurations/sections.py index 13d4a49d..985ed7ee 100644 --- a/xcore/configurations/sections.py +++ b/xcore/configurations/sections.py @@ -33,6 +33,59 @@ class DatabaseConfig: max_connections: int | None = None # Redis +@dataclass +class WorkerConfig: + enabled: bool = False + module: str = "extensions.xworker.main:WorkerService" + broker_url: str = "redis://localhost:6379/0" + result_backend: str = "redis://localhost:6379/1" + concurrency: int = 4 + task_soft_time_limit: int = 300 + task_time_limit: int = 360 + broker_connection_retry_on_startup: bool = True + task_serializer: str = "json" + result_serializer: str = "json" + accept_content: list[str] = field(default_factory=lambda: ["json"]) + result_expires: int = 86400 + queues: list[str] = field(default_factory=lambda: ["default"]) + modules: list[str] = field(default_factory=list) + task_list: list[str] = field(default_factory=list) + + def __post_init__(self) -> None: + if self.task_list and not self.modules: + self.modules = list(self.task_list) + if self.modules and not self.task_list: + self.task_list = list(self.modules) + + @classmethod + def from_dict(cls, d: dict[str, Any]) -> "WorkerConfig": + # type: ignore[attr-defined] + valid = {f.name for f in cls.__dataclass_fields__.values()} + parsed = {k: v for k, v in d.items() if k in valid} + if "task_list" not in parsed and "modules" in parsed: + parsed["task_list"] = list(parsed["modules"]) + if "modules" not in parsed and "task_list" in parsed: + parsed["modules"] = list(parsed["task_list"]) + return cls(**parsed) + + def to_payload(self) -> dict[str, Any]: + payload = { + "broker_url": self.broker_url, + "result_backend": self.result_backend, + "concurrency": self.concurrency, + "task_soft_time_limit": self.task_soft_time_limit, + "task_time_limit": self.task_time_limit, + "broker_connection_retry_on_startup": self.broker_connection_retry_on_startup, + "task_serializer": self.task_serializer, + "result_serializer": self.result_serializer, + "accept_content": self.accept_content, + "result_expires": self.result_expires, + "queues": self.queues, + "modules": self.modules or self.task_list, + } + return payload + + @dataclass class CacheConfig: backend: str = "memory" # memory | redis @@ -55,6 +108,12 @@ class ServicesConfig: cache: CacheConfig = field(default_factory=CacheConfig) scheduler: SchedulerConfig = field(default_factory=SchedulerConfig) extensions: dict[str, dict[str, Any]] = field(default_factory=dict) + xworker: WorkerConfig = field(default_factory=WorkerConfig) + celery: WorkerConfig | None = None + + def __post_init__(self) -> None: + if self.celery and not self.xworker.enabled: + self.xworker = self.celery @dataclass diff --git a/xcore/sdk/__init__.py b/xcore/sdk/__init__.py index d6fb2797..e2f15c43 100644 --- a/xcore/sdk/__init__.py +++ b/xcore/sdk/__init__.py @@ -8,6 +8,7 @@ from ..kernel.api.contract import BasePlugin, ExecutionMode, TrustedBase, error, ok from ..kernel.api.rbac import RBACChecker, require_permission, require_role +from ..services.xworker import WorkerService, task, task_registry from .adapter.asyncsql import BaseAsyncRepository from .adapter.syncsql import BaseSyncRepository from .decorators import ( @@ -45,5 +46,8 @@ "RBACChecker", "require_permission", "require_role", + "WorkerService", + "task_registry", + "task", "RouterRegistry", ] diff --git a/xcore/services/container.py b/xcore/services/container.py index 9d687f8c..30c3a60f 100644 --- a/xcore/services/container.py +++ b/xcore/services/container.py @@ -32,6 +32,7 @@ from .cache.service import CacheService from .database.adapters.async_sql import AsyncSQLAdapter from .scheduler.service import SchedulerService + from .xworker.main import WorkerService from .base import BaseService, BaseServiceProvider @@ -85,6 +86,24 @@ async def init(self, container: ServiceContainer) -> None: logger.info("Scheduler : prêt") +class XWorkerServiceProvider(BaseServiceProvider): + async def init(self, container: ServiceContainer) -> None: + cfg = container._config.xworker + if not cfg or not cfg.enabled: + return + from .xworker.main import WorkerService + + svc = WorkerService(cfg.to_payload()) + await svc.init() + container._services["worker_service"] = svc + container._raw["worker"] = svc + logger.info( + "XWorker : broker=%s queues=%s", + cfg.broker_url, + cfg.queues, + ) + + class ExtensionServiceProvider(BaseServiceProvider): async def init(self, container: ServiceContainer) -> None: if not container._config.extensions: @@ -119,11 +138,11 @@ class ServiceContainer: await container.shutdown() """ - # Pour compatibilité avec les anciens tests DEFAULT_PROVIDERS = [ DatabaseServiceProvider, CacheServiceProvider, SchedulerServiceProvider, + XWorkerServiceProvider, ExtensionServiceProvider, ] @@ -146,6 +165,7 @@ def load_default_providers(self) -> None: DatabaseServiceProvider(), CacheServiceProvider(), SchedulerServiceProvider(), + XWorkerServiceProvider(), ExtensionServiceProvider(), ] ) @@ -192,6 +212,9 @@ def get(self, name: "Literal['cache']") -> "CacheService": ... # noqa: F811 @overload def get(self, name: "Literal['scheduler']") -> "SchedulerService": ... # noqa: F811 + @overload + def get(self, name: "Literal['worker']") -> "WorkerService": ... # noqa: F811 + @overload def get(self, name: str) -> Any: ... # noqa: F811 diff --git a/xcore/services/database/migrations.py b/xcore/services/database/migrations.py index 6edee287..2bc5bd94 100644 --- a/xcore/services/database/migrations.py +++ b/xcore/services/database/migrations.py @@ -8,6 +8,7 @@ db_url=cfg.url, migrations_dir="./migrations", ) + runner.init() # génère la première révision (ignoré si déjà existante) runner.upgrade() # applique toutes les migrations en attente runner.status() # liste les migrations appliquées / en attente """ @@ -44,10 +45,71 @@ def _get_config(self): cfg.set_main_option("script_location", str(self.migrations_dir)) return cfg + def _is_async(self) -> bool: + return "+asyncpg" in self.db_url or "+aiosqlite" in self.db_url + + async def init(self, autogenerate: bool = True, message: str = "init") -> None: + """ + Génère la première révision de migration. + Ignoré silencieusement si des révisions existent déjà. + + Args: + autogenerate: Si True, Alembic détecte automatiquement les changements + de schéma depuis les modèles SQLAlchemy. + message: Message/label de la révision (ex: "init", "create_users"). + """ + from alembic import command # type: ignore + + if not self.migrations_dir.exists(): + raise MigrationError( + f"Le dossier de migrations '{self.migrations_dir}' n'existe pas. " + "Lancez d'abord `alembic init ` pour initialiser la structure." + ) + + # Vérifie si des révisions existent déjà + versions_dir = self.migrations_dir / "versions" + if versions_dir.exists(): + existing = [ + f + for f in versions_dir.iterdir() + if f.suffix == ".py" and f.name != "__init__.py" + ] + if existing: + logger.info( + f"{len(existing)} migration(s) déjà existante(s) — init ignoré. " + "Utilisez `revision()` pour créer une nouvelle révision." + ) + return + + kwargs = dict(message=message, autogenerate=autogenerate) + + if self._is_async(): + engine = create_async_engine(self.db_url) + async with engine.begin() as conn: + + def do_init(sync_conn): + cfg = self._get_config() + cfg.attributes["connection"] = sync_conn + command.revision(config=cfg, **kwargs) + + await conn.run_sync(do_init) + + await engine.dispose() + logger.info( + f"Première révision '{message}' générée (async, autogenerate={autogenerate})" + ) + return + + command.revision(config=self._get_config(), **kwargs) + logger.info( + f"Première révision '{message}' générée (sync, autogenerate={autogenerate})" + ) + async def upgrade(self, revision: str = "head") -> None: - from alembic import command + """Applique toutes les migrations en attente jusqu'à `revision`.""" + from alembic import command # type: ignore - if "+asyncpg" in self.db_url or "+aiosqlite" in self.db_url: + if self._is_async(): engine = create_async_engine(self.db_url) async with engine.begin() as conn: logger.info("Migration asynchrone demare") @@ -62,41 +124,41 @@ def do_upgrade(sync_conn): logger.info("Migration faite avec sucess") return - else: - # Mode synchrone classique - logger.info(f"Migration sync upgrade → {revision}") - command.upgrade(self._get_config(), revision) + logger.info(f"Migration sync upgrade → {revision}") + command.upgrade(self._get_config(), revision) async def downgrade(self, revision: str = "-1") -> None: + """Annule la dernière migration (ou jusqu'à `revision`).""" from alembic import command # type: ignore - if "+asyncpg" in self.db_url or "+aiosqlite" in self.db_url: + if self._is_async(): engine = create_async_engine(self.db_url) async with engine.begin() as conn: - def do_downgrade(conn): + def do_downgrade(sync_conn): cfg = self._get_config() - cfg.attributes["connection"] = conn + cfg.attributes["connection"] = sync_conn command.downgrade(config=cfg, revision=revision) await conn.run_sync(do_downgrade) await engine.dispose() - + logger.info(f"Migration downgrade → {revision}") return - logger.info(f"Migration downgrade → {revision}") + logger.info(f"Migration sync downgrade → {revision}") command.downgrade(self._get_config(), revision) - async def revison(self, **kwargs): + async def revison(self, **kwargs) -> None: + """Crée une nouvelle révision de migration.""" from alembic import command # type: ignore - if "+asyncpg" in self.db_url or "+aiosqlite" in self.db_url: + if self._is_async(): engine = create_async_engine(self.db_url) async with engine.begin() as conn: - def do_revision(conn): + def do_revision(sync_conn): cfg = self._get_config() - cfg.attributes["connection"] = conn + cfg.attributes["connection"] = sync_conn command.revision(config=cfg, **kwargs) await conn.run_sync(do_revision) @@ -106,21 +168,22 @@ def do_revision(conn): command.revision(config=self._get_config(), **kwargs) - async def status(self, **kwargs): + async def status(self, **kwargs) -> None: + """Affiche les migrations appliquées et en attente.""" from alembic import command # type: ignore - if "+asyncpg" in self.db_url or "+aiosqlite" in self.db_url: + if self._is_async(): engine = create_async_engine(self.db_url) async with engine.begin() as conn: - def do_revision(conn): + def do_status(sync_conn): cfg = self._get_config() - cfg.attributes["connection"] = conn + cfg.attributes["connection"] = sync_conn command.status(config=cfg, **kwargs) - await conn.run_sync(do_revision) + await conn.run_sync(do_status) await engine.dispose() return - command.revision(config=self._get_config(), **kwargs) + command.status(config=self._get_config(), **kwargs) diff --git a/xcore/services/xworker b/xcore/services/xworker new file mode 160000 index 00000000..9fb99da0 --- /dev/null +++ b/xcore/services/xworker @@ -0,0 +1 @@ +Subproject commit 9fb99da048be11e0eb759fff67ed5f86a94de634