feat(dream): add unified self-improvement agent

flocks/agent/agents/self_improve/agent.yaml

@@ -0,0 +1,15 @@
+name: self-improve
+description: Hidden Dream Agent that improves durable Memory and reusable user Skills.
+mode: subagent
+hidden: true
+tags: [system, evolution]
+delegatable: false
+steps: 24
+tools:
+  - read
+  - write
+  - edit
+  - glob
+  - grep
+  - bash
+  - skill_load

flocks/agent/agents/self_improve/prompt_builder.py

@@ -0,0 +1,8 @@
+"""Prompt injection for the hidden self-improve Agent."""
+
+from flocks.memory.evolution.dream import DREAM_SYSTEM_PROMPT
+
+
+def inject(agent_info, *_args) -> None:
+    """Inject the integrated Dream system prompt."""
+    agent_info.prompt = DREAM_SYSTEM_PROMPT

flocks/command/command.py

@@ -213,6 +213,16 @@ def _ensure_defaults(cls) -> None:
                 requires_existing_session=True,
                 channel_safe=True,
             ),
+            CommandDef(
+                name="dream",
+                description="Run self-improvement for Memory and Skills",
+                template="Run Dream self-improvement for Memory and Skills.",
+                execution_kind="direct",
+                allow_attachments=False,
+                visible_surfaces=ALL_SURFACES,
+                requires_existing_session=True,
+                channel_safe=True,
+            ),
             CommandDef(
                 name="model",
                 description="Change or inspect the current model",

flocks/command/direct.py

@@ -6,7 +6,7 @@
 
 from collections import defaultdict
 from dataclasses import dataclass
-from typing import Any, Optional
+from typing import Any, Awaitable, Callable, Optional
 
 from flocks.agent.agent import AvailableAgent
 from flocks.agent.registry import Agent
@@ -31,6 +31,54 @@ class DirectCommandResult:
     clear_history: bool = False
 
 
+CommandStatusCallback = Callable[[str, Optional[str]], Awaitable[None]]
+
+
+async def _publish_command_status(
+    callback: Optional[CommandStatusCallback],
+    status: str,
+    message: Optional[str] = None,
+) -> None:
+    """Publish best-effort foreground status for a long-running command."""
+    if callback is None:
+        return
+    try:
+        await callback(status, message)
+    except Exception:
+        return
+
+
+def _format_dream_result(result: Any, target_label: str) -> str:
+    """Format the visible result of one manual Dream run."""
+    changed_memory_files = tuple(getattr(result, "changed_memory_files", ()) or ())
+    changed_skills = tuple(getattr(result, "changed_skills", ()) or ())
+    memory_result = (
+        f"Updated {', '.join(changed_memory_files)}"
+        if changed_memory_files
+        else "Updated"
+        if getattr(result, "memory_changed", False)
+        else "No changes"
+    )
+    skill_result = (
+        f"Updated {', '.join(changed_skills)}"
+        if changed_skills
+        else "Updated"
+        if getattr(result, "skill_changed", False)
+        else "No changes"
+    )
+    lines = [
+        "Dream completed",
+        "",
+        f"- Target: {target_label}",
+        f"- Evidence processed: {result.processed_sources}",
+        f"- Memory: {memory_result}",
+        f"- Skill: {skill_result}",
+    ]
+    if getattr(result, "backlog", False):
+        lines.append("- Backlog: More evidence remains for a later Dream")
+    return "\n".join(lines)
+
+
 def is_agent_safe_direct_command(command: CommandInfo) -> bool:
     return (
         command.execution_kind == "direct"
@@ -136,6 +184,7 @@ async def run_direct_command(
     args_json: Optional[Any] = None,
     surface: Optional[CommandSurface] = None,
     session_id: Optional[str] = None,
+    status_callback: Optional[CommandStatusCallback] = None,
 ) -> DirectCommandResult:
     """Execute a direct command and return its result."""
     resolved = Command.resolve(name)
@@ -173,6 +222,69 @@ async def run_direct_command(
             prompt=GoalManager.goal_prompt(state.objective),
         )
 
+    if name == "dream":
+        if not session_id:
+            return DirectCommandResult(
+                handled=True,
+                success=False,
+                text="Usage: /dream requires an active session.",
+            )
+        from flocks.config import Config
+        from flocks.memory.config import resolve_memory_config
+        from flocks.memory.evolution.common import DreamTarget
+        from flocks.memory.evolution.dream import run_dream_bridge
+        from flocks.memory.paths import is_registered_project_id
+        from flocks.session.session import Session
+
+        session = await Session.get_by_id(session_id)
+        if session is None:
+            return DirectCommandResult(
+                handled=True,
+                success=False,
+                text="Session not found.",
+            )
+        memory_config = resolve_memory_config(await Config.get())
+        if not memory_config.dream.enabled:
+            return DirectCommandResult(
+                handled=True,
+                success=False,
+                text="Dream is disabled",
+            )
+        target = (
+            DreamTarget.project(session.project_id)
+            if is_registered_project_id(session.project_id)
+            else DreamTarget.global_only()
+        )
+        target_label = (
+            f"Project {target.scope_id}"
+            if is_registered_project_id(target.scope_id)
+            else "Global"
+        )
+        await _publish_command_status(
+            status_callback,
+            "dreaming",
+            f"Dream is reviewing {target_label} evidence for durable Memory and Skill updates…",
+        )
+        try:
+            result = await run_dream_bridge(
+                target,
+                parent_session_id=session.id,
+            )
+        except Exception as exc:
+            command_result = DirectCommandResult(
+                handled=True,
+                success=False,
+                text=f"Dream failed: {exc}",
+            )
+        else:
+            command_result = DirectCommandResult(
+                handled=True,
+                text=_format_dream_result(result, target_label),
+            )
+        finally:
+            await _publish_command_status(status_callback, "idle")
+        return command_result
+
     if name == "tools":
         if not args or args == "list":
             return DirectCommandResult(handled=True, text=build_tools_catalog_summary())

flocks/command/handler.py

@@ -11,6 +11,7 @@
 
 SendText = Callable[[str], Awaitable[None]]
 SendPrompt = Callable[[str], Awaitable[None]]
+SendStatus = Callable[[str, Optional[str]], Awaitable[None]]
 ClearScreen = Callable[[], Awaitable[None]]
 ClearHistory = Callable[[], Awaitable[None]]
 
@@ -21,6 +22,7 @@ async def handle_slash_command(
     parsed_command: Optional[ParsedCommand] = None,
     send_text: SendText,
     send_prompt: SendPrompt,
+    send_status: Optional[SendStatus] = None,
     clear_screen: Optional[ClearScreen] = None,
     clear_history: Optional[ClearHistory] = None,
     surface: Optional[CommandSurface] = None,
@@ -58,6 +60,7 @@ async def handle_slash_command(
         args_json=parsed.args_json,
         surface=surface,
         session_id=session_id,
+        status_callback=send_status,
     )
     if not result.handled:
         return False

flocks/config/config_writer.py

@@ -144,7 +144,7 @@ def _write_raw(
 
     @classmethod
     def ensure_memory_config(cls) -> bool:
-        """Persist the editable Memory Search config when absent."""
+        """Persist editable Memory Search and Dream config when absent."""
         path = Config.get_config_file()
         try:
             text = path.read_text(encoding="utf-8") if path.exists() else ""
@@ -175,6 +175,10 @@ def ensure_memory_config(cls) -> bool:
                     exclude_none=True,
                 ),
             },
+            "dream": default_config.dream.model_dump(
+                mode="json",
+                exclude_none=True,
+            ),
         }
         cls._write_raw(data, path=path)
         log.info("config_writer.memory_config_initialized", {"path": str(path)})

flocks/input/dispatcher.py

@@ -110,6 +110,9 @@ async def _collect_text(text: str) -> None:
         async def _collect_prompt(prompt: str) -> None:
             llm_prompts.append(prompt)
 
+        async def _publish_status(status: str, message: Optional[str]) -> None:
+            await sink.publish_command_status(event, status, message)
+
         # Pass only optional callbacks, not the bound methods on the sink: those
         # are always truthy even when no concrete callback was registered.
         clear_cb = getattr(sink, "_clear_screen", None)
@@ -119,6 +122,7 @@ async def _collect_prompt(prompt: str) -> None:
             parsed_command=parsed,
             send_text=_collect_text,
             send_prompt=_collect_prompt,
+            send_status=_publish_status,
             clear_screen=clear_cb,
             clear_history=clear_history_cb,
             surface=sink.surface,

flocks/input/output.py

@@ -10,6 +10,10 @@
 DirectResponseCallback = Callable[[UserInputEvent, str], Awaitable[None]]
 RunLlmCallback = Callable[[UserInputEvent, str, Optional[str]], Awaitable[None]]
 SessionControlCallback = Callable[[UserInputEvent, ParsedCommand], Awaitable[bool]]
+CommandStatusCallback = Callable[
+    [UserInputEvent, str, Optional[str]],
+    Awaitable[None],
+]
 SideEffectCallback = Callable[[], Awaitable[None]]
 
 
@@ -39,6 +43,14 @@ async def execute_session_control(
     ) -> bool:
         return False
 
+    async def publish_command_status(
+        self,
+        event: UserInputEvent,
+        status: str,
+        message: Optional[str] = None,
+    ) -> None:
+        return None
+
     async def clear_screen(self) -> None:
         return None
 
@@ -56,13 +68,15 @@ def __init__(
         direct_response: DirectResponseCallback,
         run_llm: RunLlmCallback,
         session_control: Optional[SessionControlCallback] = None,
+        command_status: Optional[CommandStatusCallback] = None,
         clear_screen: Optional[SideEffectCallback] = None,
         clear_history: Optional[SideEffectCallback] = None,
     ) -> None:
         super().__init__(surface)
         self._direct_response = direct_response
         self._run_llm = run_llm
         self._session_control = session_control
+        self._command_status = command_status
         self._clear_screen = clear_screen
         self._clear_history = clear_history
 
@@ -86,6 +100,15 @@ async def execute_session_control(
             return False
         return await self._session_control(event, parsed)
 
+    async def publish_command_status(
+        self,
+        event: UserInputEvent,
+        status: str,
+        message: Optional[str] = None,
+    ) -> None:
+        if self._command_status is not None:
+            await self._command_status(event, status, message)
+
     async def clear_screen(self) -> None:
         if self._clear_screen is not None:
             await self._clear_screen()

flocks/memory/__init__.py

@@ -35,6 +35,7 @@
     MemoryCacheConfig,
     MemoryBatchConfig,
     MemoryAutoFlushConfig,
+    MemoryDreamConfig,
     resolve_memory_config,
 )
 
@@ -76,6 +77,7 @@
     "MemoryCacheConfig",
     "MemoryBatchConfig",
     "MemoryAutoFlushConfig",
+    "MemoryDreamConfig",
     "resolve_memory_config",
 
     # Utils

flocks/memory/config.py

@@ -22,18 +22,10 @@ class MemoryEmbeddingConfig(BaseModel):
         "text-embedding-3-small",
         description="Embedding model name"
     )
-    api_key: Optional[str] = Field(
-        None,
-        description="API key (optional, can use env var)"
-    )
     local_model_path: Optional[str] = Field(
         None,
         description="Local model path for local provider"
     )
-    timeout_ms: int = Field(
-        60000,
-        description="Request timeout in milliseconds"
-    )
 
 
 class MemorySearchConfig(BaseModel):
@@ -217,6 +209,25 @@ class MemoryAutoFlushConfig(BaseModel):
     )
 
 
+class MemoryDreamConfig(BaseModel):
+    """Scheduled and manual Dream self-improvement configuration."""
+
+    enabled: bool = Field(
+        True,
+        description="Enable scheduled and manual Dream self-improvement",
+    )
+    interval_hours: float = Field(
+        24,
+        gt=0,
+        description="Hours between successful background Dream bridging runs",
+    )
+    recent_daily_days: int = Field(
+        7,
+        ge=0,
+        description="Number of recent daily memory files included in extraction",
+    )
+
+
 class CompactionConfig(BaseModel):
     """
     Dynamic compaction configuration.
@@ -336,6 +347,10 @@ class MemoryConfig(BaseModel):
         default_factory=MemoryAutoFlushConfig,
         description="Auto flush configuration"
     )
+    dream: MemoryDreamConfig = Field(
+        default_factory=MemoryDreamConfig,
+        description="Scheduled and manual Dream self-improvement",
+    )
     compaction: CompactionConfig = Field(
         default_factory=CompactionConfig,
         description="Dynamic compaction configuration (auto-scales to model context)"