diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index d34a4d7..0000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,224 +0,0 @@ -name: CI - -on: - push: - branches: [main] - pull_request: - schedule: - # Daily health check at 05:00 UTC; failures open/refresh an Issue. - - cron: "0 5 * * *" - workflow_dispatch: - -permissions: - contents: read - issues: write - -jobs: - backend: - runs-on: ubuntu-latest - services: - postgres: - image: postgres:16-alpine - env: - POSTGRES_USER: agentflow - POSTGRES_PASSWORD: agentflow - POSTGRES_DB: agentflow - ports: ["5432:5432"] - options: >- - --health-cmd="pg_isready -U agentflow" - --health-interval=5s - --health-timeout=5s - --health-retries=10 - redis: - image: redis:7-alpine - ports: ["6379:6379"] - env: - AGENTFLOW_DATABASE_URL: postgresql+asyncpg://agentflow:agentflow@localhost:5432/agentflow - AGENTFLOW_REDIS_URL: redis://localhost:6379/0 - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v3 - with: - enable-cache: true - # uv.lock is gitignored; key cache on project manifests instead. - cache-dependency-glob: | - **/pyproject.toml - - name: Install - working-directory: backend - run: uv sync --all-extras - - name: Lint - working-directory: backend - run: uv run ruff check . - - name: Database schema - working-directory: backend - run: uv run alembic upgrade head - - name: Test - working-directory: backend - run: uv run pytest -q - - backend-java: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: "21" - cache: maven - - name: Test - working-directory: backend-java - run: mvn -B verify - - integration: - runs-on: ubuntu-latest - needs: [backend, backend-java] - env: - AGENTFLOW_DATABASE_URL: postgresql+asyncpg://agentflow:agentflow@localhost:5432/agentflow - AGENTFLOW_REDIS_URL: redis://localhost:6379/0 - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v3 - with: - enable-cache: true - cache-dependency-glob: | - **/pyproject.toml - - name: Install Python deps - working-directory: backend - run: uv sync --all-extras - - name: Start infrastructure - run: docker compose up -d postgres redis - - name: Database schema - working-directory: backend - run: uv run alembic upgrade head - - name: Start application stack - run: docker compose --profile app up -d --build - - name: Wait for API - working-directory: backend - run: uv run python ../scripts/ci/wait_for_http.py http://localhost:8000/v1/health 240 - - name: End-to-end smoke test - working-directory: backend - run: uv run python ../scripts/ci/java_stack_smoke.py http://localhost:8000 - - frontend: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: "20" - cache: npm - cache-dependency-path: frontend/package-lock.json - - name: Install - working-directory: frontend - run: npm ci || npm install - - name: Lint - working-directory: frontend - run: npm run lint - - name: Build - working-directory: frontend - run: npm run build - - sdk: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v3 - with: - enable-cache: true - cache-dependency-path: | - **/pyproject.toml - - uses: actions/setup-node@v4 - with: - node-version: "20" - - name: Python SDK tests - working-directory: sdk/python - run: uv run --with pytest pytest -q - - name: TypeScript SDK build - working-directory: sdk/typescript - run: npm install && npm run build && npm run lint - - report-scheduled-failure: - name: Open Issue on scheduled CI failure - runs-on: ubuntu-latest - needs: [backend, backend-java, integration, frontend, sdk] - if: | - always() && - (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && - ( - needs.backend.result == 'failure' || - needs.backend-java.result == 'failure' || - needs.integration.result == 'failure' || - needs.frontend.result == 'failure' || - needs.sdk.result == 'failure' - ) - steps: - - uses: actions/checkout@v4 - - name: Write failure report and upsert Issue - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BACKEND_RESULT: ${{ needs.backend.result }} - JAVA_RESULT: ${{ needs.backend-java.result }} - INTEGRATION_RESULT: ${{ needs.integration.result }} - FRONTEND_RESULT: ${{ needs.frontend.result }} - SDK_RESULT: ${{ needs.sdk.result }} - run: | - set -euo pipefail - chmod +x scripts/ci/upsert_automation_issue.sh - REPORT="ci-failure-report.md" - RUN_URL="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" - { - echo "# Scheduled CI failure" - echo - echo "Generated: $(date -u +"%Y-%m-%d %H:%M:%S UTC")" - echo - echo "Workflow run: ${RUN_URL}" - echo - echo "| Job | Result |" - echo "|---|---|" - echo "| backend | \`${BACKEND_RESULT}\` |" - echo "| backend-java | \`${JAVA_RESULT}\` |" - echo "| integration | \`${INTEGRATION_RESULT}\` |" - echo "| frontend | \`${FRONTEND_RESULT}\` |" - echo "| sdk | \`${SDK_RESULT}\` |" - echo - echo "This issue is refreshed while scheduled/manual CI keeps failing." - echo "Close it after the failing jobs are green again." - } >"$REPORT" - - scripts/ci/upsert_automation_issue.sh \ - "[automation] Scheduled CI failure" \ - "$REPORT" \ - automation \ - ci-failure - - close-scheduled-success: - name: Close CI failure Issue when green - runs-on: ubuntu-latest - needs: [backend, backend-java, integration, frontend, sdk] - if: | - always() && - github.event_name == 'schedule' && - needs.backend.result == 'success' && - needs.backend-java.result == 'success' && - needs.integration.result == 'success' && - needs.frontend.result == 'success' && - needs.sdk.result == 'success' - steps: - - name: Close open automation CI failure issue - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -euo pipefail - TITLE="[automation] Scheduled CI failure" - EXISTING="$( - gh issue list --state open --limit 100 --json number,title \ - | jq -r --arg t "$TITLE" '.[] | select(.title == $t) | .number' \ - | head -n 1 - )" - if [[ -n "$EXISTING" ]]; then - gh issue comment "$EXISTING" --body "Scheduled CI is green again. Closing. Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" - gh issue close "$EXISTING" --reason completed - echo "Closed issue #${EXISTING}" - else - echo "No open CI failure issue." - fi diff --git a/README.md b/README.md index e56ba77..56419a5 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ See: - [docs/api-contract.md](docs/api-contract.md) — frozen `/v1` HTTP contract and the API↔worker Redis protocol. -- [docs/deployment.md](docs/deployment.md) — production runbook. +- [docs/deployment.md](docs/deployment.md) — production runbook (Compose, Helm, Terraform). - [docs/architecture.md](docs/architecture.md) and [docs/data-model.md](docs/data-model.md) — runtime topology and database model. diff --git a/backend-java/src/main/java/io/agentflow/api/dto/ThreadMessageResponse.java b/backend-java/src/main/java/io/agentflow/api/dto/ThreadMessageResponse.java index abaf852..4e5252a 100644 --- a/backend-java/src/main/java/io/agentflow/api/dto/ThreadMessageResponse.java +++ b/backend-java/src/main/java/io/agentflow/api/dto/ThreadMessageResponse.java @@ -1,5 +1,6 @@ package io.agentflow.api.dto; +import com.fasterxml.jackson.annotation.JsonIgnore; import io.agentflow.api.entity.MessageEntity; import java.time.Instant; import java.util.List; @@ -17,8 +18,15 @@ public class ThreadMessageResponse { private String toolCallId; private Map extra; private Instant createdAt; + /** Sort key for thread transcript cursors (run.created_at). */ + private Instant runCreatedAt; public static ThreadMessageResponse from(MessageEntity entity, String runId) { + return from(entity, runId, null); + } + + public static ThreadMessageResponse from( + MessageEntity entity, String runId, Instant runCreatedAt) { ThreadMessageResponse dto = new ThreadMessageResponse(); dto.id = entity.getId(); dto.runId = runId; @@ -30,6 +38,7 @@ public static ThreadMessageResponse from(MessageEntity entity, String runId) { dto.toolCallId = entity.getToolCallId(); dto.extra = entity.getExtra(); dto.createdAt = entity.getCreatedAt(); + dto.runCreatedAt = runCreatedAt; return dto; } @@ -73,6 +82,11 @@ public Instant getCreatedAt() { return createdAt; } + @JsonIgnore + public Instant getRunCreatedAt() { + return runCreatedAt; + } + public static class Page { private final List items; private final String nextCursor; diff --git a/backend-java/src/main/java/io/agentflow/api/repository/MessageRepository.java b/backend-java/src/main/java/io/agentflow/api/repository/MessageRepository.java index 37fe51b..23f30a8 100644 --- a/backend-java/src/main/java/io/agentflow/api/repository/MessageRepository.java +++ b/backend-java/src/main/java/io/agentflow/api/repository/MessageRepository.java @@ -1,9 +1,12 @@ package io.agentflow.api.repository; import io.agentflow.api.entity.MessageEntity; +import java.time.Instant; import java.util.List; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; public interface MessageRepository extends JpaRepository { @@ -14,6 +17,42 @@ public interface MessageRepository extends JpaRepository List findByRunIdAndIndexLessThanOrderByIndexDesc( String runId, int index, Pageable pageable); + /** + * Thread transcript rows as {@code [MessageEntity, RunEntity]} ordered + * newest-first for limit-based paging. + */ + @Query( + """ + SELECT m, r FROM MessageEntity m, RunEntity r + WHERE m.runId = r.id AND r.threadId = :threadId + ORDER BY r.createdAt DESC, m.index DESC, m.id DESC + """) + List findThreadMessagesNewestFirst( + @Param("threadId") String threadId, Pageable pageable); + + /** + * Messages older than the opaque cursor {@code (runCreatedAt, index, id)}, + * still newest-first within that older window. + */ + @Query( + """ + SELECT m, r FROM MessageEntity m, RunEntity r + WHERE m.runId = r.id AND r.threadId = :threadId + AND ( + r.createdAt < :cursorCreated + OR (r.createdAt = :cursorCreated AND m.index < :cursorIndex) + OR (r.createdAt = :cursorCreated AND m.index = :cursorIndex + AND m.id < :cursorId) + ) + ORDER BY r.createdAt DESC, m.index DESC, m.id DESC + """) + List findThreadMessagesOlderThan( + @Param("threadId") String threadId, + @Param("cursorCreated") Instant cursorCreated, + @Param("cursorIndex") int cursorIndex, + @Param("cursorId") String cursorId, + Pageable pageable); + long deleteByRunId(String runId); long deleteByRunIdIn(Iterable runIds); diff --git a/backend-java/src/main/java/io/agentflow/api/service/ThreadService.java b/backend-java/src/main/java/io/agentflow/api/service/ThreadService.java index 2fc6780..b8e6de2 100644 --- a/backend-java/src/main/java/io/agentflow/api/service/ThreadService.java +++ b/backend-java/src/main/java/io/agentflow/api/service/ThreadService.java @@ -13,7 +13,9 @@ import io.agentflow.api.repository.ThreadRepository; import io.agentflow.api.security.AccessControl; import io.agentflow.api.security.Role; +import java.time.Instant; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import org.springframework.data.domain.PageRequest; import org.springframework.stereotype.Service; @@ -45,9 +47,11 @@ public ThreadResponse create(ThreadCreateRequest req) { ThreadEntity thread = new ThreadEntity(); thread.setTenantId(agent.getTenantId()); thread.setAgentId(agent.getId()); - thread.setProjectId(req.getProjectId()); - thread.setUserId(req.getUserId()); - thread.setTitle(req.getTitle()); + // Prefer explicit project_id; otherwise leave null (Java AgentEntity + // does not yet mirror agents.project_id — Python create inherits it). + thread.setProjectId(blankToNull(req.getProjectId())); + thread.setUserId(blankToNull(req.getUserId())); + thread.setTitle(blankToNull(req.getTitle())); return ThreadResponse.fromEntity(threads.save(thread)); } @@ -68,22 +72,39 @@ public ThreadResponse get(String id) { @Transactional(readOnly = true) public ThreadMessageResponse.Page listMessages(String id, String cursor, int limit) { ThreadEntity thread = requireThread(id); - int capped = Math.max(1, Math.min(limit, 200)); - List threadRuns = runs.findAllByThreadIdOrderByCreatedAtAsc(thread.getId()); - List all = new ArrayList<>(); - for (RunEntity run : threadRuns) { - for (MessageEntity message : messages.findAllByRunIdOrderByIndexAsc(run.getId())) { - all.add(ThreadMessageResponse.from(message, run.getId())); - } + int capped = Math.max(1, Math.min(limit, MessagePagination.PAGE_MAX)); + PageRequest page = PageRequest.of(0, capped + 1); + + List rows; + CursorParts parsed = parseCursor(cursor); + if (parsed != null) { + rows = + messages.findThreadMessagesOlderThan( + thread.getId(), + parsed.createdAt(), + parsed.index(), + parsed.id(), + page); + } else { + rows = messages.findThreadMessagesNewestFirst(thread.getId(), page); } - if (cursor != null && !cursor.isBlank()) { - all = all.stream().filter(m -> cursorKey(m).compareTo(cursor) < 0).toList(); + + List newestFirst = new ArrayList<>(rows.size()); + for (Object[] row : rows) { + MessageEntity message = (MessageEntity) row[0]; + RunEntity run = (RunEntity) row[1]; + newestFirst.add( + ThreadMessageResponse.from(message, run.getId(), run.getCreatedAt())); } - boolean hasMore = all.size() > capped; - List page = - hasMore ? all.subList(Math.max(0, all.size() - capped), all.size()) : all; - String nextCursor = hasMore && !page.isEmpty() ? cursorKey(page.get(0)) : null; - return new ThreadMessageResponse.Page(List.copyOf(page), nextCursor, hasMore); + + boolean hasMore = newestFirst.size() > capped; + List pageDesc = + hasMore ? newestFirst.subList(0, capped) : newestFirst; + List ascending = new ArrayList<>(pageDesc); + Collections.reverse(ascending); + String nextCursor = + hasMore && !ascending.isEmpty() ? cursorKey(ascending.get(0)) : null; + return new ThreadMessageResponse.Page(List.copyOf(ascending), nextCursor, hasMore); } @Transactional(readOnly = true) @@ -103,11 +124,43 @@ ThreadEntity requireThread(String id) { .orElseThrow(() -> new ThreadNotFoundException(id)); } + /** + * Cursor uses the run's created_at (sort key) plus message index/id so + * pagination matches transcript order across runs. + */ private static String cursorKey(ThreadMessageResponse message) { - return message.getCreatedAt() + Instant sortAt = + message.getRunCreatedAt() != null + ? message.getRunCreatedAt() + : message.getCreatedAt(); + return sortAt + "|" + String.format("%08d", message.getIndex()) + "|" + message.getId(); } + + private static CursorParts parseCursor(String cursor) { + if (cursor == null || cursor.isBlank()) { + return null; + } + String[] parts = cursor.split("\\|", 3); + if (parts.length != 3) { + return null; + } + try { + return new CursorParts(Instant.parse(parts[0]), Integer.parseInt(parts[1]), parts[2]); + } catch (RuntimeException ex) { + return null; + } + } + + private static String blankToNull(String value) { + if (value == null || value.isBlank()) { + return null; + } + return value; + } + + private record CursorParts(Instant createdAt, int index, String id) {} } diff --git a/backend/app/adapters/langgraph_adapter.py b/backend/app/adapters/langgraph_adapter.py index 3d8ea7d..74ca245 100644 --- a/backend/app/adapters/langgraph_adapter.py +++ b/backend/app/adapters/langgraph_adapter.py @@ -1193,14 +1193,19 @@ def _initial_graph_state(ctx: AdapterContext) -> dict[str, Any]: "human_input": None, "route": None, } - if ctx.thread_messages: - default["messages"] = list(ctx.thread_messages) + # Prior thread turns (other runs) + this run's transcript on resume. + # run_messages alone used to drop L1 context across HITL / retry. + seed: list[dict[str, Any]] = list(ctx.thread_messages or []) + if ctx.run_messages is not None: + seed = [*seed, *ctx.run_messages] + if seed: + default["messages"] = seed if ctx.resume and ctx.resume.checkpoint_state: saved = ctx.resume.checkpoint_state.get("graph_state") if isinstance(saved, dict): merged = {**default, **saved} - if ctx.run_messages is not None: - merged["messages"] = list(ctx.run_messages) + if ctx.run_messages is not None or ctx.thread_messages: + merged["messages"] = list(default["messages"]) elif isinstance(saved.get("messages"), list): # Legacy checkpoints that still inlined messages. merged["messages"] = list(saved["messages"]) diff --git a/backend/app/schemas/thread.py b/backend/app/schemas/thread.py index db47a4a..02671ed 100644 --- a/backend/app/schemas/thread.py +++ b/backend/app/schemas/thread.py @@ -40,6 +40,6 @@ class ThreadMessagePage(BaseModel): items: list[ThreadMessageRead] next_cursor: str | None = Field( default=None, - description="Opaque cursor for older messages (created_at|index|id).", + description="Opaque cursor for older messages (run.created_at|index|id).", ) has_more: bool = False diff --git a/backend/app/services/thread_service.py b/backend/app/services/thread_service.py index 86169ac..44c4002 100644 --- a/backend/app/services/thread_service.py +++ b/backend/app/services/thread_service.py @@ -2,9 +2,10 @@ from __future__ import annotations +from datetime import datetime from typing import Any -from sqlalchemy import select +from sqlalchemy import select, tuple_ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload @@ -81,7 +82,13 @@ async def get_thread( raise ThreadNotFound(thread_id) if tenant_id is not None and thread.tenant_id != tenant_id: raise ThreadNotFound(thread_id) - if project_id is not None and thread.project_id != project_id: + # Match create_run: only reject when both sides declare a project and + # they disagree. A null thread.project_id stays visible to project keys. + if ( + project_id is not None + and thread.project_id is not None + and thread.project_id != project_id + ): raise ThreadNotFound(thread_id) if agent_id is not None and thread.agent_id != agent_id: raise ThreadNotFound(thread_id) @@ -98,7 +105,10 @@ async def list_threads( capped = max(1, min(limit, 200)) stmt = select(Thread).where(Thread.tenant_id == tenant_id) if project_id is not None: - stmt = stmt.where(Thread.project_id == project_id) + # Include unscoped threads (null project) for the same tenant. + stmt = stmt.where( + (Thread.project_id == project_id) | (Thread.project_id.is_(None)) + ) if agent_id is not None: stmt = stmt.where(Thread.agent_id == agent_id) stmt = stmt.order_by(Thread.created_at.desc()).limit(capped) @@ -123,14 +133,28 @@ async def list_thread_messages( ) capped = max(1, min(limit, get_settings().run_messages_page_max)) + # Newest page first via DESC + limit, then reverse for ASC wire order. + # Cursor keys use run.created_at (sort key), not message.created_at. stmt = ( select(Message, Run) .join(Run, Message.run_id == Run.id) .where(Run.thread_id == thread_id) - .order_by(Run.created_at.asc(), Message.index.asc(), Message.id.asc()) + .order_by(Run.created_at.desc(), Message.index.desc(), Message.id.desc()) ) - result = await self.session.execute(stmt) + if cursor: + parsed = _parse_message_cursor(cursor) + if parsed is not None: + run_created, index, msg_id = parsed + stmt = stmt.where( + tuple_(Run.created_at, Message.index, Message.id) + < (run_created, index, msg_id) + ) + + result = await self.session.execute(stmt.limit(capped + 1)) rows = list(result.all()) + has_more = len(rows) > capped + page_rows = rows[:capped] + page_rows.reverse() items = [ ThreadMessageRead( @@ -145,26 +169,17 @@ async def list_thread_messages( created_at=msg.created_at, run_id=run.id, ) - for msg, run in rows + for msg, run in page_rows ] - - # Cursor = exclusive lower bound encoded as created_at|index|id of the - # oldest item on the previous (newer) page. For simplicity we page from - # the end (newest first window) like run messages. - if cursor: - items = [m for m in items if _message_cursor_key(m) < cursor] - - # Newest page: take last N, then report has_more for older ones. - has_more = len(items) > capped - if has_more: - page = items[-capped:] - next_cursor = _message_cursor_key(page[0]) - else: - page = items - next_cursor = None + next_cursor = None + if has_more and page_rows: + oldest_msg, oldest_run = page_rows[0] + next_cursor = _message_cursor_key( + oldest_run.created_at, oldest_msg.index, oldest_msg.id + ) return ThreadMessagePage( - items=page, next_cursor=next_cursor, has_more=has_more + items=items, next_cursor=next_cursor, has_more=has_more ) async def list_thread_runs( @@ -202,17 +217,30 @@ async def load_thread_window( ) -> list[dict[str, Any]]: """Return OpenAI-style chat dicts for prior thread turns, window-trimmed. - Skips prompt_echo / system rows so adapters can attach their own system - prompt. Used by the worker when constructing ``AdapterContext``. + Seeds only complete user/assistant turns (no system / prompt_echo / tool). + Cross-run tool chains are incomplete and break model APIs, so they are + omitted from L1. Used by the worker when constructing ``AdapterContext``. """ + max_messages = get_settings().thread_messages_max + # Over-fetch slightly so role filtering still fills the cap. + fetch_limit = max_messages * 3 if max_messages > 0 else None + stmt = ( select(Message, Run) .join(Run, Message.run_id == Run.id) .where(Run.thread_id == thread_id) - .order_by(Run.created_at.asc(), Run.id.asc(), Message.index.asc(), Message.id.asc()) + .where(Message.role.in_(("user", "assistant"))) + .order_by( + Run.created_at.desc(), + Run.id.desc(), + Message.index.desc(), + Message.id.desc(), + ) ) if exclude_run_id is not None: stmt = stmt.where(Run.id != exclude_run_id) + if fetch_limit is not None: + stmt = stmt.limit(fetch_limit) result = await self.session.execute(stmt) messages: list[dict[str, Any]] = [] @@ -220,15 +248,21 @@ async def load_thread_window( extra = msg.extra or {} if extra.get("kind") == "prompt_echo": continue - if msg.role == "system": + payload = message_row_to_dict(msg) + # Drop incomplete tool-call metadata from prior runs. + payload.pop("tool_calls", None) + payload.pop("tool_call_id", None) + if not str(payload.get("content") or "").strip(): continue - messages.append(message_row_to_dict(msg)) + messages.append(payload) + + # Queried newest-first; restore chronological order for adapters. + messages.reverse() - memory_cfg = parse_memory_config(agent_config or {}) - max_messages = get_settings().thread_messages_max if max_messages > 0 and len(messages) > max_messages: messages = messages[-max_messages:] + memory_cfg = parse_memory_config(agent_config or {}) if memory_cfg.window_tokens > 0: messages = fit_messages_to_window( messages, @@ -238,5 +272,19 @@ async def load_thread_window( return messages -def _message_cursor_key(msg: ThreadMessageRead) -> str: - return f"{msg.created_at.isoformat()}|{msg.index:08d}|{msg.id}" +def _message_cursor_key(run_created_at: datetime, index: int, message_id: str) -> str: + return f"{run_created_at.isoformat()}|{index:08d}|{message_id}" + + +def _parse_message_cursor( + cursor: str, +) -> tuple[datetime, int, str] | None: + parts = cursor.split("|", 2) + if len(parts) != 3: + return None + try: + created = datetime.fromisoformat(parts[0]) + index = int(parts[1]) + except ValueError: + return None + return created, index, parts[2] diff --git a/backend/tests/test_langgraph_adapter.py b/backend/tests/test_langgraph_adapter.py index bf89adc..1c1fa18 100644 --- a/backend/tests/test_langgraph_adapter.py +++ b/backend/tests/test_langgraph_adapter.py @@ -464,3 +464,37 @@ def test_checkpoint_payload_omits_messages(): assert graph["reply"] == "done" assert graph["pending_human"] == "approve" assert "messages" not in graph + + +def test_initial_graph_state_keeps_thread_messages_on_resume(): + from app.adapters.langgraph_adapter import _initial_graph_state + + ctx = _RecordingContext( + thread_messages=[ + {"role": "user", "content": "prior turn"}, + {"role": "assistant", "content": "prior reply"}, + ], + run_messages=[{"role": "user", "content": "this run"}], + resume=RunResumeContext( + mode="resume", + checkpoint_index=0, + checkpoint_state={"graph_state": {"completed_nodes": ["draft"]}}, + ), + ) + state = _initial_graph_state(ctx) + assert state["messages"] == [ + {"role": "user", "content": "prior turn"}, + {"role": "assistant", "content": "prior reply"}, + {"role": "user", "content": "this run"}, + ] + assert state["completed_nodes"] == ["draft"] + + +def test_initial_graph_state_seeds_thread_only_without_resume(): + from app.adapters.langgraph_adapter import _initial_graph_state + + ctx = _RecordingContext( + thread_messages=[{"role": "user", "content": "hello"}], + ) + state = _initial_graph_state(ctx) + assert state["messages"] == [{"role": "user", "content": "hello"}] diff --git a/backend/tests/test_threads.py b/backend/tests/test_threads.py index 5c8115f..b676762 100644 --- a/backend/tests/test_threads.py +++ b/backend/tests/test_threads.py @@ -26,10 +26,11 @@ async def test_thread_cross_run_messages_and_seed(client, monkeypatch): """Two runs share a thread; the second AdapterContext gets prior turns.""" captured: list[list[dict]] = [] - from app.adapters.base import AdapterContext, AdapterResult, OrchestratorAdapter + from ulid import ULID + from app.adapters import register_adapter + from app.adapters.base import AdapterContext, AdapterResult, OrchestratorAdapter from app.models.run import RunStatus - from ulid import ULID adapter_name = f"thread-capture-{ULID()}" @@ -195,3 +196,119 @@ async def test_missing_thread_on_create_run_404(client): }, ) assert resp.status_code == 404 + + +@pytest.mark.asyncio +async def test_thread_message_pagination(client): + agent = await client.post( + "/v1/agents", + json={"name": "page-bot", "adapter": "echo", "config": {"delay": 0}}, + ) + agent_id = agent.json()["id"] + thread = await client.post( + "/v1/threads", + json={"agent_id": agent_id, "title": "paged"}, + ) + thread_id = thread.json()["id"] + + for i in range(3): + run = await client.post( + "/v1/runs", + json={ + "agent_id": agent_id, + "thread_id": thread_id, + "input": {"prompt": f"turn-{i}"}, + }, + ) + body = await _poll_until(client, run.json()["id"], {"succeeded", "failed"}) + assert body["status"] == "succeeded" + + first = await client.get(f"/v1/threads/{thread_id}/messages?limit=2") + assert first.status_code == 200 + page1 = first.json() + assert len(page1["items"]) == 2 + assert page1["has_more"] is True + assert page1["next_cursor"] + + second = await client.get( + f"/v1/threads/{thread_id}/messages", + params={"limit": 2, "cursor": page1["next_cursor"]}, + ) + assert second.status_code == 200 + page2 = second.json() + assert page2["items"] + ids1 = {m["id"] for m in page1["items"]} + ids2 = {m["id"] for m in page2["items"]} + assert ids1.isdisjoint(ids2) + + +@pytest.mark.asyncio +async def test_load_thread_window_skips_tool_and_prompt_echo(): + from app.db.base import Base + from app.db.session import SessionLocal, engine + from app.models import Agent, Message, Run, Thread + from app.models.run import RunStatus + from app.services.thread_service import ThreadService + + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.drop_all) + await conn.run_sync(Base.metadata.create_all) + + async with SessionLocal() as session: + agent = Agent(name="win-bot", adapter="echo", config={}) + session.add(agent) + await session.commit() + await session.refresh(agent) + + thread = Thread(tenant_id=agent.tenant_id, agent_id=agent.id, title="w") + session.add(thread) + await session.commit() + await session.refresh(thread) + + run = Run( + tenant_id=agent.tenant_id, + agent_id=agent.id, + thread_id=thread.id, + adapter="echo", + status=RunStatus.SUCCEEDED, + input={"prompt": "hi"}, + ) + session.add(run) + await session.commit() + await session.refresh(run) + + session.add_all( + [ + Message( + run_id=run.id, + index=0, + role="system", + content="sys", + extra={"kind": "prompt_echo"}, + ), + Message(run_id=run.id, index=1, role="user", content="hello"), + Message( + run_id=run.id, + index=2, + role="assistant", + content="", + extra={"tool_calls": [{"id": "c1", "name": "echo"}]}, + ), + Message( + run_id=run.id, + index=3, + role="tool", + content='{"ok":true}', + tool_call_id="c1", + ), + Message(run_id=run.id, index=4, role="assistant", content="done"), + ] + ) + await session.commit() + + service = ThreadService(session) + window = await service.load_thread_window(thread.id) + assert window == [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "done"}, + ] diff --git a/deploy/helm/agentflow/.helmignore b/deploy/helm/agentflow/.helmignore new file mode 100644 index 0000000..dbcc83a --- /dev/null +++ b/deploy/helm/agentflow/.helmignore @@ -0,0 +1,5 @@ +*.md +.gitignore +.DS_Store +*.tgz +charts/*.tgz diff --git a/deploy/helm/agentflow/Chart.yaml b/deploy/helm/agentflow/Chart.yaml new file mode 100644 index 0000000..76dda6c --- /dev/null +++ b/deploy/helm/agentflow/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: agentflow +description: AgentFlow Java API + Python workers with queue-delay autoscaling +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/deploy/helm/agentflow/templates/NOTES.txt b/deploy/helm/agentflow/templates/NOTES.txt new file mode 100644 index 0000000..5bbc565 --- /dev/null +++ b/deploy/helm/agentflow/templates/NOTES.txt @@ -0,0 +1,11 @@ +1. Apply Alembic migrations (`uv run alembic upgrade head` in the migrate Job / worker init container). +2. Confirm `GET /v1/health` on the API Service. +3. With OTel + Prometheus in place, enqueue enough runs that `agentflow_queue_consumer_delay` exceeds `worker.autoscaling.targetConsumerDelaySeconds` and watch worker replicas climb toward `maxReplicas`. +{{- if .Values.worker.autoscaling.enabled }} +{{- if eq .Values.worker.autoscaling.provider "keda" }} + +Queue-delay autoscaling is a KEDA ScaledObject on `max(agentflow_queue_consumer_delay)` (threshold {{ .Values.worker.autoscaling.targetConsumerDelaySeconds }}s). +Install KEDA and scrape worker OTLP metrics into Prometheus at: + {{ .Values.worker.autoscaling.prometheus.serverAddress }} +{{- end }} +{{- end }} diff --git a/deploy/helm/agentflow/templates/_helpers.tpl b/deploy/helm/agentflow/templates/_helpers.tpl new file mode 100644 index 0000000..886fe95 --- /dev/null +++ b/deploy/helm/agentflow/templates/_helpers.tpl @@ -0,0 +1,119 @@ +{{- define "agentflow.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "agentflow.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{- define "agentflow.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{- define "agentflow.labels" -}} +helm.sh/chart: {{ include "agentflow.chart" . }} +{{ include "agentflow.selectorLabels" . }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} + +{{- define "agentflow.selectorLabels" -}} +app.kubernetes.io/name: {{ include "agentflow.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{- define "agentflow.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "agentflow.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{- define "agentflow.jdbcUrl" -}} +{{- if .Values.postgres.enabled }} +{{- printf "jdbc:postgresql://%s-postgres:5432/%s" (include "agentflow.fullname" .) .Values.postgres.auth.database }} +{{- else }} +{{- .Values.database.jdbcUrl }} +{{- end }} +{{- end }} + +{{- define "agentflow.sqlalchemyUrl" -}} +{{- if .Values.postgres.enabled }} +{{- printf "postgresql+asyncpg://%s:%s@%s-postgres:5432/%s" .Values.postgres.auth.username .Values.postgres.auth.password (include "agentflow.fullname" .) .Values.postgres.auth.database }} +{{- else }} +{{- .Values.database.sqlalchemyUrl }} +{{- end }} +{{- end }} + +{{- define "agentflow.redisHost" -}} +{{- if .Values.redis.enabled }} +{{- printf "%s-redis" (include "agentflow.fullname" .) }} +{{- else }} +{{- .Values.redisExternal.host }} +{{- end }} +{{- end }} + +{{- define "agentflow.redisPort" -}} +{{- if .Values.redis.enabled }} +{{- "6379" }} +{{- else }} +{{- .Values.redisExternal.port | int }} +{{- end }} +{{- end }} + +{{- define "agentflow.redisUrl" -}} +{{- if .Values.redis.enabled }} +{{- printf "redis://%s-redis:6379/0" (include "agentflow.fullname" .) }} +{{- else }} +{{- .Values.redisExternal.url }} +{{- end }} +{{- end }} + +{{- define "agentflow.dbUser" -}} +{{- if .Values.postgres.enabled }} +{{- .Values.postgres.auth.username }} +{{- else }} +{{- .Values.database.username }} +{{- end }} +{{- end }} + +{{- define "agentflow.dbPassword" -}} +{{- if .Values.postgres.enabled }} +{{- .Values.postgres.auth.password }} +{{- else }} +{{- .Values.database.password }} +{{- end }} +{{- end }} + +{{- define "agentflow.migrateInitContainer" -}} +- name: migrate + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + i=0 + while [ "$i" -lt 30 ]; do + uv run alembic upgrade head && exit 0 + i=$((i + 1)) + sleep 2 + done + exit 1 + env: + - name: AGENTFLOW_DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ include "agentflow.secretName" . }} + key: sqlalchemy-url +{{- end }} diff --git a/deploy/helm/agentflow/templates/_secretname.tpl b/deploy/helm/agentflow/templates/_secretname.tpl new file mode 100644 index 0000000..e8b5c8c --- /dev/null +++ b/deploy/helm/agentflow/templates/_secretname.tpl @@ -0,0 +1,7 @@ +{{- define "agentflow.secretName" -}} +{{- if .Values.database.existingSecret }} +{{- .Values.database.existingSecret }} +{{- else }} +{{- include "agentflow.fullname" . }} +{{- end }} +{{- end }} diff --git a/deploy/helm/agentflow/templates/api.yaml b/deploy/helm/agentflow/templates/api.yaml new file mode 100644 index 0000000..241dc63 --- /dev/null +++ b/deploy/helm/agentflow/templates/api.yaml @@ -0,0 +1,130 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "agentflow.fullname" . }}-api + labels: + {{- include "agentflow.labels" . | nindent 4 }} + app.kubernetes.io/component: api +spec: + replicas: {{ .Values.api.replicaCount }} + selector: + matchLabels: + {{- include "agentflow.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: api + template: + metadata: + labels: + {{- include "agentflow.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: api + {{- with .Values.api.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "agentflow.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.migrate.enabled }} + initContainers: + {{- include "agentflow.migrateInitContainer" . | nindent 8 }} + {{- end }} + containers: + - name: api + image: "{{ .Values.api.image.repository }}:{{ .Values.api.image.tag }}" + imagePullPolicy: {{ .Values.api.image.pullPolicy }} + ports: + - name: http + containerPort: {{ .Values.api.service.port }} + env: + - name: AGENTFLOW_DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ include "agentflow.secretName" . }} + key: jdbc-url + - name: AGENTFLOW_DATABASE_USERNAME + valueFrom: + secretKeyRef: + name: {{ include "agentflow.secretName" . }} + key: {{ .Values.database.existingSecretUsernameKey | default "username" }} + - name: AGENTFLOW_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "agentflow.secretName" . }} + key: {{ .Values.database.existingSecretPasswordKey | default "password" }} + - name: AGENTFLOW_REDIS_HOST + valueFrom: + secretKeyRef: + name: {{ include "agentflow.secretName" . }} + key: redis-host + - name: AGENTFLOW_REDIS_PORT + valueFrom: + secretKeyRef: + name: {{ include "agentflow.secretName" . }} + key: redis-port + - name: AGENTFLOW_SERVER_PORT + value: {{ .Values.api.service.port | quote }} + - name: AGENTFLOW_JOBS_IMPL + value: {{ .Values.jobs.impl | quote }} + - name: AGENTFLOW_AUTH_ENABLED + value: {{ .Values.auth.enabled | quote }} + - name: AGENTFLOW_OTEL_ENABLED + value: {{ .Values.otel.enabled | quote }} + - name: AGENTFLOW_OTEL_SERVICE_NAME + value: agentflow-api + - name: AGENTFLOW_OTEL_EXPORTER_ENDPOINT + value: {{ printf "%s/v1/traces" (trimSuffix "/" .Values.otel.exporterEndpoint) | quote }} + - name: AGENTFLOW_OTEL_METRICS_ENDPOINT + value: {{ printf "%s/v1/metrics" (trimSuffix "/" .Values.otel.exporterEndpoint) | quote }} + {{- range $k, $v := .Values.api.env }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- with .Values.api.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + readinessProbe: + httpGet: + path: /v1/health + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 12 + livenessProbe: + httpGet: + path: /v1/health + port: http + periodSeconds: 15 + timeoutSeconds: 3 + resources: + {{- toYaml .Values.api.resources | nindent 12 }} + {{- with .Values.api.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.api.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.api.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "agentflow.fullname" . }}-api + labels: + {{- include "agentflow.labels" . | nindent 4 }} + app.kubernetes.io/component: api +spec: + type: {{ .Values.api.service.type }} + ports: + - name: http + port: {{ .Values.api.service.port }} + targetPort: http + selector: + {{- include "agentflow.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: api diff --git a/deploy/helm/agentflow/templates/console.yaml b/deploy/helm/agentflow/templates/console.yaml new file mode 100644 index 0000000..0764f16 --- /dev/null +++ b/deploy/helm/agentflow/templates/console.yaml @@ -0,0 +1,51 @@ +{{- if .Values.console.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "agentflow.fullname" . }}-console + labels: + {{- include "agentflow.labels" . | nindent 4 }} + app.kubernetes.io/component: console +spec: + replicas: {{ .Values.console.replicaCount }} + selector: + matchLabels: + {{- include "agentflow.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: console + template: + metadata: + labels: + {{- include "agentflow.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: console + spec: + serviceAccountName: {{ include "agentflow.serviceAccountName" . }} + containers: + - name: console + image: "{{ .Values.console.image.repository }}:{{ .Values.console.image.tag }}" + imagePullPolicy: {{ .Values.console.image.pullPolicy }} + env: + - name: AGENTFLOW_API_URL + value: {{ printf "http://%s-api:%v" (include "agentflow.fullname" .) .Values.api.service.port | quote }} + ports: + - name: http + containerPort: {{ .Values.console.service.port }} + resources: + {{- toYaml .Values.console.resources | nindent 12 }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "agentflow.fullname" . }}-console + labels: + {{- include "agentflow.labels" . | nindent 4 }} + app.kubernetes.io/component: console +spec: + type: {{ .Values.console.service.type }} + ports: + - name: http + port: {{ .Values.console.service.port }} + targetPort: http + selector: + {{- include "agentflow.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: console +{{- end }} diff --git a/deploy/helm/agentflow/templates/ingress.yaml b/deploy/helm/agentflow/templates/ingress.yaml new file mode 100644 index 0000000..bc9e270 --- /dev/null +++ b/deploy/helm/agentflow/templates/ingress.yaml @@ -0,0 +1,35 @@ +{{- if .Values.ingress.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "agentflow.fullname" . }} + labels: + {{- include "agentflow.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.className }} + ingressClassName: {{ .Values.ingress.className | quote }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- toYaml .Values.ingress.tls | nindent 4 }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + pathType: {{ .pathType }} + backend: + service: + name: {{ include "agentflow.fullname" $ }}-api + port: + name: http + {{- end }} + {{- end }} +{{- end }} diff --git a/deploy/helm/agentflow/templates/migrate-job.yaml b/deploy/helm/agentflow/templates/migrate-job.yaml new file mode 100644 index 0000000..a406068 --- /dev/null +++ b/deploy/helm/agentflow/templates/migrate-job.yaml @@ -0,0 +1,49 @@ +{{- if .Values.migrate.enabled }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "agentflow.fullname" . }}-migrate + labels: + {{- include "agentflow.labels" . | nindent 4 }} + app.kubernetes.io/component: migrate + annotations: + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "5" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded +spec: + backoffLimit: {{ .Values.migrate.backoffLimit }} + ttlSecondsAfterFinished: {{ .Values.migrate.ttlSecondsAfterFinished }} + template: + metadata: + labels: + {{- include "agentflow.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: migrate + spec: + restartPolicy: OnFailure + serviceAccountName: {{ include "agentflow.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: migrate + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + i=0 + while [ "$i" -lt 30 ]; do + uv run alembic upgrade head && exit 0 + i=$((i + 1)) + sleep 2 + done + exit 1 + env: + - name: AGENTFLOW_DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ include "agentflow.secretName" . }} + key: sqlalchemy-url +{{- end }} diff --git a/deploy/helm/agentflow/templates/postgres.yaml b/deploy/helm/agentflow/templates/postgres.yaml new file mode 100644 index 0000000..78a24b8 --- /dev/null +++ b/deploy/helm/agentflow/templates/postgres.yaml @@ -0,0 +1,87 @@ +{{- if .Values.postgres.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "agentflow.fullname" . }}-postgres + labels: + {{- include "agentflow.labels" . | nindent 4 }} + app.kubernetes.io/component: postgres +spec: + ports: + - name: postgres + port: 5432 + targetPort: postgres + selector: + {{- include "agentflow.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: postgres +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "agentflow.fullname" . }}-postgres + labels: + {{- include "agentflow.labels" . | nindent 4 }} + app.kubernetes.io/component: postgres +spec: + serviceName: {{ include "agentflow.fullname" . }}-postgres + replicas: 1 + selector: + matchLabels: + {{- include "agentflow.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: postgres + template: + metadata: + labels: + {{- include "agentflow.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: postgres + spec: + containers: + - name: postgres + image: {{ .Values.postgres.image }} + ports: + - name: postgres + containerPort: 5432 + env: + - name: POSTGRES_USER + valueFrom: + secretKeyRef: + name: {{ include "agentflow.secretName" . }} + key: username + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "agentflow.secretName" . }} + key: password + - name: POSTGRES_DB + value: {{ .Values.postgres.auth.database | quote }} + readinessProbe: + exec: + command: ["pg_isready", "-U", {{ .Values.postgres.auth.username | quote }}] + periodSeconds: 5 + livenessProbe: + exec: + command: ["pg_isready", "-U", {{ .Values.postgres.auth.username | quote }}] + periodSeconds: 10 + volumeMounts: + - name: data + mountPath: /var/lib/postgresql/data + subPath: pgdata + {{- if not .Values.postgres.persistence.enabled }} + volumes: + - name: data + emptyDir: {} + {{- end }} + {{- if .Values.postgres.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + {{- if .Values.postgres.persistence.storageClass }} + storageClassName: {{ .Values.postgres.persistence.storageClass | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.postgres.persistence.size }} + {{- end }} +{{- end }} diff --git a/deploy/helm/agentflow/templates/redis.yaml b/deploy/helm/agentflow/templates/redis.yaml new file mode 100644 index 0000000..af554df --- /dev/null +++ b/deploy/helm/agentflow/templates/redis.yaml @@ -0,0 +1,51 @@ +{{- if .Values.redis.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "agentflow.fullname" . }}-redis + labels: + {{- include "agentflow.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +spec: + ports: + - name: redis + port: 6379 + targetPort: redis + selector: + {{- include "agentflow.selectorLabels" . | nindent 4 }} + app.kubernetes.io/component: redis +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "agentflow.fullname" . }}-redis + labels: + {{- include "agentflow.labels" . | nindent 4 }} + app.kubernetes.io/component: redis +spec: + replicas: 1 + selector: + matchLabels: + {{- include "agentflow.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: redis + template: + metadata: + labels: + {{- include "agentflow.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: redis + spec: + containers: + - name: redis + image: {{ .Values.redis.image }} + ports: + - name: redis + containerPort: 6379 + readinessProbe: + exec: + command: ["redis-cli", "ping"] + periodSeconds: 5 + livenessProbe: + exec: + command: ["redis-cli", "ping"] + periodSeconds: 10 +{{- end }} diff --git a/deploy/helm/agentflow/templates/secret.yaml b/deploy/helm/agentflow/templates/secret.yaml new file mode 100644 index 0000000..a675568 --- /dev/null +++ b/deploy/helm/agentflow/templates/secret.yaml @@ -0,0 +1,17 @@ +{{- if not .Values.database.existingSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "agentflow.fullname" . }} + labels: + {{- include "agentflow.labels" . | nindent 4 }} +type: Opaque +stringData: + username: {{ include "agentflow.dbUser" . | quote }} + password: {{ include "agentflow.dbPassword" . | quote }} + jdbc-url: {{ include "agentflow.jdbcUrl" . | quote }} + sqlalchemy-url: {{ include "agentflow.sqlalchemyUrl" . | quote }} + redis-host: {{ include "agentflow.redisHost" . | quote }} + redis-port: {{ include "agentflow.redisPort" . | quote }} + redis-url: {{ include "agentflow.redisUrl" . | quote }} +{{- end }} diff --git a/deploy/helm/agentflow/templates/serviceaccount.yaml b/deploy/helm/agentflow/templates/serviceaccount.yaml new file mode 100644 index 0000000..000dd7f --- /dev/null +++ b/deploy/helm/agentflow/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "agentflow.serviceAccountName" . }} + labels: + {{- include "agentflow.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/deploy/helm/agentflow/templates/tests/test-api-health.yaml b/deploy/helm/agentflow/templates/tests/test-api-health.yaml new file mode 100644 index 0000000..c8a8dc5 --- /dev/null +++ b/deploy/helm/agentflow/templates/tests/test-api-health.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "agentflow.fullname" . }}-test-api" + labels: + {{- include "agentflow.labels" . | nindent 4 }} + annotations: + helm.sh/hook: test +spec: + restartPolicy: Never + containers: + - name: wget + image: busybox:1.36 + command: + - wget + - -qO- + - {{ printf "http://%s-api:%v/v1/health" (include "agentflow.fullname" .) .Values.api.service.port | quote }} diff --git a/deploy/helm/agentflow/templates/worker-autoscaling.yaml b/deploy/helm/agentflow/templates/worker-autoscaling.yaml new file mode 100644 index 0000000..680fc05 --- /dev/null +++ b/deploy/helm/agentflow/templates/worker-autoscaling.yaml @@ -0,0 +1,65 @@ +{{- if and .Values.worker.autoscaling.enabled (eq .Values.worker.autoscaling.provider "keda") }} +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{ include "agentflow.fullname" . }}-worker + labels: + {{- include "agentflow.labels" . | nindent 4 }} + app.kubernetes.io/component: worker +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "agentflow.fullname" . }}-worker + minReplicaCount: {{ .Values.worker.autoscaling.minReplicas }} + maxReplicaCount: {{ .Values.worker.autoscaling.maxReplicas }} + pollingInterval: {{ .Values.worker.autoscaling.pollingInterval }} + cooldownPeriod: {{ .Values.worker.autoscaling.cooldownPeriod }} + triggers: + - type: prometheus + metadata: + serverAddress: {{ .Values.worker.autoscaling.prometheus.serverAddress | quote }} + metricName: agentflow_queue_consumer_delay + query: max(agentflow_queue_consumer_delay) or vector(0) + threshold: {{ .Values.worker.autoscaling.targetConsumerDelaySeconds | quote }} + activationThreshold: {{ .Values.worker.autoscaling.activationConsumerDelaySeconds | quote }} + {{- if gt (int .Values.worker.autoscaling.backlogThreshold) 0 }} + - type: prometheus + metadata: + serverAddress: {{ .Values.worker.autoscaling.prometheus.serverAddress | quote }} + metricName: agentflow_queue_backlog + query: max(agentflow_queue_backlog) or vector(0) + threshold: {{ .Values.worker.autoscaling.backlogThreshold | quote }} + {{- end }} +{{- end }} +{{- if and .Values.worker.autoscaling.enabled (eq .Values.worker.autoscaling.provider "hpa") }} +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "agentflow.fullname" . }}-worker + labels: + {{- include "agentflow.labels" . | nindent 4 }} + app.kubernetes.io/component: worker +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "agentflow.fullname" . }}-worker + minReplicas: {{ .Values.worker.autoscaling.minReplicas }} + maxReplicas: {{ .Values.worker.autoscaling.maxReplicas }} + metrics: + - type: External + external: + metric: + name: agentflow_queue_consumer_delay + target: + type: Value + value: {{ .Values.worker.autoscaling.targetConsumerDelaySeconds | quote }} + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: {{ .Values.worker.autoscaling.cpu.targetAverageUtilization }} +{{- end }} diff --git a/deploy/helm/agentflow/templates/worker.yaml b/deploy/helm/agentflow/templates/worker.yaml new file mode 100644 index 0000000..c9105af --- /dev/null +++ b/deploy/helm/agentflow/templates/worker.yaml @@ -0,0 +1,81 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "agentflow.fullname" . }}-worker + labels: + {{- include "agentflow.labels" . | nindent 4 }} + app.kubernetes.io/component: worker +spec: + replicas: {{ if .Values.worker.autoscaling.enabled }}{{ .Values.worker.autoscaling.minReplicas }}{{ else }}{{ .Values.worker.replicaCount }}{{ end }} + selector: + matchLabels: + {{- include "agentflow.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: worker + template: + metadata: + labels: + {{- include "agentflow.selectorLabels" . | nindent 8 }} + app.kubernetes.io/component: worker + {{- with .Values.worker.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "agentflow.serviceAccountName" . }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.migrate.enabled }} + initContainers: + {{- include "agentflow.migrateInitContainer" . | nindent 8 }} + {{- end }} + containers: + - name: worker + image: "{{ .Values.worker.image.repository }}:{{ .Values.worker.image.tag }}" + imagePullPolicy: {{ .Values.worker.image.pullPolicy }} + command: ["uv", "run", "python", "-m", "app.worker"] + env: + - name: AGENTFLOW_WORKER_MODE + value: queue + - name: AGENTFLOW_WORKER_CONCURRENCY + value: {{ .Values.worker.concurrency | quote }} + - name: AGENTFLOW_DATABASE_URL + valueFrom: + secretKeyRef: + name: {{ include "agentflow.secretName" . }} + key: sqlalchemy-url + - name: AGENTFLOW_REDIS_URL + valueFrom: + secretKeyRef: + name: {{ include "agentflow.secretName" . }} + key: redis-url + - name: AGENTFLOW_JOBS_IMPL + value: {{ .Values.jobs.impl | quote }} + - name: AGENTFLOW_OTEL_ENABLED + value: {{ .Values.otel.enabled | quote }} + - name: AGENTFLOW_OTEL_SERVICE_NAME + value: agentflow-worker + - name: AGENTFLOW_OTEL_EXPORTER_ENDPOINT + value: {{ .Values.otel.exporterEndpoint | quote }} + {{- range $k, $v := .Values.worker.env }} + - name: {{ $k }} + value: {{ $v | quote }} + {{- end }} + {{- with .Values.worker.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.worker.resources | nindent 12 }} + {{- with .Values.worker.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/deploy/helm/agentflow/values-dev.yaml b/deploy/helm/agentflow/values-dev.yaml new file mode 100644 index 0000000..33bbc39 --- /dev/null +++ b/deploy/helm/agentflow/values-dev.yaml @@ -0,0 +1,15 @@ +# In-cluster Postgres/Redis for kind / minikube. Do not use in production. +postgres: + enabled: true + persistence: + enabled: false + +redis: + enabled: true + +worker: + autoscaling: + enabled: false + +otel: + enabled: false diff --git a/deploy/helm/agentflow/values.yaml b/deploy/helm/agentflow/values.yaml new file mode 100644 index 0000000..88c9810 --- /dev/null +++ b/deploy/helm/agentflow/values.yaml @@ -0,0 +1,151 @@ +nameOverride: "" +fullnameOverride: "" + +imagePullSecrets: [] + +serviceAccount: + create: true + name: "" + annotations: {} + +otel: + enabled: true + # Worker uses the OTLP HTTP base URL (no path). API appends /v1/traces and /v1/metrics. + exporterEndpoint: http://otel-collector.observability.svc.cluster.local:4318 + +jobs: + impl: streams + +auth: + enabled: false + +api: + replicaCount: 2 + image: + repository: agentflow-api + tag: "0.1.0" + pullPolicy: IfNotPresent + service: + type: ClusterIP + port: 8000 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + env: {} + extraEnv: [] + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + +worker: + replicaCount: 1 + image: + repository: agentflow-worker + tag: "0.1.0" + pullPolicy: IfNotPresent + concurrency: 1 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + env: {} + extraEnv: [] + podAnnotations: {} + nodeSelector: {} + tolerations: [] + affinity: {} + # Scale workers from Prometheus queue delay (agentflow_queue_consumer_delay). + # Requires OTel export from workers and a Prometheus that scrapes the collector. + autoscaling: + enabled: true + # keda (queue delay via Prometheus) or hpa (CPU only) + provider: keda + minReplicas: 1 + maxReplicas: 10 + pollingInterval: 15 + cooldownPeriod: 300 + targetConsumerDelaySeconds: 30 + activationConsumerDelaySeconds: 5 + prometheus: + serverAddress: http://prometheus.observability.svc.cluster.local:9090 + # Optional second trigger: undelivered + pending jobs. + backlogThreshold: 0 + cpu: + targetAverageUtilization: 70 + +migrate: + enabled: true + backoffLimit: 6 + ttlSecondsAfterFinished: 600 + +postgres: + # Demo-only in-cluster Postgres. Production should set enabled=false and + # point database.url at a managed instance. + enabled: false + image: postgres:16-alpine + auth: + username: agentflow + password: agentflow + database: agentflow + persistence: + enabled: true + size: 8Gi + storageClass: "" + +redis: + enabled: false + image: redis:7-alpine + +database: + # Used when postgres.enabled is false. + jdbcUrl: jdbc:postgresql://postgres:5432/agentflow + sqlalchemyUrl: postgresql+asyncpg://agentflow:agentflow@postgres:5432/agentflow + username: agentflow + password: "" + existingSecret: "" + existingSecretUsernameKey: username + existingSecretPasswordKey: password + +redisExternal: + # Used when redis.enabled is false. + host: redis + port: 6379 + url: redis://redis:6379/0 + +ingress: + enabled: false + className: "" + annotations: {} + hosts: + - host: agentflow.local + paths: + - path: / + pathType: Prefix + tls: [] + +# Optional console Deployment. Image is not built by default compose. +console: + enabled: false + replicaCount: 1 + image: + repository: agentflow-console + tag: "0.1.0" + pullPolicy: IfNotPresent + service: + type: ClusterIP + port: 3000 + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 500m + memory: 512Mi diff --git a/deploy/terraform/.gitignore b/deploy/terraform/.gitignore new file mode 100644 index 0000000..ec120ca --- /dev/null +++ b/deploy/terraform/.gitignore @@ -0,0 +1,9 @@ +.terraform/ +terraform.tfstate +terraform.tfstate.backup +terraform.tfvars +crash.log +override.tf +override.tf.json +*_override.tf +*_override.tf.json diff --git a/deploy/terraform/.terraform.lock.hcl b/deploy/terraform/.terraform.lock.hcl new file mode 100644 index 0000000..d221c80 --- /dev/null +++ b/deploy/terraform/.terraform.lock.hcl @@ -0,0 +1,42 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/helm" { + version = "2.17.0" + constraints = "~> 2.17" + hashes = [ + "h1:kQMkcPVvHOguOqnxoEU2sm1ND9vCHiT8TvZ2x6v/Rsw=", + "zh:06fb4e9932f0afc1904d2279e6e99353c2ddac0d765305ce90519af410706bd4", + "zh:104eccfc781fc868da3c7fec4385ad14ed183eb985c96331a1a937ac79c2d1a7", + "zh:129345c82359837bb3f0070ce4891ec232697052f7d5ccf61d43d818912cf5f3", + "zh:3956187ec239f4045975b35e8c30741f701aa494c386aaa04ebabffe7749f81c", + "zh:66a9686d92a6b3ec43de3ca3fde60ef3d89fb76259ed3313ca4eb9bb8c13b7dd", + "zh:88644260090aa621e7e8083585c468c8dd5e09a3c01a432fb05da5c4623af940", + "zh:a248f650d174a883b32c5b94f9e725f4057e623b00f171936dcdcc840fad0b3e", + "zh:aa498c1f1ab93be5c8fbf6d48af51dc6ef0f10b2ea88d67bcb9f02d1d80d3930", + "zh:bf01e0f2ec2468c53596e027d376532a2d30feb72b0b5b810334d043109ae32f", + "zh:c46fa84cc8388e5ca87eb575a534ebcf68819c5a5724142998b487cb11246654", + "zh:d0c0f15ffc115c0965cbfe5c81f18c2e114113e7a1e6829f6bfd879ce5744fbb", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.38.0" + constraints = "~> 2.35" + hashes = [ + "h1:soK8Lt0SZ6dB+HsypFRDzuX/npqlMU6M0fvyaR1yW0k=", + "zh:0af928d776eb269b192dc0ea0f8a3f0f5ec117224cd644bdacdc682300f84ba0", + "zh:1be998e67206f7cfc4ffe77c01a09ac91ce725de0abaec9030b22c0a832af44f", + "zh:326803fe5946023687d603f6f1bab24de7af3d426b01d20e51d4e6fbe4e7ec1b", + "zh:4a99ec8d91193af961de1abb1f824be73df07489301d62e6141a656b3ebfff12", + "zh:5136e51765d6a0b9e4dbcc3b38821e9736bd2136cf15e9aac11668f22db117d2", + "zh:63fab47349852d7802fb032e4f2b6a101ee1ce34b62557a9ad0f0f0f5b6ecfdc", + "zh:924fb0257e2d03e03e2bfe9c7b99aa73c195b1f19412ca09960001bee3c50d15", + "zh:b63a0be5e233f8f6727c56bed3b61eb9456ca7a8bb29539fba0837f1badf1396", + "zh:d39861aa21077f1bc899bc53e7233262e530ba8a3a2d737449b100daeb303e4d", + "zh:de0805e10ebe4c83ce3b728a67f6b0f9d18be32b25146aa89116634df5145ad4", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:faf23e45f0090eef8ba28a8aac7ec5d4fdf11a36c40a8d286304567d71c1e7db", + ] +} diff --git a/deploy/terraform/main.tf b/deploy/terraform/main.tf new file mode 100644 index 0000000..71b74a6 --- /dev/null +++ b/deploy/terraform/main.tf @@ -0,0 +1,100 @@ +resource "kubernetes_namespace_v1" "agentflow" { + count = var.create_namespace ? 1 : 0 + + metadata { + name = var.namespace + } +} + +resource "helm_release" "keda" { + count = var.install_keda ? 1 : 0 + + name = "keda" + repository = "https://kedacore.github.io/charts" + chart = "keda" + namespace = var.keda_namespace + create_namespace = true + wait = true + timeout = 300 +} + +resource "helm_release" "agentflow" { + name = var.release_name + chart = "${path.module}/../helm/agentflow" + namespace = var.namespace + + atomic = true + cleanup_on_fail = true + wait = true + timeout = 600 + + values = concat( + [ + yamlencode({ + api = { + image = { + repository = var.api_image_repository + tag = var.api_image_tag + } + } + worker = { + image = { + repository = var.worker_image_repository + tag = var.worker_image_tag + } + autoscaling = { + enabled = var.autoscaling_enabled + provider = "keda" + minReplicas = var.worker_min_replicas + maxReplicas = var.worker_max_replicas + targetConsumerDelaySeconds = var.target_consumer_delay_seconds + prometheus = { + serverAddress = var.prometheus_server_address + } + } + } + postgres = { + enabled = var.postgres_enabled + } + redis = { + enabled = var.redis_enabled + } + database = { + jdbcUrl = var.database_jdbc_url + sqlalchemyUrl = var.database_sqlalchemy_url + username = var.database_username + password = var.database_password + } + redisExternal = { + host = var.redis_host + port = var.redis_port + url = var.redis_url + } + otel = { + enabled = var.otel_enabled + exporterEndpoint = var.otel_exporter_endpoint + } + ingress = { + enabled = var.ingress_enabled + hosts = [ + { + host = var.ingress_host + paths = [ + { + path = "/" + pathType = "Prefix" + } + ] + } + ] + } + }) + ], + [for f in var.extra_helm_values : file(f)] + ) + + depends_on = [ + kubernetes_namespace_v1.agentflow, + helm_release.keda, + ] +} diff --git a/deploy/terraform/outputs.tf b/deploy/terraform/outputs.tf new file mode 100644 index 0000000..19f0806 --- /dev/null +++ b/deploy/terraform/outputs.tf @@ -0,0 +1,17 @@ +output "namespace" { + value = var.namespace +} + +output "release_name" { + value = helm_release.agentflow.name +} + +output "api_service" { + description = "Kubernetes Service for the Java API (chart fullname + '-api')." + value = helm_release.agentflow.name +} + +output "worker_scaledobject" { + description = "KEDA ScaledObject name when queue-delay autoscaling is enabled." + value = var.autoscaling_enabled ? "${helm_release.agentflow.name}-worker" : null +} diff --git a/deploy/terraform/terraform.tfvars.example b/deploy/terraform/terraform.tfvars.example new file mode 100644 index 0000000..bb22614 --- /dev/null +++ b/deploy/terraform/terraform.tfvars.example @@ -0,0 +1,29 @@ +# Copy to terraform.tfvars (not committed) and fill in image registries / DB URLs. +namespace = "agentflow" +install_keda = true +autoscaling_enabled = true + +api_image_repository = "ghcr.io/example/agentflow-api" +api_image_tag = "0.1.0" +worker_image_repository = "ghcr.io/example/agentflow-worker" +worker_image_tag = "0.1.0" + +postgres_enabled = false +redis_enabled = false + +database_jdbc_url = "jdbc:postgresql://postgres.example:5432/agentflow" +database_sqlalchemy_url = "postgresql+asyncpg://agentflow:changeme@postgres.example:5432/agentflow" +database_username = "agentflow" +database_password = "changeme" + +redis_host = "redis.example" +redis_port = 6379 +redis_url = "redis://redis.example:6379/0" + +otel_enabled = true +otel_exporter_endpoint = "http://otel-collector.observability.svc.cluster.local:4318" +prometheus_server_address = "http://prometheus.observability.svc.cluster.local:9090" + +target_consumer_delay_seconds = 30 +worker_min_replicas = 1 +worker_max_replicas = 10 diff --git a/deploy/terraform/variables.tf b/deploy/terraform/variables.tf new file mode 100644 index 0000000..ad1ccce --- /dev/null +++ b/deploy/terraform/variables.tf @@ -0,0 +1,161 @@ +variable "kubeconfig_path" { + type = string + description = "Path to a kubeconfig that can install the AgentFlow chart." + default = "~/.kube/config" +} + +variable "kubeconfig_context" { + type = string + description = "Optional kubeconfig context. Empty uses the current context." + default = null +} + +variable "namespace" { + type = string + description = "Kubernetes namespace for AgentFlow." + default = "agentflow" +} + +variable "create_namespace" { + type = bool + default = true +} + +variable "release_name" { + type = string + default = "agentflow" +} + +variable "install_keda" { + type = bool + description = "Install KEDA so workers can scale on Prometheus queue delay." + default = true +} + +variable "keda_namespace" { + type = string + default = "keda" +} + +variable "api_image_repository" { + type = string + default = "agentflow-api" +} + +variable "api_image_tag" { + type = string + default = "0.1.0" +} + +variable "worker_image_repository" { + type = string + default = "agentflow-worker" +} + +variable "worker_image_tag" { + type = string + default = "0.1.0" +} + +variable "postgres_enabled" { + type = bool + description = "Deploy the chart's demo Postgres StatefulSet. Prefer a managed database in production." + default = false +} + +variable "redis_enabled" { + type = bool + description = "Deploy the chart's demo Redis. Prefer a managed Redis in production." + default = false +} + +variable "database_jdbc_url" { + type = string + default = "jdbc:postgresql://postgres:5432/agentflow" +} + +variable "database_sqlalchemy_url" { + type = string + default = "postgresql+asyncpg://agentflow:agentflow@postgres:5432/agentflow" + sensitive = true +} + +variable "database_username" { + type = string + default = "agentflow" +} + +variable "database_password" { + type = string + default = "" + sensitive = true +} + +variable "redis_host" { + type = string + default = "redis" +} + +variable "redis_port" { + type = number + default = 6379 +} + +variable "redis_url" { + type = string + default = "redis://redis:6379/0" + sensitive = true +} + +variable "otel_enabled" { + type = bool + default = true +} + +variable "otel_exporter_endpoint" { + type = string + default = "http://otel-collector.observability.svc.cluster.local:4318" +} + +variable "prometheus_server_address" { + type = string + description = "Prometheus base URL used by the KEDA queue-delay trigger." + default = "http://prometheus.observability.svc.cluster.local:9090" +} + +variable "worker_min_replicas" { + type = number + default = 1 +} + +variable "worker_max_replicas" { + type = number + default = 10 +} + +variable "target_consumer_delay_seconds" { + type = number + description = "Scale out when agentflow_queue_consumer_delay meets or exceeds this many seconds." + default = 30 +} + +variable "autoscaling_enabled" { + type = bool + default = true +} + +variable "ingress_enabled" { + type = bool + default = false +} + +variable "ingress_host" { + type = string + default = "agentflow.local" +} + +variable "extra_helm_values" { + type = list(string) + description = "Additional Helm -f values files (absolute or module-relative)." + default = [] +} diff --git a/deploy/terraform/versions.tf b/deploy/terraform/versions.tf new file mode 100644 index 0000000..63c0ffb --- /dev/null +++ b/deploy/terraform/versions.tf @@ -0,0 +1,26 @@ +terraform { + required_version = ">= 1.5.0" + + required_providers { + helm = { + source = "hashicorp/helm" + version = "~> 2.17" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.35" + } + } +} + +provider "kubernetes" { + config_path = var.kubeconfig_path + config_context = var.kubeconfig_context +} + +provider "helm" { + kubernetes { + config_path = var.kubeconfig_path + config_context = var.kubeconfig_context + } +} diff --git a/docker/prometheus/alerts.yml b/docker/prometheus/alerts.yml index a46f0aa..123ffb9 100644 --- a/docker/prometheus/alerts.yml +++ b/docker/prometheus/alerts.yml @@ -80,6 +80,17 @@ groups: - name: agentflow_reliability rules: + - alert: AgentFlowQueueConsumerDelayHigh + expr: agentflow_queue_consumer_delay > 30 + for: 5m + labels: + severity: warning + annotations: + summary: Run-job queue consumer delay is high + description: > + Oldest lagging or pending job has waited {{ $value | humanizeDuration }} + (threshold 30s). Worker replicas should scale out via KEDA/HPA. + - alert: AgentFlowDlqGrowing expr: agentflow_queue_dlq_length > 0 for: 10m diff --git a/docs/api-contract.md b/docs/api-contract.md index 7b76f20..2bad5a7 100644 --- a/docs/api-contract.md +++ b/docs/api-contract.md @@ -148,7 +148,8 @@ Response: one `Thread`. 404 if missing or not visible (including cross-tenant). Merged transcript across all Runs in the thread, ordered by run time then message index. Each item includes `run_id` plus the usual Message fields. -`cursor` is an opaque string for older pages. 404 if the thread is not visible. +The first response is the newest page; `next_cursor` is an opaque +`run.created_at|index|id` key for older pages. 404 if the thread is not visible. ### `GET /v1/threads/{id}/runs?limit=50` → 200 diff --git a/docs/deployment.md b/docs/deployment.md index aaaff2a..15863c3 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -94,9 +94,10 @@ docker compose --profile observability --profile app up --build export AGENTFLOW_OTEL_ENABLED=true ``` -Prometheus scrapes the OTel collector on `:8889` and evaluates p95 latency rules in -`docker/prometheus/alerts.yml` (worker job p95 > 60s, HTTP p95 > 5s, both sustained -for 5 minutes). View firing alerts at `http://localhost:9090/alerts`. +Prometheus scrapes the OTel collector on `:8889` and evaluates rules in +`docker/prometheus/alerts.yml` (worker job p95 > 60s, HTTP p95 > 5s, queue +consumer delay > 30s, all sustained for 5 minutes). View firing alerts at +`http://localhost:9090/alerts`. Import `docker/grafana/dashboards/agentflow-observability.json` into Grafana (or add a Grafana service to compose) for worker utilization, queue depth, consumer delay, and @@ -125,6 +126,43 @@ worker. Scale workers horizontally by running additional `worker` containers Health check: `GET /v1/health` should return `{"status":"ok",...}`. +## Helm + Terraform + +The Compose stack is the local reference. Production packaging lives under +`deploy/`: + +| Path | Role | +| --- | --- | +| `deploy/helm/agentflow` | Chart: Java API, Python workers, Alembic migrate, optional in-cluster Postgres/Redis | +| `deploy/terraform` | Installs the chart (and optionally [KEDA](https://keda.sh)) against an existing cluster | + +Worker autoscaling is driven by the Prometheus gauge +`agentflow_queue_consumer_delay` (OTel `agentflow.queue.consumer_delay`, seconds +of wait on the oldest lagging or pending Redis Stream job). Default trigger: +KEDA `ScaledObject` with `max(agentflow_queue_consumer_delay) or vector(0)` and +`targetConsumerDelaySeconds: 30`. Set `worker.autoscaling.provider: hpa` only +when Prometheus Adapter already publishes that name as an External metric. + +Workers must run with `AGENTFLOW_OTEL_ENABLED=true` (chart `otel.enabled`) and +Prometheus must scrape the OTel collector, or the scaler sees `vector(0)` and +stays at `minReplicas`. + +```bash +# Dev cluster (in-chart Postgres/Redis, autoscaling off — no KEDA required) +helm upgrade --install agentflow deploy/helm/agentflow \ + -f deploy/helm/agentflow/values-dev.yaml \ + --set api.image.repository=... \ + --set worker.image.repository=... + +# Production: Terraform applies the chart + KEDA +cd deploy/terraform +cp terraform.tfvars.example terraform.tfvars +terraform init +terraform apply +``` + +`helm lint deploy/helm/agentflow` and `terraform validate` are covered in CI. + ## Startup order 1. Postgres and Redis @@ -150,9 +188,10 @@ GitHub Actions job `integration` (see `.github/workflows/ci.yml`) runs: `AGENTFLOW_REDIS_QUEUE_IMPL=streams`). Switching to the LIST protocol requires rolling API and workers together and clearing the old key — see [api-contract.md](api-contract.md). -- **Horizontal scale:** Add worker replicas; keep a single Java API tier (or - put it behind a load balancer — SSE subscribers stick to one instance unless - you add shared pub/sub bridging). +- **Horizontal scale:** Add worker replicas (Helm `worker.replicaCount`, or KEDA + on `agentflow_queue_consumer_delay`). Keep a single Java API tier (or put it + behind a load balancer — SSE subscribers stick to one instance unless you add + shared pub/sub bridging). - **Backups:** Postgres holds all durable state; Redis is ephemeral coordination. - **Logs:** Java API logs Spring Boot output; worker logs structlog from `app.worker`. Look for `queue.metrics` (baseline depth/lag), `queue.consumer_delay_alert` diff --git a/docs/plan.md b/docs/plan.md index 245cbcc..83e036f 100644 --- a/docs/plan.md +++ b/docs/plan.md @@ -66,7 +66,7 @@ - [ ] cancel/resume 审计 - [ ] 人工审批 UI(`waiting_human` + 通知) - [x] Temporal(或 Restate)集成超长 Run -- [ ] Helm + Terraform;按队列延迟自动扩缩 worker +- [x] Helm + Terraform;按队列延迟自动扩缩 worker - [ ] Agent 级 token/成本配额 **验收:** 双租户演示;审批门控生效;24h+ 工作流在 worker 重启后仍可恢复。