From 3f86fac8e02af62857345a0fa609cd42a1a1d8ab Mon Sep 17 00:00:00 2001 From: Pranav Janakiraman Date: Tue, 16 Jun 2026 20:16:33 -0700 Subject: [PATCH 1/3] Update Google ADK package README --- google-adk/README.md | 345 +++++++++++++++++++++++++++++++------- google-adk/pyproject.toml | 2 +- 2 files changed, 286 insertions(+), 61 deletions(-) diff --git a/google-adk/README.md b/google-adk/README.md index d00f5e9..c9cbb27 100644 --- a/google-adk/README.md +++ b/google-adk/README.md @@ -1,6 +1,16 @@ -# TinyFish Web Agent — Google ADK Integration +# tinyfish-adk -Automate any website using natural language with [TinyFish Web Agent](https://tinyfish.ai), integrated as function tools for [Google Agent Development Kit (ADK)](https://google.github.io/adk-docs/). +[![PyPI version](https://img.shields.io/pypi/v/tinyfish-adk)](https://pypi.org/project/tinyfish-adk/) +[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) + +[TinyFish](https://tinyfish.ai) gives AI agents every level of web access through one platform. Four primitives, each built for a different layer: + +- **Search**: live web results, structured and ready for LLM consumption. Free. +- **Fetch**: renders pages in a real browser, returns clean markdown, HTML, or JSON. Token-efficient for LLM pipelines. Free. +- **Web Agent**: autonomous agents that navigate, authenticate, and extract from real websites. Uses credits. +- **Browser**: remote Chromium sessions with full CDP access. Bring your own Playwright or Puppeteer scripts. Uses credits. + +This package wraps all four as function tools for [Google Agent Development Kit (ADK)](https://google.github.io/adk-docs/). ADK auto-discovers tool schemas from function signatures and docstrings. [Get your API key ->](https://agent.tinyfish.ai/api-keys) ## Installation @@ -8,117 +18,332 @@ Automate any website using natural language with [TinyFish Web Agent](https://ti pip install tinyfish-adk ``` -Or install from source: - ```bash -git clone https://github.com/tinyfish-io/tinyfish-web-agent-integrations.git -cd tinyfish-web-agent-integrations/google-adk -pip install -e . +export TINYFISH_API_KEY="your-api-key" +export GOOGLE_API_KEY="your-google-api-key" ``` -## Setup +`tinyfish-adk` automatically tags SDK requests with `TF_API_INTEGRATION=google-adk`. You do not need to set this yourself. -Get your API key at [agent.tinyfish.ai/api-keys](https://agent.tinyfish.ai/api-keys): +## Tools -```bash -export TINYFISH_API_KEY="your-api-key" -export GOOGLE_API_KEY="your-gemini-key" +All seven tools can be imported and passed directly to an ADK `Agent`: + +```python +from tinyfish_adk import ( + tinyfish_search, + tinyfish_fetch, + tinyfish_web_agent, + tinyfish_queue_run, + tinyfish_get_run, + tinyfish_list_runs, + tinyfish_create_browser_session, +) ``` -`tinyfish-adk` automatically tags TinyFish SDK requests as originating from -`google-adk`. You do not need to set `TF_API_INTEGRATION` yourself. +Search, Fetch, and Browser Session return JSON strings produced from TinyFish SDK responses. Web Agent returns a JSON string for completed results, or a human-readable status/error message. Run-management tools return formatted status text for agent consumption. -## Tools +### Search + +Search the web and get structured results with titles, snippets, and URLs. Free, no credits required. -| Tool | Description | -|------|-------------| -| `tinyfish_web_agent` | Run a browser automation synchronously. Best for quick tasks. | -| `tinyfish_queue_run` | Start an automation asynchronously. Returns a `run_id`. | -| `tinyfish_get_run` | Check status and get results of a run by `run_id`. | -| `tinyfish_list_runs` | List recent automation runs, optionally filtered by status. | -| `tinyfish_search` | Search the web and return structured results. | -| `tinyfish_fetch` | Fetch clean content from one or more URLs. | -| `tinyfish_create_browser_session` | Create a remote browser session and return connection URLs. | +```python +from tinyfish_adk import tinyfish_search + +result = tinyfish_search( + query="latest LLM benchmarks 2025", + location="US", +) +print(result) +``` -## Usage +Supported inputs: -### Basic agent +| Parameter | Required | Description | +|-----------|----------|-------------| +| `query` | Yes | Search query | +| `location` | No | Optional location/country scope, such as `"US"` | +| `language` | No | Optional language code, such as `"en"` | + +### Fetch + +Extract clean content from up to 10 URLs at once. Returns markdown, HTML, or JSON. Free, no credits required. + +```python +from tinyfish_adk import tinyfish_fetch + +result = tinyfish_fetch( + urls=["https://docs.tinyfish.ai"], + format="markdown", + links=True, +) +print(result) +``` + +Supported inputs: + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `urls` | Yes | List of 1-10 URLs | +| `format` | No | `"markdown"`, `"html"`, or `"json"`; defaults to `"markdown"` | +| `links` | No | Include extracted page links | +| `image_links` | No | Include extracted image links | + +### Web Agent + +Run complex, goal-oriented tasks on live websites. TinyFish handles navigation, anti-bot protection, and returns structured JSON. Uses credits. ```python -from google.adk import Agent from tinyfish_adk import tinyfish_web_agent -agent = Agent( +result = tinyfish_web_agent( + url="https://finance.yahoo.com/quote/NVDA/", + goal="Extract the current stock price of NVIDIA", +) +print(result) +``` + +Supported inputs: + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `url` | Yes | Target website URL; use `http://` or `https://` | +| `goal` | Yes | Natural-language instructions for what to do and what to return | +| `browser_profile` | No | `"lite"` (default, fast) or `"stealth"` (anti-detection for bot-protected sites) | +| `proxy_country` | No | Route through a proxy: `US`, `GB`, `CA`, `DE`, `FR`, `JP`, `AU`. Empty for no proxy | + +### Queue Run + +Start an automation asynchronously and get a `run_id` back. Best for long-running tasks. Uses credits. + +```python +from tinyfish_adk import tinyfish_queue_run + +result = tinyfish_queue_run( + url="https://example.com/products", + goal="Extract all product names, prices, and ratings as JSON", +) +print(result) +# Returns: "Automation started. run_id: " +``` + +Supported inputs: + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `url` | Yes | Target website URL | +| `goal` | Yes | Natural-language instructions for what to do | +| `browser_profile` | No | `"lite"` (default) or `"stealth"` | +| `proxy_country` | No | Optional proxy country code | + +### Get Run + +Check status and retrieve results of a run by `run_id`. + +```python +from tinyfish_adk import tinyfish_get_run + +result = tinyfish_get_run(run_id="your-run-id") +print(result) +# Returns status, live view URL, result data, and/or error details +``` + +Supported inputs: + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `run_id` | Yes | The unique run ID to look up | + +### List Runs + +List recent automation runs, optionally filtered by status. + +```python +from tinyfish_adk import tinyfish_list_runs + +result = tinyfish_list_runs(status="COMPLETED", limit=10) +print(result) +``` + +Supported inputs: + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `status` | No | Filter by status: `PENDING`, `RUNNING`, `COMPLETED`, `FAILED`, or `CANCELLED`. Empty for all | +| `limit` | No | Maximum number of runs to return; 1-100, default 20 | + +### Browser Session + +Launch a remote Chromium browser and get a CDP (Chrome DevTools Protocol) URL to connect with Playwright or Puppeteer. Sessions include remote browser infrastructure for direct CDP control. Uses credits. + +```python +from tinyfish_adk import tinyfish_create_browser_session + +result = tinyfish_create_browser_session( + url="https://example.com", + timeout_seconds=300, +) +print(result) +# Returns a JSON string with: {"session_id": "...", "cdp_url": "wss://...", "base_url": "https://..."} +``` + +Supported inputs: + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `url` | No | Optional URL to open when the session starts | +| `timeout_seconds` | No | Optional inactivity timeout in seconds. Use 0 for plan default | + +## With an ADK Agent + +Give your agent access to TinyFish tools. The agent decides which primitive to use based on the task. + +```python +from google.adk.agents import LlmAgent +from google.adk.runners import Runner +from google.adk.sessions import InMemorySessionService +from google.genai import types +from tinyfish_adk import tinyfish_search, tinyfish_fetch, tinyfish_web_agent + +agent = LlmAgent( name="web_researcher", - model="gemini-3.5-flash", - instruction="Use tinyfish_web_agent to browse and extract data from websites.", - tools=[tinyfish_web_agent], + model="gemini-flash-latest", + instruction="Use TinyFish tools to search the web, fetch page content, and automate websites.", + tools=[tinyfish_search, tinyfish_fetch, tinyfish_web_agent], +) + +APP_NAME = "tinyfish_app" +USER_ID = "user1" +SESSION_ID = "session1" + +session_service = InMemorySessionService() +session_service.create_session( + app_name=APP_NAME, + user_id=USER_ID, + session_id=SESSION_ID, ) +runner = Runner( + agent=agent, + app_name=APP_NAME, + session_service=session_service, +) + +content = types.Content( + role="user", + parts=[ + types.Part( + text="Find the top 3 results for 'best open source LLMs' and fetch the full content of the first result" + ) + ], +) +events = runner.run( + user_id=USER_ID, + session_id=SESSION_ID, + new_message=content, +) + +for event in events: + if event.is_final_response() and event.content: + print(event.content.parts[0].text) ``` -### Async workflow +### Async Workflow Agent + +For long-running tasks, give the agent queue and poll tools so it can start automations and check back for results: ```python -from google.adk import Agent -from tinyfish_adk import tinyfish_queue_run, tinyfish_get_run +from google.adk.agents import LlmAgent +from tinyfish_adk import tinyfish_queue_run, tinyfish_get_run, tinyfish_list_runs -agent = Agent( +agent = LlmAgent( name="async_scraper", - model="gemini-3.5-flash", - instruction="Queue long-running scrapes and poll for results.", - tools=[tinyfish_queue_run, tinyfish_get_run], + model="gemini-flash-latest", + instruction="Queue long-running scrapes with tinyfish_queue_run and poll for results with tinyfish_get_run.", + tools=[tinyfish_queue_run, tinyfish_get_run, tinyfish_list_runs], ) ``` -### All tools +### All Tools ```python +from google.adk.agents import LlmAgent from tinyfish_adk import ( + tinyfish_search, + tinyfish_fetch, tinyfish_web_agent, tinyfish_queue_run, tinyfish_get_run, tinyfish_list_runs, - tinyfish_search, - tinyfish_fetch, tinyfish_create_browser_session, ) -agent = Agent( +agent = LlmAgent( name="web_automation_agent", - model="gemini-3.5-flash", + model="gemini-flash-latest", + instruction="Use TinyFish tools to search, fetch, automate, and control browsers.", tools=[ + tinyfish_search, + tinyfish_fetch, tinyfish_web_agent, tinyfish_queue_run, tinyfish_get_run, tinyfish_list_runs, - tinyfish_search, - tinyfish_fetch, tinyfish_create_browser_session, ], ) ``` +## Stealth Mode and Proxies + +For Web Agent runs on sites with bot protection (Cloudflare, CAPTCHAs, etc.), pass stealth and proxy options directly to the tool: + +```python +from tinyfish_adk import tinyfish_web_agent + +result = tinyfish_web_agent( + url="https://protected-site.com/data", + goal="Extract all pricing tiers as JSON", + browser_profile="stealth", + proxy_country="US", # Also: GB, CA, DE, FR, JP, AU +) +``` + +`browser_profile` and `proxy_country` are accepted by `tinyfish_web_agent` and `tinyfish_queue_run`. Search, Fetch, Get Run, List Runs, and Browser Session tools do not use these parameters. + ## Configuration -All tools read `TINYFISH_API_KEY` from the environment. The package also sets -`TF_API_INTEGRATION=google-adk` internally so requests are attributed -automatically. Each tool also accepts optional parameters: +All tools read `TINYFISH_API_KEY` from the environment. Web Agent tools accept additional parameters: -| Parameter | Description | -|-----------|-------------| -| `browser_profile` | `"lite"` (default, fast) or `"stealth"` (anti-detection) | -| `proxy_country` | Route through a proxy: `US`, `GB`, `CA`, `DE`, `FR`, `JP`, `AU` | +| Parameter | Default | Description | +|-----------|---------|-------------| +| `browser_profile` | `"lite"` | Web Agent browser profile: `"lite"` or `"stealth"` | +| `proxy_country` | `""` (disabled) | Proxy exit country for Web Agent runs: `US`, `GB`, `CA`, `DE`, `FR`, `JP`, `AU` | -## Example goals +## Development -```text -"Extract all product names, prices, and ratings from this page" -"Fill the contact form with name 'Jane Doe' and email 'jane@example.com', then submit" -"Click 'Next Page' 3 times, extracting all listings from each page" +```bash +# Install package + dev dependencies +pip install -e . +pip install -r requirements-dev.txt + +# Run tests +make test + +# Run linter +make lint + +# Format code +make format ``` +## Resources + +- [TinyFish Documentation](https://docs.tinyfish.ai) +- [TinyFish Website](https://tinyfish.ai) +- [API Keys](https://agent.tinyfish.ai/api-keys) +- [Google ADK Documentation](https://google.github.io/adk-docs/) +- [Discord Community](https://discord.com/invite/tinyfish) + ## Support -- [TinyFish Docs](https://docs.tinyfish.ai) -- [Google ADK Docs](https://google.github.io/adk-docs/) -- [Discord](https://discord.gg/agentql) +Questions or issues? Reach out at [support@tinyfish.ai](mailto:support@tinyfish.ai) or join our [Discord](https://discord.com/invite/tinyfish). diff --git a/google-adk/pyproject.toml b/google-adk/pyproject.toml index 3f5ca63..2da1f31 100644 --- a/google-adk/pyproject.toml +++ b/google-adk/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tinyfish-adk" -version = "0.1.2" +version = "0.1.3" description = "TinyFish Web Agent tools for Google Agent Development Kit (ADK)" readme = "README.md" license = { text = "MIT" } From 1e0c65906f808cc3e22d6405033a078f9895b7a8 Mon Sep 17 00:00:00 2001 From: Pranav Janakiraman Date: Fri, 26 Jun 2026 18:56:53 -0700 Subject: [PATCH 2/3] Add CrewAI TinyFish integration Add `crewai-tinyfish`: official TinyFish tools for CrewAI agents, wrapping the Search, Fetch, Agent, and Browser API surfaces as CrewAI BaseTools. - src/crewai_tinyfish: four tools with lazy SDK import, version-skew-tolerant kwargs, and TF_API_INTEGRATION="crewai" request attribution - tests: fully offline (SDK faked), 11 passing - examples: direct-tool quickstart and a Search+Fetch research crew - CI + CD workflows mirroring the langchain/google-adk packages (lint/test/build on PR, version-gated PyPI publish on merge to main) - README in the house style of the sibling integrations --- .github/workflows/crewai-ci.yml | 41 ++++ .github/workflows/crewai-publish.yml | 106 +++++++++ crewai/.env.example | 5 + crewai/.gitignore | 26 +++ crewai/LICENSE | 21 ++ crewai/Makefile | 16 ++ crewai/README.md | 208 +++++++++++++++++ crewai/UPSTREAM.md | 57 +++++ crewai/examples/quickstart.py | 16 ++ crewai/examples/research_crew.py | 69 ++++++ crewai/pyproject.toml | 62 +++++ crewai/requirements-dev.txt | 4 + crewai/src/crewai_tinyfish/__init__.py | 30 +++ crewai/src/crewai_tinyfish/_client.py | 78 +++++++ crewai/src/crewai_tinyfish/_serde.py | 47 ++++ .../crewai_tinyfish/tinyfish_agent_tool.py | 220 ++++++++++++++++++ .../crewai_tinyfish/tinyfish_browser_tool.py | 102 ++++++++ .../crewai_tinyfish/tinyfish_fetch_tool.py | 149 ++++++++++++ .../crewai_tinyfish/tinyfish_search_tool.py | 134 +++++++++++ crewai/tests/conftest.py | 46 ++++ crewai/tests/test_agent_tool.py | 78 +++++++ crewai/tests/test_browser_tool.py | 40 ++++ crewai/tests/test_fetch_tool.py | 66 ++++++ crewai/tests/test_search_tool.py | 60 +++++ 24 files changed, 1681 insertions(+) create mode 100644 .github/workflows/crewai-ci.yml create mode 100644 .github/workflows/crewai-publish.yml create mode 100644 crewai/.env.example create mode 100644 crewai/.gitignore create mode 100644 crewai/LICENSE create mode 100644 crewai/Makefile create mode 100644 crewai/README.md create mode 100644 crewai/UPSTREAM.md create mode 100644 crewai/examples/quickstart.py create mode 100644 crewai/examples/research_crew.py create mode 100644 crewai/pyproject.toml create mode 100644 crewai/requirements-dev.txt create mode 100644 crewai/src/crewai_tinyfish/__init__.py create mode 100644 crewai/src/crewai_tinyfish/_client.py create mode 100644 crewai/src/crewai_tinyfish/_serde.py create mode 100644 crewai/src/crewai_tinyfish/tinyfish_agent_tool.py create mode 100644 crewai/src/crewai_tinyfish/tinyfish_browser_tool.py create mode 100644 crewai/src/crewai_tinyfish/tinyfish_fetch_tool.py create mode 100644 crewai/src/crewai_tinyfish/tinyfish_search_tool.py create mode 100644 crewai/tests/conftest.py create mode 100644 crewai/tests/test_agent_tool.py create mode 100644 crewai/tests/test_browser_tool.py create mode 100644 crewai/tests/test_fetch_tool.py create mode 100644 crewai/tests/test_search_tool.py diff --git a/.github/workflows/crewai-ci.yml b/.github/workflows/crewai-ci.yml new file mode 100644 index 0000000..bba16f2 --- /dev/null +++ b/.github/workflows/crewai-ci.yml @@ -0,0 +1,41 @@ +name: CrewAI CI + +on: + pull_request: + branches: [main] + paths: + - ".github/workflows/crewai-ci.yml" + - "crewai/**" + push: + branches: [main] + paths: + - ".github/workflows/crewai-ci.yml" + - "crewai/**" + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + + defaults: + run: + working-directory: crewai + + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - run: python -m pip install --upgrade pip + + - run: python -m pip install -e . -r requirements-dev.txt + + - run: make lint + + - run: make test + + - run: python -m build diff --git a/.github/workflows/crewai-publish.yml b/.github/workflows/crewai-publish.yml new file mode 100644 index 0000000..927bc39 --- /dev/null +++ b/.github/workflows/crewai-publish.yml @@ -0,0 +1,106 @@ +name: CrewAI CD - Publish to PyPI + +on: + push: + branches: [main] + paths: + - ".github/workflows/crewai-publish.yml" + - "crewai/**" + +jobs: + build: + name: Build distribution + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + version-exists: ${{ steps.version.outputs.exists }} + package-version: ${{ steps.version.outputs.version }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + + - name: Check PyPI version availability + id: version + working-directory: ./crewai + run: | + PACKAGE=$(python -c "import tomllib; f=open('pyproject.toml','rb'); d=tomllib.load(f); print(d['project']['name'])") + VERSION=$(python -c "import tomllib; f=open('pyproject.toml','rb'); d=tomllib.load(f); print(d['project']['version'])") + echo "package=${PACKAGE}" >> "$GITHUB_OUTPUT" + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + RESULT=$(python - "$PACKAGE" "$VERSION" <<'PY' + import sys + import urllib.error + import urllib.parse + import urllib.request + + package = sys.argv[1] + version = sys.argv[2] + url = f"https://pypi.org/pypi/{urllib.parse.quote(package)}/{version}/json" + try: + with urllib.request.urlopen(url, timeout=10): + pass + except urllib.error.HTTPError as exc: + if exc.code == 404: + print("missing") + sys.exit(0) + print(f"PyPI returned HTTP {exc.code} while checking {url}", file=sys.stderr) + sys.exit(2) + except Exception as exc: + print(f"Failed to check PyPI for {url}: {exc}", file=sys.stderr) + sys.exit(2) + print("exists") + PY + ) + case "$RESULT" in + exists) + echo "exists=true" >> "$GITHUB_OUTPUT" + echo "${PACKAGE} ${VERSION} already exists on PyPI; skipping publish." + ;; + missing) + echo "exists=false" >> "$GITHUB_OUTPUT" + echo "${PACKAGE} ${VERSION} is available on PyPI; building distribution." + ;; + *) + echo "Unexpected PyPI availability result: $RESULT" + exit 1 + ;; + esac + + - name: Build package + if: steps.version.outputs.exists == 'false' + working-directory: ./crewai + run: uv build + + - name: Upload distribution artifact + if: steps.version.outputs.exists == 'false' + uses: actions/upload-artifact@v7 + with: + name: python-package-distributions + path: crewai/dist/ + + publish-pypi: + name: Publish to PyPI + needs: build + if: needs.build.outputs.version-exists == 'false' + runs-on: ubuntu-latest + steps: + - name: Download distributions + uses: actions/download-artifact@v8 + with: + name: python-package-distributions + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: dist/ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/crewai/.env.example b/crewai/.env.example new file mode 100644 index 0000000..10a217c --- /dev/null +++ b/crewai/.env.example @@ -0,0 +1,5 @@ +# TinyFish API key — create one at https://agent.tinyfish.ai/api-keys +TINYFISH_API_KEY=your_api_key_here + +# An LLM key for CrewAI agents (any provider CrewAI supports). Example: +# OPENAI_API_KEY=sk-... diff --git a/crewai/.gitignore b/crewai/.gitignore new file mode 100644 index 0000000..41d1b10 --- /dev/null +++ b/crewai/.gitignore @@ -0,0 +1,26 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +build/ +dist/ +.eggs/ + +# Virtual environments +.venv/ +venv/ +env/ + +# Tooling caches +.pytest_cache/ +.ruff_cache/ +.mypy_cache/ + +# Secrets / local config +.env +.env.local + +# Editor / OS +.DS_Store +.idea/ +.vscode/ diff --git a/crewai/LICENSE b/crewai/LICENSE new file mode 100644 index 0000000..cede851 --- /dev/null +++ b/crewai/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 TinyFish + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crewai/Makefile b/crewai/Makefile new file mode 100644 index 0000000..3e27702 --- /dev/null +++ b/crewai/Makefile @@ -0,0 +1,16 @@ +.PHONY: format lint test clean + +format: + ruff format src tests + ruff check --fix src tests + +lint: + ruff check src tests + ruff format --check src tests + +test: + pytest tests -v + +clean: + rm -rf build dist *.egg-info .pytest_cache .ruff_cache + find . -type d -name __pycache__ -exec rm -rf {} + diff --git a/crewai/README.md b/crewai/README.md new file mode 100644 index 0000000..b75fe28 --- /dev/null +++ b/crewai/README.md @@ -0,0 +1,208 @@ +# crewai-tinyfish + +[![PyPI version](https://img.shields.io/pypi/v/crewai-tinyfish)](https://pypi.org/project/crewai-tinyfish/) +[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) + +[TinyFish](https://tinyfish.ai) gives AI agents every level of web access through one platform. Four primitives, each built for a different layer: + +- **Search**: live web results, structured and ready for LLM consumption. +- **Fetch**: renders pages in a real browser, returns clean markdown, HTML, or JSON. Token-efficient for LLM pipelines. +- **Web Agent**: autonomous agents that navigate, authenticate, and extract from real websites. +- **Browser**: remote Chromium sessions with full CDP access. Bring your own Playwright or Puppeteer scripts. + +This package wraps all four as CrewAI tools (`BaseTool`). [Get your API key ->](https://agent.tinyfish.ai/api-keys) + +## Installation + +```bash +pip install "crewai-tinyfish[tinyfish]" +``` + +```bash +export TINYFISH_API_KEY="your-api-key" +``` + +The `tinyfish` extra pulls in the official TinyFish Python SDK. It is kept optional so the base install stays light — the tools lazy-import it and raise a clear error if it is missing. + +## Tools + +All four tools can be used standalone or passed to a CrewAI agent: + +```python +from crewai_tinyfish import ( + TinyFishSearchTool, + TinyFishFetchTool, + TinyFishAgentTool, + TinyFishBrowserSessionTool, +) +``` + +Tool calls return strings. Successful responses are JSON strings produced from the TinyFish SDK response. + +### Search + +Search the web and get structured results with titles, snippets, and URLs. + +```python +from crewai_tinyfish import TinyFishSearchTool + +search = TinyFishSearchTool() +results = search.run(query="latest LLM benchmarks 2025", location="US") +print(results) +``` + +Supported inputs: + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `query` | Yes | Search query | +| `location` | No | Optional location/country scope, such as `"US"` | +| `language` | No | Optional language code, such as `"en"` | +| `page` | No | Zero-indexed result page for pagination (0-10) | +| `max_results` | No | Maximum number of results to return; 1-50, default 10 | + +### Fetch + +Extract clean content from up to 10 URLs at once. Returns markdown, HTML, or JSON. + +```python +from crewai_tinyfish import TinyFishFetchTool + +fetch = TinyFishFetchTool() +content = fetch.run(urls=["https://docs.tinyfish.ai"], format="markdown", links=True) +print(content) +``` + +Supported inputs: + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `urls` | Yes | List of 1-10 URLs | +| `format` | No | `"markdown"`, `"html"`, or `"json"`; defaults to `"markdown"` | +| `links` | No | Include extracted page links | +| `image_links` | No | Include extracted image links | +| `ttl` | No | Cache freshness tolerance in seconds; `0` forces a live fetch | +| `max_chars_per_url` | No | Truncate each page's extracted text to N characters | + +### Web Agent + +Run complex, goal-oriented tasks on live websites. TinyFish handles navigation, anti-bot protection, and returns structured JSON. + +```python +from crewai_tinyfish import TinyFishAgentTool + +agent_tool = TinyFishAgentTool() +result = agent_tool.run( + url="https://finance.yahoo.com/quote/NVDA/", + goal="Extract the current stock price of NVIDIA", +) +print(result) +``` + +Supported inputs: + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `url` | Yes | Target website URL; use `http://` or `https://` | +| `goal` | Yes | Natural-language instructions for what to do and what to return | +| `output_schema` | No | Optional JSON Schema constraining the structured result | +| `browser_profile` | No | `"lite"` (default, fast) or `"stealth"` (anti-detection for bot-protected sites) | +| `use_proxy` | No | Route the run through TinyFish proxy infrastructure | +| `proxy_country` | No | Proxy exit country when `use_proxy` is set: `US`, `GB`, `CA`, `DE`, `FR`, `JP`, `AU` | +| `use_profile` | No | Reuse a saved Browser Context Profile (logged-in state) | +| `profile_id` | No | Specific Browser Context Profile id; requires `use_profile` | +| `use_vault` | No | Allow TinyFish to log in using connected credentials | +| `credential_item_ids` | No | Scope the run to specific vault credentials; requires `use_vault` | + +### Browser Session + +Launch a remote Chromium browser and get a CDP (Chrome DevTools Protocol) URL to connect with Playwright or Puppeteer. Sessions include remote browser infrastructure for direct CDP control. + +```python +from crewai_tinyfish import TinyFishBrowserSessionTool + +browser = TinyFishBrowserSessionTool() +session = browser.run(url="https://example.com", timeout_seconds=300) +print(session) +# Returns a JSON string with: {"session_id": "...", "cdp_url": "wss://...", "base_url": "https://..."} +``` + +Supported inputs: + +| Parameter | Required | Description | +|-----------|----------|-------------| +| `url` | No | Optional URL to open when the session starts | +| `timeout_seconds` | No | Optional inactivity timeout in seconds (5-86400) | + +## With a CrewAI Crew + +Give your agent access to multiple TinyFish tools. The agent decides which primitive to use based on the task. + +```python +from crewai import Agent, Task, Crew, Process, LLM +from crewai_tinyfish import TinyFishSearchTool, TinyFishFetchTool, TinyFishAgentTool + +llm = LLM(model="gpt-5.5") + +researcher = Agent( + role="Web Researcher", + goal="Find and summarize current information from the web", + backstory="An analyst who grounds every answer in live sources.", + tools=[TinyFishSearchTool(), TinyFishFetchTool(), TinyFishAgentTool()], + llm=llm, +) + +task = Task( + description=( + "Find the top 3 results for 'best open source LLMs' and extract the " + "full content of the first result." + ), + expected_output="A 3-bullet summary, each with a source URL.", + agent=researcher, +) + +crew = Crew(agents=[researcher], tasks=[task], process=Process.sequential) +print(crew.kickoff()) +``` + +CrewAI resolves an LLM from your environment by default (for example, `OPENAI_API_KEY`). Pass an explicit `LLM(...)` to choose a specific model or provider — CrewAI uses LiteLLM, so any supported provider works (OpenAI, Anthropic, Google, OpenRouter, local models, and more). + +## Stealth Mode and Proxies + +For Web Agent runs on sites with bot protection (Cloudflare, CAPTCHAs, etc.), pass stealth and proxy options directly to the tool: + +```python +from crewai_tinyfish import TinyFishAgentTool + +result = TinyFishAgentTool().run( + url="https://protected-site.com/data", + goal="Extract all pricing tiers as JSON", + browser_profile="stealth", + use_proxy=True, + proxy_country="US", # Also: GB, CA, DE, FR, JP, AU +) +``` + +`browser_profile`, `use_proxy`, and `proxy_country` are used by `TinyFishAgentTool`. Search, Fetch, and Browser Session tools do not use these parameters. + +## Development + +```bash +uv venv --python 3.12 && source .venv/bin/activate +uv pip install -e ".[dev]" +pytest +``` + +Tests run fully offline — the TinyFish client is faked, so no API key or network is required. + +## Resources + +- [TinyFish Documentation](https://docs.tinyfish.ai) +- [TinyFish Website](https://tinyfish.ai) +- [API Keys](https://agent.tinyfish.ai/api-keys) +- [CrewAI Documentation](https://docs.crewai.com) +- [Discord Community](https://discord.com/invite/tinyfish) + +## Support + +Questions or issues? Reach out at [support@tinyfish.ai](mailto:support@tinyfish.ai) or join our [Discord](https://discord.com/invite/tinyfish). diff --git a/crewai/UPSTREAM.md b/crewai/UPSTREAM.md new file mode 100644 index 0000000..3742c08 --- /dev/null +++ b/crewai/UPSTREAM.md @@ -0,0 +1,57 @@ +# Contributing this integration upstream to `crewai-tools` + +This package is intentionally built to match the official +[`crewai-tools` BUILDING_TOOLS.md](https://github.com/crewAIInc/crewai-tools/blob/main/BUILDING_TOOLS.md) +conventions, so it can be contributed into the main repository with minimal changes. + +## Conventions already followed + +- Each tool subclasses `crewai.tools.BaseTool` with a Pydantic `args_schema`. +- `name` / `description` are agent-facing and explain *when* to use the tool. +- Optional SDK dependency declared via `package_dependencies = ["tinyfish"]` and + **lazy-imported** so the base install stays light. +- `env_vars = [EnvVar(name="TINYFISH_API_KEY", required=True)]`. +- Clear, graceful error strings instead of raised exceptions inside `_run`. +- `_arun` provided, delegating to `_run`. +- Tests in `tests/` mock the client (no network, no key required in CI). +- SDK calls are version-skew tolerant (`supported_kwargs`) so a param the + installed SDK doesn't accept is dropped instead of crashing the agent. + +## Mapping into the crewai-tools tree + +In `crewai-tools`, tools live at `crewai_tools/tools//`. Suggested layout: + +``` +crewai_tools/tools/tinyfish_search_tool/ + __init__.py + tinyfish_search_tool.py # from src/crewai_tinyfish/tinyfish_search_tool.py + README.md +crewai_tools/tools/tinyfish_fetch_tool/ + ... +crewai_tools/tools/tinyfish_agent_tool/ + ... +crewai_tools/tools/tinyfish_browser_tool/ + ... +``` + +Steps: + +1. Copy each `tinyfish_*_tool.py` into its own folder under `crewai_tools/tools/`. +2. Inline the helpers from `_client.py` / `_serde.py` into each tool (the upstream + repo prefers self-contained tool modules), or add them as a small shared + `crewai_tools/tools/tinyfish_*/_shared.py`. +3. Export the classes from `crewai_tools/tools/__init__.py` and + `crewai_tools/__init__.py`. +4. Register the optional dependency in the repo's `pyproject.toml` under + `[project.optional-dependencies]`: + ```toml + tinyfish = ["tinyfish>=0.2.6"] + ``` +5. Add a short `README.md` per tool (the sections in this repo's README can be split). +6. `uv run pytest` and run the repo's lint/type checks before opening the PR. + +## Standalone distribution + +Until/unless merged upstream, this repo publishes as `crewai-tinyfish` on PyPI and +is used via `from crewai_tinyfish import TinyFishSearchTool`. The import path is the +only difference from the upstream `from crewai_tools import TinyFishSearchTool`. diff --git a/crewai/examples/quickstart.py b/crewai/examples/quickstart.py new file mode 100644 index 0000000..c28cb78 --- /dev/null +++ b/crewai/examples/quickstart.py @@ -0,0 +1,16 @@ +"""Minimal direct-tool smoke test (no LLM, no Crew). + +Calls each tool once so you can confirm your TINYFISH_API_KEY works. + + export TINYFISH_API_KEY="your_api_key_here" + python examples/quickstart.py +""" + +from crewai_tinyfish import TinyFishFetchTool, TinyFishSearchTool + +if __name__ == "__main__": + print("== Search ==") + print(TinyFishSearchTool().run(query="TinyFish web agent", max_results=3)) + + print("\n== Fetch ==") + print(TinyFishFetchTool().run(urls=["https://www.tinyfish.ai/"], max_chars_per_url=400)) diff --git a/crewai/examples/research_crew.py b/crewai/examples/research_crew.py new file mode 100644 index 0000000..4f07fc9 --- /dev/null +++ b/crewai/examples/research_crew.py @@ -0,0 +1,69 @@ +"""Example: a CrewAI research crew grounded on TinyFish Search + Fetch. + +Run: + export TINYFISH_API_KEY="your_api_key_here" + export OPENAI_API_KEY="..." # or configure any CrewAI-supported LLM + python examples/research_crew.py + +CrewAI uses LiteLLM, so swap the model below for any supported provider +(Anthropic, Google, OpenRouter, local models, and more). +""" + +from crewai import LLM, Agent, Crew, Process, Task + +from crewai_tinyfish import TinyFishFetchTool, TinyFishSearchTool + + +def build_crew(topic: str) -> Crew: + search = TinyFishSearchTool() + fetch = TinyFishFetchTool() + llm = LLM(model="gpt-5.5") + + researcher = Agent( + role="Web Researcher", + goal=f"Gather accurate, current information about {topic} from live web sources.", + backstory=( + "A meticulous analyst who always searches first, then reads the most promising " + "sources in full before drawing conclusions. Never invents facts." + ), + tools=[search, fetch], + llm=llm, + verbose=True, + ) + + writer = Agent( + role="Briefing Writer", + goal="Turn research notes into a crisp, well-cited briefing.", + backstory="A writer who values clarity and always attributes claims to a source URL.", + llm=llm, + verbose=True, + ) + + research_task = Task( + description=( + f"Research '{topic}'. Use TinyFish Web Search to find the best sources, then use " + "TinyFish Web Fetch to read the most relevant 2-3 pages in full. " + "Collect the key facts with their source URLs." + ), + expected_output="A bullet list of key findings, each with a source URL.", + agent=researcher, + ) + + write_task = Task( + description="Write a 5-bullet executive briefing from the research findings.", + expected_output="A 5-bullet briefing; every bullet cites a source URL.", + agent=writer, + context=[research_task], + ) + + return Crew( + agents=[researcher, writer], + tasks=[research_task, write_task], + process=Process.sequential, + verbose=True, + ) + + +if __name__ == "__main__": + crew = build_crew("the latest TinyFish product announcements") + print(crew.kickoff()) diff --git a/crewai/pyproject.toml b/crewai/pyproject.toml new file mode 100644 index 0000000..4028b8b --- /dev/null +++ b/crewai/pyproject.toml @@ -0,0 +1,62 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "crewai-tinyfish" +version = "0.1.0" +description = "Official TinyFish tools for CrewAI — Search, Fetch, Agent, and Browser web infrastructure for AI agents." +readme = "README.md" +requires-python = ">=3.10,<3.14" +license = { text = "MIT" } +authors = [{ name = "TinyFish", email = "support@tinyfish.io" }] +keywords = ["crewai", "crewai-tools", "tinyfish", "web-agent", "search", "fetch", "browser-automation", "ai-agents"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Libraries", + "Topic :: Internet :: WWW/HTTP :: Browsers", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] + +dependencies = [ + "crewai>=0.100.0", + "pydantic>=2.4.2", +] + +[project.optional-dependencies] +# TinyFish SDK is an optional extra so the base install stays light (lazy imports). +tinyfish = ["tinyfish>=0.2.5"] +# Direct browser control over the CDP url returned by TinyFishBrowserSessionTool. +browser = ["playwright>=1.40.0"] +dev = [ + "tinyfish>=0.2.5", + "pytest>=8.0.0", + "pytest-asyncio>=0.23.0", +] + +[project.urls] +Homepage = "https://www.tinyfish.ai" +Documentation = "https://docs.tinyfish.ai" +Repository = "https://github.com/tinyfish-io/tinyfish-web-agent-integrations" +"TinyFish API Keys" = "https://agent.tinyfish.ai/api-keys" + +[tool.hatch.build.targets.wheel] +packages = ["src/crewai_tinyfish"] + +[tool.ruff] +target-version = "py310" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "F", "I", "T201", "W"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] diff --git a/crewai/requirements-dev.txt b/crewai/requirements-dev.txt new file mode 100644 index 0000000..2d0f9fb --- /dev/null +++ b/crewai/requirements-dev.txt @@ -0,0 +1,4 @@ +build>=1.0,<2.0 +pytest>=8.0,<9.0 +pytest-asyncio>=0.23,<1.0 +ruff>=0.4,<1.0 diff --git a/crewai/src/crewai_tinyfish/__init__.py b/crewai/src/crewai_tinyfish/__init__.py new file mode 100644 index 0000000..e2a01a9 --- /dev/null +++ b/crewai/src/crewai_tinyfish/__init__.py @@ -0,0 +1,30 @@ +"""crewai-tinyfish: official TinyFish tools for CrewAI agents. + +Exposes one CrewAI ``BaseTool`` per TinyFish API surface: + +- :class:`TinyFishSearchTool` — Search API: structured ranked web results. +- :class:`TinyFishFetchTool` — Fetch API: clean extracted page content. +- :class:`TinyFishAgentTool` — Agent API: goal-based web automation on live sites. +- :class:`TinyFishBrowserSessionTool` — Browser API: remote CDP browser sessions. + +All tools read the ``TINYFISH_API_KEY`` environment variable (get one at +https://agent.tinyfish.ai/api-keys). +""" + +from .tinyfish_agent_tool import TinyFishAgentTool, TinyFishAgentToolInput +from .tinyfish_browser_tool import TinyFishBrowserSessionTool, TinyFishBrowserSessionToolInput +from .tinyfish_fetch_tool import TinyFishFetchTool, TinyFishFetchToolInput +from .tinyfish_search_tool import TinyFishSearchTool, TinyFishSearchToolInput + +__version__ = "0.1.0" + +__all__ = [ + "TinyFishSearchTool", + "TinyFishSearchToolInput", + "TinyFishFetchTool", + "TinyFishFetchToolInput", + "TinyFishAgentTool", + "TinyFishAgentToolInput", + "TinyFishBrowserSessionTool", + "TinyFishBrowserSessionToolInput", +] diff --git a/crewai/src/crewai_tinyfish/_client.py b/crewai/src/crewai_tinyfish/_client.py new file mode 100644 index 0000000..a10f811 --- /dev/null +++ b/crewai/src/crewai_tinyfish/_client.py @@ -0,0 +1,78 @@ +"""Shared helpers for the TinyFish CrewAI tools. + +Centralizes the optional-dependency handling, environment-variable checks, and +SDK client construction so every tool stays small and consistent. TinyFish is +kept an *optional* dependency (lazy import) per the crewai-tools conventions, so +the base install stays light and the error message is actionable. +""" + +from __future__ import annotations + +import os +from typing import Any, Optional + +#: Environment variable the TinyFish SDK reads automatically. +TINYFISH_API_KEY_ENV = "TINYFISH_API_KEY" + +#: Integration tag the TinyFish SDK forwards for request attribution. +TF_API_INTEGRATION_ENV = "TF_API_INTEGRATION" +_INTEGRATION_NAME = "crewai" + +_INSTALL_HINT = ( + "Missing optional dependency 'tinyfish'. Install it with:\n" + " uv add crewai-tinyfish --extra tinyfish\n" + "or\n" + " pip install tinyfish\n" +) + +_MISSING_KEY_HINT = ( + f"Environment variable {TINYFISH_API_KEY_ENV} is required. " + "Create a key at https://agent.tinyfish.ai/api-keys and export it:\n" + f' export {TINYFISH_API_KEY_ENV}="your_api_key_here"' +) + + +def ensure_integration_tag() -> None: + """Tag SDK requests as originating from this integration (for attribution). + + Uses ``setdefault`` so an explicit ``TF_API_INTEGRATION`` set by the caller + always wins. + """ + os.environ.setdefault(TF_API_INTEGRATION_ENV, _INTEGRATION_NAME) + + +def ensure_sdk_installed() -> None: + """Raise an actionable ImportError if the ``tinyfish`` SDK is missing.""" + try: + import tinyfish # noqa: F401 + except ImportError as exc: # pragma: no cover - exercised via install hint + raise ImportError(_INSTALL_HINT) from exc + + +def ensure_api_key(api_key: Optional[str] = None) -> None: + """Raise a clear ValueError if no API key is available. + + A key is considered available if it is passed explicitly (``api_key``) or + present in the ``TINYFISH_API_KEY`` environment variable. + """ + if api_key: + return + if not os.environ.get(TINYFISH_API_KEY_ENV): + raise ValueError(_MISSING_KEY_HINT) + + +def build_client(api_key: Optional[str] = None) -> Any: + """Construct an authenticated ``tinyfish.TinyFish`` client. + + The SDK reads ``TINYFISH_API_KEY`` from the environment automatically; an + explicit ``api_key`` takes precedence when provided. + """ + ensure_sdk_installed() + ensure_api_key(api_key) + ensure_integration_tag() + + from tinyfish import TinyFish # local import keeps base install light + + if api_key: + return TinyFish(api_key=api_key) + return TinyFish() diff --git a/crewai/src/crewai_tinyfish/_serde.py b/crewai/src/crewai_tinyfish/_serde.py new file mode 100644 index 0000000..c284c8e --- /dev/null +++ b/crewai/src/crewai_tinyfish/_serde.py @@ -0,0 +1,47 @@ +"""Small helpers to read fields off TinyFish SDK responses or plain dicts. + +The TinyFish SDK returns typed objects, but tests (and older/newer SDK builds) +may hand back dicts. These accessors tolerate both so the tools stay robust. +""" + +from __future__ import annotations + +import inspect +from typing import Any, Callable, Dict + + +def attr(obj: Any, key: str, default: Any = None) -> Any: + """Read ``key`` from a dict or an object attribute.""" + if isinstance(obj, dict): + return obj.get(key, default) + return getattr(obj, key, default) + + +def supported_kwargs(func: Callable[..., Any], params: Dict[str, Any]) -> Dict[str, Any]: + """Drop kwargs the target callable doesn't accept (guards against SDK version skew). + + If the callable accepts ``**kwargs`` we pass everything through; otherwise we keep + only the names present in its signature. If the signature can't be read, pass through. + """ + try: + sig = inspect.signature(func) + except (TypeError, ValueError): + return dict(params) + + if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in sig.parameters.values()): + return dict(params) + + allowed = set(sig.parameters) + return {k: v for k, v in params.items() if k in allowed} + + +def as_list(value: Any) -> list: + """Coerce ``value`` into a list (None -> []).""" + if value is None: + return [] + if isinstance(value, list): + return value + try: + return list(value) + except TypeError: + return [value] diff --git a/crewai/src/crewai_tinyfish/tinyfish_agent_tool.py b/crewai/src/crewai_tinyfish/tinyfish_agent_tool.py new file mode 100644 index 0000000..120b3f3 --- /dev/null +++ b/crewai/src/crewai_tinyfish/tinyfish_agent_tool.py @@ -0,0 +1,220 @@ +"""TinyFish Agent tool for CrewAI. + +Wraps the TinyFish Agent API (``POST https://agent.tinyfish.ai/v1/automation/run``), +which takes a natural-language goal and autonomously drives a real browser on a +target site to complete it and return structured results. +""" + +from __future__ import annotations + +import json +from typing import Any, Dict, List, Optional, Type + +from crewai.tools import BaseTool, EnvVar +from pydantic import BaseModel, Field, PrivateAttr + +from ._client import TINYFISH_API_KEY_ENV, build_client, ensure_api_key, ensure_sdk_installed +from ._serde import attr, supported_kwargs + + +class TinyFishAgentToolInput(BaseModel): + """Input schema for TinyFishAgentTool.""" + + url: str = Field(..., description="Target website URL the agent should operate on.") + goal: str = Field( + ..., + description=( + "Natural-language instruction. Be specific about the exact fields to extract and " + "the desired output format, e.g. 'Extract the first 5 product names and prices. " + "Return JSON.'" + ), + ) + output_schema: Optional[Dict[str, Any]] = Field( + None, + description="Optional JSON Schema (object) constraining the structured result.", + ) + browser_profile: Optional[str] = Field( + None, + description="Browser runtime mode: 'lite' (default) or 'stealth' for bot-protected sites.", + ) + use_proxy: bool = Field( + False, + description=( + "Route the run through TinyFish proxy infrastructure (helps with geo/anti-bot)." + ), + ) + proxy_country: Optional[str] = Field( + None, + description="Proxy country when use_proxy is true: US, GB, CA, DE, FR, JP, or AU.", + ) + use_profile: bool = Field( + False, + description=( + "Reuse the default saved Browser Context Profile (logged-in state) for this run." + ), + ) + profile_id: Optional[str] = Field( + None, + description="Specific Browser Context Profile id. Requires use_profile=true.", + ) + use_vault: bool = Field( + False, + description="Allow TinyFish to log in using your connected password-manager credentials.", + ) + credential_item_ids: Optional[List[str]] = Field( + None, + description="Scope the run to specific vault credential URIs. Requires use_vault=true.", + ) + + +class TinyFishAgentTool(BaseTool): + """Run a goal-based web automation on a live site via TinyFish's web agent.""" + + name: str = "TinyFish Web Agent" + description: str = ( + "Give TinyFish a natural-language goal and a target URL, and it autonomously drives a real " + "browser to complete the task (navigate, click, fill forms, extract data) and returns a " + "structured result. Use this for multi-step workflows that plain search/fetch can't do, " + "such as logging in, filling forms, or extracting data behind interactions." + ) + args_schema: Type[BaseModel] = TinyFishAgentToolInput + package_dependencies: List[str] = ["tinyfish"] + env_vars: List[EnvVar] = [ + EnvVar( + name=TINYFISH_API_KEY_ENV, + description="TinyFish API key from https://agent.tinyfish.ai/api-keys", + required=True, + ), + ] + + _api_key: Optional[str] = PrivateAttr(default=None) + _client: Any = PrivateAttr(default=None) + + def __init__(self, api_key: Optional[str] = None, **kwargs: Any) -> None: + super().__init__(**kwargs) + ensure_sdk_installed() + ensure_api_key(api_key) + self._api_key = api_key + + def _get_client(self) -> Any: + if self._client is None: + self._client = build_client(self._api_key) + return self._client + + def _build_kwargs( + self, + url: str, + goal: str, + output_schema: Optional[Dict[str, Any]], + browser_profile: Optional[str], + use_proxy: bool, + proxy_country: Optional[str], + use_profile: bool, + profile_id: Optional[str], + use_vault: bool, + credential_item_ids: Optional[List[str]], + ) -> Dict[str, Any]: + kwargs: Dict[str, Any] = {"url": url, "goal": goal} + if output_schema is not None: + kwargs["output_schema"] = output_schema + + if browser_profile is not None: + kwargs["browser_profile"] = _coerce_browser_profile(browser_profile) + + if use_proxy: + kwargs["proxy_config"] = _coerce_proxy_config(proxy_country) + + if use_profile: + kwargs["use_profile"] = True + if profile_id is not None: + kwargs["profile_id"] = profile_id + if use_vault: + kwargs["use_vault"] = True + if credential_item_ids: + kwargs["credential_item_ids"] = credential_item_ids + + return kwargs + + def _run( + self, + url: str, + goal: str, + output_schema: Optional[Dict[str, Any]] = None, + browser_profile: Optional[str] = None, + use_proxy: bool = False, + proxy_country: Optional[str] = None, + use_profile: bool = False, + profile_id: Optional[str] = None, + use_vault: bool = False, + credential_item_ids: Optional[List[str]] = None, + **_: Any, + ) -> str: + kwargs = self._build_kwargs( + url, + goal, + output_schema, + browser_profile, + use_proxy, + proxy_country, + use_profile, + profile_id, + use_vault, + credential_item_ids, + ) + + client = self._get_client() + try: + run = client.agent.run(**supported_kwargs(client.agent.run, kwargs)) + except Exception as exc: # noqa: BLE001 - surface a clear message to the agent + return f"TinyFish Agent run failed: {exc}" + + status = _status_str(attr(run, "status")) + payload: Dict[str, Any] = { + "run_id": attr(run, "run_id"), + "status": status, + "num_of_steps": attr(run, "num_of_steps"), + "result": attr(run, "result"), + } + error = attr(run, "error") + if error is not None: + payload["error"] = { + "code": attr(error, "code"), + "message": attr(error, "message"), + "category": attr(error, "category"), + } + return json.dumps(payload, ensure_ascii=False, indent=2, default=str) + + async def _arun(self, *args: Any, **kwargs: Any) -> str: + return self._run(*args, **kwargs) + + +def _coerce_browser_profile(value: str) -> Any: + """Return the SDK BrowserProfile enum when available, else the raw string.""" + try: + from tinyfish import BrowserProfile # type: ignore + + return BrowserProfile(value.lower()) + except Exception: # noqa: BLE001 - SDK may accept plain strings + return value + + +def _coerce_proxy_config(country: Optional[str]) -> Any: + """Build a ProxyConfig via the SDK when available, else a plain dict.""" + try: + from tinyfish import ProxyConfig, ProxyCountryCode # type: ignore + + if country: + return ProxyConfig(enabled=True, country_code=ProxyCountryCode(country.upper())) + return ProxyConfig(enabled=True) + except Exception: # noqa: BLE001 - fall back to documented JSON body + config: Dict[str, Any] = {"enabled": True, "type": "tetra"} + if country: + config["country_code"] = country.upper() + return config + + +def _status_str(status: Any) -> Any: + """Normalize a RunStatus enum (or string) into a plain string.""" + if status is None: + return None + return getattr(status, "value", status) diff --git a/crewai/src/crewai_tinyfish/tinyfish_browser_tool.py b/crewai/src/crewai_tinyfish/tinyfish_browser_tool.py new file mode 100644 index 0000000..a806e3f --- /dev/null +++ b/crewai/src/crewai_tinyfish/tinyfish_browser_tool.py @@ -0,0 +1,102 @@ +"""TinyFish Browser session tool for CrewAI. + +Wraps the TinyFish Browser API (``POST https://api.browser.tinyfish.ai``), which +provisions a remote, isolated browser session and returns a CDP WebSocket URL +you can drive directly with Playwright/Puppeteer. Use this when an agent needs +direct, programmatic browser control rather than goal-based automation. +""" + +from __future__ import annotations + +import json +from typing import Any, List, Optional, Type + +from crewai.tools import BaseTool, EnvVar +from pydantic import BaseModel, Field, PrivateAttr + +from ._client import TINYFISH_API_KEY_ENV, build_client, ensure_api_key, ensure_sdk_installed +from ._serde import attr, supported_kwargs + + +class TinyFishBrowserSessionToolInput(BaseModel): + """Input schema for TinyFishBrowserSessionTool.""" + + url: Optional[str] = Field( + None, + description="URL to navigate to on startup. Omit to start at about:blank.", + ) + timeout_seconds: Optional[int] = Field( + None, + ge=5, + le=86_400, + description="Inactivity timeout in seconds (5-86400). Defaults to your plan maximum.", + ) + + +class TinyFishBrowserSessionTool(BaseTool): + """Create a remote browser session and return its CDP connection details.""" + + name: str = "TinyFish Browser Session" + description: str = ( + "Create a remote, isolated browser session via TinyFish and get back a CDP WebSocket " + "URL (cdp_url), session_id, and base_url. Connect to cdp_url with Playwright/Puppeteer " + "to drive the browser directly. Use when you need low-level browser control instead of " + "goal-based " + "automation. Session creation can take 10-30s." + ) + args_schema: Type[BaseModel] = TinyFishBrowserSessionToolInput + package_dependencies: List[str] = ["tinyfish"] + env_vars: List[EnvVar] = [ + EnvVar( + name=TINYFISH_API_KEY_ENV, + description="TinyFish API key from https://agent.tinyfish.ai/api-keys", + required=True, + ), + ] + + _api_key: Optional[str] = PrivateAttr(default=None) + _client: Any = PrivateAttr(default=None) + + def __init__(self, api_key: Optional[str] = None, **kwargs: Any) -> None: + super().__init__(**kwargs) + ensure_sdk_installed() + ensure_api_key(api_key) + self._api_key = api_key + + def _get_client(self) -> Any: + if self._client is None: + self._client = build_client(self._api_key) + return self._client + + def _run( + self, + url: Optional[str] = None, + timeout_seconds: Optional[int] = None, + **_: Any, + ) -> str: + params: dict[str, Any] = {} + if url is not None: + params["url"] = url + if timeout_seconds is not None: + params["timeout_seconds"] = timeout_seconds + + client = self._get_client() + try: + session = client.browser.sessions.create( + **supported_kwargs(client.browser.sessions.create, params) + ) + except Exception as exc: # noqa: BLE001 - surface a clear message to the agent + return f"TinyFish Browser session creation failed: {exc}" + + return json.dumps( + { + "session_id": attr(session, "session_id"), + "cdp_url": attr(session, "cdp_url"), + "base_url": attr(session, "base_url"), + }, + ensure_ascii=False, + indent=2, + ) + + async def _arun(self, *args: Any, **kwargs: Any) -> str: + return self._run(*args, **kwargs) diff --git a/crewai/src/crewai_tinyfish/tinyfish_fetch_tool.py b/crewai/src/crewai_tinyfish/tinyfish_fetch_tool.py new file mode 100644 index 0000000..3c8e161 --- /dev/null +++ b/crewai/src/crewai_tinyfish/tinyfish_fetch_tool.py @@ -0,0 +1,149 @@ +"""TinyFish Fetch tool for CrewAI. + +Wraps the TinyFish Fetch API (``POST https://api.fetch.tinyfish.ai``), which +renders pages in a real browser (JS-heavy sites included) and returns clean +extracted content. Up to 10 URLs per call, each processed independently. +""" + +from __future__ import annotations + +import json +from typing import Any, List, Optional, Type + +from crewai.tools import BaseTool, EnvVar +from pydantic import BaseModel, Field, PrivateAttr + +from ._client import TINYFISH_API_KEY_ENV, build_client, ensure_api_key, ensure_sdk_installed +from ._serde import as_list, attr, supported_kwargs + + +class TinyFishFetchToolInput(BaseModel): + """Input schema for TinyFishFetchTool.""" + + urls: List[str] = Field( + ..., + max_length=10, + description="1-10 http(s) URLs to fetch and extract. Each URL is processed independently.", + ) + format: str = Field( + "markdown", + description="Output format for extracted text: 'markdown' (default), 'html', or 'json'.", + ) + links: bool = Field( + False, + description="When true, also return all links found on each page.", + ) + image_links: bool = Field( + False, + description="When true, also return all URLs found on each page.", + ) + ttl: Optional[int] = Field( + None, + ge=0, + description=( + "Cache freshness tolerance in seconds. Omit to accept any cached entry, 0 to force a " + "live fetch, or N to accept entries younger than N seconds." + ), + ) + max_chars_per_url: Optional[int] = Field( + None, + ge=1, + description="Optionally truncate each page's extracted text to this many characters.", + ) + + +class TinyFishFetchTool(BaseTool): + """Fetch one or more URLs via TinyFish and return clean extracted content.""" + + name: str = "TinyFish Web Fetch" + description: str = ( + "Fetch and extract clean content from one or more URLs (up to 10) using TinyFish. " + "Renders JavaScript-heavy pages in a real browser and returns markdown/html/json. " + "Use this when you already have URLs and need their page content." + ) + args_schema: Type[BaseModel] = TinyFishFetchToolInput + package_dependencies: List[str] = ["tinyfish"] + env_vars: List[EnvVar] = [ + EnvVar( + name=TINYFISH_API_KEY_ENV, + description="TinyFish API key from https://agent.tinyfish.ai/api-keys", + required=True, + ), + ] + + _api_key: Optional[str] = PrivateAttr(default=None) + _client: Any = PrivateAttr(default=None) + + def __init__(self, api_key: Optional[str] = None, **kwargs: Any) -> None: + super().__init__(**kwargs) + ensure_sdk_installed() + ensure_api_key(api_key) + self._api_key = api_key + + def _get_client(self) -> Any: + if self._client is None: + self._client = build_client(self._api_key) + return self._client + + def _run( + self, + urls: List[str], + format: str = "markdown", + links: bool = False, + image_links: bool = False, + ttl: Optional[int] = None, + max_chars_per_url: Optional[int] = None, + **_: Any, + ) -> str: + if not urls: + return "TinyFish Fetch failed: at least one URL is required." + + params: dict[str, Any] = {"urls": urls, "format": format} + if links: + params["links"] = links + if image_links: + params["image_links"] = image_links + if ttl is not None: + params["ttl"] = ttl + + client = self._get_client() + try: + response = client.fetch.get_contents( + **supported_kwargs(client.fetch.get_contents, params) + ) + except Exception as exc: # noqa: BLE001 - surface a clear message to the agent + return f"TinyFish Fetch failed: {exc}" + + pages = [] + for page in as_list(attr(response, "results")): + text = attr(page, "text") + if isinstance(text, str) and max_chars_per_url and len(text) > max_chars_per_url: + text = text[:max_chars_per_url] + "…[truncated]" + pages.append( + { + "url": attr(page, "url"), + "final_url": attr(page, "final_url"), + "title": attr(page, "title"), + "description": attr(page, "description"), + "language": attr(page, "language"), + "format": attr(page, "format"), + "text": text, + "links": attr(page, "links") if links else None, + "image_links": attr(page, "image_links") if image_links else None, + } + ) + + errors = [] + for err in as_list(attr(response, "errors")): + errors.append( + { + "url": attr(err, "url"), + "error": attr(err, "error"), + "status": attr(err, "status"), + } + ) + + return json.dumps({"results": pages, "errors": errors}, ensure_ascii=False, indent=2) + + async def _arun(self, *args: Any, **kwargs: Any) -> str: + return self._run(*args, **kwargs) diff --git a/crewai/src/crewai_tinyfish/tinyfish_search_tool.py b/crewai/src/crewai_tinyfish/tinyfish_search_tool.py new file mode 100644 index 0000000..a474cfb --- /dev/null +++ b/crewai/src/crewai_tinyfish/tinyfish_search_tool.py @@ -0,0 +1,134 @@ +"""TinyFish Search tool for CrewAI. + +Wraps the TinyFish Search API (``GET https://api.search.tinyfish.ai``), which +returns structured, ranked web results (title, snippet, url) ready for LLM +consumption. +""" + +from __future__ import annotations + +import json +from typing import Any, List, Optional, Type + +from crewai.tools import BaseTool, EnvVar +from pydantic import BaseModel, Field, PrivateAttr + +from ._client import TINYFISH_API_KEY_ENV, build_client, ensure_api_key, ensure_sdk_installed +from ._serde import as_list, attr, supported_kwargs + + +class TinyFishSearchToolInput(BaseModel): + """Input schema for TinyFishSearchTool.""" + + query: str = Field( + ..., + description=( + "Search query string. Supports operators, e.g. " + "'python tutorial site:docs.python.org' or 'recipes -site:youtube.com'." + ), + ) + location: Optional[str] = Field( + None, + description=( + "Country code for geo-targeted results (e.g. 'US', 'GB', 'FR'). Defaults to US." + ), + ) + language: Optional[str] = Field( + None, + description="Language code for result language (e.g. 'en', 'fr', 'de'). Defaults to en.", + ) + page: Optional[int] = Field( + None, + ge=0, + le=10, + description="Zero-indexed result page for pagination (0-10).", + ) + max_results: int = Field( + 10, + ge=1, + le=50, + description="Maximum number of results to return to the agent.", + ) + + +class TinyFishSearchTool(BaseTool): + """Run a web search via TinyFish and return structured ranked results.""" + + name: str = "TinyFish Web Search" + description: str = ( + "Search the web with TinyFish and get back structured, ranked results " + "(title, url, snippet, source). Use this when you need to discover URLs or " + "get current information for a query." + ) + args_schema: Type[BaseModel] = TinyFishSearchToolInput + package_dependencies: List[str] = ["tinyfish"] + env_vars: List[EnvVar] = [ + EnvVar( + name=TINYFISH_API_KEY_ENV, + description="TinyFish API key from https://agent.tinyfish.ai/api-keys", + required=True, + ), + ] + + _api_key: Optional[str] = PrivateAttr(default=None) + _client: Any = PrivateAttr(default=None) + + def __init__(self, api_key: Optional[str] = None, **kwargs: Any) -> None: + super().__init__(**kwargs) + ensure_sdk_installed() + ensure_api_key(api_key) + self._api_key = api_key + + def _get_client(self) -> Any: + if self._client is None: + self._client = build_client(self._api_key) + return self._client + + def _run( + self, + query: str, + location: Optional[str] = None, + language: Optional[str] = None, + page: Optional[int] = None, + max_results: int = 10, + **_: Any, + ) -> str: + params: dict[str, Any] = {"query": query} + if location is not None: + params["location"] = location + if language is not None: + params["language"] = language + if page is not None: + params["page"] = page + + client = self._get_client() + try: + response = client.search.query(**supported_kwargs(client.search.query, params)) + except Exception as exc: # noqa: BLE001 - surface a clear message to the agent + return f"TinyFish Search failed: {exc}" + + results = as_list(attr(response, "results")) + trimmed = [] + for item in results[:max_results]: + trimmed.append( + { + "position": attr(item, "position"), + "title": attr(item, "title"), + "url": attr(item, "url"), + "snippet": attr(item, "snippet"), + "site_name": attr(item, "site_name"), + } + ) + + return json.dumps( + { + "query": query, + "total_results": attr(response, "total_results"), + "results": trimmed, + }, + ensure_ascii=False, + indent=2, + ) + + async def _arun(self, *args: Any, **kwargs: Any) -> str: + return self._run(*args, **kwargs) diff --git a/crewai/tests/conftest.py b/crewai/tests/conftest.py new file mode 100644 index 0000000..cef5154 --- /dev/null +++ b/crewai/tests/conftest.py @@ -0,0 +1,46 @@ +"""Shared pytest fixtures for the crewai-tinyfish tools. + +Tests run without the real ``tinyfish`` SDK or network access: we stub out the +SDK install check and inject a fake client into each tool module. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import pytest + +TOOL_MODULES = [ + "crewai_tinyfish.tinyfish_search_tool", + "crewai_tinyfish.tinyfish_fetch_tool", + "crewai_tinyfish.tinyfish_agent_tool", + "crewai_tinyfish.tinyfish_browser_tool", +] + + +@pytest.fixture +def api_key_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("TINYFISH_API_KEY", "test-key") + + +@pytest.fixture +def no_sdk_check(monkeypatch: pytest.MonkeyPatch) -> None: + """Make every tool module's SDK-install check a no-op.""" + import importlib + + for mod_name in TOOL_MODULES: + mod = importlib.import_module(mod_name) + monkeypatch.setattr(mod, "ensure_sdk_installed", lambda *a, **k: None) + + +def inject_client(monkeypatch: pytest.MonkeyPatch, module_name: str, client: Any) -> None: + """Replace ``build_client`` in a tool module so it returns ``client``.""" + import importlib + + mod = importlib.import_module(module_name) + monkeypatch.setattr(mod, "build_client", lambda *a, **k: client) + + +def ns(**kwargs: Any) -> SimpleNamespace: + return SimpleNamespace(**kwargs) diff --git a/crewai/tests/test_agent_tool.py b/crewai/tests/test_agent_tool.py new file mode 100644 index 0000000..29a16d4 --- /dev/null +++ b/crewai/tests/test_agent_tool.py @@ -0,0 +1,78 @@ +import json + +import pytest +from conftest import inject_client, ns + +MODULE = "crewai_tinyfish.tinyfish_agent_tool" + + +def test_requires_api_key(monkeypatch, no_sdk_check): + monkeypatch.delenv("TINYFISH_API_KEY", raising=False) + from crewai_tinyfish import TinyFishAgentTool + + with pytest.raises(ValueError): + TinyFishAgentTool() + + +def test_happy_path_builds_expected_kwargs(monkeypatch, api_key_env, no_sdk_check): + captured = {} + + class FakeAgent: + def run(self, **kwargs): + captured.update(kwargs) + return ns( + run_id="run_123", + status="COMPLETED", + num_of_steps=3, + result={"price": "$799"}, + error=None, + ) + + inject_client(monkeypatch, MODULE, ns(agent=FakeAgent())) + + from crewai_tinyfish import TinyFishAgentTool + + tool = TinyFishAgentTool() + out = tool.run( + url="https://scrapeme.live/shop", + goal="Extract first product price. Return JSON.", + use_proxy=True, + proxy_country="us", + use_profile=True, + profile_id="prof_abc", + ) + data = json.loads(out) + + assert captured["url"] == "https://scrapeme.live/shop" + assert captured["use_profile"] is True + assert captured["profile_id"] == "prof_abc" + # proxy_config is a real tinyfish ProxyConfig when the SDK is installed, or the + # documented dict fallback otherwise. Read fields tolerantly for both shapes. + proxy = captured["proxy_config"] + enabled = proxy["enabled"] if isinstance(proxy, dict) else getattr(proxy, "enabled") + country = proxy["country_code"] if isinstance(proxy, dict) else getattr(proxy, "country_code") + assert enabled is True + assert str(getattr(country, "value", country)).upper() == "US" + assert data["status"] == "COMPLETED" + assert data["result"]["price"] == "$799" + + +def test_failed_run_surfaces_error(monkeypatch, api_key_env, no_sdk_check): + class FakeAgent: + def run(self, **kwargs): + return ns( + run_id="run_x", + status="FAILED", + num_of_steps=1, + result=None, + error={"code": "SITE_BLOCKED", "message": "blocked", "category": "AGENT_FAILURE"}, + ) + + inject_client(monkeypatch, MODULE, ns(agent=FakeAgent())) + + from crewai_tinyfish import TinyFishAgentTool + + out = TinyFishAgentTool().run(url="https://x.com", goal="do it") + data = json.loads(out) + assert data["status"] == "FAILED" + assert data["error"]["code"] == "SITE_BLOCKED" diff --git a/crewai/tests/test_browser_tool.py b/crewai/tests/test_browser_tool.py new file mode 100644 index 0000000..91b9e62 --- /dev/null +++ b/crewai/tests/test_browser_tool.py @@ -0,0 +1,40 @@ +import json + +import pytest +from conftest import inject_client, ns + +MODULE = "crewai_tinyfish.tinyfish_browser_tool" + + +def test_requires_api_key(monkeypatch, no_sdk_check): + monkeypatch.delenv("TINYFISH_API_KEY", raising=False) + from crewai_tinyfish import TinyFishBrowserSessionTool + + with pytest.raises(ValueError): + TinyFishBrowserSessionTool() + + +def test_happy_path(monkeypatch, api_key_env, no_sdk_check): + captured = {} + + class FakeSessions: + def create(self, **params): + captured.update(params) + return ns( + session_id="br-123", + cdp_url="wss://example.tinyfish.io/cdp", + base_url="https://example.tinyfish.io", + ) + + inject_client(monkeypatch, MODULE, ns(browser=ns(sessions=FakeSessions()))) + + from crewai_tinyfish import TinyFishBrowserSessionTool + + tool = TinyFishBrowserSessionTool() + out = tool.run(url="https://scrapeme.live/shop", timeout_seconds=300) + data = json.loads(out) + + assert captured["url"] == "https://scrapeme.live/shop" + assert captured["timeout_seconds"] == 300 + assert data["cdp_url"] == "wss://example.tinyfish.io/cdp" + assert data["session_id"] == "br-123" diff --git a/crewai/tests/test_fetch_tool.py b/crewai/tests/test_fetch_tool.py new file mode 100644 index 0000000..5b6f3be --- /dev/null +++ b/crewai/tests/test_fetch_tool.py @@ -0,0 +1,66 @@ +import json + +import pytest +from conftest import inject_client, ns + +MODULE = "crewai_tinyfish.tinyfish_fetch_tool" + + +def test_requires_api_key(monkeypatch, no_sdk_check): + monkeypatch.delenv("TINYFISH_API_KEY", raising=False) + from crewai_tinyfish import TinyFishFetchTool + + with pytest.raises(ValueError): + TinyFishFetchTool() + + +def test_happy_path_and_truncation(monkeypatch, api_key_env, no_sdk_check): + captured = {} + + class FakeFetch: + def get_contents(self, **params): + captured.update(params) + return ns( + results=[ + { + "url": "https://tinyfish.ai", + "final_url": "https://tinyfish.ai", + "title": "TinyFish", + "description": "infra", + "language": "en", + "format": "markdown", + "text": "x" * 100, + } + ], + errors=[], + ) + + inject_client(monkeypatch, MODULE, ns(fetch=FakeFetch())) + + from crewai_tinyfish import TinyFishFetchTool + + tool = TinyFishFetchTool() + out = tool.run(urls=["https://tinyfish.ai"], max_chars_per_url=10) + data = json.loads(out) + + assert captured["urls"] == ["https://tinyfish.ai"] + assert captured["format"] == "markdown" + assert data["results"][0]["text"].endswith("…[truncated]") + assert data["errors"] == [] + + +def test_per_url_errors_passthrough(monkeypatch, api_key_env, no_sdk_check): + class FakeFetch: + def get_contents(self, **params): + return ns( + results=[], + errors=[{"url": "https://bad.example", "error": "bot_blocked", "status": None}], + ) + + inject_client(monkeypatch, MODULE, ns(fetch=FakeFetch())) + + from crewai_tinyfish import TinyFishFetchTool + + out = TinyFishFetchTool().run(urls=["https://bad.example"]) + data = json.loads(out) + assert data["errors"][0]["error"] == "bot_blocked" diff --git a/crewai/tests/test_search_tool.py b/crewai/tests/test_search_tool.py new file mode 100644 index 0000000..cfb060a --- /dev/null +++ b/crewai/tests/test_search_tool.py @@ -0,0 +1,60 @@ +import json + +import pytest +from conftest import inject_client, ns + +MODULE = "crewai_tinyfish.tinyfish_search_tool" + + +def test_requires_api_key(monkeypatch, no_sdk_check): + monkeypatch.delenv("TINYFISH_API_KEY", raising=False) + from crewai_tinyfish import TinyFishSearchTool + + with pytest.raises(ValueError): + TinyFishSearchTool() + + +def test_happy_path(monkeypatch, api_key_env, no_sdk_check): + captured = {} + + class FakeSearch: + def query(self, **params): + captured.update(params) + return ns( + query=params["query"], + total_results=1, + results=[ + { + "position": 1, + "title": "TinyFish", + "url": "https://tinyfish.ai", + "snippet": "web agent infra", + "site_name": "tinyfish.ai", + } + ], + ) + + inject_client(monkeypatch, MODULE, ns(search=FakeSearch())) + + from crewai_tinyfish import TinyFishSearchTool + + tool = TinyFishSearchTool() + out = tool.run(query="web automation tools", location="US", max_results=5) + data = json.loads(out) + + assert captured["query"] == "web automation tools" + assert captured["location"] == "US" + assert data["results"][0]["url"] == "https://tinyfish.ai" + + +def test_errors_are_returned_as_strings(monkeypatch, api_key_env, no_sdk_check): + class FakeSearch: + def query(self, **params): + raise RuntimeError("boom") + + inject_client(monkeypatch, MODULE, ns(search=FakeSearch())) + + from crewai_tinyfish import TinyFishSearchTool + + out = TinyFishSearchTool().run(query="x") + assert "TinyFish Search failed" in out From 42508bb4c5aa079366d6d770066989b8bcaa9aa1 Mon Sep 17 00:00:00 2001 From: Pranav Janakiraman Date: Fri, 26 Jun 2026 19:22:40 -0700 Subject: [PATCH 3/3] Address CodeRabbit review feedback - Offload sync SDK calls to a worker thread in every tool's _arun (asyncio.to_thread) so async crews don't block the event loop during slow Agent/Browser calls - Harden _serde.as_list to wrap str/bytes/dict as single items instead of splitting them into characters/keys - Align dev tooling: add ruff + build to the [dev] extra so `.[dev]` is a complete toolchain - UPSTREAM.md: tag the directory-tree fence as text (markdownlint MD040) and align the documented tinyfish floor to >=0.2.5 --- crewai/UPSTREAM.md | 4 ++-- crewai/pyproject.toml | 2 ++ crewai/src/crewai_tinyfish/_serde.py | 6 +++++- crewai/src/crewai_tinyfish/tinyfish_agent_tool.py | 3 ++- crewai/src/crewai_tinyfish/tinyfish_browser_tool.py | 3 ++- crewai/src/crewai_tinyfish/tinyfish_fetch_tool.py | 3 ++- crewai/src/crewai_tinyfish/tinyfish_search_tool.py | 3 ++- 7 files changed, 17 insertions(+), 7 deletions(-) diff --git a/crewai/UPSTREAM.md b/crewai/UPSTREAM.md index 3742c08..d614b97 100644 --- a/crewai/UPSTREAM.md +++ b/crewai/UPSTREAM.md @@ -21,7 +21,7 @@ conventions, so it can be contributed into the main repository with minimal chan In `crewai-tools`, tools live at `crewai_tools/tools//`. Suggested layout: -``` +```text crewai_tools/tools/tinyfish_search_tool/ __init__.py tinyfish_search_tool.py # from src/crewai_tinyfish/tinyfish_search_tool.py @@ -45,7 +45,7 @@ Steps: 4. Register the optional dependency in the repo's `pyproject.toml` under `[project.optional-dependencies]`: ```toml - tinyfish = ["tinyfish>=0.2.6"] + tinyfish = ["tinyfish>=0.2.5"] ``` 5. Add a short `README.md` per tool (the sections in this repo's README can be split). 6. `uv run pytest` and run the repo's lint/type checks before opening the PR. diff --git a/crewai/pyproject.toml b/crewai/pyproject.toml index 4028b8b..d3b481c 100644 --- a/crewai/pyproject.toml +++ b/crewai/pyproject.toml @@ -39,6 +39,8 @@ dev = [ "tinyfish>=0.2.5", "pytest>=8.0.0", "pytest-asyncio>=0.23.0", + "ruff>=0.4.0", + "build>=1.0.0", ] [project.urls] diff --git a/crewai/src/crewai_tinyfish/_serde.py b/crewai/src/crewai_tinyfish/_serde.py index c284c8e..5fb5157 100644 --- a/crewai/src/crewai_tinyfish/_serde.py +++ b/crewai/src/crewai_tinyfish/_serde.py @@ -36,11 +36,15 @@ def supported_kwargs(func: Callable[..., Any], params: Dict[str, Any]) -> Dict[s def as_list(value: Any) -> list: - """Coerce ``value`` into a list (None -> []).""" + """Coerce ``value`` into a list (None -> [], scalars -> ``[value]``).""" if value is None: return [] if isinstance(value, list): return value + # Treat str/bytes/dict as scalars so they wrap instead of splitting into + # characters or keys. + if isinstance(value, (str, bytes, dict)): + return [value] try: return list(value) except TypeError: diff --git a/crewai/src/crewai_tinyfish/tinyfish_agent_tool.py b/crewai/src/crewai_tinyfish/tinyfish_agent_tool.py index 120b3f3..27e2e84 100644 --- a/crewai/src/crewai_tinyfish/tinyfish_agent_tool.py +++ b/crewai/src/crewai_tinyfish/tinyfish_agent_tool.py @@ -7,6 +7,7 @@ from __future__ import annotations +import asyncio import json from typing import Any, Dict, List, Optional, Type @@ -185,7 +186,7 @@ def _run( return json.dumps(payload, ensure_ascii=False, indent=2, default=str) async def _arun(self, *args: Any, **kwargs: Any) -> str: - return self._run(*args, **kwargs) + return await asyncio.to_thread(self._run, *args, **kwargs) def _coerce_browser_profile(value: str) -> Any: diff --git a/crewai/src/crewai_tinyfish/tinyfish_browser_tool.py b/crewai/src/crewai_tinyfish/tinyfish_browser_tool.py index a806e3f..8d7b26f 100644 --- a/crewai/src/crewai_tinyfish/tinyfish_browser_tool.py +++ b/crewai/src/crewai_tinyfish/tinyfish_browser_tool.py @@ -8,6 +8,7 @@ from __future__ import annotations +import asyncio import json from typing import Any, List, Optional, Type @@ -99,4 +100,4 @@ def _run( ) async def _arun(self, *args: Any, **kwargs: Any) -> str: - return self._run(*args, **kwargs) + return await asyncio.to_thread(self._run, *args, **kwargs) diff --git a/crewai/src/crewai_tinyfish/tinyfish_fetch_tool.py b/crewai/src/crewai_tinyfish/tinyfish_fetch_tool.py index 3c8e161..96d99f5 100644 --- a/crewai/src/crewai_tinyfish/tinyfish_fetch_tool.py +++ b/crewai/src/crewai_tinyfish/tinyfish_fetch_tool.py @@ -7,6 +7,7 @@ from __future__ import annotations +import asyncio import json from typing import Any, List, Optional, Type @@ -146,4 +147,4 @@ def _run( return json.dumps({"results": pages, "errors": errors}, ensure_ascii=False, indent=2) async def _arun(self, *args: Any, **kwargs: Any) -> str: - return self._run(*args, **kwargs) + return await asyncio.to_thread(self._run, *args, **kwargs) diff --git a/crewai/src/crewai_tinyfish/tinyfish_search_tool.py b/crewai/src/crewai_tinyfish/tinyfish_search_tool.py index a474cfb..a613659 100644 --- a/crewai/src/crewai_tinyfish/tinyfish_search_tool.py +++ b/crewai/src/crewai_tinyfish/tinyfish_search_tool.py @@ -7,6 +7,7 @@ from __future__ import annotations +import asyncio import json from typing import Any, List, Optional, Type @@ -131,4 +132,4 @@ def _run( ) async def _arun(self, *args: Any, **kwargs: Any) -> str: - return self._run(*args, **kwargs) + return await asyncio.to_thread(self._run, *args, **kwargs)