From 23182824d4a5542e166b9e59c5c9f6e60651540b Mon Sep 17 00:00:00 2001 From: Ethan Hanlon Date: Fri, 17 Jul 2026 18:13:08 -0700 Subject: [PATCH 1/6] [ethan] feat: add spotify integration. not completely reliable yet. --- backend/apps/agents/core/mcp_preflight.py | 5 + backend/apps/spotify_mcp_shim/__main__.py | 5 + backend/apps/spotify_mcp_shim/handlers.py | 222 ++++++++++++++++++ backend/apps/spotify_mcp_shim/server.py | 71 ++++++ backend/apps/spotify_mcp_shim/tools.py | 22 ++ backend/apps/tools_lib/mcp_config.py | 2 +- backend/requirements.txt | 4 +- frontend/src/app/pages/Tools/integrations.tsx | 13 + 8 files changed, 342 insertions(+), 2 deletions(-) create mode 100644 backend/apps/spotify_mcp_shim/__main__.py create mode 100644 backend/apps/spotify_mcp_shim/handlers.py create mode 100644 backend/apps/spotify_mcp_shim/server.py create mode 100644 backend/apps/spotify_mcp_shim/tools.py diff --git a/backend/apps/agents/core/mcp_preflight.py b/backend/apps/agents/core/mcp_preflight.py index a9383302f..2b5745a2a 100644 --- a/backend/apps/agents/core/mcp_preflight.py +++ b/backend/apps/agents/core/mcp_preflight.py @@ -26,6 +26,11 @@ "title": "Google Workspace", "description": "Gmail, Calendar, Drive, Docs, Sheets, Slides; for reading/sending email, checking the user's schedule, and pulling context from their documents.", }, + { + "id": "Spotify", + "title": "Spotify", + "description": "Control playback, search music, and manage playlists; when the task involves music or audio." + }, { "id": "Microsoft 365", "title": "Microsoft 365", diff --git a/backend/apps/spotify_mcp_shim/__main__.py b/backend/apps/spotify_mcp_shim/__main__.py new file mode 100644 index 000000000..ba566a590 --- /dev/null +++ b/backend/apps/spotify_mcp_shim/__main__.py @@ -0,0 +1,5 @@ +"""Module-level entrypoint so `python -m backend.apps.spotify_mcp_shim` works.""" +from backend.apps.spotify_mcp_shim.server import main + +if __name__ == "__main__": + main() diff --git a/backend/apps/spotify_mcp_shim/handlers.py b/backend/apps/spotify_mcp_shim/handlers.py new file mode 100644 index 000000000..c6d99148b --- /dev/null +++ b/backend/apps/spotify_mcp_shim/handlers.py @@ -0,0 +1,222 @@ +from typing import Optional, Any, Dict +import psutil +import json +import urllib.request +import urllib.error +import os + +# Bypass the OpenSwarm proxy for local Chrome CDP connections +os.environ["no_proxy"] = "*" + + +import subprocess +import time + +def p_ensure_chrome_cdp(): + """Ensure Chrome is running with CDP on port 9223. + If not, we launch a dedicated profile so we don't conflict with the user's main Chrome, + and we leave it running in the background so music keeps playing after the script exits! + """ + try: + req = urllib.request.Request("http://127.0.0.1:9223/json/version") + with urllib.request.urlopen(req, timeout=0.5) as response: + if response.status == 200: + return True + except Exception: + pass + + # CDP not responding. Let's auto-launch a dedicated Chrome instance! + profile_dir = os.path.expanduser("~/.openswarm/spotify_chrome_profile") + os.makedirs(profile_dir, exist_ok=True) + + subprocess.Popen([ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "--remote-debugging-port=9223", + f"--user-data-dir={profile_dir}", + "--no-first-run", + "--no-default-browser-check" + ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + # Wait for it to spin up + for _ in range(10): + time.sleep(0.5) + try: + req = urllib.request.Request("http://127.0.0.1:9223/json/version") + with urllib.request.urlopen(req, timeout=0.5) as response: + if response.status == 200: + return True + except Exception: + continue + + return False + +def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]: + """ + Using the Playwright MCP, we'll: + + 1. Auto-launch a dedicated Chrome instance (with CDP) if it's not already running. + 2. Navigate to Spotify. If this is the user's first time on this dedicated profile, they will need to log in. + 3. Search for the song name. + 4. Play the song! + """ + + if not p_ensure_chrome_cdp(): + return { + "is_error": True, + "is_human_intervention": False, + "message": "Failed to auto-start Chrome with remote debugging on port 9223." + } + + import asyncio + + def run_async_play_track(): + async def inner(): + from playwright.async_api import async_playwright + import urllib.parse + import base64 + from io import BytesIO + from PIL import Image + + from backend.apps.settings.settings import load_settings + from backend.apps.settings.credentials import get_anthropic_client_for_model + from backend.apps.agents.providers.registry import resolve_aux_model + + settings = load_settings() + try: + model_id, _ = await resolve_aux_model(settings, preferred_tier="haiku") + client = get_anthropic_client_for_model(settings, model_id) + except Exception as e: + return { + "is_error": True, + "is_human_intervention": False, + "message": f"Could not load AI client for screenshot analysis: {e}" + } + + try: + async with async_playwright() as p: + browser = await p.chromium.connect_over_cdp("http://127.0.0.1:9223") + context = browser.contexts[0] + + page = None + for p_obj in context.pages: + if "spotify.com" in p_obj.url: + page = p_obj + break + + if not page: + page = await context.new_page() + + query = track_name + if artist: + query += f" {artist}" + + search_url = f"https://open.spotify.com/search/{urllib.parse.quote(query)}" + await page.goto(search_url) + await page.wait_for_load_state("networkidle") + await asyncio.sleep(2) # Give elements time to render + + prompt = f""" +I want to play the track '{query}' on Spotify. Look at this screenshot of the Spotify web UI. +Determine the state and return ONLY a valid JSON object. Do not include markdown formatting or backticks, just the raw JSON. +You must choose one of the following exact JSON structures: + +1. If a login prompt or overlay is blocking the UI: +{{ + "action": "human_intervention", + "message": "User needs to log in" +}} + +2. If the track '{query}' is already playing (pause button is visible): +{{ + "action": "done", + "message": "Successfully started playback" +}} + +3. Otherwise, find the Play button for the top search result and return its exact center coordinates as percentages (0 to 100) of the image width and height: +{{ + "action": "click", + "x_percent": 50.5, + "y_percent": 25.0, + "message": "Clicking play button" +}} + +CRITICAL: Return strictly valid JSON. Double check your quotes and commas. +""".strip() + + for _ in range(3): + screenshot_bytes = await page.screenshot() + + img = Image.open(BytesIO(screenshot_bytes)) + max_width = 1024 + if img.width > max_width: + ratio = max_width / img.width + img = img.resize((max_width, int(img.height * ratio)), Image.LANCZOS) + buf = BytesIO() + img.convert("RGB").save(buf, format="JPEG", quality=45) + b64_img = base64.b64encode(buf.getvalue()).decode("utf-8") + + response = await client.messages.create( + model=model_id, + max_tokens=256, + messages=[{ + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": b64_img}} + ] + }] + ) + + try: + # Parse JSON from response + resp_text = response.content[0].text.strip() + if resp_text.startswith("```json"): + resp_text = resp_text[7:-3].strip() + elif resp_text.startswith("```"): + resp_text = resp_text[3:-3].strip() + + action_data = json.loads(resp_text) + + if action_data["action"] == "human_intervention": + return { + "is_error": False, + "is_human_intervention": True, + "message": action_data.get("message", "Human intervention needed.") + } + elif action_data["action"] == "done": + return { + "is_error": False, + "is_human_intervention": False, + "message": action_data.get("message", "Task completed.") + } + elif action_data["action"] == "click": + vp_w = await page.evaluate("window.innerWidth") + vp_h = await page.evaluate("window.innerHeight") + click_x = (float(action_data["x_percent"]) / 100.0) * vp_w + click_y = (float(action_data["y_percent"]) / 100.0) * vp_h + await page.mouse.click(click_x, click_y) + await asyncio.sleep(2) # wait for playback to start + # loop continues to verify + except Exception as e: + return { + "is_error": True, + "is_human_intervention": False, + "message": f"Failed to parse AI response: {e}\nResponse was: {response.content[0].text}" + } + + return { + "is_error": True, + "is_human_intervention": False, + "message": "AI failed to start playback after multiple attempts." + } + + except Exception as e: + return { + "is_error": True, + "is_human_intervention": False, + "message": f"An error occurred: {str(e)}" + } + + return asyncio.run(inner()) + + return run_async_play_track() \ No newline at end of file diff --git a/backend/apps/spotify_mcp_shim/server.py b/backend/apps/spotify_mcp_shim/server.py new file mode 100644 index 000000000..925b8baf4 --- /dev/null +++ b/backend/apps/spotify_mcp_shim/server.py @@ -0,0 +1,71 @@ +import sys +import json + +from backend.apps.spotify_mcp_shim.tools import TOOLS + +def p_send(id_, result=None, error=None): + msg = {"jsonrpc": "2.0", "id": id_} + if error is not None: + msg["error"] = error + else: + msg["result"] = result + sys.stdout.write(json.dumps(msg) + "\n") + sys.stdout.flush() + + +def p_err(text: str) -> dict: + return {"content": [{"type": "text", "text": f"Error: {text}"}], "isError": True} + +def p_ok(payload) -> dict: + if isinstance(payload, str): + return {"content": [{"type": "text", "text": payload}]} + return {"content": [{"type": "text", "text": json.dumps(payload, indent=2, default=str)}]} + +from backend.apps.spotify_mcp_shim.handlers import play_track + +def handle_tool_call(name: str, args: dict) -> dict: + match name: + case "play_track": + return p_ok(play_track(**args)) + + return p_err(f"Unknown tool: {name}") + +def main(): + for line in sys.stdin: + line = line.strip() + if not line: + continue + + try: + msg = json.loads(line) + except json.JSONDecodeError: + continue + + method = msg.get("method") + id_ = msg.get("id") + params = msg.get("params", {}) or {} + + if method == "initialize": + p_send(id_, { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "openswarm-spotify", "version": "1.0.0"}, + }) + elif method == "notifications/initialized": + pass + elif method == "tools/list": + p_send(id_, {"tools": TOOLS}) + elif method == "tools/call": + name = params.get("name", "") + args = params.get("arguments", {}) or {} + try: + p_send(id_, handle_tool_call(name, args)) + except Exception as e: + p_send(id_, p_err(f"shim crashed: {e!r}")) + elif method == "ping": + p_send(id_, {}) + elif id_ is not None: + p_send(id_, error={"code": -32601, "message": f"Method not found: {method}"}) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/backend/apps/spotify_mcp_shim/tools.py b/backend/apps/spotify_mcp_shim/tools.py new file mode 100644 index 000000000..2b0510cf6 --- /dev/null +++ b/backend/apps/spotify_mcp_shim/tools.py @@ -0,0 +1,22 @@ +"""MCP tool surface for Spotify: the things a logged-in human does. + +Plays tracks. This is accomplished via a Playwright session, which may require +the user to manually input their credentials or solve a CAPTCHA. +""" + +OBJ = "object" + +TOOLS = [ + { + "name": "play_track", + "description": "Opens a browser and plays the requested song.", + "inputSchema": { + "type": OBJ, + "properties": { + "track_name": {"type": "string"}, + "artist": {"type": "string"} + }, + "required": ["track_name"] + } + } +] \ No newline at end of file diff --git a/backend/apps/tools_lib/mcp_config.py b/backend/apps/tools_lib/mcp_config.py index 898ba531f..aa6977fba 100644 --- a/backend/apps/tools_lib/mcp_config.py +++ b/backend/apps/tools_lib/mcp_config.py @@ -152,7 +152,7 @@ def derive_mcp_config(tool: ToolDefinition) -> Optional[dict]: env["PYTHONPATH"] = (p_project_root + os.pathsep + existing_pp) if existing_pp else p_project_root # The session-borrow social shims (reddit/x/tiktok) each run as a Python shim that borrows the user's live browser session via the backend's cookie bridge, so they need the localhost port + auth token, plus PYTHONPATH to import themselves. - if tool.name.lower() in {"reddit", "x", "tiktok"} and config.get("type") == "stdio": + if tool.name.lower() in {"reddit", "x", "tiktok", "spotify"} and config.get("type") == "stdio": from backend.auth import get_auth_token env = config.setdefault("env", {}) env["OPENSWARM_PORT"] = os.environ.get("OPENSWARM_PORT", "8324") diff --git a/backend/requirements.txt b/backend/requirements.txt index a9b7fea04..0901942bf 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -24,4 +24,6 @@ swarm-analytics==0.1.1 tzlocal==5.3.1 # Test deps (pytest, pytest-asyncio) live in requirements-dev.txt — they # never ship to production users and shaved ~3 MB / ~200 files off the -# Mac DMG when removed from the prod env. \ No newline at end of file +# Mac DMG when removed from the prod env. +psutil==7.2.2 +playwright==1.61.0 \ No newline at end of file diff --git a/frontend/src/app/pages/Tools/integrations.tsx b/frontend/src/app/pages/Tools/integrations.tsx index f1fe529cb..25191cb77 100644 --- a/frontend/src/app/pages/Tools/integrations.tsx +++ b/frontend/src/app/pages/Tools/integrations.tsx @@ -40,6 +40,19 @@ export const INTEGRATIONS: Integration[] = [ ), }, + { + id: 'spotify', + name: "Spotify", + description: 'Control playback, search tracks, and manage playlists.', + mcp_config: { type: 'stdio', command: 'python', args: ['-m', 'backend.apps.spotify_mcp_shim'] }, + color: '#1DB954', + website: 'https://open.spotify.com', + icon: ( + + + + ), + }, { id: 'tiktok', name: 'TikTok', From a6664a693d787154da76af62b7f3455d52da5257 Mon Sep 17 00:00:00 2001 From: Ethan Hanlon Date: Sat, 18 Jul 2026 23:25:18 -0700 Subject: [PATCH 2/6] [ethan] fix: issues preventing playback, login persistence --- backend/apps/spotify_mcp_shim/handlers.py | 288 ++++++++++-------- backend/apps/spotify_mcp_shim/server.py | 18 +- frontend/src/app/pages/Tools/integrations.tsx | 6 +- 3 files changed, 179 insertions(+), 133 deletions(-) diff --git a/backend/apps/spotify_mcp_shim/handlers.py b/backend/apps/spotify_mcp_shim/handlers.py index c6d99148b..5efb05a54 100644 --- a/backend/apps/spotify_mcp_shim/handlers.py +++ b/backend/apps/spotify_mcp_shim/handlers.py @@ -1,8 +1,7 @@ from typing import Optional, Any, Dict -import psutil +import asyncio import json import urllib.request -import urllib.error import os # Bypass the OpenSwarm proxy for local Chrome CDP connections @@ -34,7 +33,9 @@ def p_ensure_chrome_cdp(): "--remote-debugging-port=9223", f"--user-data-dir={profile_dir}", "--no-first-run", - "--no-default-browser-check" + "--no-default-browser-check", + "--restore-last-session", + "--password-store=basic" ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) # Wait for it to spin up @@ -50,7 +51,7 @@ def p_ensure_chrome_cdp(): return False -def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]: +async def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]: """ Using the Playwright MCP, we'll: @@ -67,156 +68,191 @@ def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]: "message": "Failed to auto-start Chrome with remote debugging on port 9223." } - import asyncio + from playwright.async_api import async_playwright + import urllib.parse + import base64 + from io import BytesIO + from PIL import Image - def run_async_play_track(): - async def inner(): - from playwright.async_api import async_playwright - import urllib.parse - import base64 - from io import BytesIO - from PIL import Image + from backend.apps.settings.settings import load_settings + from backend.apps.settings.credentials import get_anthropic_client_for_model + from backend.apps.agents.providers.registry import resolve_aux_model + + settings = load_settings() + try: + model_id, _ = await resolve_aux_model(settings, preferred_tier="haiku") + client = get_anthropic_client_for_model(settings, model_id) + except Exception as e: + return { + "is_error": True, + "is_human_intervention": False, + "message": f"Could not load AI client for screenshot analysis: {e}" + } + + try: + async with async_playwright() as p: + browser = await p.chromium.connect_over_cdp("http://127.0.0.1:9223") + context = browser.contexts[0] - from backend.apps.settings.settings import load_settings - from backend.apps.settings.credentials import get_anthropic_client_for_model - from backend.apps.agents.providers.registry import resolve_aux_model + page = None + for p_obj in context.pages: + if "spotify.com" in p_obj.url: + page = p_obj + break - settings = load_settings() - try: - model_id, _ = await resolve_aux_model(settings, preferred_tier="haiku") - client = get_anthropic_client_for_model(settings, model_id) - except Exception as e: - return { - "is_error": True, - "is_human_intervention": False, - "message": f"Could not load AI client for screenshot analysis: {e}" - } + if not page: + page = await context.new_page() - try: - async with async_playwright() as p: - browser = await p.chromium.connect_over_cdp("http://127.0.0.1:9223") - context = browser.contexts[0] - - page = None - for p_obj in context.pages: - if "spotify.com" in p_obj.url: - page = p_obj - break - - if not page: - page = await context.new_page() - - query = track_name - if artist: - query += f" {artist}" - - search_url = f"https://open.spotify.com/search/{urllib.parse.quote(query)}" - await page.goto(search_url) - await page.wait_for_load_state("networkidle") - await asyncio.sleep(2) # Give elements time to render - - prompt = f""" + query = track_name + if artist: + query += f" {artist}" + + search_url = f"https://open.spotify.com/search/{urllib.parse.quote(query)}" + await page.goto(search_url) + await page.wait_for_load_state("networkidle") + await asyncio.sleep(4) # Give elements time to render + + prompt = f""" I want to play the track '{query}' on Spotify. Look at this screenshot of the Spotify web UI. Determine the state and return ONLY a valid JSON object. Do not include markdown formatting or backticks, just the raw JSON. You must choose one of the following exact JSON structures: 1. If a login prompt or overlay is blocking the UI: {{ - "action": "human_intervention", - "message": "User needs to log in" +"action": "human_intervention", +"message": "User needs to log in" }} -2. If the track '{query}' is already playing (pause button is visible): +2. If the track '{query}' is already playing (a pause button is visible, or the now-playing bar at the bottom shows the track playing): {{ - "action": "done", - "message": "Successfully started playback" +"action": "done", +"message": "Successfully started playback" }} 3. Otherwise, find the Play button for the top search result and return its exact center coordinates as percentages (0 to 100) of the image width and height: {{ - "action": "click", - "x_percent": 50.5, - "y_percent": 25.0, - "message": "Clicking play button" +"action": "click", +"x_percent": 50.5, +"y_percent": 25.0, +"message": "Clicking play button" }} CRITICAL: Return strictly valid JSON. Double check your quotes and commas. """.strip() - for _ in range(3): - screenshot_bytes = await page.screenshot() - - img = Image.open(BytesIO(screenshot_bytes)) - max_width = 1024 - if img.width > max_width: - ratio = max_width / img.width - img = img.resize((max_width, int(img.height * ratio)), Image.LANCZOS) - buf = BytesIO() - img.convert("RGB").save(buf, format="JPEG", quality=45) - b64_img = base64.b64encode(buf.getvalue()).decode("utf-8") + for attempt in range(4): + if attempt > 0: + # Wait a bit longer between clicks/checks to let the stream start + await asyncio.sleep(4) + + screenshot_bytes = await page.screenshot() + + img = Image.open(BytesIO(screenshot_bytes)) + max_width = 1024 + if img.width > max_width: + ratio = max_width / img.width + img = img.resize((max_width, int(img.height * ratio)), Image.LANCZOS) + buf = BytesIO() + img.convert("RGB").save(buf, format="JPEG", quality=45) + b64_img = base64.b64encode(buf.getvalue()).decode("utf-8") + + response = await client.messages.create( + model=model_id, + max_tokens=256, + messages=[{ + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": b64_img}} + ] + }] + ) + + try: + # Parse JSON from response safely, handling both Anthropic and OpenAI wrapper formats + resp_text = "" + if getattr(response, "content", None): + content_block = response.content[0] + resp_text = getattr(content_block, "text", str(content_block)).strip() + else: + choices = getattr(response, "choices", None) + if not choices and hasattr(response, "model_dump"): + choices = response.model_dump().get("choices") + if not choices and hasattr(response, "dict"): + choices = response.dict().get("choices") + if choices and len(choices) > 0: + choice = choices[0] + if isinstance(choice, dict): + msg = choice.get("message", {}) + if isinstance(msg, dict): + resp_text = msg.get("content", "") + else: + resp_text = getattr(msg, "content", "") + else: + msg = getattr(choice, "message", None) + resp_text = getattr(msg, "content", "") + + if not resp_text: + raise ValueError(f"AI returned empty content or unsupported format! Full response: {response}") + + if resp_text.startswith("```json"): + resp_text = resp_text[7:-3].strip() + elif resp_text.startswith("```"): + resp_text = resp_text[3:-3].strip() - response = await client.messages.create( - model=model_id, - max_tokens=256, - messages=[{ - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": b64_img}} - ] - }] - ) + action_data = json.loads(resp_text) + + if action_data["action"] == "human_intervention": + return { + "is_error": False, + "is_human_intervention": True, + "message": action_data.get("message", "Human intervention needed.") + } + elif action_data["action"] == "done": + return { + "is_error": False, + "is_human_intervention": False, + "message": action_data.get("message", "Task completed.") + } + elif action_data["action"] == "click": + vp_w = await page.evaluate("window.innerWidth") + vp_h = await page.evaluate("window.innerHeight") - try: - # Parse JSON from response - resp_text = response.content[0].text.strip() - if resp_text.startswith("```json"): - resp_text = resp_text[7:-3].strip() - elif resp_text.startswith("```"): - resp_text = resp_text[3:-3].strip() - - action_data = json.loads(resp_text) + # Use exact image dimensions to calculate percentages, then map to CSS pixels + if "x_percent" in action_data and "y_percent" in action_data: + x_pct = float(action_data["x_percent"]) + y_pct = float(action_data["y_percent"]) + # If the AI accidentally returned pixels instead of percentages, cap them + if x_pct > 100: x_pct = (x_pct / img.width) * 100 + if y_pct > 100: y_pct = (y_pct / img.height) * 100 - if action_data["action"] == "human_intervention": - return { - "is_error": False, - "is_human_intervention": True, - "message": action_data.get("message", "Human intervention needed.") - } - elif action_data["action"] == "done": - return { - "is_error": False, - "is_human_intervention": False, - "message": action_data.get("message", "Task completed.") - } - elif action_data["action"] == "click": - vp_w = await page.evaluate("window.innerWidth") - vp_h = await page.evaluate("window.innerHeight") - click_x = (float(action_data["x_percent"]) / 100.0) * vp_w - click_y = (float(action_data["y_percent"]) / 100.0) * vp_h - await page.mouse.click(click_x, click_y) - await asyncio.sleep(2) # wait for playback to start - # loop continues to verify - except Exception as e: - return { - "is_error": True, - "is_human_intervention": False, - "message": f"Failed to parse AI response: {e}\nResponse was: {response.content[0].text}" - } + click_x = (x_pct / 100.0) * vp_w + click_y = (y_pct / 100.0) * vp_h + else: + # Fallback if AI returned raw pixels (x, y) + click_x = float(action_data["x"]) * (vp_w / img.width) + click_y = float(action_data["y"]) * (vp_h / img.height) + await page.mouse.click(click_x, click_y) + # sleep is now handled at the start of the next loop iteration + except Exception as e: + raw_resp = getattr(response, 'content', 'No content attribute') return { "is_error": True, "is_human_intervention": False, - "message": "AI failed to start playback after multiple attempts." + "message": f"Failed to parse AI response: {e}\nRaw content was: {raw_resp}" } - except Exception as e: - return { - "is_error": True, - "is_human_intervention": False, - "message": f"An error occurred: {str(e)}" - } - - return asyncio.run(inner()) - - return run_async_play_track() \ No newline at end of file + return { + "is_error": True, + "is_human_intervention": False, + "message": "AI failed to start playback after multiple attempts." + } + + except Exception as e: + import traceback + return { + "is_error": True, + "is_human_intervention": False, + "message": f"An error occurred: {str(e)}\n\nTraceback:\n{traceback.format_exc()}" + } \ No newline at end of file diff --git a/backend/apps/spotify_mcp_shim/server.py b/backend/apps/spotify_mcp_shim/server.py index 925b8baf4..1adfd3721 100644 --- a/backend/apps/spotify_mcp_shim/server.py +++ b/backend/apps/spotify_mcp_shim/server.py @@ -21,14 +21,20 @@ def p_ok(payload) -> dict: return {"content": [{"type": "text", "text": payload}]} return {"content": [{"type": "text", "text": json.dumps(payload, indent=2, default=str)}]} -from backend.apps.spotify_mcp_shim.handlers import play_track +import asyncio +from backend.apps.spotify_mcp_shim import handlers -def handle_tool_call(name: str, args: dict) -> dict: - match name: - case "play_track": - return p_ok(play_track(**args)) +def execute_tool_function(func, args: dict): + if asyncio.iscoroutinefunction(func): + return asyncio.run(func(**args)) + return func(**args) - return p_err(f"Unknown tool: {name}") +def handle_tool_call(name: str, args: dict) -> dict: + handler = getattr(handlers, name, None) + if not handler or not callable(handler): + return p_err(f"Unknown tool: {name}") + + return p_ok(execute_tool_function(handler, args)) def main(): for line in sys.stdin: diff --git a/frontend/src/app/pages/Tools/integrations.tsx b/frontend/src/app/pages/Tools/integrations.tsx index 25191cb77..10c1e04af 100644 --- a/frontend/src/app/pages/Tools/integrations.tsx +++ b/frontend/src/app/pages/Tools/integrations.tsx @@ -43,10 +43,14 @@ export const INTEGRATIONS: Integration[] = [ { id: 'spotify', name: "Spotify", - description: 'Control playback, search tracks, and manage playlists.', + description: 'Control playback, search tracks, and manage playlists. (Opens an external Chrome window to bypass DRM)', mcp_config: { type: 'stdio', command: 'python', args: ['-m', 'backend.apps.spotify_mcp_shim'] }, color: '#1DB954', website: 'https://open.spotify.com', + authType: 'browser_login', + connectLabel: 'Instructions', + loginUrl: '#', + connectInstructions: 'IMPORTANT: Spotify DRM blocks internal browsers. When you run a command for the first time, an external Google Chrome window will automatically open. Please sign into Spotify in that external window. Do NOT log in via the internal OpenSwarm browser, as playback will fail.', icon: ( From 7c1fcd84583f98cd5db02d1454f36961a9578ea1 Mon Sep 17 00:00:00 2001 From: Ethan Hanlon Date: Sun, 19 Jul 2026 11:20:06 -0700 Subject: [PATCH 3/6] [ethan] feat: add tools for playlist, album, and artist playback. fix overconfidence bugs and logo. --- .../apps/spotify_mcp_shim/Spotify_icon.svg | 5 + backend/apps/spotify_mcp_shim/handlers.py | 219 ++++++++++++------ backend/apps/spotify_mcp_shim/tools.py | 36 ++- frontend/src/app/pages/Tools/integrations.tsx | 5 +- 4 files changed, 188 insertions(+), 77 deletions(-) create mode 100644 backend/apps/spotify_mcp_shim/Spotify_icon.svg diff --git a/backend/apps/spotify_mcp_shim/Spotify_icon.svg b/backend/apps/spotify_mcp_shim/Spotify_icon.svg new file mode 100644 index 000000000..4f2e80329 --- /dev/null +++ b/backend/apps/spotify_mcp_shim/Spotify_icon.svg @@ -0,0 +1,5 @@ + + + + + \ No newline at end of file diff --git a/backend/apps/spotify_mcp_shim/handlers.py b/backend/apps/spotify_mcp_shim/handlers.py index 5efb05a54..7c28853c5 100644 --- a/backend/apps/spotify_mcp_shim/handlers.py +++ b/backend/apps/spotify_mcp_shim/handlers.py @@ -1,7 +1,7 @@ from typing import Optional, Any, Dict -import asyncio import json import urllib.request +import asyncio import os # Bypass the OpenSwarm proxy for local Chrome CDP connections @@ -11,10 +11,23 @@ import subprocess import time -def p_ensure_chrome_cdp(): - """Ensure Chrome is running with CDP on port 9223. - If not, we launch a dedicated profile so we don't conflict with the user's main Chrome, - and we leave it running in the background so music keeps playing after the script exits! +def get_browser_executable() -> str: + """Returns the path to a browser that is both CDP and Spotify compatibile.""" + browsers = [ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", + "/Applications/Opera.app/Contents/MacOS/Opera", + "/Applications/Opera GX.app/Contents/MacOS/Opera GX", + ] + for b in browsers: + if os.path.exists(b): + return b + raise FileNotFoundError("No CDP-compatible browser found (Chrome, Edge, Opera).") + +def ensure_browser_cdp() -> bool: + """Ensure a CDP-compatible browser is running with CDP on port 9223. + If not, we launch a dedicated profile so we don't conflict with the user's main browser, + and we leave it running in the background. """ try: req = urllib.request.Request("http://127.0.0.1:9223/json/version") @@ -24,12 +37,18 @@ def p_ensure_chrome_cdp(): except Exception: pass - # CDP not responding. Let's auto-launch a dedicated Chrome instance! + # CDP not responding. Let's auto-launch a dedicated profile! + try: + executable = get_browser_executable() + except FileNotFoundError as e: + print(f"Error: {e}") + return False + profile_dir = os.path.expanduser("~/.openswarm/spotify_chrome_profile") os.makedirs(profile_dir, exist_ok=True) subprocess.Popen([ - "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + executable, "--remote-debugging-port=9223", f"--user-data-dir={profile_dir}", "--no-first-run", @@ -51,22 +70,34 @@ def p_ensure_chrome_cdp(): return False -async def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]: +async def get_browser_page(p, target_url_substring: Optional[str] = None): """ - Using the Playwright MCP, we'll: - - 1. Auto-launch a dedicated Chrome instance (with CDP) if it's not already running. - 2. Navigate to Spotify. If this is the user's first time on this dedicated profile, they will need to log in. - 3. Search for the song name. - 4. Play the song! + Connects to the CDP browser and returns a (browser, context, page) tuple. + If target_url_substring is provided, it will try to find and return an existing tab + that matches the substring. Otherwise, it will open a new tab. """ + if not ensure_browser_cdp(): + raise RuntimeError("Failed to auto-start browser with remote debugging on port 9223.") + + browser = await p.chromium.connect_over_cdp("http://127.0.0.1:9223") + context = browser.contexts[0] + + page = None + if target_url_substring: + for p_obj in context.pages: + if target_url_substring in p_obj.url: + page = p_obj + break + + if not page: + page = await context.new_page() + + return browser, context, page - if not p_ensure_chrome_cdp(): - return { - "is_error": True, - "is_human_intervention": False, - "message": "Failed to auto-start Chrome with remote debugging on port 9223." - } +async def play_media(query: str, media_type: str) -> Dict[str, Any]: + """ + Shared helper for play_track, play_album, and play_playlist. + """ from playwright.async_api import async_playwright import urllib.parse @@ -91,29 +122,44 @@ async def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]: try: async with async_playwright() as p: - browser = await p.chromium.connect_over_cdp("http://127.0.0.1:9223") - context = browser.contexts[0] - - page = None - for p_obj in context.pages: - if "spotify.com" in p_obj.url: - page = p_obj - break - - if not page: - page = await context.new_page() - - query = track_name - if artist: - query += f" {artist}" + try: + browser, context, page = await get_browser_page(p, target_url_substring="spotify.com") + except RuntimeError as e: + return { + "is_error": True, + "is_human_intervention": False, + "message": str(e) + } - search_url = f"https://open.spotify.com/search/{urllib.parse.quote(query)}" + search_url = f"https://open.spotify.com/search/{urllib.parse.quote(query, safe='')}" await page.goto(search_url) - await page.wait_for_load_state("networkidle") - await asyncio.sleep(4) # Give elements time to render - prompt = f""" -I want to play the track '{query}' on Spotify. Look at this screenshot of the Spotify web UI. + try: + await page.wait_for_load_state("networkidle", timeout=2000) + except Exception: + pass + + try: + # Scientifically wait for the play buttons OR the login button to render in the DOM + await page.wait_for_selector('button[data-testid="play-button"], [data-testid="login-button"]', timeout=4000) + except Exception: + pass + + action_history = [] + for attempt in range(4): + screenshot_bytes = await page.screenshot() + + img = Image.open(BytesIO(screenshot_bytes)) + max_width = 1024 + if img.width > max_width: + ratio = max_width / img.width + img = img.resize((max_width, int(img.height * ratio)), Image.LANCZOS) + buf = BytesIO() + img.convert("RGB").save(buf, format="JPEG", quality=45) + b64_img = base64.b64encode(buf.getvalue()).decode("utf-8") + + prompt = f""" +I want to play the {media_type} '{query}' on Spotify. Look at this screenshot of the Spotify web UI. Determine the state and return ONLY a valid JSON object. Do not include markdown formatting or backticks, just the raw JSON. You must choose one of the following exact JSON structures: @@ -123,38 +169,20 @@ async def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]: "message": "User needs to log in" }} -2. If the track '{query}' is already playing (a pause button is visible, or the now-playing bar at the bottom shows the track playing): -{{ -"action": "done", -"message": "Successfully started playback" -}} - -3. Otherwise, find the Play button for the top search result and return its exact center coordinates as percentages (0 to 100) of the image width and height: +3. Otherwise, if the {media_type} is not currently playing, you must start it. +- Look at the screen. If you are currently on a search results page, find the best match for '{query}' that is explicitly labeled as a {media_type} (e.g. look for the subtitle 'Album', 'Playlist', 'Song', or 'Artist'). + - If the best match has a prominent green Play button (e.g., the Top Result card), return its coordinates. + - If the best match does NOT have a visible Play button (e.g., it is a row in a list), return the coordinates of its text title to click on it. This will navigate to its dedicated page where a Play button will be visible on the next step. +- If you are ALREADY on a dedicated media page (e.g., you already clicked a title in a previous step), return the coordinates of the main green Play button on the page. {{ "action": "click", "x_percent": 50.5, "y_percent": 25.0, -"message": "Clicking play button" +"message": "Clicking play button (or title to navigate)" }} CRITICAL: Return strictly valid JSON. Double check your quotes and commas. """.strip() - - for attempt in range(4): - if attempt > 0: - # Wait a bit longer between clicks/checks to let the stream start - await asyncio.sleep(4) - - screenshot_bytes = await page.screenshot() - - img = Image.open(BytesIO(screenshot_bytes)) - max_width = 1024 - if img.width > max_width: - ratio = max_width / img.width - img = img.resize((max_width, int(img.height * ratio)), Image.LANCZOS) - buf = BytesIO() - img.convert("RGB").save(buf, format="JPEG", quality=45) - b64_img = base64.b64encode(buf.getvalue()).decode("utf-8") response = await client.messages.create( model=model_id, @@ -201,6 +229,7 @@ async def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]: resp_text = resp_text[3:-3].strip() action_data = json.loads(resp_text) + action_history.append(action_data) if action_data["action"] == "human_intervention": return { @@ -208,12 +237,6 @@ async def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]: "is_human_intervention": True, "message": action_data.get("message", "Human intervention needed.") } - elif action_data["action"] == "done": - return { - "is_error": False, - "is_human_intervention": False, - "message": action_data.get("message", "Task completed.") - } elif action_data["action"] == "click": vp_w = await page.evaluate("window.innerWidth") vp_h = await page.evaluate("window.innerHeight") @@ -233,8 +256,36 @@ async def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]: click_x = float(action_data["x"]) * (vp_w / img.width) click_y = float(action_data["y"]) * (vp_h / img.height) - await page.mouse.click(click_x, click_y) - # sleep is now handled at the start of the next loop iteration + await page.mouse.click(click_x, click_y, delay=100) + + # Deterministically wait for playback to start + success = False + for _ in range(10): # 5 seconds + try: + # Check React UI + pause_btn = await page.query_selector('button[data-testid="control-button-pause"]') + if pause_btn: + success = True + break + + # Check HTML5 Media + is_playing = await page.evaluate("() => Array.from(document.querySelectorAll('audio, video')).some(el => !el.paused && el.duration > 0)") + if is_playing: + success = True + break + except Exception: + pass + + await asyncio.sleep(0.5) + + if success: + return { + "is_error": False, + "is_human_intervention": False, + "message": "Task completed. Playback verified programmatically." + } + + # If not successful, the loop will just continue to the next attempt! except Exception as e: raw_resp = getattr(response, 'content', 'No content attribute') return { @@ -246,7 +297,7 @@ async def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]: return { "is_error": True, "is_human_intervention": False, - "message": "AI failed to start playback after multiple attempts." + "message": f"AI failed to start playback after multiple attempts.\nAction history:\n{json.dumps(action_history, indent=2)}" } except Exception as e: @@ -255,4 +306,24 @@ async def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]: "is_error": True, "is_human_intervention": False, "message": f"An error occurred: {str(e)}\n\nTraceback:\n{traceback.format_exc()}" - } \ No newline at end of file + } + +async def play_track(track_name: str, artist: Optional[str] = None) -> Dict[str, Any]: + query = track_name + if artist: + query += f" {artist}" + return await play_media(query, "track") + +async def play_album(album_name: str, artist: Optional[str] = None) -> Dict[str, Any]: + query = album_name + if artist: + query += f" {artist}" + return await play_media(query, "album") + +async def play_playlist(playlist_name: str) -> Dict[str, Any]: + query = playlist_name + return await play_media(query, "playlist") + +async def play_artist(artist_name: str) -> Dict[str, Any]: + query = artist_name + return await play_media(query, "artist") \ No newline at end of file diff --git a/backend/apps/spotify_mcp_shim/tools.py b/backend/apps/spotify_mcp_shim/tools.py index 2b0510cf6..c7a3692b9 100644 --- a/backend/apps/spotify_mcp_shim/tools.py +++ b/backend/apps/spotify_mcp_shim/tools.py @@ -9,7 +9,7 @@ TOOLS = [ { "name": "play_track", - "description": "Opens a browser and plays the requested song.", + "description": "Opens an external browser (like Chrome or Edge) to bypass DRM and plays the requested song. NOTE: The user will see a separate browser window open.", "inputSchema": { "type": OBJ, "properties": { @@ -18,5 +18,39 @@ }, "required": ["track_name"] } + }, + { + "name": "play_album", + "description": "Opens an external browser (like Chrome or Edge) to bypass DRM and plays the requested album. NOTE: The user will see a separate browser window open.", + "inputSchema": { + "type": OBJ, + "properties": { + "album_name": {"type": "string"}, + "artist": {"type": "string"} + }, + "required": ["album_name"] + } + }, + { + "name": "play_playlist", + "description": "Opens an external browser (like Chrome or Edge) to bypass DRM and plays the requested playlist. NOTE: The user will see a separate browser window open.", + "inputSchema": { + "type": OBJ, + "properties": { + "playlist_name": {"type": "string"} + }, + "required": ["playlist_name"] + } + }, + { + "name": "play_artist", + "description": "Opens an external browser (like Chrome or Edge) to bypass DRM and plays the requested artist's top tracks. NOTE: The user will see a separate browser window open.", + "inputSchema": { + "type": OBJ, + "properties": { + "artist_name": {"type": "string"} + }, + "required": ["artist_name"] + } } ] \ No newline at end of file diff --git a/frontend/src/app/pages/Tools/integrations.tsx b/frontend/src/app/pages/Tools/integrations.tsx index 10c1e04af..af0be2f50 100644 --- a/frontend/src/app/pages/Tools/integrations.tsx +++ b/frontend/src/app/pages/Tools/integrations.tsx @@ -52,8 +52,9 @@ export const INTEGRATIONS: Integration[] = [ loginUrl: '#', connectInstructions: 'IMPORTANT: Spotify DRM blocks internal browsers. When you run a command for the first time, an external Google Chrome window will automatically open. Please sign into Spotify in that external window. Do NOT log in via the internal OpenSwarm browser, as playback will fail.', icon: ( - - + + + ), }, From e776b2d6fa427e18dd87c539fd809739964169ce Mon Sep 17 00:00:00 2001 From: Ethan Hanlon Date: Sun, 19 Jul 2026 11:58:06 -0700 Subject: [PATCH 4/6] [ethan] fix: remove gray circles from connection logos. repair some bad logos. --- .../app/pages/Tools/cards/CustomToolCard.tsx | 2 +- .../Tools/cards/IntegrationGalleryCard.tsx | 2 +- .../src/app/pages/Tools/integrationIcons.tsx | 94 +++++++++++++++++++ frontend/src/app/pages/Tools/integrations.tsx | 93 +++--------------- 4 files changed, 111 insertions(+), 80 deletions(-) create mode 100644 frontend/src/app/pages/Tools/integrationIcons.tsx diff --git a/frontend/src/app/pages/Tools/cards/CustomToolCard.tsx b/frontend/src/app/pages/Tools/cards/CustomToolCard.tsx index d6c7da399..571eb681d 100644 --- a/frontend/src/app/pages/Tools/cards/CustomToolCard.tsx +++ b/frontend/src/app/pages/Tools/cards/CustomToolCard.tsx @@ -111,7 +111,7 @@ const CustomToolCard: React.FC = ({ {ig && ( {ig.icon} diff --git a/frontend/src/app/pages/Tools/cards/IntegrationGalleryCard.tsx b/frontend/src/app/pages/Tools/cards/IntegrationGalleryCard.tsx index 5b1bcbc12..b4cbbc77e 100644 --- a/frontend/src/app/pages/Tools/cards/IntegrationGalleryCard.tsx +++ b/frontend/src/app/pages/Tools/cards/IntegrationGalleryCard.tsx @@ -27,7 +27,7 @@ const IntegrationGalleryCard: React.FC = ({ integra {ig.icon} diff --git a/frontend/src/app/pages/Tools/integrationIcons.tsx b/frontend/src/app/pages/Tools/integrationIcons.tsx new file mode 100644 index 000000000..a07c4cc7c --- /dev/null +++ b/frontend/src/app/pages/Tools/integrationIcons.tsx @@ -0,0 +1,94 @@ +import React from 'react'; + +export const XIcon = ( + + + +); + +export const SpotifyIcon = ( + + + + +); + +export const TikTokIcon = ( + + + +); + +export const RedditIcon = ( + + + + +); + +export const YouTubeIcon = ( + + + + +); + +export const GitHubIcon = ( + + + +); + +export const GoogleWorkspaceIcon = ( + + + + + + +); + +export const Microsoft365Icon = ( + + + +); + +export const NotionIcon = ( + + + + +); + +export const AirtableIcon = ( + + + + + + +); + +export const HubSpotIcon = ( + + + +); + +export const SlackIcon = ( + + + + + + +); + +export const DiscordIcon = ( + + + +); diff --git a/frontend/src/app/pages/Tools/integrations.tsx b/frontend/src/app/pages/Tools/integrations.tsx index af0be2f50..c928a1a3b 100644 --- a/frontend/src/app/pages/Tools/integrations.tsx +++ b/frontend/src/app/pages/Tools/integrations.tsx @@ -1,4 +1,5 @@ import React from 'react'; +import { AirtableIcon, DiscordIcon, GitHubIcon, GoogleWorkspaceIcon, HubSpotIcon, Microsoft365Icon, NotionIcon, RedditIcon, SlackIcon, SpotifyIcon, TikTokIcon, XIcon, YouTubeIcon } from './integrationIcons'; export interface CredentialField { key: string; @@ -34,11 +35,7 @@ export const INTEGRATIONS: Integration[] = [ connectLabel: 'Sign in to X', loginUrl: 'https://x.com/i/flow/login', connectInstructions: 'Uses your own X account: open x.com in an OpenSwarm browser card and sign in once. Nothing is stored, the integration borrows your live session per request and paces itself to stay within human limits.', - icon: ( - - - - ), + icon: XIcon, }, { id: 'spotify', @@ -50,13 +47,8 @@ export const INTEGRATIONS: Integration[] = [ authType: 'browser_login', connectLabel: 'Instructions', loginUrl: '#', - connectInstructions: 'IMPORTANT: Spotify DRM blocks internal browsers. When you run a command for the first time, an external Google Chrome window will automatically open. Please sign into Spotify in that external window. Do NOT log in via the internal OpenSwarm browser, as playback will fail.', - icon: ( - - - - - ), + connectInstructions: 'IMPORTANT: Spotify DRM blocks internal browsers. When you run a command for the first time, an external browser window will automatically open. Please sign into Spotify in that external window. Do NOT log in via the internal OpenSwarm browser, as playback will fail.', + icon: SpotifyIcon, }, { id: 'tiktok', @@ -69,11 +61,7 @@ export const INTEGRATIONS: Integration[] = [ connectLabel: 'Sign in to TikTok', loginUrl: 'https://www.tiktok.com/login', connectInstructions: 'Uses your own TikTok account: open tiktok.com in an OpenSwarm browser card and sign in once. Nothing is stored. Note: TikTok signs every request, so signed writes and uploads route to the OpenSwarm browser agent (also free, using your real session).', - icon: ( - - - - ), + icon: TikTokIcon, }, { id: 'reddit', @@ -86,12 +74,7 @@ export const INTEGRATIONS: Integration[] = [ connectLabel: 'Sign in to Reddit', loginUrl: 'https://www.reddit.com/login', connectInstructions: 'Uses your own Reddit account: open reddit.com in an OpenSwarm browser card and sign in once. Nothing is stored, the integration borrows your live session per request and paces itself to stay within human limits.', - icon: ( - - - - - ), + icon: RedditIcon, }, { id: 'youtube', @@ -100,12 +83,7 @@ export const INTEGRATIONS: Integration[] = [ mcp_config: { type: 'stdio', command: 'npx', args: ['-y', '@kirbah/mcp-youtube'] }, color: '#FF0000', website: 'https://www.npmjs.com/package/@kirbah/mcp-youtube', - icon: ( - - - - - ), + icon: YouTubeIcon, }, { id: 'github', @@ -114,11 +92,7 @@ export const INTEGRATIONS: Integration[] = [ mcp_config: { type: 'http', url: 'https://api.githubcopilot.com/mcp/x/all' }, color: '#181717', website: 'https://github.com/github/github-mcp-server', - icon: ( - - - - ), + icon: GitHubIcon, authType: 'oauth2', }, { @@ -128,14 +102,7 @@ export const INTEGRATIONS: Integration[] = [ mcp_config: { type: 'stdio', command: 'uvx', args: ['--from', 'google-workspace-mcp', 'google-workspace-worker'] }, color: '#4285F4', website: 'https://developers.google.com/gemini-api/docs/mcp', - icon: ( - - - - - - - ), + icon: GoogleWorkspaceIcon, authType: 'oauth2', }, { @@ -145,11 +112,7 @@ export const INTEGRATIONS: Integration[] = [ mcp_config: { type: 'stdio', command: 'npx', args: ['-y', '@softeria/ms-365-mcp-server'] }, color: '#0078D4', website: 'https://www.npmjs.com/package/@softeria/ms-365-mcp-server', - icon: ( - - - - ), + icon: Microsoft365Icon, authType: 'device_code', }, { @@ -159,11 +122,7 @@ export const INTEGRATIONS: Integration[] = [ mcp_config: { type: 'stdio', command: 'npx', args: ['-y', '@notionhq/notion-mcp-server'] }, color: '#000000', website: 'https://www.npmjs.com/package/@notionhq/notion-mcp-server', - icon: ( - - - - ), + icon: NotionIcon, authType: 'oauth2', }, { @@ -173,14 +132,7 @@ export const INTEGRATIONS: Integration[] = [ mcp_config: { type: 'http', url: 'https://mcp.airtable.com/mcp' }, color: '#18BFFF', website: 'https://airtable.com/developers/web/api/introduction', - icon: ( - - - - - - - ), + icon: AirtableIcon, authType: 'oauth2', }, { @@ -190,11 +142,7 @@ export const INTEGRATIONS: Integration[] = [ mcp_config: { type: 'stdio', command: 'npx', args: ['-y', '@hubspot/mcp-server'] }, color: '#FF7A59', website: 'https://developers.hubspot.com/docs/guides/apps/developer-platform/build-apps/integrate-with-the-remote-hubspot-mcp-server', - icon: ( - - - - ), + icon: HubSpotIcon, authType: 'oauth2', }, { @@ -204,14 +152,7 @@ export const INTEGRATIONS: Integration[] = [ mcp_config: { type: 'stdio', command: 'npx', args: ['-y', 'slack-mcp-server@latest', '--transport', 'stdio'], env: { SLACK_MCP_ADD_MESSAGE_TOOL: 'true' } }, color: '#4A154B', website: 'https://github.com/korotovsky/slack-mcp-server', - icon: ( - - - - - - - ), + icon: SlackIcon, connectLabel: 'Connect Slack', connectInstructions: 'On macOS with Chrome, tokens are auto-extracted. Otherwise: open app.slack.com in Chrome → F12 → Console → type `JSON.stringify({token: boot_data.api_token, cookie: document.cookie.match(/d=([^;]+)/)?.[1]})` → copy the result.', credentialFields: [ @@ -226,11 +167,7 @@ export const INTEGRATIONS: Integration[] = [ mcp_config: { type: 'stdio', command: 'python', args: ['-m', 'backend.apps.discord_mcp_shim'] }, color: '#5865F2', website: 'https://github.com/barryyip0625/mcp-discord', - icon: ( - - - - ), + icon: DiscordIcon, authType: 'oauth2', }, ]; From cafa415f501bd6e219c07257c5e14adbda265e75 Mon Sep 17 00:00:00 2001 From: Ethan Hanlon Date: Sun, 19 Jul 2026 13:26:02 -0700 Subject: [PATCH 5/6] [ethan] feat: Add Safari, Edge, Opera support --- backend/apps/spotify_mcp_shim/handlers.py | 630 ++++++++++++++++------ backend/apps/spotify_mcp_shim/server.py | 91 ++-- backend/apps/spotify_mcp_shim/tools.py | 32 +- 3 files changed, 545 insertions(+), 208 deletions(-) diff --git a/backend/apps/spotify_mcp_shim/handlers.py b/backend/apps/spotify_mcp_shim/handlers.py index 7c28853c5..e9e50862e 100644 --- a/backend/apps/spotify_mcp_shim/handlers.py +++ b/backend/apps/spotify_mcp_shim/handlers.py @@ -1,5 +1,24 @@ from typing import Optional, Any, Dict import json +import os + +def set_default_engine(engine: str) -> Dict[str, Any]: + config_path = os.path.expanduser("~/.openswarm/spotify_engine.json") + os.makedirs(os.path.dirname(config_path), exist_ok=True) + with open(config_path, "w") as f: + json.dump({"engine": engine}, f) + return {"message": f"Successfully set default engine to {engine}"} + +def get_default_engine() -> str: + config_path = os.path.expanduser("~/.openswarm/spotify_engine.json") + if os.path.exists(config_path): + try: + with open(config_path, "r") as f: + return json.load(f).get("engine", "safari_applescript") + except: + pass + return "safari_applescript" + import urllib.request import asyncio import os @@ -70,40 +89,84 @@ def ensure_browser_cdp() -> bool: return False -async def get_browser_page(p, target_url_substring: Optional[str] = None): +_playwright_mgr = None +_playwright = None +_webkit_context = None + +async def get_playwright(): + global _playwright_mgr, _playwright + if _playwright is None: + from playwright.async_api import async_playwright + _playwright_mgr = async_playwright() + _playwright = await _playwright_mgr.start() + return _playwright + +async def get_browser_page(target_url_substring: Optional[str] = None, engine: Optional[str] = None): """ - Connects to the CDP browser and returns a (browser, context, page) tuple. + Connects to the CDP browser (Chromium) or persistent context (WebKit) and returns a (browser, context, page) tuple. If target_url_substring is provided, it will try to find and return an existing tab that matches the substring. Otherwise, it will open a new tab. """ - if not ensure_browser_cdp(): - raise RuntimeError("Failed to auto-start browser with remote debugging on port 9223.") - - browser = await p.chromium.connect_over_cdp("http://127.0.0.1:9223") - context = browser.contexts[0] + p = await get_playwright() - page = None - if target_url_substring: - for p_obj in context.pages: - if target_url_substring in p_obj.url: - page = p_obj - break - - if not page: - page = await context.new_page() + if engine == "webkit": + global _webkit_context + if _webkit_context is None: + profile_dir = os.path.expanduser("~/.openswarm/spotify_webkit_profile") + os.makedirs(profile_dir, exist_ok=True) + _webkit_context = await p.webkit.launch_persistent_context( + user_data_dir=profile_dir, + headless=False + ) + + page = None + if target_url_substring: + for p_obj in _webkit_context.pages: + if target_url_substring in p_obj.url: + page = p_obj + break + + if not page: + if len(_webkit_context.pages) == 1 and _webkit_context.pages[0].url == "about:blank": + page = _webkit_context.pages[0] + else: + page = await _webkit_context.new_page() + + return None, _webkit_context, page - return browser, context, page + else: + if not ensure_browser_cdp(): + raise RuntimeError("Failed to auto-start Chromium browser with remote debugging on port 9223.") + + browser = await p.chromium.connect_over_cdp("http://127.0.0.1:9223") + context = browser.contexts[0] + + page = None + if target_url_substring: + for p_obj in context.pages: + if target_url_substring in p_obj.url: + page = p_obj + break + + if not page: + if len(context.pages) == 1 and context.pages[0].url == "about:blank": + page = context.pages[0] + else: + page = await context.new_page() + + return browser, context, page -async def play_media(query: str, media_type: str) -> Dict[str, Any]: +async def play_media(query: str, media_type: str, engine: Optional[str] = None) -> Dict[str, Any]: """ Shared helper for play_track, play_album, and play_playlist. """ + if not engine: + engine = get_default_engine() - from playwright.async_api import async_playwright import urllib.parse - import base64 - from io import BytesIO - from PIL import Image + import asyncio + import subprocess + import json from backend.apps.settings.settings import load_settings from backend.apps.settings.credentials import get_anthropic_client_for_model @@ -117,118 +180,178 @@ async def play_media(query: str, media_type: str) -> Dict[str, Any]: return { "is_error": True, "is_human_intervention": False, - "message": f"Could not load AI client for screenshot analysis: {e}" + "message": f"Could not load AI client: {e}" } - try: - async with async_playwright() as p: - try: - browser, context, page = await get_browser_page(p, target_url_substring="spotify.com") - except RuntimeError as e: - return { - "is_error": True, - "is_human_intervention": False, - "message": str(e) - } - - search_url = f"https://open.spotify.com/search/{urllib.parse.quote(query, safe='')}" - await page.goto(search_url) + if engine == "safari_applescript": + search_url = f"https://open.spotify.com/search/{urllib.parse.quote(query, safe='')}" + + # JS extraction script for Set-of-Mark + js_payload = f''' + (function() {{ + let container = document.querySelector('.main-view-container') || document.querySelector('main') || document.body; + let interactables = container.querySelectorAll('button, a'); - try: - await page.wait_for_load_state("networkidle", timeout=2000) - except Exception: - pass + if (interactables.length === 0) {{ + return JSON.stringify({{action: "wait"}}); + }} + + let elements = []; + let metadata = {{}}; + let counter = 1; + + for (let i = 0; i < interactables.length; i++) {{ + let el = interactables[i]; + if (el.offsetWidth === 0 || el.offsetHeight === 0) continue; - try: - # Scientifically wait for the play buttons OR the login button to render in the DOM - await page.wait_for_selector('button[data-testid="play-button"], [data-testid="login-button"]', timeout=4000) - except Exception: - pass + let id = counter++; + + let ariaLabel = el.getAttribute('aria-label') || ''; + let testId = el.getAttribute('data-testid') || ''; + let text = el.innerText.substring(0, 50).trim().replace(/\n/g, ' '); + let tagName = el.tagName.toLowerCase(); + + let tagStr = `<${{tagName}} id="${{id}}"`; + if (ariaLabel) tagStr += ` aria-label="${{ariaLabel}}"`; + if (testId) tagStr += ` data-testid="${{testId}}"`; + tagStr += `>${{text}}`; + + elements.push(tagStr); + metadata[id] = {{tagName: tagName, testId: testId, ariaLabel: ariaLabel, text: text}}; + }} + return JSON.stringify({{action: "dom_extracted", html: elements.join('\n'), meta: metadata}}); + }})() + ''' + + apple_script_nav = f''' + tell application "Safari" + activate + if (count of windows) = 0 then + make new document with properties {{URL:"{search_url}"}} + else + set foundTab to false + repeat with w in windows + repeat with t in tabs of w + if URL of t contains "spotify.com" then + set current tab of w to t + set index of w to 1 + set URL of t to "{search_url}" + set foundTab to true + exit repeat + end if + end repeat + if foundTab then exit repeat + end repeat + + if not foundTab then + tell window 1 + make new tab with properties {{URL:"{search_url}"}} + set current tab to result + end tell + end if + end if + delay 3 + end tell + ''' + + try: + # Step 1: Navigate to page + res_nav = subprocess.run(["osascript", "-e", apple_script_nav], capture_output=True, text=True) + if res_nav.returncode != 0 and "-29004" in res_nav.stderr: + return { + "is_error": False, + "is_human_intervention": True, + "message": "To allow me to play music for you, I need permission to interact with Safari.\n\n1. Open Safari and press `Option-Command-,` (or go to **Safari > Settings > Advanced**).\n2. Check the box at the bottom for **Show features for web developers**.\n3. In the menu bar at the top of your screen, click **Develop > Allow JavaScript from Apple Events**.\n\n(You might see a scary Apple warning about malicious programs. Don't worry, this is just a standard warning because you are giving an AI permission to click buttons on web pages on your behalf!)\n\nIf you'd prefer not to do this, you can just ask me to use **Google Chrome** instead!" + } + action_history = [] + success = False + + # Action loop (extract DOM -> AI -> click -> repeat) for attempt in range(4): - screenshot_bytes = await page.screenshot() - - img = Image.open(BytesIO(screenshot_bytes)) - max_width = 1024 - if img.width > max_width: - ratio = max_width / img.width - img = img.resize((max_width, int(img.height * ratio)), Image.LANCZOS) - buf = BytesIO() - img.convert("RGB").save(buf, format="JPEG", quality=45) - b64_img = base64.b64encode(buf.getvalue()).decode("utf-8") + apple_script_extract = f''' + tell application "Safari" + set jsStr to "{js_payload.replace('"', '\\"').replace('\n', ' ')}" + set resultJSON to do JavaScript jsStr in current tab of front window + return resultJSON + end tell + ''' + res_extract = subprocess.run(["osascript", "-e", apple_script_extract], capture_output=True, text=True) + if res_extract.returncode != 0: + break + + out = res_extract.stdout.strip() + try: + payload = json.loads(out) + except: + continue + + if payload.get("action") == "human_intervention": + return { + "is_error": False, + "is_human_intervention": True, + "message": "User needs to log in to Spotify in Safari." + } + elif payload.get("action") == "wait": + await asyncio.sleep(2) + continue + + dom_snippet = payload.get("html", "") + if not dom_snippet: + await asyncio.sleep(2) + continue + + dom_meta = payload.get("meta", {}) + # Send to AI prompt = f""" -I want to play the {media_type} '{query}' on Spotify. Look at this screenshot of the Spotify web UI. +I want to play the {media_type} '{query}' on Spotify. Look at this extracted HTML of interactive elements from the screen. Determine the state and return ONLY a valid JSON object. Do not include markdown formatting or backticks, just the raw JSON. -You must choose one of the following exact JSON structures: -1. If a login prompt or overlay is blocking the UI: +If you see a login/signup modal popup overlaying the content, a "Log in to Spotify" wall, or a large "Sign up free" button, you MUST return: {{ "action": "human_intervention", "message": "User needs to log in" }} -3. Otherwise, if the {media_type} is not currently playing, you must start it. -- Look at the screen. If you are currently on a search results page, find the best match for '{query}' that is explicitly labeled as a {media_type} (e.g. look for the subtitle 'Album', 'Playlist', 'Song', or 'Artist'). - - If the best match has a prominent green Play button (e.g., the Top Result card), return its coordinates. - - If the best match does NOT have a visible Play button (e.g., it is a row in a list), return the coordinates of its text title to click on it. This will navigate to its dedicated page where a Play button will be visible on the next step. -- If you are ALREADY on a dedicated media page (e.g., you already clicked a title in a previous step), return the coordinates of the main green Play button on the page. +Otherwise, if the {media_type} is not currently playing, you must start it. +- Find the best match for '{query}' that is explicitly labeled as a {media_type} (or similar). +- If the best match has a Play button, return its ID. +- If the best match does NOT have a visible Play button (e.g. it is a list item), return the ID of its text title link to click on it. This will navigate to its dedicated page where a Play button will be visible on the next step. +- If you are ALREADY on a dedicated media page, return the ID of the main Play button on the page. + +Return this JSON if clicking is required: {{ "action": "click", -"x_percent": 50.5, -"y_percent": 25.0, -"message": "Clicking play button (or title to navigate)" +"id": "" }} -CRITICAL: Return strictly valid JSON. Double check your quotes and commas. -""".strip() - +HTML SNIPPET: +{dom_snippet} +""" response = await client.messages.create( model=model_id, - max_tokens=256, - messages=[{ - "role": "user", - "content": [ - {"type": "text", "text": prompt}, - {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": b64_img}} - ] - }] + max_tokens=300, + system="You are an autonomous browser automation agent. Return ONLY raw JSON without backticks.", + messages=[ + { + "role": "user", + "content": [{"type": "text", "text": prompt}] + } + ] ) try: - # Parse JSON from response safely, handling both Anthropic and OpenAI wrapper formats - resp_text = "" - if getattr(response, "content", None): - content_block = response.content[0] - resp_text = getattr(content_block, "text", str(content_block)).strip() - else: - choices = getattr(response, "choices", None) - if not choices and hasattr(response, "model_dump"): - choices = response.model_dump().get("choices") - if not choices and hasattr(response, "dict"): - choices = response.dict().get("choices") - if choices and len(choices) > 0: - choice = choices[0] - if isinstance(choice, dict): - msg = choice.get("message", {}) - if isinstance(msg, dict): - resp_text = msg.get("content", "") - else: - resp_text = getattr(msg, "content", "") - else: - msg = getattr(choice, "message", None) - resp_text = getattr(msg, "content", "") - - if not resp_text: - raise ValueError(f"AI returned empty content or unsupported format! Full response: {response}") - - if resp_text.startswith("```json"): - resp_text = resp_text[7:-3].strip() - elif resp_text.startswith("```"): - resp_text = resp_text[3:-3].strip() + content = response.content[0].text.strip() + if content.startswith("```json"): + content = content[7:] + if content.startswith("```"): + content = content[3:] + if content.endswith("```"): + content = content[:-3] - action_data = json.loads(resp_text) + action_data = json.loads(content) action_history.append(action_data) if action_data["action"] == "human_intervention": @@ -238,68 +361,253 @@ async def play_media(query: str, media_type: str) -> Dict[str, Any]: "message": action_data.get("message", "Human intervention needed.") } elif action_data["action"] == "click": - vp_w = await page.evaluate("window.innerWidth") - vp_h = await page.evaluate("window.innerHeight") + target_id = str(action_data["id"]) + meta = dom_meta.get(target_id, {}) - # Use exact image dimensions to calculate percentages, then map to CSS pixels - if "x_percent" in action_data and "y_percent" in action_data: - x_pct = float(action_data["x_percent"]) - y_pct = float(action_data["y_percent"]) - # If the AI accidentally returned pixels instead of percentages, cap them - if x_pct > 100: x_pct = (x_pct / img.width) * 100 - if y_pct > 100: y_pct = (y_pct / img.height) * 100 - - click_x = (x_pct / 100.0) * vp_w - click_y = (y_pct / 100.0) * vp_h - else: - # Fallback if AI returned raw pixels (x, y) - click_x = float(action_data["x"]) * (vp_w / img.width) - click_y = float(action_data["y"]) * (vp_h / img.height) - - await page.mouse.click(click_x, click_y, delay=100) + t_id = meta.get("testId", "") + a_lbl = meta.get("ariaLabel", "").replace("'", "\'") + tag = meta.get("tagName", "button") + + js_click = f""" + let el = null; + let t = '{t_id}'; + let a = '{a_lbl}'; + let tag = '{tag}'; + + let query = tag; + if (t) query += `[data-testid="${{t}}"]`; + if (a) query += `[aria-label="${{a}}"]`; + + if (query !== tag) {{ + el = document.querySelector('.main-view-container ' + query) || document.querySelector(query); + }} + + if (!el) {{ + let container = document.querySelector('.main-view-container') || document.querySelector('main') || document.body; + let interactables = container.querySelectorAll('button, a'); + let interactablesArray = []; + for (let i = 0; i < interactables.length; i++) {{ + if (interactables[i].offsetWidth > 0 && interactables[i].offsetHeight > 0) {{ + interactablesArray.push(interactables[i]); + }} + }} + let idx = parseInt('{target_id}') - 1; + if (idx >= 0 && idx < interactablesArray.length) {{ + el = interactablesArray[idx]; + }} + }} - # Deterministically wait for playback to start - success = False + if (el) {{ + el.dispatchEvent(new PointerEvent('pointerdown', {{bubbles: true}})); + el.dispatchEvent(new MouseEvent('mousedown', {{bubbles: true}})); + el.dispatchEvent(new PointerEvent('pointerup', {{bubbles: true}})); + el.dispatchEvent(new MouseEvent('mouseup', {{bubbles: true}})); + el.click(); + }} + """ + click_script = f''' + tell application "Safari" + do JavaScript "{js_click.replace('"', '\\"').replace('\n', ' ')}" in current tab of front window + end tell + ''' + subprocess.run(["osascript", "-e", click_script], capture_output=True, text=True) + + # Verification for _ in range(10): # 5 seconds - try: - # Check React UI - pause_btn = await page.query_selector('button[data-testid="control-button-pause"]') - if pause_btn: - success = True - break - - # Check HTML5 Media - is_playing = await page.evaluate("() => Array.from(document.querySelectorAll('audio, video')).some(el => !el.paused && el.duration > 0)") - if is_playing: - success = True - break - except Exception: - pass - + state_script = ''' + tell application "Safari" + do JavaScript "navigator.mediaSession.playbackState" in current tab of front window + end tell + ''' + res_state = subprocess.run(["osascript", "-e", state_script], capture_output=True, text=True) + if res_state.returncode == 0 and "playing" in res_state.stdout: + success = True + break await asyncio.sleep(0.5) if success: return { "is_error": False, "is_human_intervention": False, - "message": "Task completed. Playback verified programmatically." + "message": "Task completed. Safari playback verified programmatically via DOM interpretation." } - - # If not successful, the loop will just continue to the next attempt! except Exception as e: - raw_resp = getattr(response, 'content', 'No content attribute') - return { - "is_error": True, - "is_human_intervention": False, - "message": f"Failed to parse AI response: {e}\nRaw content was: {raw_resp}" - } + pass # Continue loop if AI failed to parse + await asyncio.sleep(2) + return { "is_error": True, "is_human_intervention": False, - "message": f"AI failed to start playback after multiple attempts.\nAction history:\n{json.dumps(action_history, indent=2)}" + "message": f"AI failed to start playback in Safari after multiple attempts.\nAction history:\n{json.dumps(action_history, indent=2)}" } + except Exception as e: + return { + "is_error": True, + "is_human_intervention": False, + "message": f"Error executing Safari workflow: {e}" + } + + # --- CHROMIUM LOGIC --- + from playwright.async_api import async_playwright + import base64 + from io import BytesIO + from PIL import Image + + try: + try: + browser, context, page = await get_browser_page(target_url_substring="spotify.com", engine=engine) + except RuntimeError as e: + return { + "is_error": True, + "is_human_intervention": False, + "message": str(e) + } + + search_url = f"https://open.spotify.com/search/{urllib.parse.quote(query, safe='')}" + await page.goto(search_url) + + try: + await page.wait_for_load_state("networkidle", timeout=2000) + except Exception: + pass + + try: + # Scientifically wait for the play buttons OR the login button to render in the DOM + await page.wait_for_selector('button[data-testid="play-button"], [data-testid="login-button"]', timeout=4000) + except Exception: + pass + + action_history = [] + for attempt in range(4): + screenshot_bytes = await page.screenshot() + + img = Image.open(BytesIO(screenshot_bytes)) + max_width = 1024 + if img.width > max_width: + ratio = max_width / img.width + img = img.resize((max_width, int(img.height * ratio)), Image.LANCZOS) + buf = BytesIO() + img.convert("RGB").save(buf, format="JPEG", quality=45) + b64_img = base64.b64encode(buf.getvalue()).decode("utf-8") + + prompt = f""" +I want to play the {media_type} '{query}' on Spotify. Look at this screenshot of the Spotify web UI. +Determine the state and return ONLY a valid JSON object. Do not include markdown formatting or backticks, just the raw JSON. +You must choose one of the following exact JSON structures: + +1. If you see a login/signup modal popup overlaying the content, a "Log in to Spotify" wall, or a large "Sign up free" button, the main UI is not visible because you need to log in. You MUST return: +{{ +"action": "human_intervention", +"message": "User needs to log in" +}} + +3. Otherwise, if the {media_type} is not currently playing, you must start it. +- Look at the screen. If you are currently on a search results page, find the best match for '{query}' that is explicitly labeled as a {media_type} (e.g. look for the subtitle 'Album', 'Playlist', 'Song', or 'Artist'). + - If the best match has a prominent green Play button (e.g., the Top Result card), return its coordinates. + - If the best match does NOT have a visible Play button (e.g., it is a row in a list), return the coordinates of its text title to click on it. This will navigate to its dedicated page where a Play button will be visible on the next step. +- If you are ALREADY on a dedicated media page (e.g., you already clicked a title in a previous step), return the coordinates of the main green Play button on the page. + +Return this JSON if clicking is required: +{{ +"action": "click", +"x_percent": , +"y_percent": +}} +""" + + response = await client.messages.create( + model=model_id, + max_tokens=300, + system="You are an autonomous browser automation agent. Return ONLY raw JSON without backticks.", + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": b64_img}} + ] + } + ] + ) + + try: + content = response.content[0].text.strip() + if content.startswith("```json"): + content = content[7:] + if content.startswith("```"): + content = content[3:] + if content.endswith("```"): + content = content[:-3] + + action_data = json.loads(content) + action_history.append(action_data) + + if action_data["action"] == "human_intervention": + return { + "is_error": False, + "is_human_intervention": True, + "message": action_data.get("message", "Human intervention needed.") + } + elif action_data["action"] == "click": + vp_w = await page.evaluate("window.innerWidth") + vp_h = await page.evaluate("window.innerHeight") + + if "x_percent" in action_data and "y_percent" in action_data: + x_pct = float(action_data["x_percent"]) + y_pct = float(action_data["y_percent"]) + if x_pct > 100: x_pct = (x_pct / img.width) * 100 + if y_pct > 100: y_pct = (y_pct / img.height) * 100 + + click_x = (x_pct / 100.0) * vp_w + click_y = (y_pct / 100.0) * vp_h + else: + click_x = float(action_data["x"]) * (vp_w / img.width) + click_y = float(action_data["y"]) * (vp_h / img.height) + + await page.mouse.click(click_x, click_y, delay=100) + + success = False + for _ in range(10): # 5 seconds + try: + pause_btn = await page.query_selector('button[data-testid="control-button-pause"], button[aria-label^="Pause"]') + if pause_btn: + success = True + break + + # Use the official MediaSession API to check if the browser is actually playing media + # This ignores silent Canvas animations and background videos natively! + playback_state = await page.evaluate("() => navigator.mediaSession.playbackState") + if playback_state == "playing": + success = True + break + except Exception: + pass + + await asyncio.sleep(0.5) + + if success: + return { + "is_error": False, + "is_human_intervention": False, + "message": "Task completed. Playback verified programmatically." + } + + except Exception as e: + raw_resp = getattr(response, 'content', 'No content attribute') + return { + "is_error": True, + "is_human_intervention": False, + "message": f"Failed to parse AI response: {e}\nRaw content was: {raw_resp}" + } + + return { + "is_error": True, + "is_human_intervention": False, + "message": f"AI failed to start playback after multiple attempts.\nAction history:\n{json.dumps(action_history, indent=2)}" + } + except Exception as e: import traceback return { @@ -308,22 +616,22 @@ async def play_media(query: str, media_type: str) -> Dict[str, Any]: "message": f"An error occurred: {str(e)}\n\nTraceback:\n{traceback.format_exc()}" } -async def play_track(track_name: str, artist: Optional[str] = None) -> Dict[str, Any]: +async def play_track(track_name: str, artist: Optional[str] = None, engine: Optional[str] = None) -> Dict[str, Any]: query = track_name if artist: query += f" {artist}" - return await play_media(query, "track") + return await play_media(query, "track", engine=engine) -async def play_album(album_name: str, artist: Optional[str] = None) -> Dict[str, Any]: +async def play_album(album_name: str, artist: Optional[str] = None, engine: Optional[str] = None) -> Dict[str, Any]: query = album_name if artist: query += f" {artist}" - return await play_media(query, "album") + return await play_media(query, "album", engine=engine) -async def play_playlist(playlist_name: str) -> Dict[str, Any]: +async def play_playlist(playlist_name: str, engine: Optional[str] = None) -> Dict[str, Any]: query = playlist_name - return await play_media(query, "playlist") + return await play_media(query, "playlist", engine=engine) -async def play_artist(artist_name: str) -> Dict[str, Any]: +async def play_artist(artist_name: str, engine: Optional[str] = None) -> Dict[str, Any]: query = artist_name - return await play_media(query, "artist") \ No newline at end of file + return await play_media(query, "artist", engine=engine) \ No newline at end of file diff --git a/backend/apps/spotify_mcp_shim/server.py b/backend/apps/spotify_mcp_shim/server.py index 1adfd3721..4007551f8 100644 --- a/backend/apps/spotify_mcp_shim/server.py +++ b/backend/apps/spotify_mcp_shim/server.py @@ -1,5 +1,6 @@ import sys import json +import inspect from backend.apps.spotify_mcp_shim.tools import TOOLS @@ -24,54 +25,68 @@ def p_ok(payload) -> dict: import asyncio from backend.apps.spotify_mcp_shim import handlers -def execute_tool_function(func, args: dict): - if asyncio.iscoroutinefunction(func): - return asyncio.run(func(**args)) +async def execute_tool_function(func, args: dict): + if inspect.iscoroutinefunction(func): + return await func(**args) return func(**args) -def handle_tool_call(name: str, args: dict) -> dict: +async def handle_tool_call(name: str, args: dict) -> dict: handler = getattr(handlers, name, None) if not handler or not callable(handler): return p_err(f"Unknown tool: {name}") - return p_ok(execute_tool_function(handler, args)) + return p_ok(await execute_tool_function(handler, args)) -def main(): - for line in sys.stdin: - line = line.strip() - if not line: - continue - +async def process_line(line: str): + line = line.strip() + if not line: + return + + try: + msg = json.loads(line) + except json.JSONDecodeError: + return + + method = msg.get("method") + id_ = msg.get("id") + params = msg.get("params", {}) or {} + + if method == "initialize": + p_send(id_, { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "openswarm-spotify", "version": "1.0.0"}, + }) + elif method == "notifications/initialized": + pass + elif method == "tools/list": + p_send(id_, {"tools": TOOLS}) + elif method == "tools/call": + name = params.get("name", "") + args = params.get("arguments", {}) or {} try: - msg = json.loads(line) - except json.JSONDecodeError: - continue + res = await handle_tool_call(name, args) + p_send(id_, res) + except Exception as e: + p_send(id_, p_err(f"shim crashed: {e!r}")) + elif method == "ping": + p_send(id_, {}) + elif id_ is not None: + p_send(id_, error={"code": -32601, "message": f"Method not found: {method}"}) - method = msg.get("method") - id_ = msg.get("id") - params = msg.get("params", {}) or {} +async def async_main(): + loop = asyncio.get_running_loop() + while True: + line = await loop.run_in_executor(None, sys.stdin.readline) + if not line: + break + await process_line(line) - if method == "initialize": - p_send(id_, { - "protocolVersion": "2024-11-05", - "capabilities": {"tools": {}}, - "serverInfo": {"name": "openswarm-spotify", "version": "1.0.0"}, - }) - elif method == "notifications/initialized": - pass - elif method == "tools/list": - p_send(id_, {"tools": TOOLS}) - elif method == "tools/call": - name = params.get("name", "") - args = params.get("arguments", {}) or {} - try: - p_send(id_, handle_tool_call(name, args)) - except Exception as e: - p_send(id_, p_err(f"shim crashed: {e!r}")) - elif method == "ping": - p_send(id_, {}) - elif id_ is not None: - p_send(id_, error={"code": -32601, "message": f"Method not found: {method}"}) +def main(): + try: + asyncio.run(async_main()) + except KeyboardInterrupt: + pass if __name__ == "__main__": main() \ No newline at end of file diff --git a/backend/apps/spotify_mcp_shim/tools.py b/backend/apps/spotify_mcp_shim/tools.py index c7a3692b9..9c8ac2689 100644 --- a/backend/apps/spotify_mcp_shim/tools.py +++ b/backend/apps/spotify_mcp_shim/tools.py @@ -5,16 +5,16 @@ """ OBJ = "object" - TOOLS = [ { "name": "play_track", "description": "Opens an external browser (like Chrome or Edge) to bypass DRM and plays the requested song. NOTE: The user will see a separate browser window open.", "inputSchema": { - "type": OBJ, + "type": "object", "properties": { "track_name": {"type": "string"}, - "artist": {"type": "string"} + "artist": {"type": "string"}, + "engine": {"type": "string", "enum": ["chromium", "safari_applescript"]} }, "required": ["track_name"] } @@ -23,10 +23,11 @@ "name": "play_album", "description": "Opens an external browser (like Chrome or Edge) to bypass DRM and plays the requested album. NOTE: The user will see a separate browser window open.", "inputSchema": { - "type": OBJ, + "type": "object", "properties": { "album_name": {"type": "string"}, - "artist": {"type": "string"} + "artist": {"type": "string"}, + "engine": {"type": "string", "enum": ["chromium", "safari_applescript"]} }, "required": ["album_name"] } @@ -35,9 +36,10 @@ "name": "play_playlist", "description": "Opens an external browser (like Chrome or Edge) to bypass DRM and plays the requested playlist. NOTE: The user will see a separate browser window open.", "inputSchema": { - "type": OBJ, + "type": "object", "properties": { - "playlist_name": {"type": "string"} + "playlist_name": {"type": "string"}, + "engine": {"type": "string", "enum": ["chromium", "safari_applescript"]} }, "required": ["playlist_name"] } @@ -46,11 +48,23 @@ "name": "play_artist", "description": "Opens an external browser (like Chrome or Edge) to bypass DRM and plays the requested artist's top tracks. NOTE: The user will see a separate browser window open.", "inputSchema": { - "type": OBJ, + "type": "object", "properties": { - "artist_name": {"type": "string"} + "artist_name": {"type": "string"}, + "engine": {"type": "string", "enum": ["chromium", "safari_applescript"]} }, "required": ["artist_name"] } + }, + { + "name": "set_default_engine", + "description": "Sets the user's preferred browser engine for Spotify playback. Trigger this *only* if the user requests Safari or a Chromium browser, and this tool hasn't been called yet.", + "inputSchema": { + "type": "object", + "properties": { + "engine": {"type": "string", "enum": ["chromium", "safari_applescript"]} + }, + "required": ["engine"] + } } ] \ No newline at end of file From e77a97afee571d4e9e7efe82cb26f99c0fba884e Mon Sep 17 00:00:00 2001 From: Ethan Hanlon Date: Sun, 19 Jul 2026 14:44:25 -0700 Subject: [PATCH 6/6] [ethan] fix: typo in Webkit permissions prompt --- backend/apps/spotify_mcp_shim/Spotify_icon.svg | 5 ----- backend/apps/spotify_mcp_shim/handlers.py | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) delete mode 100644 backend/apps/spotify_mcp_shim/Spotify_icon.svg diff --git a/backend/apps/spotify_mcp_shim/Spotify_icon.svg b/backend/apps/spotify_mcp_shim/Spotify_icon.svg deleted file mode 100644 index 4f2e80329..000000000 --- a/backend/apps/spotify_mcp_shim/Spotify_icon.svg +++ /dev/null @@ -1,5 +0,0 @@ - - - - - \ No newline at end of file diff --git a/backend/apps/spotify_mcp_shim/handlers.py b/backend/apps/spotify_mcp_shim/handlers.py index e9e50862e..19cf6e50d 100644 --- a/backend/apps/spotify_mcp_shim/handlers.py +++ b/backend/apps/spotify_mcp_shim/handlers.py @@ -262,7 +262,7 @@ async def play_media(query: str, media_type: str, engine: Optional[str] = None) return { "is_error": False, "is_human_intervention": True, - "message": "To allow me to play music for you, I need permission to interact with Safari.\n\n1. Open Safari and press `Option-Command-,` (or go to **Safari > Settings > Advanced**).\n2. Check the box at the bottom for **Show features for web developers**.\n3. In the menu bar at the top of your screen, click **Develop > Allow JavaScript from Apple Events**.\n\n(You might see a scary Apple warning about malicious programs. Don't worry, this is just a standard warning because you are giving an AI permission to click buttons on web pages on your behalf!)\n\nIf you'd prefer not to do this, you can just ask me to use **Google Chrome** instead!" + "message": "To allow me to play music for you, I need permission to interact with Safari.\n\n1. Open Safari and press `Command-,` (or go to **Safari > Settings > Advanced**).\n2. Check the box at the bottom for **Show features for web developers**.\n3. In the menu bar at the top of your screen, click **Develop > Allow JavaScript from Apple Events**.\n\n(You might see a scary Apple warning about malicious programs. Don't worry, this is just a standard warning because you are giving an AI permission to click buttons on web pages on your behalf!)\n\nIf you'd prefer not to do this, you can just ask me to use **Google Chrome** instead!" } action_history = []