From 3fea96f1ab70e6c42d83a9eb07ca90b6c2cd9492 Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Wed, 12 Mar 2025 19:13:30 -0400
Subject: [PATCH 01/20] Adapt for ubicloud.

---
 sampler/deep_research.py         | 414 +++++++++++++++++++++++++++++++
 sampler/deep_research_sampler.py |  80 ++++++
 sampler/ubicloud_sampler.py      |  82 ++++++
 simple_evals.py                  |  21 +-
 simpleqa_eval.py                 |  16 +-
 5 files changed, 607 insertions(+), 6 deletions(-)
 create mode 100644 sampler/deep_research.py
 create mode 100644 sampler/deep_research_sampler.py
 create mode 100644 sampler/ubicloud_sampler.py

diff --git a/sampler/deep_research.py b/sampler/deep_research.py
new file mode 100644
index 00000000..d9c9038a
--- /dev/null
+++ b/sampler/deep_research.py
@@ -0,0 +1,414 @@
+import argparse
+import io
+import os
+import re
+import json
+import functools
+import logging
+from enum import Enum
+from datetime import datetime
+from typing import Any, Dict, Optional, List, TypedDict
+
+import requests
+import openai
+from bs4 import BeautifulSoup
+from dotenv import load_dotenv
+from duckduckgo_search import DDGS
+from tavily import TavilyClient
+from markdown_pdf import MarkdownPdf, Section
+import PyPDF2
+
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Load UBICLOUD_API_KEY from .env file
+load_dotenv()
+UBICLOUD_API_KEY = os.getenv("UBICLOUD_API_KEY")
+TAVILY_API_KEY = os.getenv("TAVILY_API_KEY")
+
+# Model configurations
+SUMMARIZATION_MODEL = "mistral-small-3"
+SUMMARIZATION_CONTENT_CUTOFF = 50000
+REASONING_MODEL = "ds-r1-qwen-32b"
+WRITING_MODEL = "mistral-small-3"
+JSON_MODEL = "ds-r1-qwen-32b"
+
+
+class InferenceMode(Enum):
+    """Enum for different inference modes."""
+    SUMMARIZATION = 1
+    REASONING = 2
+    WRITING = 3
+    JSON = 4
+
+    def get_model(self) -> str:
+        """Retrieve the corresponding model name for the inference mode."""
+        return globals().get(f"{self.name}_MODEL")
+
+
+@functools.lru_cache(maxsize=128)
+def get_search_client(search_engine: str):
+    if search_engine == "duckduckgo":
+        return DDGS()
+    elif search_engine == "tavily":
+        return TavilyClient(TAVILY_API_KEY)
+    raise Exception("Unsupported search engine")
+
+
+@functools.lru_cache(maxsize=128)
+def get_inference_client(model: str) -> openai.OpenAI:
+    """Get the inference client for the specified model."""
+    base_url = f"https://{model}.ai.ubicloud.com/v1/"
+    return openai.OpenAI(api_key=UBICLOUD_API_KEY, base_url=base_url)
+
+
+def extract_json(content: str) -> Optional[Dict[str, Any]]:
+    """Look for code block (```...```) and parse as JSON."""
+    match = re.search(r"```(?:\w+)?\s*(.*?)\s*```", content, re.DOTALL)
+    if match:
+        json_str = match.group(1).strip()
+        try:
+            return json.loads(json_str)
+        except json.JSONDecodeError as e:
+            logger.error("JSON decoding failed: %s", e)
+            return None
+    return None
+
+
+def inference(messages: List[Dict[str, str]], mode: InferenceMode) -> Any:
+    """Perform inference using the specified messages and inference mode."""
+    model = mode.get_model()
+    inference_client = get_inference_client(model)
+    params = {
+        "model": model,
+        "messages": messages,
+    }
+    logger.debug(json.dumps(params, indent=2))
+    completion = inference_client.chat.completions.create(
+        model=model,
+        messages=messages,
+    )
+    content = completion.choices[0].message.content
+    logger.debug(content)
+    # Remove everything before the </think> from the content
+    content = re.sub(r'.*?</think>', '', content, flags=re.DOTALL).strip()
+    if mode == InferenceMode.JSON:
+        return extract_json(content)
+    return content
+
+
+def create_system_message(content: str) -> Dict[str, str]:
+    """Create a structured system message for chat completions."""
+    return {"role": "system", "content": content}
+
+
+def create_user_message(content: Any) -> Dict[str, str]:
+    """Create a structured user message for chat completions."""
+    if isinstance(content, dict):
+        # Filter out keys that start with '_'
+        content_str = json.dumps({
+            key: value for key, value in content.items() if not key.startswith("_")
+        }, indent=2)
+    else:
+        content_str = str(content)
+    return {"role": "user", "content": content_str}
+
+
+@functools.lru_cache(maxsize=128)
+def search(query: str, search_engine: str) -> List[Dict[str, Any]]:
+    """Search for a query on the web and return the top results."""
+    truncated_query = query[:200]
+    logger.info(f"Searching for \"{truncated_query}\"")
+    client = get_search_client(search_engine)
+    if search_engine == "duckduckgo":
+        search_results = client.text(truncated_query, max_results=5)
+    elif search_engine == "tavily":
+        search_results = client.search(query=truncated_query)["results"]
+    logger.debug(json.dumps(search_results, indent=2))
+    return search_results
+
+
+def read_pdf(pdf_file: io.BytesIO) -> str:
+    """Extract text from a PDF file."""
+    reader = PyPDF2.PdfReader(pdf_file)
+    return "".join(page.extract_text() or "" for page in reader.pages)
+
+
+@functools.lru_cache(maxsize=128)
+def fetch_url(url: str) -> Optional[str]:
+    """Fetch content from a URL."""
+    HEADERS = {
+        "User-Agent": (
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/120.0.0.0 Safari/537.36"
+        )
+    }
+    try:
+        response = requests.get(url, headers=HEADERS, timeout=30)
+        if response.status_code != 200:
+            logger.error(
+                f"Error fetching URL: {url} returned {response.status_code}")
+            return None
+        content_type = response.headers.get("Content-Type", "").lower()
+        if "application/pdf" in content_type or url.lower().endswith(".pdf"):
+            pdf_file = io.BytesIO(response.content)
+            return read_pdf(pdf_file)
+        soup = BeautifulSoup(response.content, 'html.parser')
+        return soup.get_text()
+    except Exception as e:
+        logger.error(f"Error fetching URL: {url}. Exception: {e}")
+        return None
+
+
+class ResearchState(TypedDict):
+    """Represents the research state."""
+    topic: str
+    search_results: List[Dict[str, Any]]
+    outline: str
+    report: str
+    _visited: List[str]
+
+
+@functools.lru_cache(maxsize=128)
+def summarize(title: str, content: str) -> Optional[str]:
+    """Summarize the given content based on the provided title."""
+    try:
+        return inference([
+            create_system_message("""Summarize the content.
+Only include information that is relevant to the title.
+Be succinct. Summarize directly, don't repeat the title."""),
+            create_user_message({
+                "title": title,
+                "content": content[:SUMMARIZATION_CONTENT_CUTOFF]
+            })
+        ], mode=InferenceMode.SUMMARIZATION)
+    except Exception as e:
+        logger.error(f"Error summarizing title '{title}': {e}")
+        return None
+
+
+def fetch_and_summarize(topic: str, search_result: Dict[str, Any]) -> Optional[str]:
+    """Fetch and summarize content from a search result URL."""
+    title = search_result.get("title", "No Title")
+    url = search_result.get("url")
+    logger.info(
+        f"Reading \"{title}\" with {InferenceMode.SUMMARIZATION.get_model()}")
+    content = fetch_url(url)
+    if not content:
+        logger.warning(f"No content fetched from URL: {url}")
+        return None
+    return summarize(title, content)
+
+
+def gather_information(topic: str, state: ResearchState, search_engine: str) -> ResearchState:
+    """Gather and update research information about a topic."""
+    logger.info(f"Gathering information about \"{topic}\"")
+    raw_search_results = search(topic, search_engine)
+    search_results: List[Dict[str, Any]] = state.get("search_results", [])
+    visited: List[str] = state.get("_visited", [])
+
+    for raw_search_result in raw_search_results:
+        # Some search engine stores the url as href. We copy href to url.
+        url = raw_search_result.get("href") or raw_search_result.get("url")
+        raw_search_result["url"] = url
+        if url in visited:
+            continue
+        visited.append(url)
+        summary = fetch_and_summarize(topic, raw_search_result)
+        if summary:
+            search_results.append({
+                "title": raw_search_result.get("title", "No Title"),
+                "url": url,
+                "summary": summary,
+                "index": len(search_results) + 1
+            })
+    logger.debug(str(fetch_url.cache_info()))
+    updated_state: ResearchState = {
+        **state,
+        "search_results": search_results,
+        "_visited": visited
+    }
+    return updated_state
+
+
+def thinking(state: ResearchState) -> ResearchState:
+    """Think about the topic."""
+    logger.info(
+        f"Thinking about \"{state['topic']}\" with {InferenceMode.REASONING.get_model()}"
+    )
+    # Remove any existing reasoning keys to ensure a fresh response
+    state.pop("thinking", None)
+    state.pop("outline", None)
+    state.pop("report", None)
+    thinking = inference([
+        create_system_message("""
+Take a deep breath, go through the search results, think through the topic step by step, and provide a well-reasoned answer.
+Use information from the search results as needed."""),
+                            #   Be objective, reasonable, and comprehensive."""),
+        create_user_message(state)
+    ], mode=InferenceMode.REASONING)
+    return {**state, "thinking": thinking}
+
+
+def deep_dive(state: ResearchState, search_engine: str) -> ResearchState:
+    """Deep dive into a few areas."""
+    subtopics = inference([
+        create_system_message("""
+Identify 3 key areas for deeper exploration on the given topic and thinking.
+Return a JSON array of strings.
+Each string should be a well-structured search engine query."""),
+        create_user_message({
+            "topic": state["topic"],
+            "thinking": state.get("thinking", "")
+        })
+    ], InferenceMode.JSON)
+    if subtopics is None:
+        logger.warning("No JSON object found. Retrying deep_dive...")
+        return deep_dive(state, search_engine)
+    for subtopic in subtopics:
+        state = gather_information(subtopic, state, search_engine)
+    return state
+
+
+def create_outline(state: ResearchState) -> ResearchState:
+    """Create an outline of the report."""
+    logger.info(
+        f"Creating an outline of the report on \"{state['topic']}\" with {InferenceMode.REASONING.get_model()}"
+    )
+    state.pop("outline", None)
+    state.pop("report", None)
+    outline = inference([
+        create_system_message("""
+Generate an outline of a professional report on the given topic and thinking.
+Think step by step. Use search results as needed."""),
+        create_user_message(state)
+    ], InferenceMode.REASONING)
+    return {**state, "outline": outline}
+
+def write_answer(state: ResearchState) -> ResearchState:
+    """Write out the answer."""
+    logger.info(
+        f"Generate the answer on \"{state['topic']}\" with {InferenceMode.REASONING.get_model()}"
+    )
+    state.pop("answer", None)
+    state.pop("outline", None)
+    state.pop("report", None)
+    outline = inference([
+        create_system_message("""
+Using the provided topic, search results, and thinking, generate a clear, accurate, and concise answer addressing the topic."""),
+        create_user_message(state)
+    ], InferenceMode.REASONING)
+    return {**state, "answer": outline}
+
+
+def write_report(state: ResearchState) -> ResearchState:
+    """Write the report."""
+    logger.info(
+        f"Writing a report on \"{state['topic']}\" with {InferenceMode.WRITING.get_model()}"
+    )
+    state.pop("report", None)
+    report = inference([
+        create_system_message(
+            """Generate an extremely detailed professional report based on the provided topic, thinking, and outline.
+Use search results as needed.
+Ensure it is well-organized and each section is well-developed.
+Use subsections and lists as needed.
+Include the topic as the title. Include an executive summary at the beginning.
+Refer to search results by their index as needed. Do not include the list of references at the end.
+Use heading level 1 for the title.
+Do not include figures."""),
+        create_user_message(state)
+    ], InferenceMode.WRITING)
+    references = [
+        f"1. {result["title"]}. (n.d.). Retrieved from {result['url']}"
+        for result in state["search_results"]]
+    report += "\n\n## References\n\n" + \
+        "\n".join(references) + "\n\nUbicloud AI"
+    return {**state, "report": report}
+
+
+CUSTOM_MARKDOWN_PDF_CSS = """
+body { font-family: sans-serif; line-height: 1.5; margin: 1em; color: #333; }
+h1, h2, h3 { color: #2c3e50; margin: 1em 0 0.5em; }
+h1 { font-size: 2em; border-bottom: 1px solid #ccc; padding-bottom: 0.2em; }
+p { margin: 0.8em 0; }
+code, pre code { font-family: monospace; background: #f4f4f4; padding: 0.2em 0.4em; border-radius: 3px; }
+pre code { background: #2d2d2d; color: #fff; padding: 1em; overflow-x: auto; }
+blockquote { border-left: 4px solid #ddd; padding-left: 1em; color: #666; margin: 1em 0; }
+ul, ol { margin: 0.8em 0; padding-left: 1.2em; }
+table { width: 100%; border-collapse: collapse; margin: 1em 0; }
+th, td { border: 1px solid #ddd; padding: 0.5em; }
+th { background: #f9f9f9; }
+"""
+
+
+def save_pdf(topic: str, report: str, timestamp: str) -> None:
+    """Save the generated report as a PDF file with a custom CSS style."""
+    pdf = MarkdownPdf(toc_level=2)
+    pdf.add_section(Section(report), user_css=CUSTOM_MARKDOWN_PDF_CSS)
+    pdf.meta["title"] = topic
+    filename = f"{topic}_{timestamp}.pdf"
+    pdf.save(filename)
+    logger.info(f"Saved PDF: {filename}")
+
+
+def save_state_json(topic: str, state: dict, timestamp: str) -> None:
+    """Save the research state as a JSON file."""
+    filename = f"{topic}_{timestamp}.json"
+    with open(filename, "w", encoding="utf-8") as f:
+        json.dump(state, f, ensure_ascii=False, indent=2)
+    logger.info(f"Saved state JSON: {filename}")
+
+
+def deep_research(topic: str, depth: int, search_engine: str, initial_state: Optional[dict]) -> None:
+    """Conduct deep research on a given topic and generate a PDF report"""
+    # timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    state = initial_state or {}
+    state["topic"] = topic
+    state = gather_information(topic, state, search_engine)
+    state = thinking(state)
+    # save_state_json(topic, state, timestamp)
+    for _ in range(depth - 1):
+        state = deep_dive(state, search_engine)
+        state = thinking(state)
+        # save_state_json(topic, state, timestamp)
+    # state = create_outline(state)
+    state = write_answer(state)
+    return state["answer"]
+    # state = write_report(state)
+    # return state["report"]
+    # save_pdf(topic, state["report"], timestamp)
+    # save_state_json(topic, state, timestamp)
+
+
+def main() -> None:
+    """Parse command-line arguments and start the deep research process."""
+    parser = argparse.ArgumentParser(
+        description="Perform deep research on a given topic."
+    )
+    parser.add_argument("topic", type=str, help="The topic to research")
+    parser.add_argument("--depth", type=int, default=3,
+                        help="The depth of the research")
+    parser.add_argument("--resume", type=str,
+                        help="Path to a JSON file to resume state from")
+    parser.add_argument("--search_engine", type=str, default="duckduckgo",
+                        help="The search engine to use, either DuckDuckGo or Tavily.")
+    args = parser.parse_args()
+
+    initial_state: Optional[dict] = None
+    if args.resume:
+        try:
+            with open(args.resume, "r", encoding="utf-8") as f:
+                initial_state = json.load(f)
+            logger.info(f"Resumed state loaded from {args.resume}")
+        except Exception as e:
+            logger.error(
+                f"Failed to load resume state from {args.resume}: {e}")
+
+    deep_research(args.topic, args.depth, args.search_engine, initial_state)
+
+
+if __name__ == "__main__":
+    main()
\ No newline at end of file
diff --git a/sampler/deep_research_sampler.py b/sampler/deep_research_sampler.py
new file mode 100644
index 00000000..f6d53ca1
--- /dev/null
+++ b/sampler/deep_research_sampler.py
@@ -0,0 +1,80 @@
+import base64
+import time
+from typing import Any
+import os
+
+import openai
+from openai import OpenAI
+
+from ..types import MessageList, SamplerBase
+from . import deep_research
+
+OPENAI_SYSTEM_MESSAGE_API = "You are a helpful assistant."
+OPENAI_SYSTEM_MESSAGE_CHATGPT = (
+    "You are ChatGPT, a large language model trained by OpenAI, based on the GPT-4 architecture."
+    + "\nKnowledge cutoff: 2023-12\nCurrent date: 2024-04-01"
+)
+
+
+class DeepRsearchCompletionSampler(SamplerBase):
+    """
+    Sample from OpenAI's chat completion API
+    """
+
+    def __init__(
+        self,
+        model: str = "llama-3-3-70b-turbo",
+        system_message: str | None = None,
+        temperature: float = 0.5,
+        max_tokens: int = 1024,
+        depth: int = 3,
+    ):
+        self.api_key_name = "OPENAI_API_KEY"
+        self.client = OpenAI(base_url=f"https://{model}.ai.ubicloud.com/v1/")
+        # using api_key=os.environ.get("OPENAI_API_KEY")  # please set your API_KEY
+        self.model = model
+        self.system_message = system_message
+        self.temperature = temperature
+        self.max_tokens = max_tokens
+        self.image_format = "url"
+        self.depth = depth
+
+    def _handle_image(
+        self, image: str, encoding: str = "base64", format: str = "png", fovea: int = 768
+    ):
+        new_image = {
+            "type": "image_url",
+            "image_url": {
+                "url": f"data:image/{format};{encoding},{image}",
+            },
+        }
+        return new_image
+
+    def _handle_text(self, text: str):
+        return {"type": "text", "text": text}
+
+    def _pack_message(self, role: str, content: Any):
+        return {"role": str(role), "content": content}
+
+    def __call__(self, message_list: MessageList) -> str:
+        if self.system_message:
+            message_list = [self._pack_message("system", self.system_message)] + message_list
+        trial = 0
+        while True:
+            try:
+                print("message_list", message_list)
+                return deep_research.deep_research(
+                    message_list[0]["content"], depth=self.depth, search_engine="tavily", initial_state=None)
+            # NOTE: BadRequestError is triggered once for MMMU, please uncomment if you are reruning MMMU
+            except openai.BadRequestError as e:
+                print("Bad Request Error", e)
+                return ""
+            except Exception as e:
+                exception_backoff = 2**trial  # expontial back off
+                print(
+                    f"Rate limit exception so wait and retry {trial} after {exception_backoff} sec",
+                    e,
+                )
+                time.sleep(exception_backoff)
+                trial += 1
+            # unknown error shall throw exception
diff --git a/sampler/ubicloud_sampler.py b/sampler/ubicloud_sampler.py
new file mode 100644
index 00000000..f5b45a4f
--- /dev/null
+++ b/sampler/ubicloud_sampler.py
@@ -0,0 +1,82 @@
+import base64
+import time
+from typing import Any
+import os
+
+import openai
+from openai import OpenAI
+
+from ..types import MessageList, SamplerBase
+
+OPENAI_SYSTEM_MESSAGE_API = "You are a helpful assistant."
+OPENAI_SYSTEM_MESSAGE_CHATGPT = (
+    "You are ChatGPT, a large language model trained by OpenAI, based on the GPT-4 architecture."
+    + "\nKnowledge cutoff: 2023-12\nCurrent date: 2024-04-01"
+)
+
+
+class UbicloudCompletionSampler(SamplerBase):
+    """
+    Sample from OpenAI's chat completion API
+    """
+
+    def __init__(
+        self,
+        model: str = "llama-3-3-70b-turbo",
+        system_message: str | None = None,
+        temperature: float = 0.5,
+        max_tokens: int = 1024,
+    ):
+        self.api_key_name = "OPENAI_API_KEY"
+        self.client = OpenAI(base_url=f"https://{model}.ai.ubicloud.com/v1/")
+        # using api_key=os.environ.get("OPENAI_API_KEY")  # please set your API_KEY
+        self.model = model
+        self.system_message = system_message
+        self.temperature = temperature
+        self.max_tokens = max_tokens
+        self.image_format = "url"
+
+    def _handle_image(
+        self, image: str, encoding: str = "base64", format: str = "png", fovea: int = 768
+    ):
+        new_image = {
+            "type": "image_url",
+            "image_url": {
+                "url": f"data:image/{format};{encoding},{image}",
+            },
+        }
+        return new_image
+
+    def _handle_text(self, text: str):
+        return {"type": "text", "text": text}
+
+    def _pack_message(self, role: str, content: Any):
+        return {"role": str(role), "content": content}
+
+    def __call__(self, message_list: MessageList) -> str:
+        if self.system_message:
+            message_list = [self._pack_message("system", self.system_message)] + message_list
+        trial = 0
+        while True:
+            try:
+                # print("message_list", message_list)
+                response = self.client.chat.completions.create(
+                    model=self.model,
+                    messages=message_list,
+                    temperature=self.temperature,
+                    max_tokens=self.max_tokens,
+                )
+                return response.choices[0].message.content
+            # NOTE: BadRequestError is triggered once for MMMU, please uncomment if you are reruning MMMU
+            except openai.BadRequestError as e:
+                print("Bad Request Error", e)
+                return ""
+            except Exception as e:
+                exception_backoff = 2**trial  # expontial back off
+                print(
+                    f"Rate limit exception so wait and retry {trial} after {exception_backoff} sec",
+                    e,
+                )
+                time.sleep(exception_backoff)
+                trial += 1
+            # unknown error shall throw exception
diff --git a/simple_evals.py b/simple_evals.py
index ceaa8c2a..27491d46 100644
--- a/simple_evals.py
+++ b/simple_evals.py
@@ -16,7 +16,8 @@
 )
 from .sampler.o_chat_completion_sampler import OChatCompletionSampler
 from .sampler.claude_sampler import ClaudeCompletionSampler, CLAUDE_SYSTEM_MESSAGE_LMSYS
-
+from .sampler.ubicloud_sampler import UbicloudCompletionSampler
+from .sampler.deep_research_sampler import DeepRsearchCompletionSampler
 
 def main():
     parser = argparse.ArgumentParser(
@@ -99,6 +100,19 @@ def main():
             model="claude-3-opus-20240229",
             system_message=CLAUDE_SYSTEM_MESSAGE_LMSYS,
         ),
+        # ubicloud models:
+        "llama-3-3-70b-turbo": UbicloudCompletionSampler(
+            model="llama-3-3-70b-turbo"
+        ),
+        "deep-research-1": DeepRsearchCompletionSampler(
+            depth=1
+        ), 
+        "deep-research-2": DeepRsearchCompletionSampler(
+            depth=2
+        ), 
+        "deep-research-3": DeepRsearchCompletionSampler(
+            depth=3
+        ), 
     }
 
     if args.list_models:
@@ -113,7 +127,7 @@ def main():
             return
         models = {args.model: models[args.model]}
 
-    grading_sampler = ChatCompletionSampler(model="gpt-4o")
+    grading_sampler = UbicloudCompletionSampler(model="llama-3-3-70b-turbo")
     equality_checker = ChatCompletionSampler(model="gpt-4-turbo-preview")
     # ^^^ used for fuzzy matching, just for math
 
@@ -154,7 +168,8 @@ def get_evals(eval_name, debug_mode):
 
     evals = {
         eval_name: get_evals(eval_name, args.debug)
-        for eval_name in ["simpleqa", "mmlu", "math", "gpqa", "mgsm", "drop", "humaneval"]
+        # for eval_name in ["simpleqa", "mmlu", "math", "gpqa", "mgsm", "drop", "humaneval"]
+        for eval_name in ["simpleqa"]
     }
     print(evals)
     debug_suffix = "_DEBUG" if args.debug else ""
diff --git a/simpleqa_eval.py b/simpleqa_eval.py
index 7ec3640b..49f2aaf9 100644
--- a/simpleqa_eval.py
+++ b/simpleqa_eval.py
@@ -7,6 +7,8 @@
 import random 
 import re
 import pandas
+import requests
+from io import StringIO
 from . import common
 from .types import Eval, EvalResult, SamplerBase, SingleEvalResult
 
@@ -98,9 +100,17 @@
 
 class SimpleQAEval(Eval):
     def __init__(self, grader_model: SamplerBase, num_examples: int | None = None, n_repeats: int = 1):
-        df = pandas.read_csv(
-            f"https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv"
-        )
+        # df = pandas.read_csv(
+        #     f"https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv",
+        #     storage_options={'http_timeout': 60}
+        # )
+        url = "https://openaipublic.blob.core.windows.net/simple-evals/simple_qa_test_set.csv"
+        response = requests.get(url, timeout=60)
+        response.raise_for_status()  # Raise an error on a bad status
+
+        # Use StringIO to simulate a file for pandas
+        data = StringIO(response.text)
+        df = pandas.read_csv(data)
         examples = [row.to_dict() for _, row in df.iterrows()]
         if num_examples:
             assert n_repeats == 1, "n_repeats only supported when max_examples = None"

From 0bd4b82786b207bc80c24d33e73bbc70d8b53d4a Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Wed, 12 Mar 2025 19:33:25 -0400
Subject: [PATCH 02/20] Emphasize wiki.

---
 sampler/deep_research.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sampler/deep_research.py b/sampler/deep_research.py
index d9c9038a..6d59d7ad 100644
--- a/sampler/deep_research.py
+++ b/sampler/deep_research.py
@@ -258,7 +258,8 @@ def deep_dive(state: ResearchState, search_engine: str) -> ResearchState:
         create_system_message("""
 Identify 3 key areas for deeper exploration on the given topic and thinking.
 Return a JSON array of strings.
-Each string should be a well-structured search engine query."""),
+Each string should be a well-structured search engine query.
+At least one of them should explicitly ask for wiki sources."""),
         create_user_message({
             "topic": state["topic"],
             "thinking": state.get("thinking", "")

From 569b7e7a5339497d28148153c917f1c1c4a7ec3d Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Wed, 12 Mar 2025 19:43:23 -0400
Subject: [PATCH 03/20] Emphasize wiki.

---
 sampler/deep_research.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sampler/deep_research.py b/sampler/deep_research.py
index 6d59d7ad..1fd704ad 100644
--- a/sampler/deep_research.py
+++ b/sampler/deep_research.py
@@ -245,7 +245,7 @@ def thinking(state: ResearchState) -> ResearchState:
     thinking = inference([
         create_system_message("""
 Take a deep breath, go through the search results, think through the topic step by step, and provide a well-reasoned answer.
-Use information from the search results as needed."""),
+Use information from the search results as needed, prioritizing trustworthy sources such as Wikipedia."""),
                             #   Be objective, reasonable, and comprehensive."""),
         create_user_message(state)
     ], mode=InferenceMode.REASONING)

From b40b68a3e73effd5317f087daefc48afb79e30b0 Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Wed, 12 Mar 2025 19:54:43 -0400
Subject: [PATCH 04/20] .

---
 sampler/deep_research.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sampler/deep_research.py b/sampler/deep_research.py
index 1fd704ad..2a52fc4d 100644
--- a/sampler/deep_research.py
+++ b/sampler/deep_research.py
@@ -245,7 +245,7 @@ def thinking(state: ResearchState) -> ResearchState:
     thinking = inference([
         create_system_message("""
 Take a deep breath, go through the search results, think through the topic step by step, and provide a well-reasoned answer.
-Use information from the search results as needed, prioritizing trustworthy sources such as Wikipedia."""),
+Use information from the search results as needed, prioritizing trustworthy sources such as Wikipedia. Ignore unrelated or unreliable sources."""),
                             #   Be objective, reasonable, and comprehensive."""),
         create_user_message(state)
     ], mode=InferenceMode.REASONING)

From 092542ff774c37c09f5a0d3344436a7115db8a2f Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Wed, 12 Mar 2025 19:56:56 -0400
Subject: [PATCH 05/20] .

---
 sampler/deep_research.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sampler/deep_research.py b/sampler/deep_research.py
index 2a52fc4d..8f717124 100644
--- a/sampler/deep_research.py
+++ b/sampler/deep_research.py
@@ -256,7 +256,8 @@ def deep_dive(state: ResearchState, search_engine: str) -> ResearchState:
     """Deep dive into a few areas."""
     subtopics = inference([
         create_system_message("""
-Identify 3 key areas for deeper exploration on the given topic and thinking.
+Identify 3 key areas for deeper exploration on the given topic.
+Pay attention to areas that may be missing or incorrect from the thinking.
 Return a JSON array of strings.
 Each string should be a well-structured search engine query.
 At least one of them should explicitly ask for wiki sources."""),

From 3c5246805ebe4ba0511b25e535ef5ce957656d85 Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Wed, 12 Mar 2025 19:58:13 -0400
Subject: [PATCH 06/20] .

---
 sampler/deep_research.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sampler/deep_research.py b/sampler/deep_research.py
index 8f717124..161b0313 100644
--- a/sampler/deep_research.py
+++ b/sampler/deep_research.py
@@ -299,7 +299,7 @@ def write_answer(state: ResearchState) -> ResearchState:
     state.pop("report", None)
     outline = inference([
         create_system_message("""
-Using the provided topic, search results, and thinking, generate a clear, accurate, and concise answer addressing the topic."""),
+Using the provided topic, search results, and thinking, generate a clear, accurate, and concise answer for the topic question."""),
         create_user_message(state)
     ], InferenceMode.REASONING)
     return {**state, "answer": outline}

From 44e5c6a9ecdd3db9e5da9b0ecde8c1fef40b87d2 Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Wed, 12 Mar 2025 19:58:26 -0400
Subject: [PATCH 07/20] .

---
 sampler/deep_research.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sampler/deep_research.py b/sampler/deep_research.py
index 161b0313..d8e75690 100644
--- a/sampler/deep_research.py
+++ b/sampler/deep_research.py
@@ -299,7 +299,7 @@ def write_answer(state: ResearchState) -> ResearchState:
     state.pop("report", None)
     outline = inference([
         create_system_message("""
-Using the provided topic, search results, and thinking, generate a clear, accurate, and concise answer for the topic question."""),
+Using the provided topic question, search results, and thinking, generate a clear, accurate, and concise answer for the topic question."""),
         create_user_message(state)
     ], InferenceMode.REASONING)
     return {**state, "answer": outline}

From 7370de8b9bf95b47eaff9fbf06ed18a3f025ef65 Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Wed, 12 Mar 2025 20:05:18 -0400
Subject: [PATCH 08/20] .

---
 sampler/deep_research.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sampler/deep_research.py b/sampler/deep_research.py
index d8e75690..9b5fee8c 100644
--- a/sampler/deep_research.py
+++ b/sampler/deep_research.py
@@ -199,7 +199,7 @@ def fetch_and_summarize(topic: str, search_result: Dict[str, Any]) -> Optional[s
     if not content:
         logger.warning(f"No content fetched from URL: {url}")
         return None
-    return summarize(title, content)
+    return summarize(topic, content)
 
 
 def gather_information(topic: str, state: ResearchState, search_engine: str) -> ResearchState:

From 0c60729dd80805012e0b4e3dcd9693c358274cab Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Wed, 12 Mar 2025 20:05:31 -0400
Subject: [PATCH 09/20] .

---
 sampler/deep_research.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/sampler/deep_research.py b/sampler/deep_research.py
index 9b5fee8c..78d1576f 100644
--- a/sampler/deep_research.py
+++ b/sampler/deep_research.py
@@ -256,8 +256,7 @@ def deep_dive(state: ResearchState, search_engine: str) -> ResearchState:
     """Deep dive into a few areas."""
     subtopics = inference([
         create_system_message("""
-Identify 3 key areas for deeper exploration on the given topic.
-Pay attention to areas that may be missing or incorrect from the thinking.
+Identify 3 key areas for further exploration or verification on the given topic.
 Return a JSON array of strings.
 Each string should be a well-structured search engine query.
 At least one of them should explicitly ask for wiki sources."""),

From 9faca65abaa6ef2829c4b2e6093a93f0f01da63e Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Wed, 12 Mar 2025 20:33:09 -0400
Subject: [PATCH 10/20] .

---
 sampler/deep_research.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sampler/deep_research.py b/sampler/deep_research.py
index 78d1576f..44035815 100644
--- a/sampler/deep_research.py
+++ b/sampler/deep_research.py
@@ -371,7 +371,7 @@ def deep_research(topic: str, depth: int, search_engine: str, initial_state: Opt
     state = gather_information(topic, state, search_engine)
     state = thinking(state)
     # save_state_json(topic, state, timestamp)
-    for _ in range(depth - 1):
+    for _ in range(depth):
         state = deep_dive(state, search_engine)
         state = thinking(state)
         # save_state_json(topic, state, timestamp)

From df519eb1b6b1037fe4bf3355a4bb9f864f2a5838 Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Wed, 12 Mar 2025 20:35:05 -0400
Subject: [PATCH 11/20] .

---
 simple_evals.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/simple_evals.py b/simple_evals.py
index 27491d46..21000891 100644
--- a/simple_evals.py
+++ b/simple_evals.py
@@ -104,6 +104,9 @@ def main():
         "llama-3-3-70b-turbo": UbicloudCompletionSampler(
             model="llama-3-3-70b-turbo"
         ),
+        "deep-research-1": DeepRsearchCompletionSampler(
+            depth=0
+        ), 
         "deep-research-1": DeepRsearchCompletionSampler(
             depth=1
         ), 

From 425faf24725b17c2784abe897cc17763a632fcba Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Wed, 12 Mar 2025 20:35:32 -0400
Subject: [PATCH 12/20] .

---
 simple_evals.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/simple_evals.py b/simple_evals.py
index 21000891..771ec205 100644
--- a/simple_evals.py
+++ b/simple_evals.py
@@ -104,7 +104,7 @@ def main():
         "llama-3-3-70b-turbo": UbicloudCompletionSampler(
             model="llama-3-3-70b-turbo"
         ),
-        "deep-research-1": DeepRsearchCompletionSampler(
+        "deep-research-0": DeepRsearchCompletionSampler(
             depth=0
         ), 
         "deep-research-1": DeepRsearchCompletionSampler(

From 6af608232c7a099bb465faf97aab3ac237b9e5a8 Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Wed, 12 Mar 2025 20:56:00 -0400
Subject: [PATCH 13/20] .

---
 sampler/deep_research.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sampler/deep_research.py b/sampler/deep_research.py
index 44035815..d536dcb1 100644
--- a/sampler/deep_research.py
+++ b/sampler/deep_research.py
@@ -28,7 +28,7 @@
 TAVILY_API_KEY = os.getenv("TAVILY_API_KEY")
 
 # Model configurations
-SUMMARIZATION_MODEL = "mistral-small-3"
+SUMMARIZATION_MODEL = ""
 SUMMARIZATION_CONTENT_CUTOFF = 50000
 REASONING_MODEL = "ds-r1-qwen-32b"
 WRITING_MODEL = "mistral-small-3"
@@ -256,7 +256,7 @@ def deep_dive(state: ResearchState, search_engine: str) -> ResearchState:
     """Deep dive into a few areas."""
     subtopics = inference([
         create_system_message("""
-Identify 3 key areas for further exploration or verification on the given topic.
+Identify 2 key areas for further exploration or verification on the given topic.
 Return a JSON array of strings.
 Each string should be a well-structured search engine query.
 At least one of them should explicitly ask for wiki sources."""),

From bc6d6f6735cb8e41cec650197b45251f2a04508d Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Wed, 12 Mar 2025 21:07:31 -0400
Subject: [PATCH 14/20] .

---
 sampler/deep_research.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sampler/deep_research.py b/sampler/deep_research.py
index d536dcb1..b56ff072 100644
--- a/sampler/deep_research.py
+++ b/sampler/deep_research.py
@@ -28,10 +28,10 @@
 TAVILY_API_KEY = os.getenv("TAVILY_API_KEY")
 
 # Model configurations
-SUMMARIZATION_MODEL = ""
+SUMMARIZATION_MODEL = "ds-r1-qwen-32b"
 SUMMARIZATION_CONTENT_CUTOFF = 50000
 REASONING_MODEL = "ds-r1-qwen-32b"
-WRITING_MODEL = "mistral-small-3"
+WRITING_MODEL = "ds-r1-qwen-32b"
 JSON_MODEL = "ds-r1-qwen-32b"
 
 

From 5006d101f9dfa8eef75e66442c7126b94bf620cd Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Wed, 12 Mar 2025 21:26:23 -0400
Subject: [PATCH 15/20] .

---
 common.py                | 2 +-
 sampler/deep_research.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/common.py b/common.py
index b6b4c0e1..74e5d58e 100644
--- a/common.py
+++ b/common.py
@@ -202,7 +202,7 @@ def aggregate_results(
     )
 
 
-def map_with_progress(f: callable, xs: list[Any], num_threads: int = 50):
+def map_with_progress(f: callable, xs: list[Any], num_threads: int = 10):
     """
     Apply f to each element of xs, using a ThreadPool, and show progress.
     """
diff --git a/sampler/deep_research.py b/sampler/deep_research.py
index b56ff072..1d70037a 100644
--- a/sampler/deep_research.py
+++ b/sampler/deep_research.py
@@ -29,7 +29,7 @@
 
 # Model configurations
 SUMMARIZATION_MODEL = "ds-r1-qwen-32b"
-SUMMARIZATION_CONTENT_CUTOFF = 50000
+SUMMARIZATION_CONTENT_CUTOFF = 40000
 REASONING_MODEL = "ds-r1-qwen-32b"
 WRITING_MODEL = "ds-r1-qwen-32b"
 JSON_MODEL = "ds-r1-qwen-32b"

From fd3a2d254922272567af8bc4a5c6c2daf381a820 Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Thu, 13 Mar 2025 09:59:55 -0400
Subject: [PATCH 16/20] .

---
 sampler/deep_research.py | 56 ++++++++++++++++++++++++++--------------
 1 file changed, 37 insertions(+), 19 deletions(-)

diff --git a/sampler/deep_research.py b/sampler/deep_research.py
index 1d70037a..056d7730 100644
--- a/sampler/deep_research.py
+++ b/sampler/deep_research.py
@@ -175,15 +175,21 @@ class ResearchState(TypedDict):
 def summarize(title: str, content: str) -> Optional[str]:
     """Summarize the given content based on the provided title."""
     try:
-        return inference([
-            create_system_message("""Summarize the content.
-Only include information that is relevant to the title.
-Be succinct. Summarize directly, don't repeat the title."""),
-            create_user_message({
-                "title": title,
-                "content": content[:SUMMARIZATION_CONTENT_CUTOFF]
-            })
-        ], mode=InferenceMode.SUMMARIZATION)
+        cutoff = SUMMARIZATION_CONTENT_CUTOFF
+        chunks = [content[i:i+cutoff] for i in range(0, len(content), cutoff)]
+        summaries = []
+        for chunk in chunks:
+            summary = inference([
+                create_system_message("""Summarize the content.
+    Only include information that is relevant to the title.
+    Be succinct. Summarize directly, don't repeat the title."""),
+                create_user_message({
+                    "title": title,
+                    "content": chunk
+                })
+            ], mode=InferenceMode.SUMMARIZATION)
+            summaries.append(summary)
+        return "\n\n".join(summaries)
     except Exception as e:
         logger.error(f"Error summarizing title '{title}': {e}")
         return None
@@ -245,7 +251,7 @@ def thinking(state: ResearchState) -> ResearchState:
     thinking = inference([
         create_system_message("""
 Take a deep breath, go through the search results, think through the topic step by step, and provide a well-reasoned answer.
-Use information from the search results as needed, prioritizing trustworthy sources such as Wikipedia. Ignore unrelated or unreliable sources."""),
+Use information from the search results as needed, prioritizing trustworthy sources, such as wiki, gov, bookforum, or fandom. Ignore unrelated or unreliable sources."""),
                             #   Be objective, reasonable, and comprehensive."""),
         create_user_message(state)
     ], mode=InferenceMode.REASONING)
@@ -256,10 +262,10 @@ def deep_dive(state: ResearchState, search_engine: str) -> ResearchState:
     """Deep dive into a few areas."""
     subtopics = inference([
         create_system_message("""
-Identify 2 key areas for further exploration or verification on the given topic.
+Generate 1 to 2 search queries on the given topic.
 Return a JSON array of strings.
 Each string should be a well-structured search engine query.
-At least one of them should explicitly ask for wiki sources."""),
+At least one of them should explicitly ask for trustworthy sources, such as wiki, gov, bookforum, or fandom."""),
         create_user_message({
             "topic": state["topic"],
             "thinking": state.get("thinking", "")
@@ -298,7 +304,14 @@ def write_answer(state: ResearchState) -> ResearchState:
     state.pop("report", None)
     outline = inference([
         create_system_message("""
-Using the provided topic question, search results, and thinking, generate a clear, accurate, and concise answer for the topic question."""),
+Using the provided topic question, search results, and thinking, generate a clear, accurate, and concise answer for the topic question.
+For example:
+What is the month, day, and year that ChatGPT is initially released?
+Answer: November, 30, 2022
+
+Who was the CEO of OpenAI when ChatGPT was initially released?
+Answer: Sam Altman
+"""),
         create_user_message(state)
     ], InferenceMode.REASONING)
     return {**state, "answer": outline}
@@ -363,20 +376,24 @@ def save_state_json(topic: str, state: dict, timestamp: str) -> None:
     logger.info(f"Saved state JSON: {filename}")
 
 
-def deep_research(topic: str, depth: int, search_engine: str, initial_state: Optional[dict]) -> None:
+def deep_research(
+        topic: str, depth: int, search_engine: str, initial_state: Optional[dict], debug: bool) -> None:
     """Conduct deep research on a given topic and generate a PDF report"""
-    # timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
     state = initial_state or {}
     state["topic"] = topic
-    state = gather_information(topic, state, search_engine)
-    state = thinking(state)
+    # state = gather_information(topic, state, search_engine)
+    # state = thinking(state)
     # save_state_json(topic, state, timestamp)
     for _ in range(depth):
         state = deep_dive(state, search_engine)
         state = thinking(state)
-        # save_state_json(topic, state, timestamp)
+        if debug:
+            save_state_json(topic, state, timestamp)
     # state = create_outline(state)
     state = write_answer(state)
+    if debug:
+        save_state_json(topic, state, timestamp)
     return state["answer"]
     # state = write_report(state)
     # return state["report"]
@@ -396,6 +413,7 @@ def main() -> None:
                         help="Path to a JSON file to resume state from")
     parser.add_argument("--search_engine", type=str, default="duckduckgo",
                         help="The search engine to use, either DuckDuckGo or Tavily.")
+    parser.add_argument("--debug", type=bool, default=False, help="Run in debug mode")
     args = parser.parse_args()
 
     initial_state: Optional[dict] = None
@@ -408,7 +426,7 @@ def main() -> None:
             logger.error(
                 f"Failed to load resume state from {args.resume}: {e}")
 
-    deep_research(args.topic, args.depth, args.search_engine, initial_state)
+    deep_research(args.topic, args.depth, args.search_engine, initial_state, args.debug)
 
 
 if __name__ == "__main__":

From 7feb0f670bb5fb8e780168b88e8bf85e572e1b50 Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Thu, 13 Mar 2025 10:04:19 -0400
Subject: [PATCH 17/20] .

---
 sampler/deep_research.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sampler/deep_research.py b/sampler/deep_research.py
index 056d7730..68b11440 100644
--- a/sampler/deep_research.py
+++ b/sampler/deep_research.py
@@ -426,8 +426,8 @@ def main() -> None:
             logger.error(
                 f"Failed to load resume state from {args.resume}: {e}")
 
-    deep_research(args.topic, args.depth, args.search_engine, initial_state, args.debug)
-
+    answer = deep_research(args.topic, args.depth, args.search_engine, initial_state, args.debug)
+    logger.info(f"Answer: {answer}")
 
 if __name__ == "__main__":
     main()
\ No newline at end of file

From 97329bd125c6a754418443d79793cf6bc36a440b Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Thu, 13 Mar 2025 10:30:37 -0400
Subject: [PATCH 18/20] .

---
 sampler/deep_research.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sampler/deep_research.py b/sampler/deep_research.py
index 68b11440..98596d35 100644
--- a/sampler/deep_research.py
+++ b/sampler/deep_research.py
@@ -309,8 +309,8 @@ def write_answer(state: ResearchState) -> ResearchState:
 What is the month, day, and year that ChatGPT is initially released?
 Answer: November, 30, 2022
 
-Who was the CEO of OpenAI when ChatGPT was initially released?
-Answer: Sam Altman
+Who was the other founder of Google besides Larry Page?
+Answer: Sergey Brin
 """),
         create_user_message(state)
     ], InferenceMode.REASONING)

From 44ff62f823b22c08602f4158103ca1736be0d07a Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Thu, 13 Mar 2025 10:33:50 -0400
Subject: [PATCH 19/20] .

---
 sampler/deep_research_sampler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sampler/deep_research_sampler.py b/sampler/deep_research_sampler.py
index f6d53ca1..7f85eceb 100644
--- a/sampler/deep_research_sampler.py
+++ b/sampler/deep_research_sampler.py
@@ -64,7 +64,7 @@ def __call__(self, message_list: MessageList) -> str:
             try:
                 print("message_list", message_list)
                 return deep_research.deep_research(
-                    message_list[0]["content"], depth=self.depth, search_engine="tavily", initial_state=None)
+                    message_list[0]["content"], depth=self.depth, search_engine="tavily", initial_state=None, debug=False)
             # NOTE: BadRequestError is triggered once for MMMU, please uncomment if you are reruning MMMU
             except openai.BadRequestError as e:
                 print("Bad Request Error", e)

From ebdc86eb6f55bc4aeebb564f12dd7ab02a6d96e7 Mon Sep 17 00:00:00 2001
From: Junhao Li <junhao@ubicloud.com>
Date: Thu, 13 Mar 2025 10:35:31 -0400
Subject: [PATCH 20/20] .

---
 common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/common.py b/common.py
index 74e5d58e..4dc8fc7b 100644
--- a/common.py
+++ b/common.py
@@ -202,7 +202,7 @@ def aggregate_results(
     )
 
 
-def map_with_progress(f: callable, xs: list[Any], num_threads: int = 10):
+def map_with_progress(f: callable, xs: list[Any], num_threads: int = 20):
     """
     Apply f to each element of xs, using a ThreadPool, and show progress.
     """