Skip to content

refactor: Restructure AgentQL Extract Web Data component #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wjwjtf wants to merge 3 commits into
base: agentql/agentq_for_documents
Choose a base branch
from
Open

refactor: Restructure AgentQL Extract Web Data component #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b8d3559
tweaks 2
wjwjtf Jun 20, 2025
58595bf
edit agentql_api.py
wjwjtf Jun 20, 2025
e44bd53
Merge branch 'agentql/agentq_for_documents' into wenjie/edit_agentql
wjwjtf Jun 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions src/backend/base/langflow/components/agentql/agentql_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,13 @@ class AgentQL(Component):
]

def build_output(self) -> Data:
if not self.prompt and not self.query:
self.status = MISSING_REQUIRED_INPUTS_MESSAGE
raise ValueError(self.status)
if self.prompt and self.query:
self.status = TOO_MANY_INPUTS_MESSAGE
raise ValueError(self.status)

endpoint = "https://api.agentql.com/v1/query-data"
headers = {
"X-API-Key": self.api_key,
Expand All @@ -130,13 +137,6 @@ def build_output(self) -> Data:
},
}

if not self.prompt and not self.query:
self.status = MISSING_REQUIRED_INPUTS_MESSAGE
raise ValueError(self.status)
if self.prompt and self.query:
self.status = TOO_MANY_INPUTS_MESSAGE
raise ValueError(self.status)

try:
response = httpx.post(endpoint, headers=headers, json=payload, timeout=self.timeout)
response.raise_for_status()
Expand Down
2 changes: 1 addition & 1 deletion src/backend/base/langflow/initial_setup/starter_projects/News Aggregator.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,7 @@
"show": true,
"title_case": false,
"type": "code",
"value": "import httpx\n\nfrom langflow.components.agentql.utils import (\n MISSING_REQUIRED_INPUTS_MESSAGE,\n QUERY_DATA_API_DOCS_URL,\n QUERY_DOCS_URL,\n TOO_MANY_INPUTS_MESSAGE,\n handle_agentql_error,\n)\nfrom langflow.custom.custom_component.component import Component\nfrom langflow.field_typing.range_spec import RangeSpec\nfrom langflow.io import (\n BoolInput,\n DropdownInput,\n IntInput,\n MessageTextInput,\n MultilineInput,\n Output,\n SecretStrInput,\n)\nfrom langflow.schema.data import Data\n\n\nclass AgentQL(Component):\n display_name = \"Extract Web Data\"\n description = \"Extracts structured data from a web page using an AgentQL query or a Natural Language description.\"\n documentation: str = QUERY_DATA_API_DOCS_URL\n icon = \"AgentQL\"\n name = \"AgentQL\"\n\n inputs = [\n SecretStrInput(\n name=\"api_key\",\n display_name=\"API Key\",\n required=True,\n password=True,\n info=\"Your AgentQL API key from dev.agentql.com\",\n ),\n MessageTextInput(\n name=\"url\",\n display_name=\"URL\",\n required=True,\n info=\"The URL of the public web page you want to extract data from.\",\n tool_mode=True,\n ),\n MultilineInput(\n name=\"query\",\n display_name=\"AgentQL Query\",\n required=False,\n info=f\"The AgentQL query to execute. Learn more at {QUERY_DOCS_URL} or use a prompt.\",\n tool_mode=True,\n ),\n MultilineInput(\n name=\"prompt\",\n display_name=\"Prompt\",\n required=False,\n info=\"A Natural Language description of the data to extract from the page. Alternative to AgentQL query.\",\n tool_mode=True,\n ),\n BoolInput(\n name=\"is_stealth_mode_enabled\",\n display_name=\"Enable Stealth Mode (Beta)\",\n info=\"Enable experimental anti-bot evasion strategies. May not work for all websites at all times.\",\n value=False,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Seconds to wait for a request.\",\n value=900,\n advanced=True,\n ),\n DropdownInput(\n name=\"mode\",\n display_name=\"Request Mode\",\n info=\"'standard' uses deep data analysis, while 'fast' trades some depth of analysis for speed.\",\n options=[\"fast\", \"standard\"],\n value=\"fast\",\n advanced=True,\n ),\n IntInput(\n name=\"wait_for\",\n display_name=\"Wait For\",\n info=\"Seconds to wait for the page to load before extracting data.\",\n value=0,\n range_spec=RangeSpec(min=0, max=10, step_type=\"int\"),\n advanced=True,\n ),\n BoolInput(\n name=\"is_scroll_to_bottom_enabled\",\n display_name=\"Enable scroll to bottom\",\n info=\"Scroll to bottom of the page before extracting data.\",\n value=False,\n advanced=True,\n ),\n BoolInput(\n name=\"is_screenshot_enabled\",\n display_name=\"Enable screenshot\",\n info=\"Take a screenshot before extracting data. Returned in 'metadata' as a Base64 string.\",\n value=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Data\", name=\"data\", method=\"build_output\"),\n ]\n\n def build_output(self) -> Data:\n endpoint = \"https://api.agentql.com/v1/query-data\"\n headers = {\n \"X-API-Key\": self.api_key,\n \"Content-Type\": \"application/json\",\n \"X-TF-Request-Origin\": \"langflow\",\n }\n\n payload = {\n \"url\": self.url,\n \"query\": self.query,\n \"prompt\": self.prompt,\n \"params\": {\n \"mode\": self.mode,\n \"wait_for\": self.wait_for,\n \"is_scroll_to_bottom_enabled\": self.is_scroll_to_bottom_enabled,\n \"is_screenshot_enabled\": self.is_screenshot_enabled,\n },\n \"metadata\": {\n \"experimental_stealth_mode_enabled\": self.is_stealth_mode_enabled,\n },\n }\n\n if not self.prompt and not self.query:\n self.status = MISSING_REQUIRED_INPUTS_MESSAGE\n raise ValueError(self.status)\n if self.prompt and self.query:\n self.status = TOO_MANY_INPUTS_MESSAGE\n raise ValueError(self.status)\n\n try:\n response = httpx.post(endpoint, headers=headers, json=payload, timeout=self.timeout)\n response.raise_for_status()\n\n response_json = response.json()\n data = Data(result=response_json[\"data\"], metadata=response_json[\"metadata\"])\n\n except httpx.HTTPStatusError as e:\n self.status = handle_agentql_error(e)\n raise ValueError(self.status) from e\n\n else:\n self.status = data\n return data\n"
"value": "import httpx\n\nfrom langflow.components.agentql.utils import (\n QUERY_DOCS_URL,\n QUERY_DATA_API_DOCS_URL,\n MISSING_REQUIRED_INPUTS_MESSAGE,\n TOO_MANY_INPUTS_MESSAGE,\n handle_agentql_error,\n)\nfrom langflow.custom import Component\nfrom langflow.field_typing.range_spec import RangeSpec\nfrom langflow.io import (\n BoolInput,\n DropdownInput,\n IntInput,\n MessageTextInput,\n MultilineInput,\n Output,\n SecretStrInput,\n)\nfrom langflow.schema import Data\n\n\nclass AgentQL(Component):\n display_name = \"Extract Web Data\"\n description = \"Extracts structured data from a web page using an AgentQL query or a Natural Language description.\"\n documentation: str = QUERY_DATA_API_DOCS_URL\n icon = \"AgentQL\"\n name = \"AgentQL\"\n\n inputs = [\n SecretStrInput(\n name=\"api_key\",\n display_name=\"API Key\",\n required=True,\n password=True,\n info=\"Your AgentQL API key from dev.agentql.com\",\n ),\n MessageTextInput(\n name=\"url\",\n display_name=\"URL\",\n required=True,\n info=\"The URL of the public web page you want to extract data from.\",\n tool_mode=True,\n ),\n MultilineInput(\n name=\"query\",\n display_name=\"AgentQL Query\",\n required=False,\n info=f\"The AgentQL query to execute. Learn more at {QUERY_DOCS_URL} or use a prompt.\",\n tool_mode=True,\n ),\n MultilineInput(\n name=\"prompt\",\n display_name=\"Prompt\",\n required=False,\n info=\"A Natural Language description of the data to extract from the page. Alternative to AgentQL query.\",\n tool_mode=True,\n ),\n BoolInput(\n name=\"is_stealth_mode_enabled\",\n display_name=\"Enable Stealth Mode (Beta)\",\n info=\"Enable experimental anti-bot evasion strategies. May not work for all websites at all times.\",\n value=False,\n advanced=True,\n ),\n IntInput(\n name=\"timeout\",\n display_name=\"Timeout\",\n info=\"Seconds to wait for a request.\",\n value=900,\n advanced=True,\n ),\n DropdownInput(\n name=\"mode\",\n display_name=\"Request Mode\",\n info=\"'standard' uses deep data analysis, while 'fast' trades some depth of analysis for speed.\",\n options=[\"fast\", \"standard\"],\n value=\"fast\",\n advanced=True,\n ),\n IntInput(\n name=\"wait_for\",\n display_name=\"Wait For\",\n info=\"Seconds to wait for the page to load before extracting data.\",\n value=0,\n range_spec=RangeSpec(min=0, max=10, step_type=\"int\"),\n advanced=True,\n ),\n BoolInput(\n name=\"is_scroll_to_bottom_enabled\",\n display_name=\"Enable scroll to bottom\",\n info=\"Scroll to bottom of the page before extracting data.\",\n value=False,\n advanced=True,\n ),\n BoolInput(\n name=\"is_screenshot_enabled\",\n display_name=\"Enable screenshot\",\n info=\"Take a screenshot before extracting data. Returned in 'metadata' as a Base64 string.\",\n value=False,\n advanced=True,\n ),\n ]\n\n outputs = [\n Output(display_name=\"Data\", name=\"data\", method=\"build_output\"),\n ]\n\n def build_output(self) -> Data:\n if not self.prompt and not self.query:\n self.status = MISSING_REQUIRED_INPUTS_MESSAGE\n raise ValueError(self.status)\n if self.prompt and self.query:\n self.status = TOO_MANY_INPUTS_MESSAGE\n raise ValueError(self.status)\n\n endpoint = \"https://api.agentql.com/v1/query-data\"\n headers = {\n \"X-API-Key\": self.api_key,\n \"Content-Type\": \"application/json\",\n \"X-TF-Request-Origin\": \"langflow\",\n }\n\n payload = {\n \"url\": self.url,\n \"query\": self.query,\n \"prompt\": self.prompt,\n \"params\": {\n \"mode\": self.mode,\n \"wait_for\": self.wait_for,\n \"is_scroll_to_bottom_enabled\": self.is_scroll_to_bottom_enabled,\n \"is_screenshot_enabled\": self.is_screenshot_enabled,\n },\n \"metadata\": {\n \"experimental_stealth_mode_enabled\": self.is_stealth_mode_enabled,\n },\n }\n\n try:\n response = httpx.post(endpoint, headers=headers, json=payload, timeout=self.timeout)\n response.raise_for_status()\n\n response_json = response.json()\n data = Data(result=response_json[\"data\"], metadata=response_json[\"metadata\"])\n\n except httpx.HTTPStatusError as e:\n self.status = handle_agentql_error(e)\n raise ValueError(self.status) from e\n\n else:\n self.status = data\n return data\n"
},
"is_screenshot_enabled": {
"_input_type": "BoolInput",
Expand Down
Loading
Loading