From 503e83e83edcf578ab7acec11275ad5695bc3c41 Mon Sep 17 00:00:00 2001 From: Sebastjan Prachovskij Date: Tue, 3 Sep 2024 18:26:11 +0300 Subject: [PATCH 01/12] Add SearchApi to search Add support for engines, improve status code error Remove changes in package, add engine to env params Improve description in env example Remove unnecessary empty line Improve text --- apps/api/.env.example | 11 ++++-- apps/api/.env.local | 2 +- apps/api/src/search/index.ts | 12 ++++++- apps/api/src/search/searchapi.ts | 60 ++++++++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 apps/api/src/search/searchapi.ts diff --git a/apps/api/.env.example b/apps/api/.env.example index f3c1dc1ba4..6ba49daa64 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -1,5 +1,5 @@ # ===== Required ENVS ====== -NUM_WORKERS_PER_QUEUE=8 +NUM_WORKERS_PER_QUEUE=8 PORT=3002 HOST=0.0.0.0 REDIS_URL=redis://redis:6379 #for self-hosting using docker, use redis://redis:6379. For running locally, use redis://localhost:6379 @@ -11,9 +11,14 @@ USE_DB_AUTHENTICATION=true # ===== Optional ENVS ====== +# SearchApi key. Head to https://searchapi.com/ to get your API key +SEARCHAPI_API_KEY= +# SearchApi engine, defaults to google. Available options: google, bing, baidu, google_news, etc. Head to https://searchapi.com/ to explore more engines +SEARCHAPI_ENGINE= + # Supabase Setup (used to support DB authentication, advanced logging, etc.) -SUPABASE_ANON_TOKEN= -SUPABASE_URL= +SUPABASE_ANON_TOKEN= +SUPABASE_URL= SUPABASE_SERVICE_TOKEN= # Other Optionals diff --git a/apps/api/.env.local b/apps/api/.env.local index 17f8593504..9fa41498b8 100644 --- a/apps/api/.env.local +++ b/apps/api/.env.local @@ -12,4 +12,4 @@ ANTHROPIC_API_KEY= BULL_AUTH_KEY= LOGTAIL_KEY= PLAYWRIGHT_MICROSERVICE_URL= - +SEARCHAPI_API_KEY= diff --git a/apps/api/src/search/index.ts b/apps/api/src/search/index.ts index f4c5b6d0ff..3bcb85d2b6 100644 --- a/apps/api/src/search/index.ts +++ b/apps/api/src/search/index.ts @@ -2,6 +2,7 @@ import { Logger } from "../../src/lib/logger"; import { SearchResult } from "../../src/lib/entities"; import { googleSearch } from "./googlesearch"; import { fireEngineMap } from "./fireEngine"; +import { searchapi_search } from "./searchapi"; import { serper_search } from "./serper"; export async function search({ @@ -30,7 +31,16 @@ export async function search({ timeout?: number; }): Promise { try { - + if (process.env.SEARCHAPI_API_KEY) { + return await searchapi_search(query, { + num_results, + tbs, + filter, + lang, + country, + location + }); + } if (process.env.SERPER_API_KEY) { return await serper_search(query, { num_results, diff --git a/apps/api/src/search/searchapi.ts b/apps/api/src/search/searchapi.ts new file mode 100644 index 0000000000..24778a7779 --- /dev/null +++ b/apps/api/src/search/searchapi.ts @@ -0,0 +1,60 @@ +import axios from "axios"; +import dotenv from "dotenv"; +import { SearchResult } from "../../src/lib/entities"; + +dotenv.config(); + +interface SearchOptions { + tbs?: string; + filter?: string; + lang?: string; + country?: string; + location?: string; + num_results: number; + page?: number; +} + +export async function searchapi_search(q: string, options: SearchOptions): Promise { + const params = { + q: q, + hl: options.lang, + gl: options.country, + location: options.location, + num: options.num_results, + page: options.page ?? 1, + engine: process.env.SEARCHAPI_ENGINE || "google", + }; + + const url = `https://www.searchapi.io/api/v1/search`; + + try { + const response = await axios.get(url, { + headers: { + "Authorization": `Bearer ${process.env.SEARCHAPI_API_KEY}`, + "Content-Type": "application/json", + "X-SearchApi-Source": "Firecrawl", + }, + params: params, + }); + + + if (response.status === 401) { + throw new Error("Unauthorized. Please check your API key."); + } + + const data = response.data; + + if (data && Array.isArray(data.organic_results)) { + return data.organic_results.map((a: any) => ({ + url: a.link, + title: a.title, + description: a.snippet, + })); + } else { + return []; + } + } catch (error) { + console.error(`There was an error searching for content: ${error.message}`); + return []; + } +} From 1355bd7c3c1b6dfaeb7341ecea7d9e13c1a06545 Mon Sep 17 00:00:00 2001 From: Isaac Cohen Date: Wed, 18 Sep 2024 11:29:45 -0400 Subject: [PATCH 02/12] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 89ed012709..a3e1dc4cb1 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- From a1861f6c10b8a5b25ff8bee655fc62b006e83870 Mon Sep 17 00:00:00 2001 From: Isaac Cohen Date: Wed, 18 Sep 2024 11:32:29 -0400 Subject: [PATCH 03/12] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a3e1dc4cb1..5c8e11ec1a 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- From 151116a789e27ea7c0f7c5584dc80ad8effbafc2 Mon Sep 17 00:00:00 2001 From: Isaac Cohen Date: Wed, 18 Sep 2024 11:35:37 -0400 Subject: [PATCH 04/12] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5c8e11ec1a..89ed012709 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@

- From 598fbd53865c607c627a0f750eb88f78c75babff Mon Sep 17 00:00:00 2001 From: Isaac Cohen Date: Wed, 18 Sep 2024 11:42:18 -0400 Subject: [PATCH 05/12] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 89ed012709..d3cde361be 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ height="200" >

-
+
License From 9874ef09c98e000d8f3fb54ce28b16b1c3f874db Mon Sep 17 00:00:00 2001 From: Isaac Cohen Date: Wed, 18 Sep 2024 11:45:22 -0400 Subject: [PATCH 06/12] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d3cde361be..f2460bf72d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ + >

From 43f07943c3b41bdad20b2e54fbf6e88cbdfc59fe Mon Sep 17 00:00:00 2001 From: Isaac Cohen Date: Wed, 18 Sep 2024 12:11:46 -0400 Subject: [PATCH 07/12] Update SELF_HOST.md --- SELF_HOST.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/SELF_HOST.md b/SELF_HOST.md index f631cf189e..f047cfd165 100644 --- a/SELF_HOST.md +++ b/SELF_HOST.md @@ -1,5 +1,5 @@ # Self-hosting Firecrawl - + #### Contributor? Welcome to [Firecrawl](https://firecrawl.dev) 🔥! Here are some instructions on how to get the project locally so you can run it on your own and contribute. @@ -176,4 +176,4 @@ By addressing these common issues, you can ensure a smoother setup and operation ## Install Firecrawl on a Kubernetes Cluster (Simple Version) -Read the [examples/kubernetes-cluster-install/README.md](https://github.com/mendableai/firecrawl/blob/main/examples/kubernetes-cluster-install/README.md) for instructions on how to install Firecrawl on a Kubernetes Cluster. \ No newline at end of file +Read the [examples/kubernetes-cluster-install/README.md](https://github.com/mendableai/firecrawl/blob/main/examples/kubernetes-cluster-install/README.md) for instructions on how to install Firecrawl on a Kubernetes Cluster. From 29eb9623eb9800e6acc58f06510e63ac6d52adcc Mon Sep 17 00:00:00 2001 From: Isaac Cohen Date: Wed, 18 Sep 2024 12:16:52 -0400 Subject: [PATCH 08/12] Update SELF_HOST.md --- SELF_HOST.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SELF_HOST.md b/SELF_HOST.md index f047cfd165..6fcdc64264 100644 --- a/SELF_HOST.md +++ b/SELF_HOST.md @@ -1,4 +1,4 @@ -# Self-hosting Firecrawl +# Self-hosting Firecrawl #### Contributor? From ba3c49abd26a4ae2a3f597be4a22a5cf1d270a3f Mon Sep 17 00:00:00 2001 From: Isaac Cohen Date: Wed, 18 Sep 2024 12:32:20 -0400 Subject: [PATCH 09/12] Update SELF_HOST.md --- SELF_HOST.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SELF_HOST.md b/SELF_HOST.md index 6fcdc64264..fbd79f96b8 100644 --- a/SELF_HOST.md +++ b/SELF_HOST.md @@ -1,4 +1,4 @@ -# Self-hosting Firecrawl +# Self-hosting Firecrawl #### Contributor? From af2d80be61d247c44be01d33e31d190fdcebef8a Mon Sep 17 00:00:00 2001 From: Isaac Cohen Date: Wed, 18 Sep 2024 13:20:01 -0400 Subject: [PATCH 10/12] Update SELF_HOST.md --- SELF_HOST.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SELF_HOST.md b/SELF_HOST.md index fbd79f96b8..822c13e907 100644 --- a/SELF_HOST.md +++ b/SELF_HOST.md @@ -1,4 +1,4 @@ -# Self-hosting Firecrawl +# Self-hosting Firecrawl #### Contributor? From 1a30ddaca01052c92a666ecf43030fb5ea995ea9 Mon Sep 17 00:00:00 2001 From: Isaac Cohen Date: Wed, 18 Sep 2024 13:24:43 -0400 Subject: [PATCH 11/12] Update SELF_HOST.md --- SELF_HOST.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SELF_HOST.md b/SELF_HOST.md index 822c13e907..6fcdc64264 100644 --- a/SELF_HOST.md +++ b/SELF_HOST.md @@ -1,4 +1,4 @@ -# Self-hosting Firecrawl +# Self-hosting Firecrawl #### Contributor? From 1144c4e9ace984690af6956955c68fdf4db45f76 Mon Sep 17 00:00:00 2001 From: Isaac Cohen Date: Wed, 18 Sep 2024 13:28:02 -0400 Subject: [PATCH 12/12] Update SELF_HOST.md --- SELF_HOST.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SELF_HOST.md b/SELF_HOST.md index 6fcdc64264..f6510cc9c2 100644 --- a/SELF_HOST.md +++ b/SELF_HOST.md @@ -1,5 +1,5 @@ # Self-hosting Firecrawl - + #### Contributor? Welcome to [Firecrawl](https://firecrawl.dev) 🔥! Here are some instructions on how to get the project locally so you can run it on your own and contribute.