diff --git a/SELF_HOST.md b/SELF_HOST.md
index f631cf189e..f6510cc9c2 100644
--- a/SELF_HOST.md
+++ b/SELF_HOST.md
@@ -1,5 +1,5 @@
-# Self-hosting Firecrawl
-
+# Self-hosting Firecrawl
+
#### Contributor?
Welcome to [Firecrawl](https://firecrawl.dev) 🔥! Here are some instructions on how to get the project locally so you can run it on your own and contribute.
@@ -176,4 +176,4 @@ By addressing these common issues, you can ensure a smoother setup and operation
## Install Firecrawl on a Kubernetes Cluster (Simple Version)
-Read the [examples/kubernetes-cluster-install/README.md](https://github.com/mendableai/firecrawl/blob/main/examples/kubernetes-cluster-install/README.md) for instructions on how to install Firecrawl on a Kubernetes Cluster.
\ No newline at end of file
+Read the [examples/kubernetes-cluster-install/README.md](https://github.com/mendableai/firecrawl/blob/main/examples/kubernetes-cluster-install/README.md) for instructions on how to install Firecrawl on a Kubernetes Cluster.
diff --git a/apps/api/.env.example b/apps/api/.env.example
index f3c1dc1ba4..6ba49daa64 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -1,5 +1,5 @@
# ===== Required ENVS ======
-NUM_WORKERS_PER_QUEUE=8
+NUM_WORKERS_PER_QUEUE=8
PORT=3002
HOST=0.0.0.0
REDIS_URL=redis://redis:6379 #for self-hosting using docker, use redis://redis:6379. For running locally, use redis://localhost:6379
@@ -11,9 +11,14 @@ USE_DB_AUTHENTICATION=true
# ===== Optional ENVS ======
+# SearchApi key. Head to https://searchapi.com/ to get your API key
+SEARCHAPI_API_KEY=
+# SearchApi engine, defaults to google. Available options: google, bing, baidu, google_news, etc. Head to https://searchapi.com/ to explore more engines
+SEARCHAPI_ENGINE=
+
# Supabase Setup (used to support DB authentication, advanced logging, etc.)
-SUPABASE_ANON_TOKEN=
-SUPABASE_URL=
+SUPABASE_ANON_TOKEN=
+SUPABASE_URL=
SUPABASE_SERVICE_TOKEN=
# Other Optionals
diff --git a/apps/api/.env.local b/apps/api/.env.local
index 17f8593504..9fa41498b8 100644
--- a/apps/api/.env.local
+++ b/apps/api/.env.local
@@ -12,4 +12,4 @@ ANTHROPIC_API_KEY=
BULL_AUTH_KEY=
LOGTAIL_KEY=
PLAYWRIGHT_MICROSERVICE_URL=
-
+SEARCHAPI_API_KEY=
diff --git a/apps/api/src/search/index.ts b/apps/api/src/search/index.ts
index f4c5b6d0ff..3bcb85d2b6 100644
--- a/apps/api/src/search/index.ts
+++ b/apps/api/src/search/index.ts
@@ -2,6 +2,7 @@ import { Logger } from "../../src/lib/logger";
import { SearchResult } from "../../src/lib/entities";
import { googleSearch } from "./googlesearch";
import { fireEngineMap } from "./fireEngine";
+import { searchapi_search } from "./searchapi";
import { serper_search } from "./serper";
export async function search({
@@ -30,7 +31,16 @@ export async function search({
timeout?: number;
}): Promise
{
try {
-
+ if (process.env.SEARCHAPI_API_KEY) {
+ return await searchapi_search(query, {
+ num_results,
+ tbs,
+ filter,
+ lang,
+ country,
+ location
+ });
+ }
if (process.env.SERPER_API_KEY) {
return await serper_search(query, {
num_results,
diff --git a/apps/api/src/search/searchapi.ts b/apps/api/src/search/searchapi.ts
new file mode 100644
index 0000000000..24778a7779
--- /dev/null
+++ b/apps/api/src/search/searchapi.ts
@@ -0,0 +1,60 @@
+import axios from "axios";
+import dotenv from "dotenv";
+import { SearchResult } from "../../src/lib/entities";
+
+dotenv.config();
+
+interface SearchOptions {
+ tbs?: string;
+ filter?: string;
+ lang?: string;
+ country?: string;
+ location?: string;
+ num_results: number;
+ page?: number;
+}
+
+export async function searchapi_search(q: string, options: SearchOptions): Promise {
+ const params = {
+ q: q,
+ hl: options.lang,
+ gl: options.country,
+ location: options.location,
+ num: options.num_results,
+ page: options.page ?? 1,
+ engine: process.env.SEARCHAPI_ENGINE || "google",
+ };
+
+ const url = `https://www.searchapi.io/api/v1/search`;
+
+ try {
+ const response = await axios.get(url, {
+ headers: {
+ "Authorization": `Bearer ${process.env.SEARCHAPI_API_KEY}`,
+ "Content-Type": "application/json",
+ "X-SearchApi-Source": "Firecrawl",
+ },
+ params: params,
+ });
+
+
+ if (response.status === 401) {
+ throw new Error("Unauthorized. Please check your API key.");
+ }
+
+ const data = response.data;
+
+ if (data && Array.isArray(data.organic_results)) {
+ return data.organic_results.map((a: any) => ({
+ url: a.link,
+ title: a.title,
+ description: a.snippet,
+ }));
+ } else {
+ return [];
+ }
+ } catch (error) {
+ console.error(`There was an error searching for content: ${error.message}`);
+ return [];
+ }
+}
+ >
-