From 80357c5608a1889d06aacc55f651b726da6731f0 Mon Sep 17 00:00:00 2001
From: olestrausss <o@olestrauss.de>
Date: Mon, 8 Apr 2024 23:41:29 -0400
Subject: [PATCH] Refactor code in free_ask_internet.py

---
 free_ask_internet.py | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/free_ask_internet.py b/free_ask_internet.py
index 7c78a37..3c1bbee 100644
--- a/free_ask_internet.py
+++ b/free_ask_internet.py
@@ -1,24 +1,21 @@
 # -*- coding: utf-8 -*-
-
-import json
-import os 
 from pprint import pprint
 import requests
 import trafilatura
 from trafilatura import bare_extraction
 from concurrent.futures import ThreadPoolExecutor
 import concurrent
-import requests
-import openai
-import time 
-from datetime import datetime
+import openai                 
 from urllib.parse import urlparse
 import tldextract
-import platform
 import urllib.parse
+from typing import List, Dict, Any, Optional, Generator
 
  
-def extract_url_content(url):
+UrlStr = str
+Query = str
+
+def extract_url_content(url: UrlStr) -> Dict[str, Any]:
     downloaded = trafilatura.fetch_url(url)
     content =  trafilatura.extract(downloaded)
     
@@ -27,7 +24,7 @@ def extract_url_content(url):
 
  
 
-def search_web_ref(query:str, debug=False):
+def search_web_ref(query: Query, debug: Optional[bool] = False) -> List[Dict[str, Any]]:
  
     content_list = []
 
@@ -56,7 +53,7 @@ def search_web_ref(query:str, debug=False):
                 if url:
                     url_parsed = urlparse(url)
                     domain = url_parsed.netloc
-                    icon_url =  url_parsed.scheme + '://' + url_parsed.netloc + '/favicon.ico'
+                    icon_url = f'{url_parsed.scheme}://{domain}/favicon.ico'
                     site_name = tldextract.extract(url).domain
  
                 conv_links.append({
@@ -94,12 +91,13 @@ def search_web_ref(query:str, debug=False):
                     print("URL: {}".format(url))
                     print("=================")
  
-        return  content_list
+        return content_list
     except Exception as ex:
         raise ex
 
 
-def gen_prompt(question,content_list, context_length_limit=11000,debug=False):
+def gen_prompt(question: Query, content_list: List[Dict[str, Any]], 
+               context_length_limit: Optional[int] = 11000, debug: Optional[bool] = False) -> str:
     
     limit_len = (context_length_limit - 2000)
     if len(question) > limit_len:
@@ -138,7 +136,7 @@ def gen_prompt(question,content_list, context_length_limit=11000,debug=False):
 
 
 
-def chat(prompt, stream=True, debug=False):
+def chat(prompt: str, stream: Optional[bool] = True, debug: Optional[bool] = False) -> Generator[str, None, None]:
     openai.base_url = "http://freegpt35:3040/v1/"
     openai.api_key = "EMPTY"
     total_content = ""
@@ -164,7 +162,7 @@ def chat(prompt, stream=True, debug=False):
 
  
     
-def ask_internet(query:str,  debug=False):
+def ask_internet(query: Query, debug: Optional[bool] = False) -> Generator[str, None, None]:
   
     content_list = search_web_ref(query,debug=debug)
     if debug: