diff --git a/app/llm-poc-variant-01/README.md b/app/llm-poc-variant-01/README.md index b84c06a..fac4c4d 100644 --- a/app/llm-poc-variant-01/README.md +++ b/app/llm-poc-variant-01/README.md @@ -8,15 +8,14 @@ It's simple and runs on the local machine with smaller sized and free LLMs. > Note: credits to this program goes to the original authors of [langchain-python-rag-privategpt](https://github.com/jmorganca/ollama/tree/main/examples/langchain-python-rag-privategpt) from Ivan Martinez who contributed to an example on [jmorganca/ollama](https://github.com/jmorganca/ollama). - ## Table of Contents - [Requirements](#requirements) - [Installation](#installation) - [Setup](#setup) - - [Downloading Learning Path Index datasets](#downloading-learning-path-index-datasets) - - [Ingesting files](#ingesting-files) - - [via native shell CLI](#via-native-shell-cli) + - [Downloading Learning Path Index datasets](#downloading-learning-path-index-datasets) + - [Ingesting files](#ingesting-files) + - [via native shell CLI](#via-native-shell-cli) - [Usage](#usage) - [Ask questions](#ask-questions) - [via native shell CLI](#via-native-shell-cli-1) @@ -42,9 +41,9 @@ List out the key requirements needed to run the project, such as: - or venv - or [pipenv](https://pipenv.pypa.io/en/latest/) - Docker (optional) -- Ollama ([Download & Install(https://ollama.com/download)) +- Ollama ([Download & Install](https://ollama.com/download)) - Windows: - - Microsoft Visual C++ 14.0 or greater is required (needed when installing ```hnswlib``` ) + - Microsoft Visual C++ 14.0 or greater is required (needed when installing `hnswlib` ) ## Installation @@ -91,7 +90,6 @@ and start the Ollama server ollama serve ``` - ## Setup ### Downloading Learning Path Index datasets @@ -181,10 +179,11 @@ If this occurs then use the Docker container to run your commands, instructions #### via native shell CLI -Before running ```lpiGPT.py```you need to specify the base URL for the Ollama API or the local instance of Ollama running on your machine. By default this will return a ```None``` value. +Before running `lpiGPT.py`you need to specify the base URL for the Ollama API or the local instance of Ollama running on your machine. By default this will return a `None` value. - Windows: - This is typically http://localhost:11434 and can be set by using the following in command line: + ```shell set OLLAMA_HOST=http://localhost:11434 ``` @@ -218,8 +217,7 @@ Note: The results will be displayed in a table format with columns for Course Na To exit the GPT prompt, press Ctrl-C or Ctrl-D and it will return to the Linux/Command-prompt. - -```bash +```bash > python3 lpiGPT.py --help usage: lpiGPT.py [-h] [--chat-model CHAT_MODEL] [--embeddings-model-name EMBEDDINGS_MODEL_NAME] [--persist-directory PERSIST_DIRECTORY] [--target-source-chunks TARGET_SOURCE_CHUNKS] [--hide-source] [--mute-stream] @@ -265,7 +263,6 @@ root@[your machine name]:/home#: in there, type the same commands as in the **via native shell CLI** sections of [Ingesting files](#ingesting-files) and [Ask questions](#ask-questions) respectively. - ### Try a different model ```shell @@ -280,7 +277,7 @@ Put any and all your files into the `source_documents` directory The supported extensions are: - `.csv`: CSV -and others, we have trimmed them off from here to keep this example simple and concise. + and others, we have trimmed them off from here to keep this example simple and concise. [back to ToC](#table-of-contents) @@ -301,6 +298,17 @@ For chat models, have a look at [this list](https://github.com/jmorganca/ollama/ _Please share your resources on either or both of the Embeddings and Chat models with us_ +## Running the Chainlit UI + +![Preview of the Chainlit app](deploy/gcp/chainlit-app-demo.gif "Preview of the Chainlit app") + +Chainlit allows us to quickly spin up an LLM friendly UI to interact with different models. +To start the app: + +```bash +chainlit run chainlit_app.py --host 0.0.0.0 --port 8000 +``` + ## Contributing We are open to any or all of the below from your side in terms of contributions: diff --git a/app/llm-poc-variant-01/docker/run-docker-container.sh b/app/llm-poc-variant-01/docker/run-docker-container.sh index a23bd35..6069e72 100755 --- a/app/llm-poc-variant-01/docker/run-docker-container.sh +++ b/app/llm-poc-variant-01/docker/run-docker-container.sh @@ -9,9 +9,9 @@ FULL_DOCKER_TAG_NAME="python-3.10-docker-env" echo "Running image ${FULL_DOCKER_TAG_NAME}"; echo "" pullImage() { - FULL_DOCKER_TAG_NAME="${DOCKER_USER_NAME}/${FULL_DOCKER_TAG_NAME}" - - docker pull ${FULL_DOCKER_TAG_NAME} || true + FULL_DOCKER_TAG_NAME="${DOCKER_USER_NAME}/${FULL_DOCKER_TAG_NAME}" + + docker pull ${FULL_DOCKER_TAG_NAME} || true } WORKDIR="/home/" @@ -23,16 +23,16 @@ HF_CACHE_SHARED="--volume ${LOCAL_MODEL_FOLDER}/.cache:/root/.cache" set -x # pullImage -time docker run --rm -it --network="host" \ - --platform="linux/amd64" \ - --network="host" \ - --add-host=host.docker.internal:host-gateway \ - --workdir "${WORKDIR}" \ - --env OLLAMA_HOST="http://host.docker.internal:11434" \ - ${HF_CACHE_SHARED} \ - ${MODEL_VOLUME_SHARED} \ - ${OLLAMA_VOLUME_SHARED} \ - "${FULL_DOCKER_TAG_NAME}" +time docker run --rm -it \ +--platform="linux/amd64" \ +--network="host" \ +--add-host=host.docker.internal:host-gateway \ +--workdir "${WORKDIR}" \ +--env OLLAMA_HOST="http://host.docker.internal:11434" \ +${HF_CACHE_SHARED} \ +${MODEL_VOLUME_SHARED} \ +${OLLAMA_VOLUME_SHARED} \ +"${FULL_DOCKER_TAG_NAME}" set +x echo "* Finished running docker image ${FULL_DOCKER_TAG_NAME}" \ No newline at end of file diff --git a/app/llm-poc-variant-01/requirements.txt b/app/llm-poc-variant-01/requirements.txt index 251e43a..109d463 100644 --- a/app/llm-poc-variant-01/requirements.txt +++ b/app/llm-poc-variant-01/requirements.txt @@ -16,7 +16,7 @@ chainlit==1.1.404 charset-normalizer==3.4.0 chevron==0.14.0 chroma-hnswlib==0.7.6 -chromadb==0.5.20 +chromadb==0.3.26 click==8.1.7 coloredlogs==15.0.1 dataclasses-json==0.5.14 diff --git a/app/qooper-scraper/.env.example b/app/qooper-scraper/.env.example new file mode 100644 index 0000000..dd51229 --- /dev/null +++ b/app/qooper-scraper/.env.example @@ -0,0 +1,5 @@ +QOOPER_EMAIL=user@email.com +QOOPER_PASSWORD=password + +CHROME_DRIVER_PATH=/chromedriver-linux64/chromedriver +CHROME_PATH=/usr/bin/google-chrome \ No newline at end of file diff --git a/app/qooper-scraper/.gitignore b/app/qooper-scraper/.gitignore new file mode 100644 index 0000000..2a7e40b --- /dev/null +++ b/app/qooper-scraper/.gitignore @@ -0,0 +1,3 @@ +output/ +chromedriver* +qooper-token.txt diff --git a/app/qooper-scraper/README.md b/app/qooper-scraper/README.md new file mode 100644 index 0000000..84b6508 --- /dev/null +++ b/app/qooper-scraper/README.md @@ -0,0 +1,20 @@ +- Setup Selenium (Chrome Driver, Chrome install) +Grab the Chrome installer and driver from https://googlechromelabs.github.io/chrome-for-testing/ + +```bash +wget https://storage.googleapis.com/chrome-for-testing-public/131.0.6778.85/linux64/chromedriver-linux64.zip +``` + +Selenium and Chrome in WSL +```bash +https://www.gregbrisebois.com/posts/chromedriver-in-wsl2/ +``` + +- Setup environment variables +.env.example is a template to become .env + + +- Works by signing in to the site using Selenium (sign-in involves multiple APIs and services, and so would be difficult to achieve with vanilla web scrapping) + +- Uses the tokens retrieved to query Qooper API + diff --git a/app/qooper-scraper/get_token.py b/app/qooper-scraper/get_token.py new file mode 100644 index 0000000..e44e3dc --- /dev/null +++ b/app/qooper-scraper/get_token.py @@ -0,0 +1,107 @@ +import os +import sys +import json + +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.chrome.service import Service +from selenium.webdriver.common.keys import Keys +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.remote.webdriver import BaseWebDriver +from selenium.webdriver.support.wait import WebDriverWait +from dotenv import load_dotenv + +load_dotenv() + + +QOOPER_HOME_PAGE = "https://mentoring.qooper.io/" + +try: + CHROME_DRIVER_PATH = os.environ.get( + "CHROME_DRIVER_PATH", r"chromedriver-linux64/chromedriver" + ) + CHROME_PATH = os.environ.get("CHROME_PATH", r"/usr/bin/google-chrome") + QOOPER_EMAIL = os.environ["QOOPER_EMAIL"] + QOOPER_PASSWORD = os.environ["QOOPER_PASSWORD"] +except KeyError as error: + print(f"Set environment variables: {error.with_traceback()}") + sys.exit(1) + + +def wait_for_element_visible( + driver: BaseWebDriver, element_identifier, by: By = By.XPATH, timeout=10 +): + return WebDriverWait(driver, timeout).until( + EC.presence_of_element_located((by, element_identifier)) + ) + + +def wait_for_element_clickable( + driver: BaseWebDriver, element_identifier, by: By = By.XPATH, timeout=10 +): + return WebDriverWait(driver, timeout).until( + EC.element_to_be_clickable((by, element_identifier)) + ) + + +def login_to_qooper(driver: webdriver.Chrome): + driver.get(QOOPER_HOME_PAGE) + + input_email = wait_for_element_visible( + driver, + "input[type=email]", + By.CSS_SELECTOR, + ) + input_email.send_keys(QOOPER_EMAIL) + next_button = driver.find_element(By.CSS_SELECTOR, "button.btn.bg-blue") + next_button.click() + + input_password = wait_for_element_visible( + driver, + "input[type=password]", + By.CSS_SELECTOR, + ) + input_password.send_keys(QOOPER_PASSWORD) + next_button = driver.find_element(By.CSS_SELECTOR, "button.btn.bg-blue") + next_button.click() + + radio_input_kagglex = wait_for_element_clickable( + driver, + 'label[for="KaggleX Fellowship Program - Cohort 4"]', + By.CSS_SELECTOR, + timeout=30, + ) + + radio_input_kagglex.click() + next_button = driver.find_element(By.CSS_SELECTOR, 'button[label="Continue"]') + next_button.click() + + wait_for_element_visible( + driver, + "p", + By.TAG_NAME, + ) + + auth_token = driver.execute_script("return localStorage.getItem('qooper_atfu');") + auth_token = json.loads(auth_token) + print('Auth token: ', f"{auth_token[:4]}...{auth_token[-4:]}") + with open('qooper-token.txt', 'w') as f: + f.write(auth_token) + + +options = webdriver.ChromeOptions() +options.page_load_strategy = "eager" +options.add_argument("--no-sandbox") +service = Service(executable_path=CHROME_DRIVER_PATH) + +browser = webdriver.Chrome( + options=options, + service=service, +) + +try: + login_to_qooper(browser) + +finally: + browser.quit() + diff --git a/app/qooper-scraper/requirements.txt b/app/qooper-scraper/requirements.txt new file mode 100644 index 0000000..959ae45 --- /dev/null +++ b/app/qooper-scraper/requirements.txt @@ -0,0 +1,21 @@ +annotated-types==0.7.0 +attrs==24.2.0 +certifi==2024.8.30 +charset-normalizer==3.4.0 +h11==0.14.0 +idna==3.10 +outcome==1.3.0.post0 +pydantic==2.10.2 +pydantic_core==2.27.1 +PySocks==1.7.1 +python-dotenv==1.0.1 +requests==2.32.3 +selenium==4.26.1 +sniffio==1.3.1 +sortedcontainers==2.4.0 +trio==0.27.0 +trio-websocket==0.11.1 +typing_extensions==4.12.2 +urllib3==2.2.3 +websocket-client==1.8.0 +wsproto==1.2.0 diff --git a/app/qooper-scraper/schemas.py b/app/qooper-scraper/schemas.py new file mode 100644 index 0000000..3c971b3 --- /dev/null +++ b/app/qooper-scraper/schemas.py @@ -0,0 +1,79 @@ +from pydantic import BaseModel, RootModel +from typing import Optional, Generic, TypeVar + +T = TypeVar("T") + + +class Group(BaseModel, Generic[T]): + id: int + name: str + description: Optional[str] = "" + image_url: Optional[str] + kind: str + tags: list[str] = [] + is_joined: bool = True + members_count: int = 0 + resources_count: int = 0 + discussions_count: int = 0 + past_events_count: int = 0 + upcoming_events_count: int = 0 + created_at: str + + +class GroupList(RootModel): + root: list[Group] + + +class GroupMember(BaseModel, Generic[T]): + id: int + first_name: str + last_name: str + image_url: Optional[str] + current_position: Optional[str] + current_organization: Optional[str] + roles: list[str] = [] + + +class GroupMemberList(RootModel): + root: list[GroupMember] + + +class GroupEvent(BaseModel, Generic[T]): + id: int + title: str + timezone: str + start_time: str + end_time: str + image_url: Optional[str] + updated_at: str + created_at: str + address: Optional[str] + # When scraping discussions or events, + # store their publishers in a separate group member table, + # Then keep only the ID on the relevant record + publisher: GroupMember["id"] + + +class GroupEventList(RootModel): + root: list[GroupEvent] + + +class Discussion(BaseModel, Generic[T]): + id: int + # If comments_count > 2, + # query the discussion detail page to get all comments + comments_count: Optional[int] = None + upvotes_count: Optional[int] = None + attachments_count: Optional[int] = None + title: Optional[str] = None + description: Optional[str] = None + created_at: str + publisher: GroupMember["id"] + tags: list[str] = [] + upvotes_user_ids: list[int] = [] + comments: list[int] = [] + is_comment: bool = False + + +class DiscussionList(RootModel): + root: list[Discussion] diff --git a/app/qooper-scraper/scrape.py b/app/qooper-scraper/scrape.py new file mode 100644 index 0000000..a0869a5 --- /dev/null +++ b/app/qooper-scraper/scrape.py @@ -0,0 +1,110 @@ +from datetime import datetime + +import requests +from schemas import ( + DiscussionList, + GroupEventList, + GroupList, + GroupMemberList, +) + + +class Endpoints: + Programs = "https://user-api.qooper.io/v1/programs" + Groups = "https://user-api.qooper.io/v1/groups" + GroupsJoined = "https://user-api.qooper.io/v1/groups/joined" + Members = "https://user-api.qooper.io/v1/groups/{group_id}/members" + GroupDiscussions = "https://user-api.qooper.io/v1/groups/{group_id}/discussions" + GroupEventsUpcoming = ( + "https://user-api.qooper.io/v1/groups/{group_id}/events?filter=upcoming" + ) + GroupEventsPast = ( + "https://user-api.qooper.io/v1/groups/{group_id}/events?filter=past" + ) + + +with open("./qooper-token.txt") as f: + qooper_token = f.read().strip() + +session = requests.Session() +session.headers.update({"Authorization": f"Token token={qooper_token}"}) + +# Get all group IDs +groups = session.get(Endpoints.Groups).json()["data"] +groups_joined = session.get(Endpoints.GroupsJoined).json()["data"] +raw_groups = groups + groups_joined + +print(f'All Groups: {[group['name'] for group in groups]}') +print( + f'Retrieving discussions from the following groups: {[group['name'] for group in groups]}' +) + + +all_groups = [] +for group in raw_groups: + group = session.get(Endpoints.Groups + f'/{group["id"]}').json()["data"] + all_groups.append(group) +all_groups = GroupList(all_groups) + +# For group, get all members +all_members = [] +for group in all_groups.root: + members: list[dict] = session.get( + Endpoints.Members.format(group_id=group.id) + ).json()["data"] + all_members += members + + +# For group, get all discussions +# NOTE: Only gets messages from groups the token grants access to +all_discussions = [] +for group in groups_joined: + discussions: list[dict] = session.get( + Endpoints.GroupDiscussions.format(group_id=group["id"]) + ).json()["data"] + + raw_comments = [] + # Retrieve all comments first + for discussion in discussions: + if len(discussion["comments"]) >= 3: + discussion.update( + session.get( + Endpoints.GroupDiscussions.format( + group_id=group["id"] + f'/{discussion['id']}' + ) + ).json()["data"] + ) + + raw_comments.extend(discussion["comments"]) + discussion["comments"] = [comment["id"] for comment in discussion["comments"]] + + all_discussions += discussions + all_discussions += raw_comments + + +# For group, get all events +all_events = [] +for group in groups_joined: + events: list[dict] = ( + session.get(Endpoints.GroupEventsUpcoming.format(group_id=group["id"])).json()[ + "data" + ] + + session.get(Endpoints.GroupEventsPast.format(group_id=group["id"])).json()[ + "data" + ] + ) + all_events += events + +file_timestamp = datetime.now().strftime(r"%Y%m%d-%H%M%S") + +with open(f"output/qooper-groups-{file_timestamp}.json", "w") as groups_file: + groups_file.write(all_groups.model_dump_json(indent=2)) + +with open(f"output/qooper-members-{file_timestamp}.json", "w") as members_file: + members_file.write(GroupMemberList(all_members).model_dump_json(indent=2)) + +with open(f"output/qooper-discussions-{file_timestamp}.json", "w") as discussions_file: + discussions_file.write(DiscussionList(all_discussions).model_dump_json(indent=2)) + +with open(f"output/qooper-events-{file_timestamp}.json", "w") as events_file: + events_file.write(GroupEventList(all_events).model_dump_json(indent=2))