selenium-guitar-parser/selenium_parser.py at main · Dmytro-bit/selenium-guitar-parser · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
from random import randint

import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait


class SeleniumParser:
    def __init__(self):
        self._driver = webdriver.Firefox()
        self._wait = WebDriverWait(self._driver, 10)

    def __del__(self):
        self._driver.quit()

    def get_products_urls(self, url: str) -> tuple[list[str], str]:
        self._driver.get(url)
        self._wait.until(
            EC.presence_of_element_located((By.CLASS_NAME, "fx-product-list-entry"))
        )
        items = self._driver.find_elements(By.CLASS_NAME, "fx-product-list-entry")
        links_to_product = []

        file_title = (
            self._driver.find_element(By.CLASS_NAME, "header-headline__label")
            .text.strip()
            .replace("/", "__")
        )

        for item in items:
            product_url = item.find_element(By.TAG_NAME, "a").get_attribute("href")
            links_to_product.append(product_url)
        return links_to_product, file_title

    def generate_data_file(
            self, file_name: str, links_to_product: list[str], params_keys: list[str]
    ) -> None:
        data = []
        for index, url in enumerate(links_to_product):
            item_data = {}

            try:
                self._driver.get(url)
                self._wait.until(EC.presence_of_element_located((By.CLASS_NAME, "price")))
                name_div = self._driver.find_element(
                    By.CLASS_NAME, "fx-content-product__main"
                )
                name = name_div.find_element(By.TAG_NAME, "h1").text
                stage_links = self._driver.find_elements(By.CLASS_NAME, "stages__link")

                item_data["name"] = name
                item_data["brand"] = stage_links[-2].text

                price_text = self._driver.find_element(By.CLASS_NAME, "price").text[1:].replace(",", "")
                price_value = float(price_text)
                item_data["price"] = f"{price_value:.2f}" if '.' in price_text else f"{int(price_value)}"

                item_data["quantity"] = randint(1, 30)

                try:
                    item_data["rating"] = self._driver.find_element(
                        By.CLASS_NAME, "rating__value"
                    ).text
                except:
                    item_data["rating"] = None

                features = self._driver.find_elements(By.CLASS_NAME, "keyfeature")
                features_dict = {}
                for feature in features:
                    try:
                        feature_name = feature.find_element(
                            By.CLASS_NAME, "keyfeature__label"
                        ).text
                        feature_value = feature.find_element(
                            By.CLASS_NAME, "fx-text--bold"
                        ).text.lower().strip()

                        if feature_name in params_keys:
                            features_dict[feature_name] = feature_value
                    except:
                        continue

                item_data["features"] = features_dict
                del features_dict

                images = self._driver.find_elements(By.CLASS_NAME, "ZoomImagePicture")

                images_list = []
                for image in images:
                    image_url = image.find_element(By.TAG_NAME, "img").get_attribute(
                        "src"
                    )
                    images_list.append(image_url)

                item_data["images"] = images_list
                del images_list

                data.append(item_data)

                print(f"Scraped: {name}")

            except Exception as e:
                print(f"Skipping item {index + 1} due to error: {e}")

        df = pd.DataFrame(data)
        df.to_csv(f"shop_data/{file_name}.csv", index=False)