ProjectScript/news.py at main · HansungMetro/ProjectScript · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
import requests as req
from fake_useragent import UserAgent
import time
import datetime as dt
from bs4 import BeautifulSoup as bs
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager as chromeD

class newsData: #기사class
    def __init__(self, type, date, text):
        self.type = type
        self.date = date
        self.text = text

def NC_CrawlingSiteIn(url): #홈페이지 게시판 내 공지사항 글 크롤링
    headers = {'User-Agent' : UserAgent().random}
    res = req.get(url, headers=headers)

    soup = bs(res.text, 'html.parser')

    newsText = soup.find('div', class_="textarea-area").find('div', class_="txc-textbox")

    return newsText.get_text()

def NC_CrawlingSite(): #홈페이지 크롤링
    headers = {'User-Agent' : UserAgent().random}
    res = req.get("http://www.seoulmetro.co.kr/kr/board.do?menuIdx=546", headers=headers)

    soup = bs(res.text, 'html.parser')

    newsSite = []

    urlList = soup.find('tbody').find_all('a')
    newsDate = soup.find('tbody').find_all('td', class_="t-disn bd5")

    for i in range(1,8):
        url = "http://www.seoulmetro.co.kr/kr/%s" %(urlList[i].get('href'))
        newsText = NC_CrawlingSiteIn(url)
        newsSite.append(newsData('홈페이지', dt.datetime.fromisoformat(newsDate[i].get_text()), newsText))

    return newsSite

def NC_CrawlingTwitter(): #트위터 크롤링
    driver = webdriver.Chrome(chromeD().install())
    driver.get("https://twitter.com/seoul_metro")
    time.sleep(5)
    html = driver.page_source
    driver.close()

    soup = bs(html, 'html.parser')

    newsTwitter = []

    newsDate = soup.find('div', class_="css-1dbjc4n").find_all('time')
    newsText = soup.find('div', class_="css-1dbjc4n").find_all('div', dir="auto")

    for i in range (8):
        dates = newsDate[i].get('datetime')
        dates = dates[0:10]
        newsTwitter.append(newsData('트위터', dt.datetime.fromisoformat(dates), newsText[i+3].get_text()))

    return newsTwitter

def NC_SortNews(newsT, newsS, numNews): #홈페이지, 트위터에서 크롤링한 기사 날짜순 정렬
    newsList = []
    j = 0
    k = 0

    for i in range(numNews):
        if newsT[j].date > newsS[k].date:
            newsList.append(newsT[j])
            j += 1
        elif newsT[j].date < newsS[k].date:
            newsList.append(newsS[k])
            k += 1
        else:
            newsList.append(newsT[j])
            newsList.append(newsS[k])
            i += 1
            j += 1
            k += 1

    return newsList

def NC_PrintNews(numNews): #기사 출력 numNews:출력 기사 갯수(최대8)
    if numNews >= 8:
        numNews = 8
    newsT = NC_CrawlingTwitter()
    newsS = NC_CrawlingSite()
    news = NC_SortNews(newsT, newsS, numNews)
    for i in news:
        print("───────────────────────────────────────────")
        print(f"[{i.type}]\t{str(i.date)[0:10]}\n")
        print(i.text)

    print("───────────────────────────────────────────")

if __name__ == '__main__':
    NC_PrintNews(8)