diff --git a/logparser/common.py b/logparser/common.py index 53ee7e9..b1e9697 100644 --- a/logparser/common.py +++ b/logparser/common.py @@ -36,6 +36,9 @@ retry_logs=r'[ ][Rr]etrying[ ]<', # DEBUG: Retrying + latest_crawl='Crawled (', # Crawled (200) (referer: None) + latest_stat=' pages (at', # Crawled 3 pages (at 0 pages/min), scraped 2 items (at 0 items/min) +) + # 2019-01-01 00:00:01 [scrapy.core.scraper] DEBUG: Scraped from <200 http://httpbin.org/headers> LATEST_SCRAPE_ITEM_PATTERN = re.compile(r"""\n ({time_}[ ][^\n]+?{pattern}[^\n]+?)\r?\n({{.*?) @@ -139,7 +168,11 @@ class Common(object): DATETIME_PATTERN = DATETIME_PATTERN DATAS_PATTERN = DATAS_PATTERN LOG_CATEGORIES_PATTERN_DICT = LOG_CATEGORIES_PATTERN_DICT + LOG_CATEGORIES_LITERAL_DICT = LOG_CATEGORIES_LITERAL_DICT + LOG_CATEGORIES_FIRSTLINE_PATTERN_DICT = LOG_CATEGORIES_FIRSTLINE_PATTERN_DICT + LOG_ENTRY_NEXT_LINE_PATTERN = LOG_ENTRY_NEXT_LINE_PATTERN LATEST_MATCHES_PATTERN_DICT = LATEST_MATCHES_PATTERN_DICT + LATEST_MATCHES_LITERAL_DICT = LATEST_MATCHES_LITERAL_DICT LATEST_SCRAPE_ITEM_PATTERN = LATEST_SCRAPE_ITEM_PATTERN SIGTERM_PATTERN = SIGTERM_PATTERN diff --git a/logparser/scrapylogparser.py b/logparser/scrapylogparser.py index d38a865..180d069 100644 --- a/logparser/scrapylogparser.py +++ b/logparser/scrapylogparser.py @@ -48,8 +48,11 @@ def main(self): self.data['logparser_version'] = __version__ return self.data - def re_search_final_match(self, pattern, default='', step=-1): - for line in self.lines[::step]: + def re_search_final_match(self, pattern, default='', step=-1, contains=None): + lines = self.lines if step == 1 else reversed(self.lines) + for line in lines: + if contains is not None and contains not in line: + continue if re.search(pattern, line): return line return default @@ -85,10 +88,16 @@ def extract_time(self): # Extract datas for chart def extract_datas(self): - datas = re.findall(self.DATAS_PATTERN, self.text) # For compatibility with Python 2, str(time_) to avoid [u'2019-01-01 00:00:01', 0, 0, 0, 0] in JavaScript - self.data['datas'] = [[str(time_), int(pages), int(pages_min), int(items), int(items_min)] - for (time_, pages, pages_min, items, items_min) in datas] + datas = [] + for line in self.lines: + if ' pages (at' not in line: + continue + m = self.DATAS_PATTERN.search('\n' + line) + if m: + time_, pages, pages_min, items, items_min = m.groups() + datas.append([str(time_), int(pages), int(pages_min), int(items), int(items_min)]) + self.data['datas'] = datas # TODO: Crawled (200) = 0: + start = self.text.rfind('\n', 0, idx) + if start < 0: + start = 0 + m = self.LATEST_SCRAPE_ITEM_PATTERN.search(self.text, start) + if m: + self.data['latest_matches']['latest_scrape'], self.data['latest_matches']['latest_item'] = m.groups() self.data['latest_matches'].setdefault('latest_scrape', '') self.data['latest_matches'].setdefault('latest_item', '') @@ -143,8 +158,12 @@ def extract_latest_matches(self): def extract_log_categories(self): self.data['log_categories'] = OrderedDict() - for level, pattern in self.LOG_CATEGORIES_PATTERN_DICT.items(): - matches = re.findall(pattern, self.text) + for level in self.LOG_CATEGORIES_PATTERN_DICT: + literal = self.LOG_CATEGORIES_LITERAL_DICT[level] + if literal not in self.text: + self.data['log_categories'][level] = dict(count=0, details=[]) + continue + matches = self.find_category_entries(literal, self.LOG_CATEGORIES_FIRSTLINE_PATTERN_DICT[level]) # DEBUG: Gave up retrying = 0] + m = None + if found_indexes: + start = self.text.rfind('\n', 0, min(found_indexes)) + 1 + m = self.PATTERN_LOG_ENDING.search(self.text, start) if not (m and m.group(3)): self.data['crawler_stats'] = {} else: