Skip to content

add video platform #36

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rom1504 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

add video platform #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions cc2dataset/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,21 @@
import time
from .spark_session_builder import build_spark_session
from io import BytesIO
from yt_dlp.extractor import gen_extractor_classes, GenericIE


def valid_video_platform_link(link):

@vinyesm vinyesm Nov 15, 2023

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can try something like this

import yt_dlp


FILTERED_EXTRACTORS = {ie.IE_NAME:ie for ie in yt_dlp.list_extractor_classes() 
                       if ie not in generic_extractors 
                       and "porn" not in ie.IE_NAME.lower()
                       and "adult" not in ie.IE_NAME.lower()
                       and "xxx" not in ie.IE_NAME.lower()
                       and "xvideos" not in ie.IE_NAME.lower()
                       and "xhamster" not in ie.IE_NAME.lower()
                       and "redtube" not in ie.IE_NAME.lower()
                       and "xtube" not in ie.IE_NAME.lower()
                       and "xstream" not in ie.IE_NAME.lower()
                       and "xfileshare" not in ie.IE_NAME.lower()
                       and "sex" not in ie.IE_NAME.lower()
                       }


# print(FILTERED_EXTRACTORS.keys())
# print(len(FILTERED_EXTRACTORS.keys()))

def is_link_valid(link, extractors):
    """Check if link is valid given a list of extractors."""
    return any([ie.suitable(link) for ie in extractors])

def valid_video_platform_link(link):
    """Check if link is a valid video platform link."""
    return link and is_link_valid(link, FILTERED_EXTRACTORS.values())


YT_URL = "https://www.youtube.com/watch?v=jLX0D8qQUBM"
DM_URL = "https://www.dailymotion.com/video/x29ryo7"
print(valid_video_platform_link(YT_URL))
print(valid_video_platform_link(DM_URL))

@vinyesm vinyesm Nov 15, 2023

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

generic_extractors = [yt_dlp.extractor.generic.GenericIE,
yt_dlp.extractor.lazy_extractors.GenericIE]

@rom1504 rom1504 Nov 15, 2023

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

tried at #49

one first problem: running these thousands of regexes is actually quite slow. I guess we need to limit the list or find a way to merge them into one to speed things up (I think that should help?)

@rom1504 rom1504 Nov 15, 2023

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

also let's try the age_limit property in yt-dlp

@rom1504 rom1504 Nov 15, 2023

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

and we can try hyperscan https://geekmonkey.org/regular-expression-matching-at-scale-with-hyperscan/ for speed

@rom1504 rom1504 Nov 15, 2023

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

or maybe https://pypi.org/project/re2/

@vinyesm vinyesm Nov 16, 2023

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks! Another option is to do it in 2 steps

  • first checks that the domain is valid among ~2000 selected domains (from yt_dlp extractors _TESTS)
  • then checks if the url is a valid video url (wwith regex from yt_dlp)

This version is more than 100x faster (but is less exhaustive)

@vinyesm vinyesm Nov 16, 2023 •
edited
Loading

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

import yt_dlp
from urllib.parse import urlparse

generic_extractors = [yt_dlp.extractor.generic.GenericIE,
yt_dlp.extractor.lazy_extractors.GenericIE]

FILTERED_EXTRACTORS = {ie.IE_NAME:ie for ie in yt_dlp.list_extractor_classes()
if ie not in generic_extractors
and "porn" not in ie.IE_NAME.lower()
and "adult" not in ie.IE_NAME.lower()
and "xxx" not in ie.IE_NAME.lower()
and "xvideos" not in ie.IE_NAME.lower()
and "xhamster" not in ie.IE_NAME.lower()
and "redtube" not in ie.IE_NAME.lower()
and "xtube" not in ie.IE_NAME.lower()
and "xstream" not in ie.IE_NAME.lower()
and "xfileshare" not in ie.IE_NAME.lower()
and "sex" not in ie.IE_NAME.lower()
}

def extract_test(extractor):
tests = []
if hasattr(extractor, "_TEST"):
tests = [extractor._TEST["url"]]
elif hasattr(extractor, "_TESTS"):
tests = [x["url"] for x in extractor._TESTS]
return tests

def normalize_domain(domain):
domain = domain.lower()
if domain.startswith("www."):
domain = domain[4:]
return domain

def extract_domain(url):
parsed_url = urlparse(url)
domain = parsed_url.netloc
return normalize_domain(domain)

DOMAIN_DICT = {}

for extractor in FILTERED_EXTRACTORS.values():
for url in extract_test(extractor):
domain = extract_domain(url)
if domain in DOMAIN_DICT:
DOMAIN_DICT[domain] = DOMAIN_DICT[domain] + [extractor]
else:
DOMAIN_DICT[domain] = [extractor]

def is_link_suitable(link, extractors):
"""Check if link is valid given an extractor."""
return any([ie.suitable(link) for ie in extractors])

def is_link_valid(link, domain_dict):
"""Check if link is valid given a list of extractors."""
is_valid = False
domain = extract_domain(link)
if domain in domain_dict:
is_valid = is_link_suitable(link, domain_dict[domain])
return is_valid

def valid_video_platform_link(link):
"""Check if link is a valid video platform link."""
return link and is_link_valid(link, DOMAIN_DICT)

YT_URL = "https://www.youtube.com/watch?v=jLX0D8qQUBM"
DM_URL = "https://www.dailymotion.com/video/x29ryo7"
DM_URL2 = "https://geo.dailymotion.com/player.html?video=x89eyek&mute=true"
print(valid_video_platform_link(YT_URL))
print(valid_video_platform_link(DM_URL))
print(valid_video_platform_link(DM_URL2))

import time

start_time = time.time()
[valid_video_platform_link(x) for x in [DM_URL] * 1000]
print("--- %s seconds ---" % (time.time() - start_time))

@rom1504 rom1504 Nov 16, 2023

Copy link
Copy Markdown
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok I put the code there #49

it's faster indeed!

if "amazon" in link.get("url", "") or "drive" in link.get("url", "") or "twitter" in link.get("url", "") or "pinterest" in link.get("url", "") or "youtube" in link.get("url", "") or "instagram" in link.get("url", ""):
return False
for ie in gen_extractor_classes():
if ie != GenericIE and ie.suitable(link.get("url", "")):
return True
return False

def extract_video_platform_from_links(links):
#links = links[:100]
filtered_links = [{"url": link["url"], "alt": link.get("text", "")} for link in links if valid_video_platform_link(link)]
return filtered_links

def valid_video_link(link):
valid_http = link.get("url", "").startswith("http")
Expand Down Expand Up @@ -105,6 +119,8 @@ def extract_documents_from_links(links, document_type):
return extract_text_from_links(links)
elif document_type == "video":
return extract_video_from_links(links)
elif document_type == "video_platform":
return extract_video_platform_from_links(links)
else:
raise ValueError(f"Unknown document type {document_type}")

Expand Down Expand Up @@ -136,6 +152,8 @@ def extract_documents_from_wat(stream, document_type):
for link in filtered_links:
link["uid"] = str(hashlib.md5((link["alt"] + link["url"]).encode()).hexdigest())
all_links.extend(filtered_links)
if len(all_links) > 100:
return all_links
except Exception as e: # pylint: disable=broad-except
logger.info(e)
logger.info("A shard failed to parse")
Expand Down
2 changes: 1 addition & 1 deletion examples/single_warc_example.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
else:
url = "https://data.commoncrawl.org/" + wat

results = process_wat(url, "image")
results = process_wat(url, "video_platform")
df = pd.DataFrame(results, columns=["uid", "url", "alt"])
df.to_parquet(os.getcwd() + "/output.parquet")
print(df)