From b14d7ed1e1019f720ce854195911c8ae981a7d78 Mon Sep 17 00:00:00 2001 From: mattkelleher Date: Mon, 12 Feb 2018 16:11:16 -0500 Subject: [PATCH 01/17] Got stream downloader to run locally. Only works for first batch. Added function to write broken camera links to file named 'deadCams.txt' --- deadCams.txt | 38 ++++++++++++++++++++++++++++++++++++++ stream_downloader.py | 13 ++++++++----- 2 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 deadCams.txt diff --git a/deadCams.txt b/deadCams.txt new file mode 100644 index 0000000..4bd69bd --- /dev/null +++ b/deadCams.txt @@ -0,0 +1,38 @@ +http://video3.earthcam.com/fecnetwork/10147.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10294.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4356.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3186.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5019.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3847fairmont.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5441.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10729.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4617.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5845.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7711.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7786.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6175.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8837.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5965.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6632.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6426.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6076.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6406.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7392.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7710.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/anguilla.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/fridays_5th.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7130.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7132.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5966.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7484.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7393.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8577.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7342.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9741.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7624.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/dceaglecam.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9602.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/paradiseon6.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8470.flv/playlist.m3u8 diff --git a/stream_downloader.py b/stream_downloader.py index c399c2f..bca2b93 100644 --- a/stream_downloader.py +++ b/stream_downloader.py @@ -11,7 +11,9 @@ This module is designed to interface with Earthcam cameras and download frames via multithreading and save them with their camera IDs and current date-time """ - +def _report_dead_cam(m3u8_target): + with open("deadCams.txt", "a") as deadFile: + deadFile.write(m3u8_target) def _get_formatted_est(): """ @@ -74,6 +76,7 @@ def download_stream_frames(m3u8_target, save_path, num_frames=1, save_img_type=" print "||||||" + _get_earthcam_id(m3u8_target) + "||||||" else: # Camera dead print "******" + _get_earthcam_id(m3u8_target) + "******" + _report_dead_cam(m3u8_target) break cam.release() @@ -136,16 +139,16 @@ def download_stream_time(m3u8_target, save_path, runtime_sec=5, save_img_type="p with open("m3u8s.txt") as stream_file: - EXPERIMENT_END_TIME = time.time() + 60 * 60 * 1 # Run for a full 8 hours - MAX_BATCH_ALLOWED_TIME = 60 * 5 # 5 minutes per batch, max - FRAMES_PER_CAM_PER_BATCH = 10000 + EXPERIMENT_END_TIME = time.time() + 60 * 3 # Run for a full 1 hours + MAX_BATCH_ALLOWED_TIME = 60 * 1 # 5 minutes per batch, max + FRAMES_PER_CAM_PER_BATCH = 15 while time.time() < EXPERIMENT_END_TIME: CURR_BATCH_START_TIME = time.time() # Create a directory for the current download batch if it doesn't already exist - MASTER_SAVE_DIR = "./OneNightOfFrames/Batch-" + _get_formatted_est() + "/" + MASTER_SAVE_DIR = "./pics/Batch-" + _get_formatted_est() + "/" print "======STARTING BATCH======" From 1beda83d482b84ed4dcb72b7af87bb30e1338da6 Mon Sep 17 00:00:00 2001 From: mattkelleher Date: Tue, 20 Feb 2018 11:35:23 -0500 Subject: [PATCH 02/17] stream_downloader will now download multiple batches --- deadCams.txt | 113 ++++++++++++++++++++++++++++ m3u8s.txt | 43 ----------- m3u8sMaster.txt | 170 +++++++++++++++++++++++++++++++++++++++++++ stream_downloader.py | 106 +++++++++++++++++---------- 4 files changed, 351 insertions(+), 81 deletions(-) create mode 100644 m3u8sMaster.txt diff --git a/deadCams.txt b/deadCams.txt index 4bd69bd..9005595 100644 --- a/deadCams.txt +++ b/deadCams.txt @@ -36,3 +36,116 @@ http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/9602.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/paradiseon6.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8470.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5441.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3847fairmont.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7130.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5845.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6632.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6175.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6076.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4617.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5019.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5966.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6406.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5965.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7393.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7710.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7392.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7786.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6426.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7132.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7484.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7624.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4356.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8837.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8577.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3186.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10729.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7711.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8470.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7342.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/dceaglecam.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9602.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/anguilla.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9741.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/fridays_5th.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/paradiseon6.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3186.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5019.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5441.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5845.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3847fairmont.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4356.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4465.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6076.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4617.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5966.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5965.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7710.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6632.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6406.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7130.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6426.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7132.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6175.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8620.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7393.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7392.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7342.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8837.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8577.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7571.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7624.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7711.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9602.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9741.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/anguilla.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7484.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/dceaglecam.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7786.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/fridays_5th.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/moscowHD1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/paradiseon6.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3847fairmont.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4135.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4356.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3186.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4465.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5441.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6076.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4617.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5019.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5845.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5965.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6406.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6426.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7130.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6175.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6632.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5966.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7132.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7342.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7392.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7484.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7393.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7624.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7571.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7711.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7786.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7710.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8577.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8620.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8837.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9602.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9741.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/anguilla.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/dceaglecam.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/fridays_5th.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/moscowHD1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/paradiseon6.flv/playlist.m3u8 diff --git a/m3u8s.txt b/m3u8s.txt index a75c830..a9229e8 100644 --- a/m3u8s.txt +++ b/m3u8s.txt @@ -1,40 +1,31 @@ http://video3.earthcam.com/fecnetwork/10072.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/10147.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/10294.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/10347.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/10551.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/10610.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/10669.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/10729.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/10874.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/3186.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/3275.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/3304.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/3635.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/3644newscafe.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/370.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/3792.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/3847fairmont.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/3983.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4017timessquare.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4049sonesta.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4054.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/4135.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4162.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4280.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4281.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4282.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4337.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4338.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/4356.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4369.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4387.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/4465.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4480.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4518.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4544.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4559.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/4617.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4644.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4646.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4717.flv/playlist.m3u8 @@ -42,7 +33,6 @@ http://video3.earthcam.com/fecnetwork/4798.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4831.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4918.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4968.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5019.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5026.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5034.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5173.flv/playlist.m3u8 @@ -54,7 +44,6 @@ http://video3.earthcam.com/fecnetwork/5324.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5358.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5359.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5423.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5441.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5465.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5536.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5595.flv/playlist.m3u8 @@ -65,21 +54,13 @@ http://video3.earthcam.com/fecnetwork/5751.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5758.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5775.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5805.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5845.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5965.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5966.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6076.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6081.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6175.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6253.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6405.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6406.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6426.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6427.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6528.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6593.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6603.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6632.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6649.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6698.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6723.flv/playlist.m3u8 @@ -88,23 +69,11 @@ http://video3.earthcam.com/fecnetwork/6961.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6975.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6998.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6999.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7130.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7132.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7300.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7322.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7342.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7382.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7384.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7392.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7393.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7484.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7524.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7571.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7624.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7710.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7711.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7786.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7928.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7945.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7947.flv/playlist.m3u8 @@ -115,17 +84,13 @@ http://video3.earthcam.com/fecnetwork/8048.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8078.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8174.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8211.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8470.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8471.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8577.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8584.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8620.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8671.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8713.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8715.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8763.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8811.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8837.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8838.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8891.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8939.flv/playlist.m3u8 @@ -134,32 +99,24 @@ http://video3.earthcam.com/fecnetwork/9115.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/9189.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/9298.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/9299.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/9602.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/9741.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/9779.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/9974.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/anguilla.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/chargingbull.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/dceaglecam.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/dealeyplaza2.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/diamondhead.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/ecfishtank01.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/eclasvegas.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/eiffelcamHD.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/fridays_5th.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/hdtimes10.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/hotelvictoria2.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/interactoneHDCam1.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/lacitytours1.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/libertyHD1.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/meprd_garch.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/miamivishd1.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/miamivishd2.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/miamivishd4.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/miamivishd5.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/miamivishd7.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/moscowHD1.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/paradiseon6.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/philippinesHD1.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/playalinda1.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/playalinda2.flv/playlist.m3u8 diff --git a/m3u8sMaster.txt b/m3u8sMaster.txt new file mode 100644 index 0000000..a75c830 --- /dev/null +++ b/m3u8sMaster.txt @@ -0,0 +1,170 @@ +http://video3.earthcam.com/fecnetwork/10072.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10147.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10294.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10347.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10551.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10610.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10669.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10729.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10874.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3186.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3275.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3304.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3635.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3644newscafe.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/370.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3792.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3847fairmont.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3983.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4017timessquare.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4049sonesta.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4054.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4135.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4162.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4280.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4281.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4282.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4337.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4338.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4356.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4369.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4387.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4465.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4480.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4518.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4544.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4559.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4617.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4644.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4646.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4717.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4798.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4831.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4918.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4968.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5019.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5026.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5034.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5173.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5187.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5204.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5230.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5288.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5324.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5358.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5359.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5423.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5441.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5465.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5536.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5595.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5663.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5718.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5719.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5751.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5758.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5775.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5805.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5845.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5965.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5966.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6076.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6081.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6175.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6253.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6405.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6406.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6426.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6427.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6528.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6593.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6603.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6632.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6649.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6698.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6723.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6954.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6961.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6975.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6998.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6999.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7130.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7132.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7300.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7322.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7342.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7382.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7384.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7392.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7393.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7484.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7524.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7571.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7624.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7710.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7711.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7786.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7928.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7945.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7947.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7949.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7962.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7988.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8048.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8078.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8174.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8211.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8470.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8471.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8577.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8584.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8620.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8671.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8713.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8715.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8763.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8811.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8837.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8838.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8891.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8939.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9106.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9115.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9189.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9298.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9299.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9602.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9741.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9779.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9974.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/anguilla.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/chargingbull.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/dceaglecam.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/dealeyplaza2.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/diamondhead.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/ecfishtank01.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/eclasvegas.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/eiffelcamHD.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/fridays_5th.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/hdtimes10.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/hotelvictoria2.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/interactoneHDCam1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/lacitytours1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/libertyHD1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/meprd_garch.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd2.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd4.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd5.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd7.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/moscowHD1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/paradiseon6.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/philippinesHD1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/playalinda1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/playalinda2.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/sedonawebcam.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/tokyo1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/warholchurch.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/windjammerHD2.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/woodmenHD.flv/playlist.m3u8 diff --git a/stream_downloader.py b/stream_downloader.py index bca2b93..b2f31fb 100644 --- a/stream_downloader.py +++ b/stream_downloader.py @@ -11,7 +11,31 @@ This module is designed to interface with Earthcam cameras and download frames via multithreading and save them with their camera IDs and current date-time """ + +def _clean_cams(): + """ + Removes dead cameras from camera list + """ + with open('deadCams.txt', 'r') as deadCamFile: + deadCamList = deadCamFile.readlines() + + with open('m3u8sMaster.txt', 'r') as masterListFile: + masterList = masterListFile.readlines() + + updatedCamList = [] + + for cam in masterList: + if cam not in deadCamList: + updatedCamList.append(cam) + + with open('m3u8s.txt', 'w') as updatedCamFile: + updatedCamFile.writelines(updatedCamList) + + def _report_dead_cam(m3u8_target): + """ + Writes dead camera links to file to be removed + """ with open("deadCams.txt", "a") as deadFile: deadFile.write(m3u8_target) @@ -128,8 +152,8 @@ def download_stream_time(m3u8_target, save_path, runtime_sec=5, save_img_type="p return failure_cnt +def get_single_batch(MAX_BAtCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): -if __name__ == "__main__": """ Downloads frames from m3u8s.txt via multithreading One thread per camera @@ -139,43 +163,49 @@ def download_stream_time(m3u8_target, save_path, runtime_sec=5, save_img_type="p with open("m3u8s.txt") as stream_file: - EXPERIMENT_END_TIME = time.time() + 60 * 3 # Run for a full 1 hours - MAX_BATCH_ALLOWED_TIME = 60 * 1 # 5 minutes per batch, max - FRAMES_PER_CAM_PER_BATCH = 15 - - while time.time() < EXPERIMENT_END_TIME: - CURR_BATCH_START_TIME = time.time() - - # Create a directory for the current download batch if it doesn't already exist - MASTER_SAVE_DIR = "./pics/Batch-" + _get_formatted_est() + "/" - - print "======STARTING BATCH======" - - for stream_link in stream_file: - cam_save_dir = MASTER_SAVE_DIR + "Cam-" + _get_earthcam_id(stream_link) + "/" - - # Create a directory for the current cam download if it doesn't already exist - if not os.path.exists(cam_save_dir): - os.makedirs(cam_save_dir) - - p = multiprocessing.Process(target=download_stream_frames, - args=(stream_link, cam_save_dir, FRAMES_PER_CAM_PER_BATCH)) - STREAM_DOWNLOADERS.append(p) - p.start() - - # Every 5 seconds, check if it's time to start a new batch - while STREAM_DOWNLOADERS is not False: # While not empty - STREAM_DOWNLOADERS = [p for p in STREAM_DOWNLOADERS if p.is_alive()] - - # Terminate early if it takes too long to finish the batch - if time.time() >= (CURR_BATCH_START_TIME + MAX_BATCH_ALLOWED_TIME): - for p in STREAM_DOWNLOADERS: - p.terminate() - p.join() - print "======Prematurely terminated batch======" - break - else: - time.sleep(5) + CURR_BATCH_START_TIME = time.time() + + # Create a directory for the current download batch if it doesn't already exist + MASTER_SAVE_DIR = "./pics/Batch-" + _get_formatted_est() + "/" + + print "======STARTING BATCH======" + + for stream_link in stream_file: + cam_save_dir = MASTER_SAVE_DIR + "Cam-" + _get_earthcam_id(stream_link) + "/" + # Create a directory for the current cam download if it doesn't already exist + if not os.path.exists(cam_save_dir): + os.makedirs(cam_save_dir) + + p = multiprocessing.Process(target=download_stream_frames, + args=(stream_link, cam_save_dir, FRAMES_PER_CAM_PER_BATCH)) + STREAM_DOWNLOADERS.append(p) + p.start() + + # Every 5 seconds, check if it's time to start a new batch + while STREAM_DOWNLOADERS is not False: # While not empty + STREAM_DOWNLOADERS = [p for p in STREAM_DOWNLOADERS if p.is_alive()] + # Terminate early if it takes too long to finish the batch + if time.time() >= (CURR_BATCH_START_TIME + MAX_BATCH_ALLOWED_TIME): + for p in STREAM_DOWNLOADERS: + p.terminate() + p.join() + print "======Prematurely terminated batch======" + break + else: + time.sleep(5) +if __name__ == "__main__": + """ + Downloads frames from m3u8s.txt via multithreading + One thread per camera + """ + EXPERIMENT_END_TIME = time.time() + 60 * 3 # Run for a full 1 hours + MAX_BATCH_ALLOWED_TIME = 60 * 1 # 1 minutes per batch, max + FRAMES_PER_CAM_PER_BATCH = 10 + + while time.time() < EXPERIMENT_END_TIME: + get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH) + + _clean_cams() print "======EXPERIMENT DONE======" From a8cee7b1c4508f6abdd59db3605e3e9f5a4e5ae8 Mon Sep 17 00:00:00 2001 From: mattkelleher Date: Tue, 6 Mar 2018 14:43:26 -0500 Subject: [PATCH 03/17] removed duplicates from deadCams.txt --- deadCams.txt | 138 ++++--------------------------------------- m3u8s.txt | 43 ++++++++++++++ stream_downloader.py | 9 ++- 3 files changed, 62 insertions(+), 128 deletions(-) diff --git a/deadCams.txt b/deadCams.txt index 9005595..ee1e847 100644 --- a/deadCams.txt +++ b/deadCams.txt @@ -1,151 +1,39 @@ -http://video3.earthcam.com/fecnetwork/10147.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/10294.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4162.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8174.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5034.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/ecfishtank01.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/eiffelcamHD.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/meprd_garch.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/sedonawebcam.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/warholchurch.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4356.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/3186.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5019.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/3847fairmont.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5441.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/10729.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/4617.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5019.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5845.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7711.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7786.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6175.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8837.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5965.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6632.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6426.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6076.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6406.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7392.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7710.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/anguilla.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/fridays_5th.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7130.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7132.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5966.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7484.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7393.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8577.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/miamivishd1.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7342.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/9741.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7624.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/dceaglecam.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/9602.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/paradiseon6.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8470.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5441.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/3847fairmont.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7130.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5845.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6632.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6175.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6076.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4617.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5019.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5966.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6406.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5965.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7393.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7710.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7392.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7786.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6426.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7132.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7484.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7624.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/4356.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8837.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8577.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/3186.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/10729.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7711.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8470.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7342.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/dceaglecam.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/9602.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/anguilla.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/9741.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/fridays_5th.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/paradiseon6.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/miamivishd1.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/3186.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5019.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5441.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5845.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/3847fairmont.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/4356.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/4465.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6076.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/4617.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5966.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5965.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7710.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6632.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6406.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7130.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6426.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7132.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6175.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8620.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7393.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7392.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7342.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8837.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8577.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7571.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7624.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7711.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/9602.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/9741.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/anguilla.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7484.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/dceaglecam.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7786.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/fridays_5th.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/miamivishd1.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/moscowHD1.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/paradiseon6.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/3847fairmont.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/4135.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/4356.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/3186.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/4465.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5441.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6076.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/4617.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5019.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5845.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5965.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6406.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6426.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7130.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6175.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6632.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5966.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7130.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7132.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7342.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7392.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7484.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7393.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7624.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7571.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7484.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7710.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7711.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7786.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7710.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8577.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8620.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8837.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/9602.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/9741.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/anguilla.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/dceaglecam.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/fridays_5th.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/miamivishd1.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/moscowHD1.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/paradiseon6.flv/playlist.m3u8 diff --git a/m3u8s.txt b/m3u8s.txt index a9229e8..a75c830 100644 --- a/m3u8s.txt +++ b/m3u8s.txt @@ -1,31 +1,40 @@ http://video3.earthcam.com/fecnetwork/10072.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10147.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10294.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/10347.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/10551.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/10610.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/10669.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10729.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/10874.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3186.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/3275.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/3304.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/3635.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/3644newscafe.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/370.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/3792.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3847fairmont.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/3983.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4017timessquare.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4049sonesta.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4054.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4135.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4162.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4280.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4281.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4282.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4337.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4338.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4356.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4369.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4387.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4465.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4480.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4518.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4544.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4559.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4617.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4644.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4646.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4717.flv/playlist.m3u8 @@ -33,6 +42,7 @@ http://video3.earthcam.com/fecnetwork/4798.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4831.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4918.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/4968.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5019.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5026.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5034.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5173.flv/playlist.m3u8 @@ -44,6 +54,7 @@ http://video3.earthcam.com/fecnetwork/5324.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5358.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5359.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5423.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5441.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5465.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5536.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5595.flv/playlist.m3u8 @@ -54,13 +65,21 @@ http://video3.earthcam.com/fecnetwork/5751.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5758.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5775.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5805.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5845.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5965.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5966.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6076.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6081.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6175.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6253.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6405.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6406.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6426.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6427.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6528.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6593.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6603.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6632.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6649.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6698.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6723.flv/playlist.m3u8 @@ -69,11 +88,23 @@ http://video3.earthcam.com/fecnetwork/6961.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6975.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6998.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6999.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7130.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7132.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7300.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7322.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7342.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7382.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7384.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7392.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7393.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7484.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7524.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7571.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7624.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7710.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7711.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7786.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7928.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7945.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7947.flv/playlist.m3u8 @@ -84,13 +115,17 @@ http://video3.earthcam.com/fecnetwork/8048.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8078.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8174.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8211.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8470.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8471.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8577.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8584.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8620.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8671.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8713.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8715.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8763.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8811.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8837.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8838.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8891.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8939.flv/playlist.m3u8 @@ -99,24 +134,32 @@ http://video3.earthcam.com/fecnetwork/9115.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/9189.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/9298.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/9299.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9602.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9741.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/9779.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/9974.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/anguilla.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/chargingbull.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/dceaglecam.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/dealeyplaza2.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/diamondhead.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/ecfishtank01.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/eclasvegas.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/eiffelcamHD.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/fridays_5th.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/hdtimes10.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/hotelvictoria2.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/interactoneHDCam1.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/lacitytours1.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/libertyHD1.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/meprd_garch.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd1.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/miamivishd2.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/miamivishd4.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/miamivishd5.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/miamivishd7.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/moscowHD1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/paradiseon6.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/philippinesHD1.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/playalinda1.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/playalinda2.flv/playlist.m3u8 diff --git a/stream_downloader.py b/stream_downloader.py index b2f31fb..67c4afc 100644 --- a/stream_downloader.py +++ b/stream_downloader.py @@ -36,8 +36,11 @@ def _report_dead_cam(m3u8_target): """ Writes dead camera links to file to be removed """ + with open("deadCams.txt", "r") as deadFile: + deadCamList = deadFile.readlines() with open("deadCams.txt", "a") as deadFile: - deadFile.write(m3u8_target) + if m3u8_target not in deadCamList: + deadFile.write(m3u8_target) def _get_formatted_est(): """ @@ -200,8 +203,8 @@ def get_single_batch(MAX_BAtCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): Downloads frames from m3u8s.txt via multithreading One thread per camera """ - EXPERIMENT_END_TIME = time.time() + 60 * 3 # Run for a full 1 hours - MAX_BATCH_ALLOWED_TIME = 60 * 1 # 1 minutes per batch, max + EXPERIMENT_END_TIME = time.time() + 60 * 5 # Run for a full 1 hours + MAX_BATCH_ALLOWED_TIME = 60 * 0.5 # 1 minutes per batch, max FRAMES_PER_CAM_PER_BATCH = 10 while time.time() < EXPERIMENT_END_TIME: From 0e7a1089511ba68f312628243bbf9f8643f15988 Mon Sep 17 00:00:00 2001 From: mattkelleher Date: Sun, 25 Mar 2018 11:15:57 -0400 Subject: [PATCH 04/17] Updated naming scheme to allow for successful save of mutliple images from one camera in 1 second --- stream_downloader.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/stream_downloader.py b/stream_downloader.py index 67c4afc..21720c1 100644 --- a/stream_downloader.py +++ b/stream_downloader.py @@ -64,7 +64,7 @@ def _get_earthcam_id(m3u8_target): return cam_id -def download_stream_frames(m3u8_target, save_path, num_frames=1, save_img_type="png"): +def download_stream_frames(m3u8_target, save_path, num_frames=1, save_img_type="jpg"): """ Worker function that downloads a specified number of frames from a target stream. WORKS FOR EARTHCAM STREAMS ONLY. @@ -77,18 +77,19 @@ def download_stream_frames(m3u8_target, save_path, num_frames=1, save_img_type=" num_frames: (int) Default 1. The desired number of frames to download from the camera. - save_img_type: (str) Default 'png'. The target image save format. + save_img_type: (str) Default 'jpg'. The target image save format. """ cam = cv2.VideoCapture(m3u8_target) failure_cnt = 0 # The number of total frame grab failures is_got_one_successfully = False - + i = 0 for _i in range(num_frames): is_success, frame = cam.read() filename = _get_earthcam_id(m3u8_target) + "--" + _get_formatted_est() + \ - "." + save_img_type + str(i) + "." + save_img_type + i += 1 if is_success: save_target = save_path + filename @@ -110,7 +111,7 @@ def download_stream_frames(m3u8_target, save_path, num_frames=1, save_img_type=" return failure_cnt -def download_stream_time(m3u8_target, save_path, runtime_sec=5, save_img_type="png"): +def download_stream_time(m3u8_target, save_path, runtime_sec=5, save_img_type="jpg"): """ Worker function that downloads a specified number of frames from a target stream. WORKS FOR EARTHCAM STREAMS ONLY. @@ -123,7 +124,7 @@ def download_stream_time(m3u8_target, save_path, runtime_sec=5, save_img_type="p runtime_sec: (int) Default 5. The desired number of seconds to run the downloader for. - save_img_type: (str) Default 'png'. The target image save format. + save_img_type: (str) Default 'jpg'. The target image save format. """ cam = cv2.VideoCapture(m3u8_target) @@ -209,6 +210,6 @@ def get_single_batch(MAX_BAtCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): while time.time() < EXPERIMENT_END_TIME: get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH) - - _clean_cams() + _clean_cams() + print "======EXPERIMENT DONE======" From 736fb450faebcc8ebe2dd4c06f635859605bf90f Mon Sep 17 00:00:00 2001 From: mattkelleher Date: Sun, 25 Mar 2018 11:21:14 -0400 Subject: [PATCH 05/17] changed variable names --- stream_downloader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stream_downloader.py b/stream_downloader.py index 21720c1..ac91644 100644 --- a/stream_downloader.py +++ b/stream_downloader.py @@ -84,12 +84,12 @@ def download_stream_frames(m3u8_target, save_path, num_frames=1, save_img_type=" failure_cnt = 0 # The number of total frame grab failures is_got_one_successfully = False - i = 0 + count = 0 for _i in range(num_frames): is_success, frame = cam.read() filename = _get_earthcam_id(m3u8_target) + "--" + _get_formatted_est() + \ - str(i) + "." + save_img_type - i += 1 + str(count) + "." + save_img_type + count += 1 if is_success: save_target = save_path + filename From d7e3a8007dac9db20a5add496d7ff0bc3d46cfaa Mon Sep 17 00:00:00 2001 From: mattkelleher Date: Sun, 25 Mar 2018 11:23:23 -0400 Subject: [PATCH 06/17] updated --- stream_downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stream_downloader.py b/stream_downloader.py index ac91644..39a33b3 100644 --- a/stream_downloader.py +++ b/stream_downloader.py @@ -89,12 +89,12 @@ def download_stream_frames(m3u8_target, save_path, num_frames=1, save_img_type=" is_success, frame = cam.read() filename = _get_earthcam_id(m3u8_target) + "--" + _get_formatted_est() + \ str(count) + "." + save_img_type - count += 1 if is_success: save_target = save_path + filename cv2.imwrite(save_target, frame) is_got_one_successfully = True + count += 1 else: failure_cnt += 1 print "FAILURE: " + filename + ", FAILED " + str(failure_cnt) From 39addc38873340691c858067a272aeeb5d4ea21e Mon Sep 17 00:00:00 2001 From: mattkelleher Date: Wed, 28 Mar 2018 11:15:50 -0400 Subject: [PATCH 07/17] Updated image file saving process and periodically reset camera list once every two hours for long jobs --- deadCams.txt | 55 ++++++++++++++++++++++++-------------------- stream_downloader.py | 25 +++++++++++++------- 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/deadCams.txt b/deadCams.txt index ee1e847..984e85c 100644 --- a/deadCams.txt +++ b/deadCams.txt @@ -1,39 +1,44 @@ -http://video3.earthcam.com/fecnetwork/4162.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8174.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5034.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/ecfishtank01.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/eiffelcamHD.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/meprd_garch.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/sedonawebcam.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/warholchurch.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/4356.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/3186.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5536.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5845.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6076.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6426.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5965.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7484.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/3847fairmont.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5441.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7928.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10551.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7342.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3186.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7624.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3792.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5019.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5845.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5441.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5359.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4617.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5026.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4356.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4162.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6175.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/5966.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/4617.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6426.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/5965.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/6076.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/6632.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7130.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6999.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7132.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7342.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7392.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7393.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7484.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7710.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/7711.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7393.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/7786.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8577.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8620.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7711.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8048.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8837.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/9741.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8838.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8620.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8577.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/anguilla.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9741.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/eiffelcamHD.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/dceaglecam.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/miamivishd1.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/paradiseon6.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/sedonawebcam.flv/playlist.m3u8 diff --git a/stream_downloader.py b/stream_downloader.py index 39a33b3..72d3dfe 100644 --- a/stream_downloader.py +++ b/stream_downloader.py @@ -88,13 +88,13 @@ def download_stream_frames(m3u8_target, save_path, num_frames=1, save_img_type=" for _i in range(num_frames): is_success, frame = cam.read() filename = _get_earthcam_id(m3u8_target) + "--" + _get_formatted_est() + \ - str(count) + "." + save_img_type + "_" + str(count) + "." + save_img_type if is_success: save_target = save_path + filename cv2.imwrite(save_target, frame) is_got_one_successfully = True - count += 1 + count = (count + 1) % 1000 else: failure_cnt += 1 print "FAILURE: " + filename + ", FAILED " + str(failure_cnt) @@ -171,7 +171,7 @@ def get_single_batch(MAX_BAtCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): CURR_BATCH_START_TIME = time.time() # Create a directory for the current download batch if it doesn't already exist - MASTER_SAVE_DIR = "./pics/Batch-" + _get_formatted_est() + "/" + MASTER_SAVE_DIR = "./pics/Batch-" + _get_formatted_est() + "/" #TODO Set MASTER_SAVE_DIR print "======STARTING BATCH======" @@ -204,12 +204,21 @@ def get_single_batch(MAX_BAtCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): Downloads frames from m3u8s.txt via multithreading One thread per camera """ - EXPERIMENT_END_TIME = time.time() + 60 * 5 # Run for a full 1 hours - MAX_BATCH_ALLOWED_TIME = 60 * 0.5 # 1 minutes per batch, max - FRAMES_PER_CAM_PER_BATCH = 10 - + EXPERIMENT_END_TIME = time.time() + 60 * 5 #TODO Set experiment run time + MAX_BATCH_ALLOWED_TIME = 60 * 0.5 #TODO Set max time per patch + FRAMES_PER_CAM_PER_BATCH = 15 #TODO Set max frames per batch + hours_remaining = (EXPERIMENT_END_TIME - time.time()) // (60) + print hours_remaining while time.time() < EXPERIMENT_END_TIME: get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH) - _clean_cams() + _clean_cams() + if((EXPERIMENT_END_TIME - time.time()) // (60) != hours_remaining): + hours_remaining = hours_remaining - 1 + print hours_remaining + with open("m3u8sMaster.txt", "r") as master: + masterList = master.readlines() + with open("m3u8s.txt", "w") as working: + working.writelines(masterList) + print "======EXPERIMENT DONE======" From 060cf52da781966bac113460b1a7cea008410623 Mon Sep 17 00:00:00 2001 From: mattkelleher Date: Wed, 28 Mar 2018 11:17:21 -0400 Subject: [PATCH 08/17] Updated image file saving process and periodically reset camera list once every two hours for long jobs --- stream_downloader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stream_downloader.py b/stream_downloader.py index 72d3dfe..87dfb92 100644 --- a/stream_downloader.py +++ b/stream_downloader.py @@ -212,8 +212,8 @@ def get_single_batch(MAX_BAtCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): while time.time() < EXPERIMENT_END_TIME: get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH) _clean_cams() - if((EXPERIMENT_END_TIME - time.time()) // (60) != hours_remaining): - hours_remaining = hours_remaining - 1 + if((EXPERIMENT_END_TIME - time.time()) // (60 * 60 * 2) != hours_remaining): + hours_remaining = hours_remaining - 2 print hours_remaining with open("m3u8sMaster.txt", "r") as master: masterList = master.readlines() From 76dc2f2edc323da26ced81453df7ebe36d79b315 Mon Sep 17 00:00:00 2001 From: mattkelleher Date: Wed, 28 Mar 2018 11:56:15 -0400 Subject: [PATCH 09/17] Fixed minor math error --- stream_downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stream_downloader.py b/stream_downloader.py index 87dfb92..2b8ce2c 100644 --- a/stream_downloader.py +++ b/stream_downloader.py @@ -207,7 +207,7 @@ def get_single_batch(MAX_BAtCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): EXPERIMENT_END_TIME = time.time() + 60 * 5 #TODO Set experiment run time MAX_BATCH_ALLOWED_TIME = 60 * 0.5 #TODO Set max time per patch FRAMES_PER_CAM_PER_BATCH = 15 #TODO Set max frames per batch - hours_remaining = (EXPERIMENT_END_TIME - time.time()) // (60) + hours_remaining = (EXPERIMENT_END_TIME - time.time()) // (60 * 60) print hours_remaining while time.time() < EXPERIMENT_END_TIME: get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH) From 88dee26b2fe8242e5c31f3e2ffdf2fef968ad57d Mon Sep 17 00:00:00 2001 From: mattkelleher Date: Tue, 19 Feb 2019 23:27:44 +0000 Subject: [PATCH 10/17] Brightness metrics evaluation script --- BrightnessMetrics.py | 95 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 BrightnessMetrics.py diff --git a/BrightnessMetrics.py b/BrightnessMetrics.py new file mode 100644 index 0000000..c5ce5d1 --- /dev/null +++ b/BrightnessMetrics.py @@ -0,0 +1,95 @@ +import os +import numpy as np +import pickle +import cv2 + + + +""" +Main program header +include programmers, project github, instructions how to use etc +""" + +""" +Do we have a cv2 img? +""" +def is_valid_img(img): + if not isinstance(img, np.ndarray): + print("Error: cv2 image not passed!") + return False + else: + return True + +""" +Perceived Brightness = Sqrt((0.241 * R^2) + (0.691 * G^2) + (0.068 * B^2)) +""" +def calc_perceived_brightness(img): + height, width, dim = img.shape + return ( np.sqrt(((np.sum(img[:, :, 2]) ** 2) * 0.241) + ((np.sum(img[:, :, 1]) ** 2) * 0.691) + ((np.sum(img[:, :, 0]) ** 2) * 0.068)) / (height * width * 255)) + + +""" +Luminance +Luminance = (0.299 * R) + (0.578 * G) + (0.114 * B) +""" +def calc_luminosity(img): + height, width, dim = img.shape + return (np.sum(img[:, :, 2]) * 0.299 + np.sum(img[:, :, 1]) * 0.578 + np.sum(img[:, :, 0]) * 0.114) / (height * width * 255) + +""" +Pixel Intensity +1.Converts image to greyscale +2. Takes the average pixel value +""" +def calc_px_intensity(img): + gImg = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + height, width= gImg.shape + return np.sum(gImg) / (height * width * 255) + +""" +Runs analysis on a single image and +""" +def run_single_analysis(working_path, fileLoc, data): + img = cv2.imread(fileLoc) + if is_valid_img(img): + lum = calc_luminosity(img) + px_int = calc_px_intensity(img) + percB = calc_perceived_brightness(img) + data.append([fileLoc.replace(working_path, "."), lum, px_int, percB]) + else: + print("ERROR: Invalid Image: " + fileLoc) + + + +""" +Goes through all .jpg or .png images in specified directory and all subdirectories +""" +def run_mass_analysis(): + + orig_path = os.path.realpath(".") + while True: + working_path = raw_input("Input path (relative or absolute) to directory to be evaluated\n Relative paths should start with './' \n Absolute paths should start with or '/' \nEnter Path: ") + try: + os.chdir(working_path) + except: + print("ERROR: Invalid directory") + else: + break + + working_path = os.path.realpath(".") + data = [["Absolute Path: ", working_path],[]] + data.append(["Image_Location", "Luminosity", "Pixel_Intensity", "Perceived_Brightness"]) + + for subDir, dirs, files in os.walk(working_path): + for file in files: + fileLoc = os.path.join(subDir,file) + if (".jpg" in fileLoc) or (".png" in fileLoc): + run_single_analysis(working_path, fileLoc, data) + with open('brightnessResults.pkl', 'wb') as fp: + pickle.dump(data, fp) + print(data) + os.chdir(orig_path) + +if __name__ == "__main__": + run_mass_analysis() + pass \ No newline at end of file From ac5c343b1f731979a0b63f6eaebf2c164862e48d Mon Sep 17 00:00:00 2001 From: mattkelleher Date: Tue, 19 Feb 2019 23:36:54 +0000 Subject: [PATCH 11/17] ready for inital experimental collection --- m3u8s_interesting.txt | 15 +++++++++++++++ stream_downloader.py | 8 ++++---- 2 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 m3u8s_interesting.txt diff --git a/m3u8s_interesting.txt b/m3u8s_interesting.txt new file mode 100644 index 0000000..c6a6a08 --- /dev/null +++ b/m3u8s_interesting.txt @@ -0,0 +1,15 @@ +http://video3.earthcam.com/fecnetwork/5751.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5775.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6427.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6593.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6603.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6698.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6975.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7571.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8174.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8470.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8471.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8584.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8763.flv/playlist.m3us +http://video3.earthcam.com/fecnetwork/10294.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10669.flv/playlist.m3u8 diff --git a/stream_downloader.py b/stream_downloader.py index 2b8ce2c..d023a51 100644 --- a/stream_downloader.py +++ b/stream_downloader.py @@ -165,7 +165,7 @@ def get_single_batch(MAX_BAtCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): STREAM_DOWNLOADERS = [] - with open("m3u8s.txt") as stream_file: + with open("m3u8s_interesting.txt") as stream_file: #TODO camera file list CURR_BATCH_START_TIME = time.time() @@ -204,9 +204,9 @@ def get_single_batch(MAX_BAtCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): Downloads frames from m3u8s.txt via multithreading One thread per camera """ - EXPERIMENT_END_TIME = time.time() + 60 * 5 #TODO Set experiment run time - MAX_BATCH_ALLOWED_TIME = 60 * 0.5 #TODO Set max time per patch - FRAMES_PER_CAM_PER_BATCH = 15 #TODO Set max frames per batch + EXPERIMENT_END_TIME = time.time() + 60 * 12 #TODO Set experiment run time (minutes) + MAX_BATCH_ALLOWED_TIME = 1 #TODO Set max time per patch (minutes) + FRAMES_PER_CAM_PER_BATCH = 10000 #TODO Set max frames per batch (minutes) hours_remaining = (EXPERIMENT_END_TIME - time.time()) // (60 * 60) print hours_remaining while time.time() < EXPERIMENT_END_TIME: From e9c784a482883f202436cc0db56f26b649a55c1e Mon Sep 17 00:00:00 2001 From: Matthew Robert Kelleher Date: Wed, 20 Feb 2019 18:27:04 -0500 Subject: [PATCH 12/17] ready for 12 hour capture --- stream_downloader.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/stream_downloader.py b/stream_downloader.py index d023a51..41a01d5 100644 --- a/stream_downloader.py +++ b/stream_downloader.py @@ -156,7 +156,7 @@ def download_stream_time(m3u8_target, save_path, runtime_sec=5, save_img_type="j return failure_cnt -def get_single_batch(MAX_BAtCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): +def get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): """ Downloads frames from m3u8s.txt via multithreading @@ -171,7 +171,7 @@ def get_single_batch(MAX_BAtCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): CURR_BATCH_START_TIME = time.time() # Create a directory for the current download batch if it doesn't already exist - MASTER_SAVE_DIR = "./pics/Batch-" + _get_formatted_est() + "/" #TODO Set MASTER_SAVE_DIR + MASTER_SAVE_DIR = "/local/b/cam2/data/1BillionDataset/12HourTest" + _get_formatted_est() + "/" #TODO Set MASTER_SAVE_DIR print "======STARTING BATCH======" @@ -204,14 +204,16 @@ def get_single_batch(MAX_BAtCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): Downloads frames from m3u8s.txt via multithreading One thread per camera """ - EXPERIMENT_END_TIME = time.time() + 60 * 12 #TODO Set experiment run time (minutes) - MAX_BATCH_ALLOWED_TIME = 1 #TODO Set max time per patch (minutes) - FRAMES_PER_CAM_PER_BATCH = 10000 #TODO Set max frames per batch (minutes) + EXPERIMENT_END_TIME = time.time() + 60 * 60 * 24 #TODO Set experiment run time (seconds) + MAX_BATCH_ALLOWED_TIME = 60 #TODO Set max time per patch (seconds) + FRAMES_PER_CAM_PER_BATCH = 100000 #TODO Set max frames per batch hours_remaining = (EXPERIMENT_END_TIME - time.time()) // (60 * 60) print hours_remaining while time.time() < EXPERIMENT_END_TIME: - get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH) - _clean_cams() + get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH) + time.sleep(60* 5) + """ + _clean_cams() if((EXPERIMENT_END_TIME - time.time()) // (60 * 60 * 2) != hours_remaining): hours_remaining = hours_remaining - 2 print hours_remaining @@ -219,6 +221,6 @@ def get_single_batch(MAX_BAtCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): masterList = master.readlines() with open("m3u8s.txt", "w") as working: working.writelines(masterList) - + """ print "======EXPERIMENT DONE======" From 3ef0701f51f7671b40546063046004aa849b837e Mon Sep 17 00:00:00 2001 From: Matthew Kelleher Date: Wed, 20 Feb 2019 18:27:51 -0500 Subject: [PATCH 13/17] Update stream_downloader.py --- stream_downloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stream_downloader.py b/stream_downloader.py index 41a01d5..177a87e 100644 --- a/stream_downloader.py +++ b/stream_downloader.py @@ -211,7 +211,7 @@ def get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): print hours_remaining while time.time() < EXPERIMENT_END_TIME: get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH) - time.sleep(60* 5) + time.sleep(60* 5) """ _clean_cams() if((EXPERIMENT_END_TIME - time.time()) // (60 * 60 * 2) != hours_remaining): From 77af09037616346215d09ddf326479b710374738 Mon Sep 17 00:00:00 2001 From: Matthew Robert Kelleher Date: Wed, 20 Mar 2019 10:44:52 -0400 Subject: [PATCH 14/17] Updated stream_downloader --- stream_downloader.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/stream_downloader.py b/stream_downloader.py index 177a87e..50e2d92 100644 --- a/stream_downloader.py +++ b/stream_downloader.py @@ -1,3 +1,5 @@ +import sys +sys.path.append('/opt/python/2.7.10/') import multiprocessing import os import time @@ -59,6 +61,9 @@ def _get_earthcam_id(m3u8_target): """ prefix = "http://video3.earthcam.com/fecnetwork/" suffix = ".flv/playlist.m3u8" + print(m3u8_target) + print(len(prefix)) + print(m3u8.target.index(suffix)) cam_id = m3u8_target[len(prefix):m3u8_target.index(suffix)] return cam_id @@ -97,13 +102,13 @@ def download_stream_frames(m3u8_target, save_path, num_frames=1, save_img_type=" count = (count + 1) % 1000 else: failure_cnt += 1 - print "FAILURE: " + filename + ", FAILED " + str(failure_cnt) + print("FAILURE: " + filename + ", FAILED " + str(failure_cnt)) if failure_cnt >= 5: if is_got_one_successfully: # Camera might work - print "||||||" + _get_earthcam_id(m3u8_target) + "||||||" + print( "||||||" + _get_earthcam_id(m3u8_target) + "||||||") else: # Camera dead - print "******" + _get_earthcam_id(m3u8_target) + "******" + print( "******" + _get_earthcam_id(m3u8_target) + "******") _report_dead_cam(m3u8_target) break cam.release() @@ -144,13 +149,13 @@ def download_stream_time(m3u8_target, save_path, runtime_sec=5, save_img_type="j is_got_one_successfully = True else: failure_cnt += 1 - print "FAILURE: " + filename + ", FAILED " + str(failure_cnt) + print( "FAILURE: " + filename + ", FAILED " + str(failure_cnt)) if failure_cnt >= 5: if is_got_one_successfully: # Camera seems to work sometimes - print "||||||" + _get_earthcam_id(m3u8_target) + "||||||" + print("||||||" + _get_earthcam_id(m3u8_target) + "||||||") else: # Camera dead - print "******" + _get_earthcam_id(m3u8_target) + "******" + print("******" + _get_earthcam_id(m3u8_target) + "******") break cam.release() @@ -171,9 +176,10 @@ def get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): CURR_BATCH_START_TIME = time.time() # Create a directory for the current download batch if it doesn't already exist - MASTER_SAVE_DIR = "/local/b/cam2/data/1BillionDataset/12HourTest" + _get_formatted_est() + "/" #TODO Set MASTER_SAVE_DIR + #TODO change to directory layout to Camera-ID/Time + MASTER_SAVE_DIR = "/local/b/cam2/data/1BillionDataset/24HourTest/" + _get_formatted_est() + "/" #TODO Set MASTER_SAVE_DIR - print "======STARTING BATCH======" + print("======STARTING BATCH======") for stream_link in stream_file: cam_save_dir = MASTER_SAVE_DIR + "Cam-" + _get_earthcam_id(stream_link) + "/" @@ -194,7 +200,7 @@ def get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): for p in STREAM_DOWNLOADERS: p.terminate() p.join() - print "======Prematurely terminated batch======" + print("======Prematurely terminated batch======") break else: time.sleep(5) @@ -208,11 +214,11 @@ def get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): MAX_BATCH_ALLOWED_TIME = 60 #TODO Set max time per patch (seconds) FRAMES_PER_CAM_PER_BATCH = 100000 #TODO Set max frames per batch hours_remaining = (EXPERIMENT_END_TIME - time.time()) // (60 * 60) - print hours_remaining + print(hours_remaining) while time.time() < EXPERIMENT_END_TIME: get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH) - time.sleep(60* 5) - """ + time.sleep(60* 5) + """ _clean_cams() if((EXPERIMENT_END_TIME - time.time()) // (60 * 60 * 2) != hours_remaining): hours_remaining = hours_remaining - 2 @@ -221,6 +227,6 @@ def get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): masterList = master.readlines() with open("m3u8s.txt", "w") as working: working.writelines(masterList) - """ + """ - print "======EXPERIMENT DONE======" + print("======EXPERIMENT DONE======") From ccd1139fe0c6ca73d847c206a3ddbb38f0a1c532 Mon Sep 17 00:00:00 2001 From: Matthew Kelleher Date: Wed, 20 Mar 2019 10:46:55 -0400 Subject: [PATCH 15/17] Update stream_downloader.py --- stream_downloader.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/stream_downloader.py b/stream_downloader.py index 50e2d92..09903fb 100644 --- a/stream_downloader.py +++ b/stream_downloader.py @@ -61,9 +61,6 @@ def _get_earthcam_id(m3u8_target): """ prefix = "http://video3.earthcam.com/fecnetwork/" suffix = ".flv/playlist.m3u8" - print(m3u8_target) - print(len(prefix)) - print(m3u8.target.index(suffix)) cam_id = m3u8_target[len(prefix):m3u8_target.index(suffix)] return cam_id From 5498fcacedcd9696967ada185efc51d73f96df77 Mon Sep 17 00:00:00 2001 From: mattkelleher Date: Wed, 20 Mar 2019 11:28:01 -0400 Subject: [PATCH 16/17] downloader from 24 hour download --- stream_downloader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stream_downloader.py b/stream_downloader.py index 177a87e..d520cc0 100644 --- a/stream_downloader.py +++ b/stream_downloader.py @@ -171,12 +171,12 @@ def get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): CURR_BATCH_START_TIME = time.time() # Create a directory for the current download batch if it doesn't already exist - MASTER_SAVE_DIR = "/local/b/cam2/data/1BillionDataset/12HourTest" + _get_formatted_est() + "/" #TODO Set MASTER_SAVE_DIR + MASTER_SAVE_DIR = "/media/kellehem/3419-1A87/1BIE/24HourTest/" #TODO Set MASTER_SAVE_DIR print "======STARTING BATCH======" for stream_link in stream_file: - cam_save_dir = MASTER_SAVE_DIR + "Cam-" + _get_earthcam_id(stream_link) + "/" + cam_save_dir = MASTER_SAVE_DIR + "Cam-" + _get_earthcam_id(stream_link) + "/" + _get_formatted_est() + "/" # Create a directory for the current cam download if it doesn't already exist if not os.path.exists(cam_save_dir): os.makedirs(cam_save_dir) @@ -211,7 +211,7 @@ def get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): print hours_remaining while time.time() < EXPERIMENT_END_TIME: get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH) - time.sleep(60* 5) + time.sleep(60* 60) """ _clean_cams() if((EXPERIMENT_END_TIME - time.time()) // (60 * 60 * 2) != hours_remaining): From 91c6e946b67179544fa13932d342af0d97d6a8c0 Mon Sep 17 00:00:00 2001 From: mattkelleher Date: Wed, 20 Mar 2019 11:28:14 -0400 Subject: [PATCH 17/17] downloader from 24 hour download --- m3u8s_interesting.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/m3u8s_interesting.txt b/m3u8s_interesting.txt index c6a6a08..e1611fb 100644 --- a/m3u8s_interesting.txt +++ b/m3u8s_interesting.txt @@ -10,6 +10,6 @@ http://video3.earthcam.com/fecnetwork/8174.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8470.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8471.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/8584.flv/playlist.m3u8 -http://video3.earthcam.com/fecnetwork/8763.flv/playlist.m3us +http://video3.earthcam.com/fecnetwork/8763.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/10294.flv/playlist.m3u8 http://video3.earthcam.com/fecnetwork/10669.flv/playlist.m3u8