diff --git a/BrightnessMetrics.py b/BrightnessMetrics.py new file mode 100644 index 0000000..c5ce5d1 --- /dev/null +++ b/BrightnessMetrics.py @@ -0,0 +1,95 @@ +import os +import numpy as np +import pickle +import cv2 + + + +""" +Main program header +include programmers, project github, instructions how to use etc +""" + +""" +Do we have a cv2 img? +""" +def is_valid_img(img): + if not isinstance(img, np.ndarray): + print("Error: cv2 image not passed!") + return False + else: + return True + +""" +Perceived Brightness = Sqrt((0.241 * R^2) + (0.691 * G^2) + (0.068 * B^2)) +""" +def calc_perceived_brightness(img): + height, width, dim = img.shape + return ( np.sqrt(((np.sum(img[:, :, 2]) ** 2) * 0.241) + ((np.sum(img[:, :, 1]) ** 2) * 0.691) + ((np.sum(img[:, :, 0]) ** 2) * 0.068)) / (height * width * 255)) + + +""" +Luminance +Luminance = (0.299 * R) + (0.578 * G) + (0.114 * B) +""" +def calc_luminosity(img): + height, width, dim = img.shape + return (np.sum(img[:, :, 2]) * 0.299 + np.sum(img[:, :, 1]) * 0.578 + np.sum(img[:, :, 0]) * 0.114) / (height * width * 255) + +""" +Pixel Intensity +1.Converts image to greyscale +2. Takes the average pixel value +""" +def calc_px_intensity(img): + gImg = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) + height, width= gImg.shape + return np.sum(gImg) / (height * width * 255) + +""" +Runs analysis on a single image and +""" +def run_single_analysis(working_path, fileLoc, data): + img = cv2.imread(fileLoc) + if is_valid_img(img): + lum = calc_luminosity(img) + px_int = calc_px_intensity(img) + percB = calc_perceived_brightness(img) + data.append([fileLoc.replace(working_path, "."), lum, px_int, percB]) + else: + print("ERROR: Invalid Image: " + fileLoc) + + + +""" +Goes through all .jpg or .png images in specified directory and all subdirectories +""" +def run_mass_analysis(): + + orig_path = os.path.realpath(".") + while True: + working_path = raw_input("Input path (relative or absolute) to directory to be evaluated\n Relative paths should start with './' \n Absolute paths should start with or '/' \nEnter Path: ") + try: + os.chdir(working_path) + except: + print("ERROR: Invalid directory") + else: + break + + working_path = os.path.realpath(".") + data = [["Absolute Path: ", working_path],[]] + data.append(["Image_Location", "Luminosity", "Pixel_Intensity", "Perceived_Brightness"]) + + for subDir, dirs, files in os.walk(working_path): + for file in files: + fileLoc = os.path.join(subDir,file) + if (".jpg" in fileLoc) or (".png" in fileLoc): + run_single_analysis(working_path, fileLoc, data) + with open('brightnessResults.pkl', 'wb') as fp: + pickle.dump(data, fp) + print(data) + os.chdir(orig_path) + +if __name__ == "__main__": + run_mass_analysis() + pass \ No newline at end of file diff --git a/deadCams.txt b/deadCams.txt new file mode 100644 index 0000000..984e85c --- /dev/null +++ b/deadCams.txt @@ -0,0 +1,44 @@ +http://video3.earthcam.com/fecnetwork/5536.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5845.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6076.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6426.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5965.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7484.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3847fairmont.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7928.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10551.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7342.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3186.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7624.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3792.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5019.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5441.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5359.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4617.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5026.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4356.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4162.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6175.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5966.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6632.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7130.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6999.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7132.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7392.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7710.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7393.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7786.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7711.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8048.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8837.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8838.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8620.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8577.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/anguilla.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9741.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/eiffelcamHD.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/dceaglecam.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/paradiseon6.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/sedonawebcam.flv/playlist.m3u8 diff --git a/m3u8sMaster.txt b/m3u8sMaster.txt new file mode 100644 index 0000000..a75c830 --- /dev/null +++ b/m3u8sMaster.txt @@ -0,0 +1,170 @@ +http://video3.earthcam.com/fecnetwork/10072.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10147.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10294.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10347.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10551.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10610.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10669.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10729.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10874.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3186.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3275.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3304.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3635.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3644newscafe.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/370.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3792.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3847fairmont.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/3983.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4017timessquare.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4049sonesta.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4054.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4135.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4162.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4280.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4281.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4282.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4337.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4338.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4356.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4369.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4387.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4465.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4480.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4518.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4544.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4559.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4617.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4644.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4646.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4717.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4798.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4831.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4918.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/4968.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5019.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5026.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5034.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5173.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5187.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5204.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5230.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5288.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5324.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5358.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5359.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5423.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5441.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5465.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5536.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5595.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5663.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5718.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5719.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5751.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5758.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5775.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5805.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5845.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5965.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5966.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6076.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6081.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6175.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6253.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6405.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6406.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6426.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6427.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6528.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6593.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6603.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6632.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6649.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6698.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6723.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6954.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6961.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6975.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6998.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6999.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7130.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7132.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7300.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7322.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7342.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7382.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7384.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7392.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7393.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7461.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7484.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7524.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7571.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7624.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7710.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7711.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7786.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7928.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7945.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7947.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7949.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7962.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7988.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8048.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8078.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8174.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8211.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8470.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8471.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8577.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8584.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8620.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8671.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8713.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8715.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8763.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8811.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8837.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8838.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8891.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8939.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9106.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9115.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9189.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9298.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9299.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9602.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9741.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9779.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/9974.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/anguilla.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/chargingbull.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/dceaglecam.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/dealeyplaza2.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/diamondhead.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/ecfishtank01.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/eclasvegas.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/eiffelcamHD.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/fridays_5th.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/hdtimes10.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/hotelvictoria2.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/interactoneHDCam1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/lacitytours1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/libertyHD1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/meprd_garch.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd2.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd4.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd5.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/miamivishd7.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/moscowHD1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/paradiseon6.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/philippinesHD1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/playalinda1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/playalinda2.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/sedonawebcam.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/tokyo1.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/warholchurch.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/windjammerHD2.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/woodmenHD.flv/playlist.m3u8 diff --git a/m3u8s_interesting.txt b/m3u8s_interesting.txt new file mode 100644 index 0000000..e1611fb --- /dev/null +++ b/m3u8s_interesting.txt @@ -0,0 +1,15 @@ +http://video3.earthcam.com/fecnetwork/5751.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/5775.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6427.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6593.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6603.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6698.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/6975.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/7571.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8174.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8470.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8471.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8584.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/8763.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10294.flv/playlist.m3u8 +http://video3.earthcam.com/fecnetwork/10669.flv/playlist.m3u8 diff --git a/stream_downloader.py b/stream_downloader.py index c399c2f..e05f504 100644 --- a/stream_downloader.py +++ b/stream_downloader.py @@ -1,3 +1,5 @@ +import sys +sys.path.append('/opt/python/2.7.10/') import multiprocessing import os import time @@ -12,6 +14,35 @@ frames via multithreading and save them with their camera IDs and current date-time """ +def _clean_cams(): + """ + Removes dead cameras from camera list + """ + with open('deadCams.txt', 'r') as deadCamFile: + deadCamList = deadCamFile.readlines() + + with open('m3u8sMaster.txt', 'r') as masterListFile: + masterList = masterListFile.readlines() + + updatedCamList = [] + + for cam in masterList: + if cam not in deadCamList: + updatedCamList.append(cam) + + with open('m3u8s.txt', 'w') as updatedCamFile: + updatedCamFile.writelines(updatedCamList) + + +def _report_dead_cam(m3u8_target): + """ + Writes dead camera links to file to be removed + """ + with open("deadCams.txt", "r") as deadFile: + deadCamList = deadFile.readlines() + with open("deadCams.txt", "a") as deadFile: + if m3u8_target not in deadCamList: + deadFile.write(m3u8_target) def _get_formatted_est(): """ @@ -35,7 +66,7 @@ def _get_earthcam_id(m3u8_target): return cam_id -def download_stream_frames(m3u8_target, save_path, num_frames=1, save_img_type="png"): +def download_stream_frames(m3u8_target, save_path, num_frames=1, save_img_type="jpg"): """ Worker function that downloads a specified number of frames from a target stream. WORKS FOR EARTHCAM STREAMS ONLY. @@ -48,39 +79,41 @@ def download_stream_frames(m3u8_target, save_path, num_frames=1, save_img_type=" num_frames: (int) Default 1. The desired number of frames to download from the camera. - save_img_type: (str) Default 'png'. The target image save format. + save_img_type: (str) Default 'jpg'. The target image save format. """ cam = cv2.VideoCapture(m3u8_target) failure_cnt = 0 # The number of total frame grab failures is_got_one_successfully = False - + count = 0 for _i in range(num_frames): is_success, frame = cam.read() filename = _get_earthcam_id(m3u8_target) + "--" + _get_formatted_est() + \ - "." + save_img_type + "_" + str(count) + "." + save_img_type if is_success: save_target = save_path + filename cv2.imwrite(save_target, frame) is_got_one_successfully = True + count = (count + 1) % 1000 else: failure_cnt += 1 - print "FAILURE: " + filename + ", FAILED " + str(failure_cnt) + print("FAILURE: " + filename + ", FAILED " + str(failure_cnt)) if failure_cnt >= 5: if is_got_one_successfully: # Camera might work - print "||||||" + _get_earthcam_id(m3u8_target) + "||||||" + print( "||||||" + _get_earthcam_id(m3u8_target) + "||||||") else: # Camera dead - print "******" + _get_earthcam_id(m3u8_target) + "******" + print( "******" + _get_earthcam_id(m3u8_target) + "******") + _report_dead_cam(m3u8_target) break cam.release() return failure_cnt -def download_stream_time(m3u8_target, save_path, runtime_sec=5, save_img_type="png"): +def download_stream_time(m3u8_target, save_path, runtime_sec=5, save_img_type="jpg"): """ Worker function that downloads a specified number of frames from a target stream. WORKS FOR EARTHCAM STREAMS ONLY. @@ -93,7 +126,7 @@ def download_stream_time(m3u8_target, save_path, runtime_sec=5, save_img_type="p runtime_sec: (int) Default 5. The desired number of seconds to run the downloader for. - save_img_type: (str) Default 'png'. The target image save format. + save_img_type: (str) Default 'jpg'. The target image save format. """ cam = cv2.VideoCapture(m3u8_target) @@ -113,20 +146,20 @@ def download_stream_time(m3u8_target, save_path, runtime_sec=5, save_img_type="p is_got_one_successfully = True else: failure_cnt += 1 - print "FAILURE: " + filename + ", FAILED " + str(failure_cnt) + print( "FAILURE: " + filename + ", FAILED " + str(failure_cnt)) if failure_cnt >= 5: if is_got_one_successfully: # Camera seems to work sometimes - print "||||||" + _get_earthcam_id(m3u8_target) + "||||||" + print("||||||" + _get_earthcam_id(m3u8_target) + "||||||") else: # Camera dead - print "******" + _get_earthcam_id(m3u8_target) + "******" + print("******" + _get_earthcam_id(m3u8_target) + "******") break cam.release() return failure_cnt +def get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH): -if __name__ == "__main__": """ Downloads frames from m3u8s.txt via multithreading One thread per camera @@ -134,45 +167,62 @@ def download_stream_time(m3u8_target, save_path, runtime_sec=5, save_img_type="p STREAM_DOWNLOADERS = [] - with open("m3u8s.txt") as stream_file: + with open("m3u8s_interesting.txt") as stream_file: #TODO camera file list - EXPERIMENT_END_TIME = time.time() + 60 * 60 * 1 # Run for a full 8 hours - MAX_BATCH_ALLOWED_TIME = 60 * 5 # 5 minutes per batch, max - FRAMES_PER_CAM_PER_BATCH = 10000 - - while time.time() < EXPERIMENT_END_TIME: - CURR_BATCH_START_TIME = time.time() - - # Create a directory for the current download batch if it doesn't already exist - MASTER_SAVE_DIR = "./OneNightOfFrames/Batch-" + _get_formatted_est() + "/" - - print "======STARTING BATCH======" - - for stream_link in stream_file: - cam_save_dir = MASTER_SAVE_DIR + "Cam-" + _get_earthcam_id(stream_link) + "/" - - # Create a directory for the current cam download if it doesn't already exist - if not os.path.exists(cam_save_dir): - os.makedirs(cam_save_dir) - - p = multiprocessing.Process(target=download_stream_frames, - args=(stream_link, cam_save_dir, FRAMES_PER_CAM_PER_BATCH)) - STREAM_DOWNLOADERS.append(p) - p.start() - - # Every 5 seconds, check if it's time to start a new batch - while STREAM_DOWNLOADERS is not False: # While not empty - STREAM_DOWNLOADERS = [p for p in STREAM_DOWNLOADERS if p.is_alive()] - - # Terminate early if it takes too long to finish the batch - if time.time() >= (CURR_BATCH_START_TIME + MAX_BATCH_ALLOWED_TIME): - for p in STREAM_DOWNLOADERS: - p.terminate() - p.join() - print "======Prematurely terminated batch======" - break - else: - time.sleep(5) - - print "======EXPERIMENT DONE======" + CURR_BATCH_START_TIME = time.time() + + # Create a directory for the current download batch if it doesn't already exist + MASTER_SAVE_DIR = "/media/kellehem/3419-1A87/1BIE/24HourTest/" #TODO Set MASTER_SAVE_DIR + + print("======STARTING BATCH======") + + for stream_link in stream_file: + cam_save_dir = MASTER_SAVE_DIR + "Cam-" + _get_earthcam_id(stream_link) + "/" + _get_formatted_est() + "/" + # Create a directory for the current cam download if it doesn't already exist + if not os.path.exists(cam_save_dir): + os.makedirs(cam_save_dir) + + p = multiprocessing.Process(target=download_stream_frames, + args=(stream_link, cam_save_dir, FRAMES_PER_CAM_PER_BATCH)) + STREAM_DOWNLOADERS.append(p) + p.start() + + # Every 5 seconds, check if it's time to start a new batch + while STREAM_DOWNLOADERS is not False: # While not empty + STREAM_DOWNLOADERS = [p for p in STREAM_DOWNLOADERS if p.is_alive()] + # Terminate early if it takes too long to finish the batch + if time.time() >= (CURR_BATCH_START_TIME + MAX_BATCH_ALLOWED_TIME): + for p in STREAM_DOWNLOADERS: + p.terminate() + p.join() + print("======Prematurely terminated batch======") + break + else: + time.sleep(5) + +if __name__ == "__main__": + """ + Downloads frames from m3u8s.txt via multithreading + One thread per camera + """ + EXPERIMENT_END_TIME = time.time() + 60 * 60 * 24 #TODO Set experiment run time (seconds) + MAX_BATCH_ALLOWED_TIME = 60 #TODO Set max time per patch (seconds) + FRAMES_PER_CAM_PER_BATCH = 100000 #TODO Set max frames per batch + hours_remaining = (EXPERIMENT_END_TIME - time.time()) // (60 * 60) + print(hours_remaining) + while time.time() < EXPERIMENT_END_TIME: + get_single_batch(MAX_BATCH_ALLOWED_TIME, FRAMES_PER_CAM_PER_BATCH) + time.sleep(60* 60) + """ + _clean_cams() + if((EXPERIMENT_END_TIME - time.time()) // (60 * 60 * 2) != hours_remaining): + hours_remaining = hours_remaining - 2 + print hours_remaining + with open("m3u8sMaster.txt", "r") as master: + masterList = master.readlines() + with open("m3u8s.txt", "w") as working: + working.writelines(masterList) + """ + + print("======EXPERIMENT DONE======")