Skip to content

Hash files asynchronously #67

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AphelionWasTaken merged 3 commits into from
Apr 23, 2026
Merged

Hash files asynchronously #67

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ef83245
Hash files asynchronously
AphelionWasTaken Apr 23, 2026
1373183
bind mousewheel on macos and windows only
AphelionWasTaken Apr 23, 2026
f148401
Remove whitespaces
AphelionWasTaken Apr 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 11 additions & 3 deletions .github/workflows/build.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,10 +103,18 @@ jobs:
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: v2.3.1
release_name: v2.3.1
tag_name: v2.4.0
release_name: v2.4.0
body: |
Set the icon to only be referenced on Windows, fixing MacOS builds until an icon is added for MacOS.
Hash files asynchronously to speed up games.yml search when many updates are owned.

Hash files in chunks to speed up hashing of large files and reduce memory usage.

Set the progress bar to 100% for files that are already owned for better visibility.

Change mousewheel scolling sensitivity for faster scrolling.

Check for correct size of firmware files that do not have a hash to check.
draft: false
prerelease: false

Expand Down
144 changes: 94 additions & 50 deletions PySN.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,32 +15,43 @@
from os import makedirs, path
from enum import Enum
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor

requests.packages.urllib3.disable_warnings()

#Checks if the download path already exists and if hashes match. Changes the buttons and status label accordingly.
def is_shit_there(self, download_path, index, fileloc, console, sha1):
if path.exists(fileloc):
hash = hashlib.sha1()
hash_match = 1
self.textbox.dlbutton_list[index].configure(text='Redownload')
self.textbox.open_button_list[index].configure(text='Open', state='normal', command=lambda: self.open_loc(download_path))
if sha1 != 'N/A':
self.textbox.dlbutton_list[index].configure(state='disabled')
self.textbox.open_button_list[index].configure(state='disabled')
self.textbox.status_list[index].configure(text_color = 'yellow', text='Checking Hash...')
with open(fileloc,'rb') as f:
if console == 'PlayStation 3' or console == 'PlayStation Vita':
data = f.read()[:-32]
else: data = f.read()
hash.update(data)
if sha1.upper() == (hash.hexdigest().upper()):
hash_match = 1
else:
hash_match = 2
def is_shit_there(self, download_path, index, fileloc, console, sha1, expected_size=None):
if not path.exists(fileloc):
return 0

self.textbox.status_list[index].configure(text_color='yellow',text='Checking Hash...')
self.textbox.dlbutton_list[index].configure(text='Redownload', state='disabled')
self.textbox.open_button_list[index].configure(
text='Open',
state='disabled',
command=lambda: self.open_loc(download_path)
)
self.after(0, lambda: self.textbox.prog_bar_list[index].set(1))

actual_size = os.path.getsize(fileloc)
if actual_size != expected_size:
self.textbox.status_list[index].configure(text_color='red', text='Size Mismatch!')
self.textbox.dlbutton_list[index].configure(state='normal')
self.textbox.open_button_list[index].configure(state='normal')
return 1

if sha1 == 'N/A':
self.textbox.status_list[index].configure(text_color='green', text='Already Owned!')
self.textbox.dlbutton_list[index].configure(state='normal')
self.textbox.open_button_list[index].configure(state='normal')
return hash_match
return 1

self.hash_executor.submit(self.hash_check_worker, index, fileloc, console, sha1)

if self.hash_executor._work_queue.qsize() > 8:
time.sleep(0.05)

return 1

#Creates a directory for the game in the download path.
def create_directories(download_path):
Expand All @@ -52,7 +63,6 @@ def resource_path(relative_path):
base_path = getattr(sys, '_MEIPASS', os.path.dirname(os.path.abspath(__file__)))
return os.path.join(base_path, relative_path)


#Handles the config file.
class ConfigSettings():
def __init__(self):
Expand Down Expand Up @@ -96,14 +106,12 @@ def check_config():

save_dir, rpcs3_dir = ConfigSettings.check_config()


#Used to send messages to the queue
class ButtonAction(Enum):
STOP = 1
PAUSE = 2
RESUME = 3


#Window with save and rpcs3/games.yml locations.
class SettingsWindow(customtkinter.CTkToplevel):
def __init__(self, *args, **kwargs):
Expand All @@ -118,7 +126,7 @@ def __init__(self, *args, **kwargs):
if sys.platform == 'win32':
self.after(200, lambda: self.iconbitmap(resource_path("AphIcon.ico")))

self.save_dir_label = customtkinter.CTkLabel(master=self, text='Download Update PKGs To This Folder:',justify = 'center', anchor='center')
self.save_dir_label = customtkinter.CTkLabel(master=self, text='Download Update PKGs To This Folder:', justify = 'center', anchor='center')
self.save_dir_label.grid(row=0, column=1, columnspan=4, padx=5, pady=(20,0), sticky='sew')

self.save_dir_field = customtkinter.CTkTextbox(master=self, height=25, wrap='none')
Expand All @@ -134,7 +142,7 @@ def __init__(self, *args, **kwargs):
self.edit_button2 = customtkinter.CTkButton(master=self, width = 50, text='Edit', command = self.button_yml_loc)
self.edit_button2.grid(row=3, padx=5, pady=(0,0), column=4, sticky='new')

self.save_button = customtkinter.CTkButton(master=self, text='Save', width = 100, command = self.button_save)
self.save_button = customtkinter.CTkButton(master=self, text='Save', width = 100, command = self.button_save)
self.save_button.grid(row=4, padx=(0,5), column=2, sticky='e')
self.cancel_button = customtkinter.CTkButton(master=self, text='Cancel', width = 100, command=self.destroy)
self.cancel_button.grid(row=4, padx=(5,135), column=3, columnspan=2, sticky='w')
Expand Down Expand Up @@ -182,7 +190,6 @@ def button_save(self):
ConfigSettings.save_config('w', save_dir , rpcs3_dir)
self.destroy()


#Download All window. I did not create the downall function as part of this class for whatever reason...
class DownloadAllWindow(customtkinter.CTkToplevel):
def __init__(self, *args, **kwargs):
Expand All @@ -205,7 +212,6 @@ def __init__(self, *args, **kwargs):
self.cancel_button = customtkinter.CTkButton(master=self, width = 100, text='Cancel', command=self.destroy)
self.cancel_button.grid(row=4, padx=(5,55), column=3, sticky='w')


#Frame where all the fun widgets go. This gets put inside the textbox.
class ScrollableLabelButtonFrame(customtkinter.CTkScrollableFrame):
def __init__(self, master, command=None, **kwargs):
Expand All @@ -220,6 +226,15 @@ def __init__(self, master, command=None, **kwargs):
self.dlbutton_list = []
self.open_button_list = []
self.prog_bar_list = []
if sys.platform in ('win32', 'darwin'):
self.bind_all("<MouseWheel>", self.on_mousewheel)

#Changes scroll speed of the mouse wheel. Default is too slow.
def on_mousewheel(self, event):
delta = event.delta
if sys.platform == "darwin":
delta = delta * 120
self._parent_canvas.yview_scroll(int(-1*(delta)), "units")

#Creates widgets within the frame, adds appropriate ones to the grid, then adds them to a list.
def add_item(self, name, title_id, ver, url, console, update_size, sha1, index, download_path, fileloc):
Expand All @@ -243,7 +258,7 @@ def add_item(self, name, title_id, ver, url, console, update_size, sha1, index,
status = customtkinter.CTkLabel(self, text='', anchor='e', width = 160)
dlbutton = customtkinter.CTkButton(self, text='Download', width=100, height=24)
open_button = customtkinter.CTkButton(self, text='Open', width=100, height=24, state = 'disabled')
prog_bar = customtkinter.CTkProgressBar(self, width=440, height=5)
prog_bar = customtkinter.CTkProgressBar(self, height=5)
prog_bar.set(0)
q = queue.Queue()

Expand All @@ -255,7 +270,7 @@ def add_item(self, name, title_id, ver, url, console, update_size, sha1, index,
title_label.grid(row=len(self.title_label_list), column=0, pady=(0, 10), sticky='w')
status.grid(row=len(self.title_label_list), column=2, pady=(0, 10),padx=(0, 0), sticky='e')
size_label.grid(row=len(self.title_label_list), column=3, padx=(10, 5), pady=(0, 10), sticky='e')
prog_bar.grid(row=len(self.dlbutton_list), column=0, columnspan=3, pady=(15, 0), sticky='w')
prog_bar.grid(row=len(self.dlbutton_list), column=0, columnspan=3, pady=(15, 0), sticky='ew')
dlbutton.grid(row=len(self.dlbutton_list), column=4, pady=(0, 10), padx=0, sticky='e')
open_button.grid(row=len(self.dlbutton_list), column=5, pady=(0, 10), padx=(0, 0), sticky='e')
else:
Expand Down Expand Up @@ -290,11 +305,11 @@ def clear_items(self):
self.open_button_list.remove(open_button)
self.prog_bar_list.remove(prog_bar)


#Main window and some button functionality. Vaguely named widgets are in order of how they appear on screen.
class App(customtkinter.CTk):
def __init__(self):
super().__init__()
self.hash_executor = ThreadPoolExecutor(max_workers=4)
self._search_thread = None
self.session = requests.Session()
self.running = True
Expand Down Expand Up @@ -334,11 +349,19 @@ def __init__(self):
self.button4 = customtkinter.CTkButton(master=self, command=self.button_settings, text='Settings', width = 125)
self.button4.grid(row=2, column=4, padx=4, pady=(0,6), sticky='ew')

#Handle closing the window while threads are running.
def on_closing(self):
self.running = False
self.session.close()

try:
self.hash_executor.shutdown(wait=False)
except Exception:
pass

self.wait_for_thread()

#Waits for thread to finish what it's doing before closing the window.
def wait_for_thread(self):
if self._search_thread is not None and self._search_thread.is_alive():
self.after(100, self.wait_for_thread)
Expand All @@ -348,6 +371,39 @@ def wait_for_thread(self):
except Exception:
pass

#Checks the hash of the file in a separate thread to avoid freezing the UI.
def hash_check_worker(self, index, fileloc, console, sha1):
try:
hash_obj = hashlib.sha1()
with open(fileloc, 'rb') as f:
if console in ('PlayStation 3', 'PlayStation Vita'):
filesize = os.path.getsize(fileloc)
remaining = max(0, filesize - 32)
while remaining > 0:
chunk = f.read(min(4 * 1024 * 1024, remaining))
if not chunk:
break
hash_obj.update(chunk)
remaining -= len(chunk)
else:
for chunk in iter(lambda: f.read(4 * 1024 * 1024), b''):
hash_obj.update(chunk)
match = (sha1.upper() == hash_obj.hexdigest().upper())

except Exception:
match = False

#Updates the UI based on the hash check result.
def update_ui():
self.textbox.dlbutton_list[index].configure(state='normal')
self.textbox.open_button_list[index].configure(state='normal')
if match:
self.textbox.status_list[index].configure(text_color='green', text='Already Owned!')
else:
self.textbox.status_list[index].configure(text_color='red', text='HASH MISMATCH DETECTED!')

self.after(0, update_ui)

#Opens the file location. Used with the open button.
def open_loc(self, download_path):
customtkinter.filedialog.askopenfilenames(initialdir=download_path)
Expand Down Expand Up @@ -384,6 +440,7 @@ def search_type(self):
title_id = index
self.search(title_id, console)
self.search_no_drm(title_id, console)

self.button1.configure(state = 'normal')
self.button2.configure(state = 'normal')
self.button3.configure(state = 'normal')
Expand Down Expand Up @@ -490,12 +547,7 @@ def search(self, title_id, console):
self.textbox.add_item(game_name, title_id, ' v' + ver, url, console, update_size, sha1, index, download_path, fileloc)

#Check the hash in case an incomplete or corrupt file already exists. Then handle errors in search results.
hash_match = is_shit_there(self, download_path, index, fileloc, console, sha1)
if hash_match == 1:
self.textbox.status_list[index].configure(text_color = 'green', text='Already Owned!')
elif hash_match == 2:
self.textbox.status_list[index].configure(text_color = 'red', text='HASH MISMATCH DETECTED!')
else: pass
is_shit_there(self, download_path, index, fileloc, console, sha1, update_size)
elif game_name == 'Invalid ID':
self.textbox.add_item('Invalid ID: ' + title_id, '', '', '', '', 0, '', '', '', '')
else: self.textbox.add_item(game_name + title_id, '', '', '', '', 0, '', '', '', '')
Expand Down Expand Up @@ -544,16 +596,12 @@ def search_no_drm(self, title_id, console):
self.textbox.add_item(game_name, title_id, ' DRM-Free v' + version, url_list[i], console, update_size_list[i], sha1_list[i], index_list[i], download_path, fileloc)

#Check the hash in case an incomplete or corrupt file already exists. Errors in search results are handled by the other search, so we just pass here.
hash_match = is_shit_there(self, download_path, index_list[i], fileloc, console, sha1_list[i])
if hash_match == 1:
self.textbox.status_list[index_list[i]].configure(text_color = 'green', text='Already Owned!')
elif hash_match == 2:
self.textbox.status_list[index_list[i]].configure(text_color = 'red', text='HASH MISMATCH DETECTED!')
else: pass
is_shit_there(self, download_path, index_list[i], fileloc, console, sha1_list[i], update_size_list[i])
i = i+1
else: pass
else: pass

#Handles users searching for PS5 updates.
def search_ps5_update(self, title_id):
name = 'PlayStation 5 title updates are not supported yet.'
self.textbox.add_item(name, '', '', '', '', 0, '', '', '', '')
Expand Down Expand Up @@ -587,6 +635,7 @@ def search_ps3_fw(self, console):
update_file = 'v' + ver + ' ' + path.basename(url)
fileloc = (download_path + '/' + update_file)
self.textbox.add_item(game_name, title_id, ' v' + ver, url, console, update_size, sha1, index, download_path, fileloc)
is_shit_there(self, download_path, index, fileloc, console, sha1, update_size)
else: self.textbox.add_item('Error Connecting to Server', '', '', '', '', 0, '', '', '', '')

#Searches for PS4 or Vita update info and populates the widgets in the frame based on that info.
Expand Down Expand Up @@ -664,6 +713,7 @@ def search_ps4_ps5_vita_fw(self, console):
update_file = 'v' + ver_list[i] + ' ' + update_data_list[i] + ' ' + path.basename(url)
fileloc = (download_path + '/' + update_file)
self.textbox.add_item(game_name, title_id, ' v' + ver_list[i], url, console, update_size_list[i], sha1, index_list[i], download_path, fileloc)
is_shit_there(self, download_path, index_list[i], fileloc, console, sha1, update_size_list[i])
i = i+1
else: self.textbox.add_item('Error Connecting to Server', '', '', '', '', 0, '', '', '', '')

Expand Down Expand Up @@ -720,7 +770,7 @@ def download_updates(self, url, download_path, size, sha1, index, title_id, name
f.flush()
i = i+(1/(size/(len(chunk))))
h = round(h+((len(chunk))/1024000),2)
self.textbox.prog_bar_list[index].set(i)
self.after(0, lambda: self.textbox.prog_bar_list[index].set(i))
self.textbox.status_list[index].configure(text_color = 'green', text= str(h) + '/' + str(round((size/1024000),2)) + 'MB' )
if self.textbox.queue_list[index].empty() == False: break
else: break
Expand All @@ -732,12 +782,7 @@ def download_updates(self, url, download_path, size, sha1, index, title_id, name
if self.textbox.status_list[index].cget('text') == 'Download Cancelled!':
os.remove(fileloc)
else:
hash_match = is_shit_there(self, download_path, index, fileloc, console, sha1)
if hash_match == 1:
self.textbox.status_list[index].configure(text_color = 'green', text='Download Complete!')
elif hash_match == 2:
self.textbox.status_list[index].configure(text_color = 'red', text='HASH MISMATCH DETECTED!')
else: pass
is_shit_there(self, download_path, index, fileloc, console, sha1, size)

#Downloads all files, or only new files based on the check box in the downall window. Pretty sure it belongs in the DownloadAllWindow class.
def downall(self):
Expand Down Expand Up @@ -797,7 +842,6 @@ def button_settings(self):
else:
self.toplevel_window.focus()


if __name__ == '__main__':
app = App()
app.mainloop()
Loading