diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9efdae5..f3a00c1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -176,14 +176,14 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - tag_name: v2.8.0 - name: v2.8.0 + tag_name: v2.8.1 + name: v2.8.1 body: | - Better sanitization of game names to prevent issues with invalid characters in file paths. - - Allows users to change download directory and file name structure without having to search again to apply these settings. - - Makes max downloads setting work as expected. + PySN now correctly identifies and downloads all pieces of a PS4 update file and then combines them into a single file. + + Previously, the app was only fetching the last piece. This was fine for many title updates that only have one piece, but many updates have multiple pieces. + + Also checks hashes and sizes of newly downloaded files more gracefully. draft: false prerelease: false files: | diff --git a/PySN.py b/PySN.py index ccb45bc..a5c999c 100644 --- a/PySN.py +++ b/PySN.py @@ -10,6 +10,7 @@ import time import sys import re +import shutil import xml.etree.ElementTree as ET from tkinter import PhotoImage from pathlib import Path @@ -575,7 +576,7 @@ def is_shit_there(self, name, title_id, download_path, index, fileloc, console, self.textbox.open_button_list[index].configure(state='normal') return 1 - if sha1 == 'N/A': + if sha1 in ('N/A', 'PS4_SPLIT', 'PS4_DELTA'): self.textbox.status_list[index].configure(text_color='green', text='Already Owned!') self.textbox.dlbutton_list[index].configure(state='normal') self.textbox.open_button_list[index].configure(state='normal') @@ -719,14 +720,35 @@ def search(self, title_id, console, root, game_name): man_url = (item.get('manifest_url')) json_url = self.session.get(man_url, stream=True, verify=False, timeout=10) json_cont = json.loads(json_url.content) - for item in (json_cont['pieces']): - url = (item.get('url')) - sha1 = (item.get('hashValue')) - update_size = (item.get('fileSize')) + pieces = [ + {'url': p.get('url'), 'offset': int(p.get('fileOffset')), + 'size': int(p.get('fileSize')), 'hash': p.get('hashValue')} + for p in json_cont['pieces'] + ] + pieces.sort(key=lambda p: p['offset']) + url = pieces + sha1 = 'PS4_SPLIT' + update_size = int(json_cont.get('originalFileSize', sum(p['size'] for p in pieces))) + + delta_elem = item.find('delta_info_set') + if delta_elem is not None: + delta_url = delta_elem.get('url') + else: + delta_url = None + delta_size = 0 + if delta_url: + delta_size = int(delta_elem.get('size', 0) or 0) + if delta_size == 0: + try: + delta_head = self.session.get(delta_url, stream=True, verify=False, timeout=10) + delta_size = int(delta_head.headers.get('Content-Length', 0)) + except Exception: + delta_size = 0 else: url = (item.get('url')) sha1 = (item.get('sha1sum')) update_size = int((item.get('size'))) + delta_url = None #Assign a download path with some handling for odd characters in the game name. Add Widgets to the textbox and check if the file already exists. name = re.sub(r'[^a-zA-Z0-9_\- \u3000-\u303F\u3040-\u30FF\u4E00-\u9FFF\uFF00-\uFFEF]', '', game_name).strip() @@ -737,12 +759,36 @@ def search(self, title_id, console, root, game_name): else: download_path = save_dir + console + '/' + name + ' [' + title_id + ']' - update_file = path.basename(url) + if console == 'PlayStation 4': + first_piece_name = path.basename(url[0]['url']).replace('_0.pkg', '.pkg') + update_file = re.sub(r'_0(\.pkg)$', r'\1', first_piece_name) + else: + update_file = path.basename(url) fileloc = (download_path + '/' + update_file) - - self.after(0, lambda n=name, gn=game_name, tid=title_id, v=ver, u=url, c=console, us=update_size, s=sha1, dp=download_path, fl=fileloc: - (self.textbox.add_item(gn, tid, ' v' + v, u, c, us, s, len(self.textbox.dlbutton_list), dp, fl, n), - self.is_shit_there(n, tid, dp, len(self.textbox.dlbutton_list) - 1, fl, c, s, us))) + + if console == 'PlayStation 4': + if delta_url: + delta_fileloc = (download_path + '/' + path.basename(delta_url)) + else: + delta_fileloc = None + + def add_ps4_items(gn=game_name, tid=title_id, v=ver, u=url, c=console, us=update_size, s=sha1, + dp=download_path, fl=fileloc, n=name, + d_url=delta_url, d_size=delta_size, d_fl=delta_fileloc): + if d_url: + d_idx = len(self.textbox.dlbutton_list) + self.textbox.add_item(gn, tid, ' Delta Patch', d_url, c, d_size, 'PS4_DELTA', d_idx, dp, d_fl, n) + self.is_shit_there(n, tid, dp, d_idx, d_fl, c, 'PS4_DELTA', d_size) + + tu_idx = len(self.textbox.dlbutton_list) + self.textbox.add_item(gn, tid, ' v' + v, u, c, us, s, tu_idx, dp, fl, n) + self.is_shit_there(n, tid, dp, tu_idx, fl, c, s, us) + + self.after(0, add_ps4_items) + else: + self.after(0, lambda n=name, gn=game_name, tid=title_id, v=ver, u=url, c=console, us=update_size, s=sha1, dp=download_path, fl=fileloc: + (self.textbox.add_item(gn, tid, ' v' + v, u, c, us, s, len(self.textbox.dlbutton_list), dp, fl, n), + self.is_shit_there(n, tid, dp, len(self.textbox.dlbutton_list) - 1, fl, c, s, us))) elif game_name == 'Invalid ID': self.after(0, lambda: self.textbox.add_item('Invalid ID: ' + title_id, '', '', '', '', 0, '', '', '', '', '')) @@ -993,7 +1039,7 @@ def try_set(widget, val): try: if path.exists(download_path) == False: create_directories(download_path) - + try_configure(status, text_color='green', text='Downloading') i=0 @@ -1002,65 +1048,147 @@ def try_set(widget, val): downloaded_bytes = 0 size_mb = f"{round(size / 1024000, 2)}" cancelled = False + download_failed = False with self._download_lock: self.total_download_size = self.total_download_size + size - #send a request to the update files URL. Handle threads and download behavior based on the button state. - with self.session.get(url, stream=True, verify=False) as r: - r.raise_for_status() - with open(fileloc,'wb') as f: - for chunk in r.iter_content(chunk_size=(1024*1024)): - try: - action = q.get_nowait() - if action == ButtonAction.PAUSE: - try_configure(status, text_color = 'yellow', text='Paused') - new_action = q.get() - if new_action == ButtonAction.STOP: - try_configure(status, text_color='red', text='Download Cancelled!') + def handle_chunk(f, chunk): + nonlocal i, h, last_ui_update, downloaded_bytes, cancelled, download_failed + + try: + action = q.get_nowait() + if action == ButtonAction.PAUSE: + try_configure(status, text_color='yellow', text='Paused') + new_action = q.get() + if new_action == ButtonAction.STOP: + try_configure(status, text_color='red', text='Download Cancelled!') + cancelled = True + return 'stop' + try_configure(status, text_color='green', text='Downloading') + elif action == ButtonAction.STOP: + try_configure(status, text_color='red', text='Download Cancelled!') + cancelled = True + return 'stop' + except queue.Empty: + pass + + if chunk: + f.write(chunk) + i = i + (1/(size/(len(chunk)))) + h = h + len(chunk) / 1024000 + downloaded_bytes = downloaded_bytes + len(chunk) + now = time.monotonic() + with self._download_lock: + self.completed_download_size = self.completed_download_size + len(chunk) + + if now - last_ui_update > 0.01: + last_ui_update = now + self.after(0, lambda v = i: try_set(prog_bar, v)) + try_configure(status, text_color = 'green', text= f"{h:.2f}/{size_mb}MB") + return None + + #Handle PS4 title update downloads since they are passed as a list of URLs. + if isinstance(url, list): + piece_paths = [] + + if len(url) > 1: + for n in range(len(url)): + filename = fileloc.replace('.pkg', f'_{n}.pkg') + piece_paths.append(filename) + else: + piece_paths.append(fileloc) + + for n, piece in enumerate(url): + if cancelled: + break + try_configure(status, text_color='green', text=f'Piece {n+1}/{len(url)}') + + piece_hash = hashlib.sha1() + bytes_before = downloaded_bytes + + with self.session.get(piece['url'], stream=True, verify=False) as r: + r.raise_for_status() + with open(piece_paths[n], 'wb') as f: + for chunk in r.iter_content(chunk_size=(1024*1024)): + if handle_chunk(f, chunk) == 'stop': break - try_configure(status, text_color='green', text='Downloading') - elif action == ButtonAction.STOP: - try_configure(status, text_color = 'red', text='Download Cancelled!') - cancelled = True - break - except queue.Empty: - pass + if chunk: + piece_hash.update(chunk) - #Download the file and update the progress bar and status label based on how much has downloaded. - if chunk: - f.write(chunk) - i = i + (1/(size/(len(chunk)))) - h = h + len(chunk) / 1024000 - downloaded_bytes = downloaded_bytes + len(chunk) - now = time.monotonic() - with self._download_lock: - self.completed_download_size = self.completed_download_size + len(chunk) - - if now - last_ui_update > 0.01: - last_ui_update = now - self.after(0, lambda v = i: try_set(prog_bar, v)) - try_configure(status, text_color = 'green', text= f"{h:.2f}/{size_mb}MB") - - #After the download loop, check if the download was cancelled. If it was, remove the file and update the completed download size. if cancelled: + break + + piece_downloaded = downloaded_bytes - bytes_before + + #Check size and hash of the downloaded pieces. + try_set(prog_bar, 1) + if piece_downloaded != piece['size']: + download_failed = True + try_configure(status, text_color='red', text=f'Piece {n+1} Size Mismatch!') + break + if piece_hash.hexdigest().lower() != (piece['hash'] or '').lower(): + download_failed = True + try_configure(status, text_color='red', text=f'Piece {n+1} HASH MISMATCH!') + break + + #Remove piece files if the download was cancelled/failed. + if cancelled or download_failed: + for p_path in piece_paths: try: - os.remove(fileloc) - except: + os.remove(p_path) + except Exception: pass - with self._download_lock: - self.completed_download_size = self.completed_download_size - downloaded_bytes - self.total_download_size = self.total_download_size - size + filename = fileloc + #Handle downloads for every other console and PS4 FW. + else: + file_hash = hashlib.sha1() + hash_limit = max(0, size - 32) if console in ('PlayStation 3', 'PlayStation Vita') else size + hashed_bytes = 0 + + with self.session.get(url, stream=True, verify=False) as r: + r.raise_for_status() + with open(fileloc, 'wb') as f: + for chunk in r.iter_content(chunk_size=(1024*1024)): + if handle_chunk(f, chunk) == 'stop': + break + if chunk and hashed_bytes < hash_limit: + take = min(len(chunk), hash_limit - hashed_bytes) + file_hash.update(chunk[:take]) + hashed_bytes += take + + #Check size and hash of the downloaded file. + if not cancelled: + try_set(prog_bar, 1) + try_configure(status, text_color='yellow', text='Checking Hash...') + try_configure(dl_button, text='Redownload', state='disabled') + try_configure(open_button, text='Open', state = 'disabled') + + if downloaded_bytes != size: + download_failed = True + try_configure(status, text_color='red', text='Size Mismatch!') + elif sha1 not in ('N/A', 'PS4_SPLIT', 'PS4_DELTA'): + if file_hash.hexdigest().lower() != (sha1 or '').lower(): + download_failed = True + try_configure(status, text_color='red', text='HASH MISMATCH DETECTED!') #After the file is downloaded, reconfigure the dl and open button behavior. - #Then remove the file if the dl was cancelled. If it completed, run is_shit_there to check the hash and configure buttons properly. - try_configure(dl_button, command=lambda: App.frame_button_download(self, name, title_id, url, console, size, sha1, index, download_path, fileloc)) - try_configure(open_button, text='Open', state = 'disabled', command=lambda: None) - - if status.cget('text') == 'Download Cancelled!': - os.remove(fileloc) + try_configure(dl_button, text='Redownload', state='normal', command=lambda: App.frame_button_download(self, name, title_id, url, console, size, sha1, index, download_path, fileloc)) + try_configure(open_button, text='Open', state = 'disabled') + + #After the download loop, check if the download was cancelled. If it was, remove the file and update the completed download size. + if cancelled or download_failed: + if os.path.exists(fileloc): + try: + os.remove(fileloc) + except Exception: + pass + with self._download_lock: + self.completed_download_size = self.completed_download_size - downloaded_bytes + self.total_download_size = self.total_download_size - size else: - self.is_shit_there(name, title_id, download_path, index, fileloc, console, sha1, size) + try_configure(status, text_color='green', text='Download Completed!') + try_configure(open_button, state = 'normal', command=lambda: self.open_loc(download_path)) finally: sem.release() else: pass @@ -1135,7 +1263,7 @@ def button_clear(self): q.put(ButtonAction.STOP) except Exception: pass - + time.sleep(0.1) self.total_download_size = 0