From 486dc9bd18138123a4b52624e0d6b43676bfd9aa Mon Sep 17 00:00:00 2001 From: Antoine Gautier Date: Thu, 15 May 2025 15:19:38 +0200 Subject: [PATCH 01/12] Fix for execution stuck in exit_test --- pyfunnel/core.py | 44 ++++++++++++++++++++++++++---------- pyfunnel/templates/plot.html | 1 - 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/pyfunnel/core.py b/pyfunnel/core.py index 66b83078..bed2ca6e 100755 --- a/pyfunnel/core.py +++ b/pyfunnel/core.py @@ -97,7 +97,7 @@ def follow(filehandler, timeout): yield line # Else: yield line. -def wait_until(somepredicate, timeout, period=0.1, *args, **kwargs): +def wait_until(somepredicate, timeout, period=0.5, *args, **kwargs): """Waits until some predicate is true or timeout is over.""" must_end = time.time() + timeout while time.time() < must_end: @@ -114,16 +114,32 @@ def exit_test(logger, list_files=None): 200: request received 304: requested resource not modified since previous transmission """ - content = logger.getvalue().decode('utf8') - if list_files is not None: - raw_pattern = r'GET.*?{}.*?(200|304)' - pattern = raw_pattern.format(re.escape(list_files[0])) # *? for non-greedy search - for file_path in list_files[1:] if len(list_files) > 1 else []: - pattern = r'{}(.*\n)*.*{}'.format( - re.escape(pattern), - re.escape(raw_pattern.format(file_path))) - return bool(re.search(pattern, content)) - else: + try: + # Use a 1MB limit for the logger content to prevent potential hanging + content_bytes = logger.getvalue() + content_size = len(content_bytes) + if content_size == 0: + return False + + if content_size > 1024*1024: # If over 1MB, truncate + print(f"Logger content too large ({content_size/1024/1024:.2f}MB), truncating for analysis") + content_bytes = content_bytes[-1024*1024:] # Take last 1MB + + content = content_bytes.decode('utf8', errors='replace') + + # Short-circuit if no files to check + if list_files is None or len(list_files) == 0: + return False + + # Check if any required file has been loaded + for file_path in list_files: + pattern = r'GET.*?{}.*?(200|304)'.format(re.escape(file_path)) + if re.search(pattern, content): + return True + + # If we get here, no match was found + return False + except Exception: return False @@ -458,7 +474,11 @@ def browse(self, *args, **kwargs): print('Server will run for {} s (or until KeyboardInterrupt) at:\n'.format(timeout) + \ 'http://localhost:{}/funnel'.format(self.server_port)) - wait_until(exit_test, timeout, 0.1, self.logger, *args) + + # Add delay to ensure browser has time to start + time.sleep(1) + wait_until(exit_test, timeout, 0.5, self.logger, *args) + except KeyboardInterrupt: print('KeyboardInterrupt') finally: diff --git a/pyfunnel/templates/plot.html b/pyfunnel/templates/plot.html index e1966dcf..d71b8c0f 100644 --- a/pyfunnel/templates/plot.html +++ b/pyfunnel/templates/plot.html @@ -46,7 +46,6 @@ }; function makePlotly(data){ - var plotDiv = document.getElementById("plot"); var traces = [ { x: data['err'].x, From a54f3e83fc8af7914ee7cf6f524d01b9246aafaf Mon Sep 17 00:00:00 2001 From: Antoine Gautier Date: Thu, 15 May 2025 16:10:01 +0200 Subject: [PATCH 02/12] Improve info message --- pyfunnel/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyfunnel/core.py b/pyfunnel/core.py index bed2ca6e..794e0856 100755 --- a/pyfunnel/core.py +++ b/pyfunnel/core.py @@ -472,8 +472,8 @@ def browse(self, *args, **kwargs): else: raise KeyboardInterrupt - print('Server will run for {} s (or until KeyboardInterrupt) at:\n'.format(timeout) + \ - 'http://localhost:{}/funnel'.format(self.server_port)) + print(f'Results available at http://localhost:{self.server_port}/funnel\n' + f'(Press Ctrl+C to shut down server and continue.)') # Add delay to ensure browser has time to start time.sleep(1) From 169d5845ec59403ad745d16bcde88c27649163d7 Mon Sep 17 00:00:00 2001 From: Antoine Gautier Date: Thu, 15 May 2025 16:24:15 +0200 Subject: [PATCH 03/12] Version and release notes --- CHANGELOG.md | 4 ++++ pyfunnel/VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f61c040..35809319 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Change Log +## Version 1.0.3 + +- Fix plotting issues ([#88](https://github.com/lbl-srg/funnel/issues/88)) + ## Version 1.0.2 - Bug fixes diff --git a/pyfunnel/VERSION b/pyfunnel/VERSION index 6d7de6e6..21e8796a 100644 --- a/pyfunnel/VERSION +++ b/pyfunnel/VERSION @@ -1 +1 @@ -1.0.2 +1.0.3 From 96c3a7236c83cb3dbce94170ae490be904ead6b4 Mon Sep 17 00:00:00 2001 From: Antoine Gautier Date: Thu, 15 May 2025 16:40:48 +0200 Subject: [PATCH 04/12] Bump Plotly version, refactor with promise and event handling --- pyfunnel/templates/plot.html | 139 +++++++++++++++++++++++++++++------ 1 file changed, 118 insertions(+), 21 deletions(-) diff --git a/pyfunnel/templates/plot.html b/pyfunnel/templates/plot.html index d71b8c0f..411546c2 100644 --- a/pyfunnel/templates/plot.html +++ b/pyfunnel/templates/plot.html @@ -1,29 +1,100 @@ - + + -
+
- \ No newline at end of file + From a8506ccef8e9865fdfffeb41cdc39360d7fc0d9b Mon Sep 17 00:00:00 2001 From: Antoine Gautier Date: Thu, 15 May 2025 16:43:09 +0200 Subject: [PATCH 05/12] Bump Ubuntu version --- .github/workflows/cmake.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cmake.yml b/.github/workflows/cmake.yml index 4a3a84c5..1bd8c1a3 100644 --- a/.github/workflows/cmake.yml +++ b/.github/workflows/cmake.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: - os: [ubuntu-20.04, windows-latest, macos-latest] + os: [ubuntu-22.04, windows-latest, macos-latest] python-version: ['3.8', '3.9'] steps: diff --git a/README.md b/README.md index e0655528..374d6414 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ By convention, the error is `max(0, y - y_up) - min(0, y - y_low)` and hence it The software is tested on the following platforms. - * Linux x64 (Ubuntu 20.04) + * Linux x64 (Ubuntu 22.04) * Windows x64 (Windows Server 2022) * macOS x64 and arm64 (macOS 12) From 268a42c1ceca3108cfe0227c92c49f70facea270 Mon Sep 17 00:00:00 2001 From: Antoine Gautier Date: Fri, 16 May 2025 11:46:47 +0200 Subject: [PATCH 06/12] Add recursive wait for plotly, format --- pyfunnel/templates/plot.html | 365 ++++++++++++++++++----------------- 1 file changed, 186 insertions(+), 179 deletions(-) diff --git a/pyfunnel/templates/plot.html b/pyfunnel/templates/plot.html index 411546c2..f8f93d22 100644 --- a/pyfunnel/templates/plot.html +++ b/pyfunnel/templates/plot.html @@ -1,216 +1,223 @@ - - - - - - -
- + + + +
+ - + From 805ba6e8a0d0fbaa2d1b482a35548ff806556076 Mon Sep 17 00:00:00 2001 From: Antoine Gautier Date: Fri, 16 May 2025 12:03:00 +0200 Subject: [PATCH 07/12] Check that server is alive before continuing --- pyfunnel/core.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pyfunnel/core.py b/pyfunnel/core.py index 794e0856..238aee46 100755 --- a/pyfunnel/core.py +++ b/pyfunnel/core.py @@ -14,6 +14,7 @@ import os import platform import re +import socket import subprocess import sys import threading @@ -365,6 +366,14 @@ def server_launch(self): self.thread.daemon = True # daemonic thread objects are terminated as soon as the main thread exits self.thread.start() + def is_server_alive(self): + """Check if the server is accepting connections.""" + try: + with socket.create_connection(("localhost", self.server_port), timeout=1) as sock: + return True + except (socket.timeout, ConnectionRefusedError): + return False + def server_close(self): # Invoke to close logger. # makes execution stall on Windows if main thread @@ -419,6 +428,12 @@ def browse(self, *args, **kwargs): try: self.server_launch() + + # Wait for server to be ready to accept connections + server_ready = wait_until(self.is_server_alive, 5, 0.1) + if not server_ready: + print("Warning: Server may not be ready to accept connections") + # Launch browser as a subprocess command to avoid web browser error into terminal. if launch_browser: with open(os.devnull, 'w') as pipe: @@ -475,8 +490,6 @@ def browse(self, *args, **kwargs): print(f'Results available at http://localhost:{self.server_port}/funnel\n' f'(Press Ctrl+C to shut down server and continue.)') - # Add delay to ensure browser has time to start - time.sleep(1) wait_until(exit_test, timeout, 0.5, self.logger, *args) except KeyboardInterrupt: From af1ea121a9c1dae9a785d1ddc0313c1b5beb5aa3 Mon Sep 17 00:00:00 2001 From: AntoineGautier Date: Wed, 17 Sep 2025 15:40:41 +0200 Subject: [PATCH 08/12] Add safe subprocess termination w/ timeout --- pyfunnel/core.py | 57 ++++++++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/pyfunnel/core.py b/pyfunnel/core.py index 238aee46..92eb98b6 100755 --- a/pyfunnel/core.py +++ b/pyfunnel/core.py @@ -7,8 +7,6 @@ from __future__ import absolute_import, division, print_function, unicode_literals -# Python standard library imports. -from ctypes import cdll, POINTER, c_double, c_int, c_char_p import io import numbers import os @@ -20,14 +18,14 @@ import threading import time import webbrowser -try: - from http.server import HTTPServer, SimpleHTTPRequestHandler # Python 3 -except ImportError: - from SimpleHTTPServer import BaseHTTPServer - HTTPServer = BaseHTTPServer.HTTPServer - from SimpleHTTPServer import SimpleHTTPRequestHandler # Python 2 + +# Python standard library imports. +from ctypes import POINTER, c_char_p, c_double, c_int, cdll +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer + # Third-party module or package imports. import six + # Code repository sub-package imports. @@ -40,7 +38,7 @@ CONFIG_PATH = os.path.join(os.path.expanduser('~'), '.pyfunnel') # os.environ.get('HOME')=None on Windows. CONFIG_DEFAULT = dict( - BROWSER=None, + BROWSER=None, # type: Optional[str] ) # Get the real browser name in case webbrowser.get(browser).name returns 'xdg-open' on Linux. @@ -342,7 +340,7 @@ def compareAndReport( ##################### -class MyHTTPServer(HTTPServer): +class MyHTTPServer(ThreadingHTTPServer): """Add custom server_launch, server_close and browse methods.""" def __init__(self, *args, **kwargs): @@ -355,7 +353,7 @@ def __init__(self, *args, **kwargs): str_html = kwargs.pop('str_html', None) url_html = kwargs.pop('url_html', None) browse_dir = kwargs.pop('browse_dir', os.getcwd()) - HTTPServer.__init__(self, *args) + ThreadingHTTPServer.__init__(self, *args) self._STR_HTML = re.sub(r'\$SERVER_PORT', str(self.server_port), str_html) self._URL_HTML = url_html self._BROWSE_DIR = browse_dir @@ -385,6 +383,15 @@ def server_close(self): except Exception as e: print('Could not close logger: {}'.format(e)) + def terminate_safely(self, process, timeout=2.0): + if process is not None and process.poll() is None: + try: + process.terminate() + process.wait(timeout=timeout) # Wait for a maximum of timeout + except subprocess.TimeoutExpired: + process.kill() # Force kill if terminate didn't work + process.wait() # Wait for kill to complete + def browse(self, *args, **kwargs): """Launch server, and web browser if available. @@ -426,6 +433,7 @@ def browse(self, *args, **kwargs): cur_dir = os.getcwd() os.chdir(self._BROWSE_DIR) + proc = None try: self.server_launch() @@ -438,6 +446,7 @@ def browse(self, *args, **kwargs): if launch_browser: with open(os.devnull, 'w') as pipe: proc = subprocess.Popen(webbrowser_cmd, stdout=pipe, stderr=pipe) + # Watch syslog for error. chrome_error = False if platform.system() == 'Linux': @@ -454,8 +463,9 @@ def browse(self, *args, **kwargs): if 'ERROR:gles2_cmd_decoder' in l: chrome_error = True break + if chrome_error: - proc.terminate() # Terminating the process does not stop Chrome in background. + self.terminate_safely(proc) # Terminating the process does not stop Chrome in background. subprocess.check_call(['pkill', 'chrome']) # This does. inp = 'y' while True: # Prompt user to retry. @@ -496,15 +506,13 @@ def browse(self, *args, **kwargs): print('KeyboardInterrupt') finally: os.chdir(cur_dir) - try: # Objects may not be defined in case of exception. - self.server_close() - proc.terminate() - except BaseException: - pass + self.server_close() + self.terminate_safely(proc) class CORSRequestHandler(SimpleHTTPRequestHandler): """Enable logging message and modify response header.""" + server: MyHTTPServer # type: ignore[override] def log_message(self, format, *args): try: @@ -520,24 +528,21 @@ def log_message(self, format, *args): print(e) def end_headers(self): - self.send_header('Access-Control-Allow-Origin'.encode('utf8'), - '*'.encode('utf8')) - self.send_header('Access-Control-Allow-Methods'.encode('utf8'), - 'GET, POST, OPTIONS'.encode('utf8')) - self.send_header('Access-Control-Allow-Headers'.encode('utf8'), - 'X-Requested-With'.encode('utf8')) + self.send_header('Access-Control-Allow-Origin', '*') + self.send_header('Access-Control-Allow-Methods', 'GET, POST, OPTIONS') + self.send_header('Access-Control-Allow-Headers', 'X-Requested-With') SimpleHTTPRequestHandler.end_headers(self) def send_head(self): if (self.server._URL_HTML is not None) and \ (self.translate_path(self.path).endswith(self.server._URL_HTML)): f = io.BytesIO() - f.write(self.server._STR_HTML.encode('utf8')) + f.write(self.server._STR_HTML.encode('utf-8')) length = f.tell() f.seek(0) self.send_response(200) - self.send_header("Content-type".encode('utf8'), "text/html".encode('utf8')) - self.send_header("Content-Length".encode('utf8'), str(length).encode('utf8')) + self.send_header("Content-type", "text/html") + self.send_header("Content-Length", str(length)) self.end_headers() return f else: From 773c4b808ce9a4003938e97c3f86f9783e319b0a Mon Sep 17 00:00:00 2001 From: AntoineGautier Date: Thu, 18 Sep 2025 10:50:07 +0200 Subject: [PATCH 09/12] Fix function name in HTML template, improve error handling --- pyfunnel/templates/plot.html | 64 ++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/pyfunnel/templates/plot.html b/pyfunnel/templates/plot.html index f8f93d22..b495034f 100644 --- a/pyfunnel/templates/plot.html +++ b/pyfunnel/templates/plot.html @@ -1,6 +1,7 @@ +