diff --git a/synth/utils/sequential_scheduler.py b/synth/utils/sequential_scheduler.py index bea6d7c..4fe43b9 100644 --- a/synth/utils/sequential_scheduler.py +++ b/synth/utils/sequential_scheduler.py @@ -27,9 +27,10 @@ def start(self): while True: try: cycle_start_time = self.run_cycle(cycle_start_time) - self.first_run = False except Exception: bt.logging.exception("Error in cycle ") + cycle_start_time = get_current_time() + self.first_run = False def run_cycle( self, diff --git a/synth/validator/miner_data_handler.py b/synth/validator/miner_data_handler.py index 3a7c328..af28720 100644 --- a/synth/validator/miner_data_handler.py +++ b/synth/validator/miner_data_handler.py @@ -107,8 +107,8 @@ def get_latest_asset(self, time_length: int) -> str | None: return None @retry( - stop=stop_after_attempt(5), - wait=wait_random_exponential(multiplier=7), + stop=stop_after_attempt(3), + wait=wait_random_exponential(multiplier=2), reraise=True, before=before_log(bt.logging._logger, logging.DEBUG), ) @@ -182,6 +182,7 @@ def save_responses( return validator_requests_id # TODO: finish this: refactor to add the validator_requests_id in the score and reward table except Exception as e: bt.logging.exception(f"in save_responses (got an exception): {e}") + raise @retry( stop=stop_after_attempt(5), @@ -406,6 +407,9 @@ def get_validator_requests_to_score( ValidatorRequest.start_time + literal_column("INTERVAL '1 second'") * ValidatorRequest.time_length + + literal_column( + "INTERVAL '1 minute'" + ) # add 1 minute to ensure that pyth has the last candle available ) query = ( diff --git a/synth/validator/price_data_provider.py b/synth/validator/price_data_provider.py index b821cbf..446107d 100644 --- a/synth/validator/price_data_provider.py +++ b/synth/validator/price_data_provider.py @@ -72,29 +72,41 @@ def fetch_data(self, validator_request: ValidatorRequest) -> list: """ asset = str(validator_request.asset) - if asset in self.HYPERLIQUID_SYMBOL_MAP: - return self.fetch_data_hyperliquid(validator_request) + prices = [] - start_time_int = from_iso_to_unix_time( - validator_request.start_time.isoformat() - ) - params = { - "symbol": self.PYTH_SYMBOL_MAP[asset], - "resolution": 1, - "from": start_time_int, - "to": start_time_int + validator_request.time_length, - } + if asset in self.HYPERLIQUID_SYMBOL_MAP: + prices = self.fetch_data_hyperliquid(validator_request) + else: + start_time_int = from_iso_to_unix_time( + validator_request.start_time.isoformat() + ) + params = { + "symbol": self.PYTH_SYMBOL_MAP[asset], + "resolution": 1, + "from": start_time_int, + "to": start_time_int + validator_request.time_length, + } + + response = requests.get(self.PYTH_BASE_URL, params=params) + response.raise_for_status() + data = response.json() + + prices = self._transform_data( + data, + start_time_int, + int(validator_request.time_increment), + int(validator_request.time_length), + ) - response = requests.get(self.PYTH_BASE_URL, params=params) - response.raise_for_status() - data = response.json() + if not prices or np.isnan(prices[-1]): + bt.logging.warning( + f"missing price data for the last timestamp for asset {asset} in request {validator_request.id}" + ) + raise ValueError( + f"missing price data for the last timestamp for asset {asset} in request {validator_request.id}" + ) - return self._transform_data( - data, - start_time_int, - int(validator_request.time_increment), - int(validator_request.time_length), - ) + return prices def fetch_data_hyperliquid( self, validator_request: ValidatorRequest @@ -194,6 +206,9 @@ def _transform_data( if len(timestamps) != int(time_length / time_increment) + 1: # Note: this part of code should never be activated; just included for precaution if len(timestamps) == int(time_length / time_increment) + 2: + bt.logging.warning( + f"Unexpected number of timestamps generated. Expected {int(time_length / time_increment) + 1} but got {len(timestamps)}. Adjusting the timestamps list by removing the extra timestamp." + ) if data["t"][-1] < timestamps[1]: timestamps = timestamps[:-1] elif data["t"][0] > timestamps[0]: diff --git a/tests/test_price_data_provider.py b/tests/test_price_data_provider.py index c346ee4..5d52b39 100644 --- a/tests/test_price_data_provider.py +++ b/tests/test_price_data_provider.py @@ -120,7 +120,7 @@ def test_fetch_data_gap_3(self): # 1739974740 - 2025-02-19T14:19:00+00:00 # 1739974800 - 2025-02-19T14:20:00+00:00 # 1739974860 - 2025-02-19T14:21:00+00:00 - # gap - 2025-02-19T14:22:00+00:00 + # 1739974920 - 2025-02-19T14:22:00+00:00 mock_response = { "t": [ 1739974320, @@ -128,6 +128,7 @@ def test_fetch_data_gap_3(self): 1739974740, 1739974800, 1739974860, + 1739974920, ], "c": [ 100000.23, @@ -135,6 +136,7 @@ def test_fetch_data_gap_3(self): 99000.23, 97123.55, 105123.345, + 107995.889, ], } @@ -144,7 +146,7 @@ def test_fetch_data_gap_3(self): validator_request_eth = ValidatorRequest( asset="ETH", start_time=datetime.fromisoformat("2025-02-19T14:12:00+00:00"), - time_length=600, + time_length=540, time_increment=120, ) @@ -156,22 +158,9 @@ def test_fetch_data_gap_3(self): np.nan, 108000.867, 97123.55, - np.nan, + 107995.889, ] - def test_fetch_data_no_prices(self): - mock_response: dict = { - "t": [], - "c": [], - } - - with patch("requests.get") as mock_get: - mock_get.return_value.json.return_value = mock_response - - result = self.dataProvider.fetch_data(validator_request) - - assert result == [] - def test_fetch_data_gap_from_start(self): # gap - 2025-02-19T14:12:00+00:00 # gap - 2025-02-19T14:13:00+00:00