update

Author: mayinghan

eval_protocol/cli_commands/upload.py

@@ -519,35 +519,35 @@ def upload_command(args: argparse.Namespace) -> int:
     description = getattr(args, "description", None)
     force = bool(getattr(args, "force", False))
 
-    # Ensure FIREWORKS_API_KEY is available to the remote by storing it as a Fireworks secret
-    try:
-        fw_account_id = get_fireworks_account_id()
-        fw_api_key_value = get_fireworks_api_key()
-        if not fw_account_id and fw_api_key_value:
-            # Attempt to verify and resolve account id from server headers
-            resolved = verify_api_key_and_get_account_id(api_key=fw_api_key_value, api_base=get_fireworks_api_base())
-            if resolved:
-                fw_account_id = resolved
-                # Propagate to environment so downstream calls use it if needed
-                os.environ["FIREWORKS_ACCOUNT_ID"] = fw_account_id
-                print(f"Resolved FIREWORKS_ACCOUNT_ID via API verification: {fw_account_id}")
-        if fw_account_id and fw_api_key_value:
-            print("Ensuring FIREWORKS_API_KEY is registered as a secret on Fireworks for rollout...")
-            if create_or_update_fireworks_secret(
-                account_id=fw_account_id,
-                key_name="FIREWORKS_API_KEY",
-                secret_value=fw_api_key_value,
-            ):
-                print("✓ FIREWORKS_API_KEY secret created/updated on Fireworks.")
-            else:
-                print("Warning: Failed to create/update FIREWORKS_API_KEY secret on Fireworks.")
-        else:
-            if not fw_account_id:
-                print("Warning: FIREWORKS_ACCOUNT_ID not found; cannot register FIREWORKS_API_KEY secret.")
-            if not fw_api_key_value:
-                print("Warning: FIREWORKS_API_KEY not found locally; cannot register secret.")
-    except Exception as e:
-        print(f"Warning: Skipped Fireworks secret registration due to error: {e}")
+    # # Ensure FIREWORKS_API_KEY is available to the remote by storing it as a Fireworks secret
+    # try:
+    #     fw_account_id = get_fireworks_account_id()
+    #     fw_api_key_value = get_fireworks_api_key()
+    #     if not fw_account_id and fw_api_key_value:
+    #         # Attempt to verify and resolve account id from server headers
+    #         resolved = verify_api_key_and_get_account_id(api_key=fw_api_key_value, api_base=get_fireworks_api_base())
+    #         if resolved:
+    #             fw_account_id = resolved
+    #             # Propagate to environment so downstream calls use it if needed
+    #             os.environ["FIREWORKS_ACCOUNT_ID"] = fw_account_id
+    #             print(f"Resolved FIREWORKS_ACCOUNT_ID via API verification: {fw_account_id}")
+    #     if fw_account_id and fw_api_key_value:
+    #         print("Ensuring FIREWORKS_API_KEY is registered as a secret on Fireworks for rollout...")
+    #         if create_or_update_fireworks_secret(
+    #             account_id=fw_account_id,
+    #             key_name="FIREWORKS_API_KEY",
+    #             secret_value=fw_api_key_value,
+    #         ):
+    #             print("✓ FIREWORKS_API_KEY secret created/updated on Fireworks.")
+    #         else:
+    #             print("Warning: Failed to create/update FIREWORKS_API_KEY secret on Fireworks.")
+    #     else:
+    #         if not fw_account_id:
+    #             print("Warning: FIREWORKS_ACCOUNT_ID not found; cannot register FIREWORKS_API_KEY secret.")
+    #         if not fw_api_key_value:
+    #             print("Warning: FIREWORKS_API_KEY not found locally; cannot register secret.")
+    # except Exception as e:
+    #     print(f"Warning: Skipped Fireworks secret registration due to error: {e}")
 
     exit_code = 0
     for i, (qualname, source_file_path) in enumerate(selected_specs):

eval_protocol/evaluation.py

@@ -606,18 +606,21 @@ def create(self, evaluator_id, display_name=None, description=None, force=False)
         if "dev.api.fireworks.ai" in self.api_base and account_id == "fireworks":
             account_id = "pyroworks-dev"
 
-        base_url = f"{self.api_base}/v1/{parent}/evaluators"
+        base_url = f"{self.api_base}/v1/{parent}/evaluatorsV2"
         headers = {
             "Authorization": f"Bearer {auth_token}",
+            "x-api-key": f"{auth_token}",
             "Content-Type": "application/json",
             "X-Fireworks-Gateway-Secret": "8b8a823c-0b29-41f4-8537-4c9f650a113c",
         }
+        
         logger.info(f"Creating evaluator '{evaluator_id}' for account '{account_id}'...")
 
         try:
             if force:
                 check_url = f"{base_url}/{evaluator_id}"
                 try:
+                    logger.info(f"check_url: {check_url}, headers: {headers}")
                     check_response = requests.get(check_url, headers=headers)
                     if check_response.status_code == 200:
                         logger.info(f"Evaluator '{evaluator_id}' already exists, deleting and recreating...")
@@ -632,12 +635,15 @@ def create(self, evaluator_id, display_name=None, description=None, force=False)
                                 )
                         except Exception as e_del:
                             logger.warning(f"Error deleting evaluator: {str(e_del)}")
+                        logger.info(f"base_url: {base_url}, payload_data: {payload_data}, headers: {headers}")
                         response = requests.post(base_url, json=payload_data, headers=headers)
                     else:
+                        print(f"base_url: {base_url}, payload_data: {payload_data}, headers: {headers}")
                         response = requests.post(base_url, json=payload_data, headers=headers)
                 except requests.exceptions.RequestException:
                     response = requests.post(base_url, json=payload_data, headers=headers)
             else:
+                logger.info(f"check_url: {base_url}, headers: {headers}, payload_data: {payload_data}")
                 response = requests.post(base_url, json=payload_data, headers=headers)
 
             response.raise_for_status()