streaming_postprocessor.py

import pandas as pd
import numpy as np
from pathlib import Path
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.metrics import accuracy_score
from cmi_2025_metric_copy_for_import import CompetitionMetric
import time


def calculate_metrics(true_labels, pred_labels, true_gestures=None):
    """Calculate accuracy and optionally H-F1 score."""
    acc = accuracy_score(true_labels, pred_labels)

    h_f1 = None
    if true_gestures is not None:
        try:
            if isinstance(pred_labels[0], (int, np.integer)):
                gesture_classes = sorted(list(set(true_gestures)))
                pred_gestures = [gesture_classes[idx] for idx in pred_labels]
            else:
                pred_gestures = pred_labels

            h_f1 = CompetitionMetric().calculate_hierarchical_f1(
                pd.DataFrame({"gesture": true_gestures}),
                pd.DataFrame({"gesture": pred_gestures}),
            )
        except Exception as e:
            print(f"Warning: Failed to calculate H-F1 score - {str(e)}")

    return acc, h_f1


def format_metric(value):
    """Format metric value, handling None values."""
    return "N/A" if value is None else f"{value:.4f}"


def analyze_gesture_orientation_combinations(train_df):
    """Analyze gesture-orientation combination counts, including missing combinations."""
    print("\nAnalyzing gesture-orientation combinations...")

    all_gestures = sorted(train_df["gesture"].unique())
    all_orientations = sorted(train_df["orientation"].unique())

    print(
        f"Found {len(all_gestures)} gestures and {len(all_orientations)} orientations"
    )
    print(f"Total possible combinations: {len(all_gestures) * len(all_orientations)}")

    actual_counts = (
        train_df.groupby(["gesture", "orientation"]).size().reset_index(name="count")
    )

    all_combinations = []
    for gesture in all_gestures:
        for orientation in all_orientations:
            existing = actual_counts[
                (actual_counts["gesture"] == gesture)
                & (actual_counts["orientation"] == orientation)
            ]

            count = existing["count"].iloc[0] if len(existing) > 0 else 0
            all_combinations.append(
                {"gesture": gesture, "orientation": orientation, "count": count}
            )

    combinations_df = pd.DataFrame(all_combinations)

    zero_count = (combinations_df["count"] == 0).sum()
    print(f"Combinations with 0 occurrences: {zero_count}")

    if zero_count > 0:
        zero_combinations = combinations_df[combinations_df["count"] == 0]
        print("Missing combinations:")
        for _, row in zero_combinations.iterrows():
            print(f"  {row['gesture']} + {row['orientation']}")

    return combinations_df


class StreamingPostprocessor:
    def __init__(
        self, gesture_classes, confidence_threshold=0.2, valid_combinations=None, starting_entries=None
    ):
        self.gesture_classes = gesture_classes
        self.confidence_threshold = confidence_threshold
        self.valid_combinations = valid_combinations
        self.gesture_predictions = {}
        self.all_subject_predictions = {}
        self.all_predictions = []
        self.displacement_log = []
        self.constraint_violation_log = []
        self._prediction_counter = 0
        self.invalid_combination_skips = 0
        self.impossible_original_redirects = 0
        self.starting_entries_count = 0
        self._applied_starting_subjects = set()
        
        # Apply starting entries if provided
        if starting_entries:
            self._apply_starting_entries(starting_entries)

    def _apply_starting_entries(self, starting_entries):
        """Apply generic starting entries to all subjects when they are first encountered."""
        self.generic_starting_entries = []
        for entry in starting_entries:
            if all(field in entry for field in ['orientation', 'gesture', 'confidence']):
                confidence = entry['confidence']
                # Note: Keep confidence values in original scale (0-100) to match prob columns
                # The prob columns in the data are already in 0-100 percentage scale
                if 0 <= confidence <= 100.0:
                    self.generic_starting_entries.append({
                        'orientation': entry['orientation'],
                        'gesture': entry['gesture'], 
                        'confidence': confidence,  # Keep original scale
                        'sequence_id': entry.get('sequence_id', f"start_{entry['orientation']}_{entry['gesture']}")
                    })

    def _apply_entries_to_subject(self, subject):
        """Apply generic starting entries to a new subject."""
        if subject in self._applied_starting_subjects or not hasattr(self, 'generic_starting_entries'):
            return
            
        applied_count = 0
        for entry in self.generic_starting_entries:
            orientation = entry['orientation']
            gesture = entry['gesture']
            confidence = entry['confidence']
            sequence_id = entry['sequence_id']
            
            # Check if combination is valid
            if self.valid_combinations and (gesture, orientation) not in self.valid_combinations:
                continue
                
            # Check if gesture is valid
            if gesture not in self.gesture_classes:
                continue
                
            key = (subject, orientation, gesture)
            
            # Add to prediction slots if space available
            if key not in self.gesture_predictions:
                self.gesture_predictions[key] = []
            
            if len(self.gesture_predictions[key]) < 2:
                self.gesture_predictions[key].append((confidence, sequence_id))
                applied_count += 1
                
                # Track in complete history
                if key not in self.all_subject_predictions:
                    self.all_subject_predictions[key] = []
                
                self._prediction_counter += 1
                self.all_subject_predictions[key].append(
                    (confidence, sequence_id, True, self._prediction_counter)
                )
        
        self._applied_starting_subjects.add(subject)
        self.starting_entries_count += applied_count

    def process_prediction(
        self,
        sequence_id,
        subject,
        orientation,
        predicted_gesture,
        confidence_scores,
        true_label,
        true_gesture,
    ):
        """Process a single prediction with confidence-based replacement and constraint enforcement."""
        self._prediction_counter += 1
        
        # Apply starting entries to new subjects
        self._apply_entries_to_subject(subject)

        original_prediction = predicted_gesture
        if (
            self.valid_combinations is not None
            and (predicted_gesture, orientation) not in self.valid_combinations
        ):
            best_alternative = self._find_best_available_gesture(
                subject, orientation, confidence_scores, exclude_gesture=None
            )

            if best_alternative is not None:
                predicted_gesture = best_alternative
                new_confidence = confidence_scores[
                    self.gesture_classes.index(predicted_gesture)
                ]
                self.impossible_original_redirects += 1

        key = (subject, orientation, predicted_gesture)
        new_confidence = confidence_scores[
            self.gesture_classes.index(predicted_gesture)
        ]

        if key not in self.all_subject_predictions:
            self.all_subject_predictions[key] = []

        current_predictions = self.gesture_predictions.get(key, [])
        confidences = [conf for conf, _ in current_predictions]

        displaced_sequence = None
        final_prediction = predicted_gesture
        was_replacement = False
        was_accepted = False

        if len(current_predictions) < 2:
            current_predictions.append((new_confidence, sequence_id))
            self.gesture_predictions[key] = current_predictions
            was_accepted = True
        else:
            min_confidence = min(confidences)

            if new_confidence > min_confidence:
                min_idx = confidences.index(min_confidence)
                displaced_confidence, displaced_sequence = current_predictions[min_idx]

                current_predictions[min_idx] = (new_confidence, sequence_id)
                self.gesture_predictions[key] = current_predictions
                was_replacement = True
                was_accepted = True

                self.displacement_log.append(
                    {
                        "displaced_sequence": displaced_sequence,
                        "displaced_confidence": displaced_confidence,
                        "replacing_sequence": sequence_id,
                        "replacing_confidence": new_confidence,
                        "gesture": predicted_gesture,
                        "subject": subject,
                        "orientation": orientation,
                        "displacement_type": "direct_replacement",
                    }
                )

                final_prediction = predicted_gesture
            else:
                best_alternative = self._find_best_available_gesture(
                    subject, orientation, confidence_scores, predicted_gesture
                )

                if best_alternative is not None:
                    alternative_confidence = confidence_scores[
                        self.gesture_classes.index(best_alternative)
                    ]
                    confidence_drop = new_confidence - alternative_confidence

                    if confidence_drop <= self.confidence_threshold:
                        final_prediction = best_alternative
                        redirect_key = (subject, orientation, final_prediction)
                        redirect_predictions = self.gesture_predictions.get(
                            redirect_key, []
                        )

                        if len(redirect_predictions) < 2:
                            redirect_predictions.append(
                                (alternative_confidence, sequence_id)
                            )
                            self.gesture_predictions[redirect_key] = (
                                redirect_predictions
                            )
                        else:
                            redirect_confidences = [
                                conf for conf, _ in redirect_predictions
                            ]
                            min_redirect_confidence = min(redirect_confidences)
                            min_redirect_idx = redirect_confidences.index(
                                min_redirect_confidence
                            )

                            displaced_alt_confidence, displaced_alt_sequence = (
                                redirect_predictions[min_redirect_idx]
                            )
                            redirect_predictions[min_redirect_idx] = (
                                alternative_confidence,
                                sequence_id,
                            )
                            self.gesture_predictions[redirect_key] = (
                                redirect_predictions
                            )

                            self.displacement_log.append(
                                {
                                    "displaced_sequence": displaced_alt_sequence,
                                    "displaced_confidence": displaced_alt_confidence,
                                    "replacing_sequence": sequence_id,
                                    "replacing_confidence": alternative_confidence,
                                    "gesture": best_alternative,
                                    "subject": subject,
                                    "orientation": orientation,
                                    "displacement_type": "redirection_displacement",
                                }
                            )
                    else:
                        final_prediction = predicted_gesture
                        current_predictions.append((new_confidence, sequence_id))
                        self.gesture_predictions[key] = current_predictions
                        was_accepted = True

                        self.constraint_violation_log.append(
                            {
                                "sequence_id": sequence_id,
                                "subject": subject,
                                "orientation": orientation,
                                "original_gesture": predicted_gesture,
                                "original_confidence": new_confidence,
                                "best_alternative": best_alternative,
                                "alternative_confidence": alternative_confidence,
                                "confidence_drop": confidence_drop,
                                "threshold": self.confidence_threshold,
                                "action_taken": "kept_original_violates_constraint",
                            }
                        )
                else:
                    final_prediction = predicted_gesture
                    current_predictions.append((new_confidence, sequence_id))
                    self.gesture_predictions[key] = current_predictions
                    was_accepted = True

                    action_reason = "kept_original_no_alternative"
                    if self.valid_combinations is not None:
                        valid_alternatives = [
                            gesture
                            for gesture in self.gesture_classes
                            if gesture != predicted_gesture
                            and (gesture, orientation) in self.valid_combinations
                        ]
                        if len(valid_alternatives) == 0:
                            action_reason = "kept_original_no_valid_combinations"
                        else:
                            action_reason = "kept_original_no_available_slots"

                    self.constraint_violation_log.append(
                        {
                            "sequence_id": sequence_id,
                            "subject": subject,
                            "orientation": orientation,
                            "original_gesture": predicted_gesture,
                            "original_confidence": new_confidence,
                            "best_alternative": None,
                            "alternative_confidence": None,
                            "confidence_drop": None,
                            "threshold": self.confidence_threshold,
                            "action_taken": action_reason,
                        }
                    )

        self.all_subject_predictions[key].append(
            (new_confidence, sequence_id, was_accepted, self._prediction_counter)
        )

        prediction_record = {
            "sequence_id": sequence_id,
            "subject": subject,
            "orientation": orientation,
            "original_prediction": original_prediction,
            "final_prediction": final_prediction,
            "true_label": true_label,
            "true_gesture": true_gesture,
            "was_redirected": final_prediction != original_prediction,
            "was_impossible_original": original_prediction != predicted_gesture,
            "was_replacement": was_replacement,
            "violates_constraint": len(
                [
                    x
                    for x in self.constraint_violation_log
                    if x["sequence_id"] == sequence_id
                ]
            )
            > 0,
            "displaced_sequence": displaced_sequence,
            "original_confidence": confidence_scores[
                self.gesture_classes.index(original_prediction)
            ],
            "final_confidence": confidence_scores[
                self.gesture_classes.index(final_prediction)
            ],
        }
        self.all_predictions.append(prediction_record)

        return final_prediction

    def _find_best_available_gesture(
        self, subject, orientation, confidence_scores, exclude_gesture
    ):
        """Find the best alternative gesture considering available slots and potential displacements."""
        best_gesture = None
        best_final_confidence = -1

        for i, gesture in enumerate(self.gesture_classes):
            if exclude_gesture is not None and gesture == exclude_gesture:
                continue

            if self.valid_combinations is not None:
                if (gesture, orientation) not in self.valid_combinations:
                    self.invalid_combination_skips += 1
                    continue

            alternative_confidence = confidence_scores[i]
            key = (subject, orientation, gesture)
            current_predictions = self.gesture_predictions.get(key, [])

            can_place = False
            final_confidence = alternative_confidence

            if len(current_predictions) < 2:
                can_place = True
                final_confidence = alternative_confidence
            elif len(current_predictions) == 2:
                existing_confidences = [conf for conf, _ in current_predictions]
                min_existing_confidence = min(existing_confidences)

                if alternative_confidence > min_existing_confidence:
                    can_place = True
                    final_confidence = alternative_confidence

            if can_place and final_confidence > best_final_confidence:
                best_final_confidence = final_confidence
                best_gesture = gesture

        return best_gesture

    def get_final_predictions_df(self):
        """Get a DataFrame of all final predictions."""
        return pd.DataFrame(self.all_predictions)

    def get_statistics(self):
        """Get statistics about the streaming postprocessing."""
        df = self.get_final_predictions_df()

        total_predictions = len(df)
        redirected_predictions = df["was_redirected"].sum()
        replacement_predictions = (
            df["was_replacement"].sum() if "was_replacement" in df.columns else 0
        )
        constraint_violations = (
            df["violates_constraint"].sum()
            if "violates_constraint" in df.columns
            else 0
        )
        redirect_rate = (
            redirected_predictions / total_predictions if total_predictions > 0 else 0
        )
        replacement_rate = (
            replacement_predictions / total_predictions if total_predictions > 0 else 0
        )
        constraint_violation_rate = (
            constraint_violations / total_predictions if total_predictions > 0 else 0
        )

        accuracy = (df["final_prediction"] == df["true_gesture"]).mean()

        total_history_predictions = sum(
            len(predictions) for predictions in self.all_subject_predictions.values()
        )
        total_accepted_in_history = sum(
            sum(1 for _, _, was_accepted, _ in predictions if was_accepted)
            for predictions in self.all_subject_predictions.values()
        )

        invalid_combination_violations = len(
            [
                v
                for v in self.constraint_violation_log
                if v.get("action_taken") == "kept_original_no_valid_combinations"
            ]
        )

        return {
            "total_predictions": total_predictions,
            "redirected_predictions": redirected_predictions,
            "replacement_predictions": replacement_predictions,
            "constraint_violations": constraint_violations,
            "redirect_rate": redirect_rate,
            "replacement_rate": replacement_rate,
            "constraint_violation_rate": constraint_violation_rate,
            "accuracy": accuracy,
            "confidence_threshold": self.confidence_threshold,
            "total_displacements": len(self.displacement_log),
            "total_constraint_violations": len(self.constraint_violation_log),
            "invalid_combination_violations": invalid_combination_violations,
            "invalid_combination_skips": self.invalid_combination_skips,
            "impossible_original_redirects": self.impossible_original_redirects,
            "total_history_predictions": total_history_predictions,
            "total_accepted_in_history": total_accepted_in_history,
            "history_acceptance_rate": total_accepted_in_history
            / total_history_predictions
            if total_history_predictions > 0
            else 0,
            "starting_entries_count": self.starting_entries_count,
            "current_occupied_slots": sum(len(preds) for preds in self.gesture_predictions.values()),
        }

    def get_complete_prediction_history(
        self, subject=None, orientation=None, gesture=None
    ):
        """Get complete prediction history, optionally filtered."""
        history = []

        for (subj, orient, gest), predictions in self.all_subject_predictions.items():
            if subject is not None and subj != subject:
                continue
            if orientation is not None and orient != orientation:
                continue
            if gesture is not None and gest != gesture:
                continue

            for confidence, sequence_id, was_accepted, timestamp in predictions:
                history.append(
                    {
                        "subject": subj,
                        "orientation": orient,
                        "gesture": gest,
                        "sequence_id": sequence_id,
                        "confidence": confidence,
                        "was_accepted": was_accepted,
                        "timestamp": timestamp,
                    }
                )

        history.sort(key=lambda x: x["timestamp"])
        return history

    def get_gesture_history_summary(self):
        """Get prediction history summary for each (subject, orientation, gesture) combination."""
        summary = {}

        for key, predictions in self.all_subject_predictions.items():
            subject, orientation, gesture = key

            if len(predictions) == 0:
                continue

            confidences = [conf for conf, _, _, _ in predictions]
            accepted_count = sum(
                1 for _, _, was_accepted, _ in predictions if was_accepted
            )

            summary[key] = {
                "subject": subject,
                "orientation": orientation,
                "gesture": gesture,
                "total_attempts": len(predictions),
                "accepted_count": accepted_count,
                "rejected_count": len(predictions) - accepted_count,
                "acceptance_rate": accepted_count / len(predictions),
                "min_confidence": min(confidences),
                "max_confidence": max(confidences),
                "mean_confidence": sum(confidences) / len(confidences),
                "confidences": confidences,
                "accepted_confidences": [
                    conf for conf, _, was_accepted, _ in predictions if was_accepted
                ],
                "rejected_confidences": [
                    conf for conf, _, was_accepted, _ in predictions if not was_accepted
                ],
            }

        return summary


def simulate_streaming_postprocessing_single_threshold(
    prepared_data, gesture_classes, valid_combinations, confidence_threshold=0.5, starting_entries=None
):
    """Simulate streaming postprocessing for a single threshold using pre-prepared data."""
    streaming_results = {}

    for sensor_type in ["full_sensors", "imu_only"]:
        processor = StreamingPostprocessor(
            gesture_classes, confidence_threshold, valid_combinations, starting_entries
        )

        prob_cols = [f"prob_{sensor_type}_{gesture}" for gesture in gesture_classes]

        for _, row in prepared_data.iterrows():
            if pd.isna(row["subject"]) or pd.isna(row["orientation"]):
                continue

            original_prediction = row[f"pred_gesture_{sensor_type}"]
            confidence_scores = [row[col] for col in prob_cols]

            processor.process_prediction(
                sequence_id=row["sequence_id"],
                subject=row["subject"],
                orientation=row["orientation"],
                predicted_gesture=original_prediction,
                confidence_scores=confidence_scores,
                true_label=row["true_label"],
                true_gesture=row["true_gesture"],
            )

        stats = processor.get_statistics()
        streaming_df = processor.get_final_predictions_df()

        streaming_results[sensor_type] = {
            "processor": processor,
            "stats": stats,
            "df": streaming_df,
        }

    return streaming_results


def prepare_streaming_data(
    ensemble_predictions_df, sequence_info, valid_combinations_df=None
):
    """Pre-process all data needed for streaming simulation to avoid redundant operations."""
    print("Preparing data for streaming simulation...")

    valid_combinations = None
    if valid_combinations_df is not None:
        valid_combinations_df_filtered = valid_combinations_df[
            valid_combinations_df["count"] > 0
        ]
        valid_combinations = set(
            zip(
                valid_combinations_df_filtered["gesture"],
                valid_combinations_df_filtered["orientation"],
            )
        )
        print(
            f"Using {len(valid_combinations)} valid gesture-orientation combinations (excluding {len(valid_combinations_df) - len(valid_combinations_df_filtered)} impossible combinations)"
        )
    else:
        print("No gesture-orientation filtering applied (all combinations allowed)")

    merged_df = ensemble_predictions_df.merge(
        sequence_info, on="sequence_id", how="left"
    )

    missing_info = merged_df[["subject", "orientation"]].isna().sum().sum()
    if missing_info > 0:
        print(
            f"Warning: {missing_info} sequences have missing subject/orientation info for streaming"
        )

    sorted_df = merged_df.sort_values(["subject", "sequence_id"]).copy()

    print(f"Prepared {len(sorted_df)} sequences for streaming simulation")

    return sorted_df, valid_combinations


def create_threshold_sweep_visualization(threshold_results, output_dir):
    """
    Create visualizations for threshold sweep results.
    """
    output_dir = Path(output_dir)

    # Prepare data for plotting
    plot_data = []
    for threshold, metrics in threshold_results.items():
        for sensor_type in ["full_sensors", "imu_only"]:
            m = metrics[sensor_type]
            plot_data.append(
                {
                    "threshold": threshold,
                    "sensor_type": sensor_type,
                    "accuracy": m["accuracy"],
                    "h_f1": m["h_f1"],
                    "constraint_violation_rate": m["constraint_violation_rate"],
                    "redirect_rate": m["redirect_rate"],
                    "replacement_rate": m["replacement_rate"],
                }
            )

    plot_df = pd.DataFrame(plot_data)

    # Create visualization
    fig, axes = plt.subplots(2, 3, figsize=(18, 12))

    # 1. Accuracy vs Threshold
    for sensor_type in ["full_sensors", "imu_only"]:
        data = plot_df[plot_df["sensor_type"] == sensor_type]
        axes[0, 0].plot(
            data["threshold"],
            data["accuracy"],
            marker="o",
            label=sensor_type.replace("_", " ").title(),
            linewidth=2,
            markersize=4,
        )
    axes[0, 0].set_xlabel("Confidence Threshold")
    axes[0, 0].set_ylabel("Accuracy")
    axes[0, 0].set_title("Accuracy vs Confidence Threshold")
    axes[0, 0].legend()
    axes[0, 0].grid(True, alpha=0.3)

    # 2. H-F1 vs Threshold
    for sensor_type in ["full_sensors", "imu_only"]:
        data = plot_df[plot_df["sensor_type"] == sensor_type]
        # Filter out None values
        valid_data = data[data["h_f1"].notna()]
        if len(valid_data) > 0:
            axes[0, 1].plot(
                valid_data["threshold"],
                valid_data["h_f1"],
                marker="s",
                label=sensor_type.replace("_", " ").title(),
                linewidth=2,
                markersize=4,
            )
    axes[0, 1].set_xlabel("Confidence Threshold")
    axes[0, 1].set_ylabel("H-F1 Score")
    axes[0, 1].set_title("H-F1 Score vs Confidence Threshold")
    axes[0, 1].legend()
    axes[0, 1].grid(True, alpha=0.3)

    # 3. Constraint Violation Rate vs Threshold
    for sensor_type in ["full_sensors", "imu_only"]:
        data = plot_df[plot_df["sensor_type"] == sensor_type]
        axes[0, 2].plot(
            data["threshold"],
            data["constraint_violation_rate"] * 100,
            marker="^",
            label=sensor_type.replace("_", " ").title(),
            linewidth=2,
            markersize=4,
        )
    axes[0, 2].set_xlabel("Confidence Threshold")
    axes[0, 2].set_ylabel("Constraint Violation Rate (%)")
    axes[0, 2].set_title("Constraint Violations vs Confidence Threshold")
    axes[0, 2].legend()
    axes[0, 2].grid(True, alpha=0.3)

    # 4. Redirect Rate vs Threshold
    for sensor_type in ["full_sensors", "imu_only"]:
        data = plot_df[plot_df["sensor_type"] == sensor_type]
        axes[1, 0].plot(
            data["threshold"],
            data["redirect_rate"] * 100,
            marker="d",
            label=sensor_type.replace("_", " ").title(),
            linewidth=2,
            markersize=4,
        )
    axes[1, 0].set_xlabel("Confidence Threshold")
    axes[1, 0].set_ylabel("Redirect Rate (%)")
    axes[1, 0].set_title("Redirect Rate vs Confidence Threshold")
    axes[1, 0].legend()
    axes[1, 0].grid(True, alpha=0.3)

    # 5. Trade-off: H-F1 vs Constraint Violations (Full Sensors Only)
    full_data = plot_df[plot_df["sensor_type"] == "full_sensors"]
    valid_full_data = full_data[full_data["h_f1"].notna()]
    if len(valid_full_data) > 0:
        scatter = axes[1, 1].scatter(
            valid_full_data["constraint_violation_rate"] * 100,
            valid_full_data["h_f1"],
            c=valid_full_data["threshold"],
            cmap="viridis",
            s=50,
        )
        axes[1, 1].set_xlabel("Constraint Violation Rate (%)")
        axes[1, 1].set_ylabel("H-F1 Score")
        axes[1, 1].set_title("H-F1 vs Constraint Violations Trade-off\n(Full Sensors)")
        plt.colorbar(scatter, ax=axes[1, 1], label="Threshold")

        # Annotate some key points
        for _, row in valid_full_data.iterrows():
            if row["threshold"] % 20 == 0:  # Annotate every 20th threshold
                axes[1, 1].annotate(
                    f"{int(row['threshold'])}",
                    (row["constraint_violation_rate"] * 100, row["h_f1"]),
                    xytext=(5, 5),
                    textcoords="offset points",
                    fontsize=8,
                )

    # 6. Summary Performance (Combined Score)
    for sensor_type in ["full_sensors", "imu_only"]:
        data = plot_df[plot_df["sensor_type"] == sensor_type]
        valid_data = data[data["h_f1"].notna()]
        if len(valid_data) > 0:
            # Create a combined score: H-F1 - penalty for constraint violations
            penalty_weight = (
                0.1  # Adjust this to balance H-F1 vs constraint satisfaction
            )
            combined_score = (
                valid_data["h_f1"]
                - penalty_weight * valid_data["constraint_violation_rate"]
            )
            axes[1, 2].plot(
                valid_data["threshold"],
                combined_score,
                marker="*",
                label=sensor_type.replace("_", " ").title(),
                linewidth=2,
                markersize=6,
            )
    axes[1, 2].set_xlabel("Confidence Threshold")
    axes[1, 2].set_ylabel("Combined Score (H-F1 - 0.1×Violation Rate)")
    axes[1, 2].set_title("Combined Performance Score")
    axes[1, 2].legend()
    axes[1, 2].grid(True, alpha=0.3)

    plt.tight_layout()
    plt.savefig(
        output_dir / "threshold_sweep_analysis.png", dpi=300, bbox_inches="tight"
    )
    plt.close()

    print(
        f"Threshold sweep visualization saved to {output_dir / 'threshold_sweep_analysis.png'}"
    )

    # Create a summary table of top performers
    full_sensor_data = plot_df[plot_df["sensor_type"] == "full_sensors"]
    valid_full_data = full_sensor_data[full_sensor_data["h_f1"].notna()]

    if len(valid_full_data) > 0:
        # Sort by H-F1 score
        top_h_f1 = valid_full_data.nlargest(5, "h_f1")[
            ["threshold", "accuracy", "h_f1", "constraint_violation_rate"]
        ]

        # Sort by combined score
        penalty_weight = 0.1
        valid_full_data_copy = valid_full_data.copy()
        valid_full_data_copy["combined_score"] = (
            valid_full_data_copy["h_f1"]
            - penalty_weight * valid_full_data_copy["constraint_violation_rate"]
        )
        top_combined = valid_full_data_copy.nlargest(5, "combined_score")[
            [
                "threshold",
                "accuracy",
                "h_f1",
                "constraint_violation_rate",
                "combined_score",
            ]
        ]

        print(f"\nTOP 5 THRESHOLDS BY H-F1 SCORE (Full Sensors):")
        print(top_h_f1.to_string(index=False, float_format=lambda x: f"{x:.4f}"))

        print(f"\nTOP 5 THRESHOLDS BY COMBINED SCORE (Full Sensors):")
        print(top_combined.to_string(index=False, float_format=lambda x: f"{x:.4f}"))


def main():
    """Run streaming postprocessor on hill climbed ensemble predictions."""
    print("=" * 60)
    print("STREAMING POSTPROCESSOR")
    print("=" * 60)

    ensemble_predictions_path = (
        "/workspace/artifacts/ensemble_results/ensemble_predictions.csv"
    )
    train_csv_path = "/workspace/data/train.csv"
    output_dir = Path("/workspace/artifacts/streaming_results")

    output_dir.mkdir(parents=True, exist_ok=True)

    if not Path(ensemble_predictions_path).exists():
        print(f"Error: Ensemble predictions not found at {ensemble_predictions_path}")
        print("Please run the hill climbing ensemble optimization first.")
        return

    print(f"Loading training data from {train_csv_path}")
    train_df = pd.read_csv(train_csv_path)
    print(f"Loaded {len(train_df)} training samples")

    sequence_info = train_df[
        ["sequence_id", "subject", "orientation"]
    ].drop_duplicates()
    print(f"Found {len(sequence_info)} unique sequences")

    print(f"Loading ensemble predictions from {ensemble_predictions_path}")
    ensemble_df = pd.read_csv(ensemble_predictions_path)
    print(f"Loaded {len(ensemble_df)} ensemble predictions")

    prob_cols = [
        col for col in ensemble_df.columns if col.startswith("prob_full_sensors_")
    ]
    gesture_classes = [col.replace("prob_full_sensors_", "") for col in prob_cols]
    print(f"Found {len(gesture_classes)} gesture classes: {gesture_classes}")

    print("\nAnalyzing valid gesture-orientation combinations...")
    combinations_df = analyze_gesture_orientation_combinations(train_df)
    combinations_df.to_csv(
        output_dir / "gesture_orientation_combinations.csv", index=False
    )
    print(
        f"Saved combination analysis to {output_dir / 'gesture_orientation_combinations.csv'}"
    )

    print("\nPreparing data for threshold sweep...")
    prepared_data, valid_combinations = prepare_streaming_data(
        ensemble_df, sequence_info, combinations_df
    )

    threshold_range = list(range(0, 101, 5))
    print(f"\nTesting {len(threshold_range)} thresholds: {threshold_range}")

    threshold_results = {}
    best_threshold = None
    best_score = -1

    print("\nRunning threshold sweep...")
    sweep_start_time = time.time()
    for threshold in threshold_range:
        print(f"--- Testing threshold: {threshold} ---", end=" ")

        streaming_results = simulate_streaming_postprocessing_single_threshold(
            prepared_data, gesture_classes, valid_combinations, confidence_threshold=threshold
        )

        threshold_metrics = {"full_sensors": {}, "imu_only": {}}

        for sensor_type in ["full_sensors", "imu_only"]:
            stats = streaming_results[sensor_type]["stats"]
            streaming_df = streaming_results[sensor_type]["df"]

            try:
                h_f1 = CompetitionMetric().calculate_hierarchical_f1(
                    pd.DataFrame({"gesture": streaming_df["true_gesture"]}),
                    pd.DataFrame({"gesture": streaming_df["final_prediction"]}),
                )
            except Exception as e:
                h_f1 = None
                print(
                    f"Warning: Could not calculate H-F1 for {sensor_type} at threshold {threshold} - {str(e)}"
                )

            threshold_metrics[sensor_type] = {
                "accuracy": stats["accuracy"],
                "h_f1": h_f1,
                "redirect_rate": stats["redirect_rate"],
                "replacement_rate": stats["replacement_rate"],
                "constraint_violation_rate": stats["constraint_violation_rate"],
                "total_constraint_violations": stats["total_constraint_violations"],
                "total_displacements": stats["total_displacements"],
                "history_acceptance_rate": stats["history_acceptance_rate"],
                "impossible_original_redirects": stats.get(
                    "impossible_original_redirects", 0
                ),
                "invalid_combination_skips": stats.get("invalid_combination_skips", 0),
            }

            if threshold % 20 == 0 or threshold == best_score:
                print(
                    f"\n  {sensor_type}: Acc={stats['accuracy']:.4f}, H-F1={format_metric(h_f1)}, "
                    f"Violations={stats['constraint_violations']}/{stats['total_predictions']} ({stats['constraint_violation_rate']:.1%})"
                )
            elif threshold == threshold_range[-1]:
                print("Done!")
            else:
                print(".", end="")

        threshold_results[threshold] = threshold_metrics

        primary_score = threshold_metrics["full_sensors"]["h_f1"]
        if primary_score is not None and primary_score > best_score:
            best_score = primary_score
            best_threshold = threshold

    sweep_end_time = time.time()
    sweep_duration = sweep_end_time - sweep_start_time
    print(f"\n" + "=" * 50)
    print("THRESHOLD SWEEP RESULTS")
    print(
        f"Completed {len(threshold_range)} thresholds in {sweep_duration:.1f} seconds ({sweep_duration / len(threshold_range):.1f}s per threshold)"
    )
    print("=" * 50)

    sweep_results = []
    for threshold, metrics in threshold_results.items():
        for sensor_type in ["full_sensors", "imu_only"]:
            m = metrics[sensor_type]
            sweep_results.append(
                {
                    "threshold": threshold,
                    "sensor_type": sensor_type,
                    "accuracy": m["accuracy"],
                    "h_f1": m["h_f1"],
                    "redirect_rate": m["redirect_rate"],
                    "replacement_rate": m["replacement_rate"],
                    "constraint_violation_rate": m["constraint_violation_rate"],
                    "total_constraint_violations": m["total_constraint_violations"],
                    "total_displacements": m["total_displacements"],
                    "history_acceptance_rate": m["history_acceptance_rate"],
                    "impossible_original_redirects": m.get(
                        "impossible_original_redirects", 0
                    ),
                    "invalid_combination_skips": m.get("invalid_combination_skips", 0),
                }
            )

    sweep_df = pd.DataFrame(sweep_results)
    sweep_df.to_csv(output_dir / "threshold_sweep_results.csv", index=False)
    print(
        f"Threshold sweep results saved to {output_dir / 'threshold_sweep_results.csv'}"
    )

    # Print best results
    print(f"\nBEST THRESHOLD: {best_threshold}")
    print(f"Best H-F1 Score (full_sensors): {best_score:.4f}")

    if best_threshold is not None:
        best_metrics = threshold_results[best_threshold]
        print("\nBest threshold detailed results:")
        for sensor_type in ["full_sensors", "imu_only"]:
            m = best_metrics[sensor_type]
            print(f"  {sensor_type}:")
            print(f"    Accuracy: {m['accuracy']:.4f}")
            print(f"    H-F1: {format_metric(m['h_f1'])}")
            print(
                f"    Constraint violations: {m['total_constraint_violations']} ({m['constraint_violation_rate']:.1%})"
            )
            print(f"    Redirects: {m['redirect_rate']:.1%}")
            print(f"    Replacements: {m['replacement_rate']:.1%}")
            print(f"    Invalid combination skips: {m['invalid_combination_skips']}")
            print(
                f"    Impossible original redirects: {m['impossible_original_redirects']}"
            )

    # Create visualization of sweep results
    create_threshold_sweep_visualization(threshold_results, output_dir)

    # Use best threshold for final detailed analysis
    print(f"\n" + "=" * 50)
    print(f"DETAILED ANALYSIS WITH BEST THRESHOLD ({best_threshold})")
    print("=" * 50)

    streaming_results = simulate_streaming_postprocessing_single_threshold(
        prepared_data,
        gesture_classes,
        valid_combinations,
        confidence_threshold=best_threshold,
    )

    # Show detailed analysis for best threshold
    for sensor_type in ["full_sensors", "imu_only"]:
        stats = streaming_results[sensor_type]["stats"]
        processor = streaming_results[sensor_type]["processor"]

        print(f"\n{sensor_type.upper()} DETAILED RESULTS:")
        print(f"  Accuracy: {stats['accuracy']:.4f}")
        print(f"  H-F1: {format_metric(best_metrics[sensor_type]['h_f1'])}")
        print(
            f"  Redirected predictions: {stats['redirected_predictions']}/{stats['total_predictions']} ({stats['redirect_rate']:.1%})"
        )
        print(
            f"  Replaced predictions: {stats['replacement_predictions']}/{stats['total_predictions']} ({stats['replacement_rate']:.1%})"
        )
        print(
            f"  Constraint violations: {stats['constraint_violations']}/{stats['total_predictions']} ({stats['constraint_violation_rate']:.1%})"
        )
        print(f"  Total displacements: {stats['total_displacements']}")
        print(f"  Invalid combination skips: {stats['invalid_combination_skips']}")
        print(
            f"  Impossible originals redirected: {stats['impossible_original_redirects']}"
        )

    # Save streaming results and show analysis
    for sensor_type in ["full_sensors", "imu_only"]:
        streaming_df = streaming_results[sensor_type]["df"]
        streaming_df.to_csv(
            output_dir / f"streaming_predictions_{sensor_type}.csv", index=False
        )

        # Save complete prediction history
        processor = streaming_results[sensor_type]["processor"]
        complete_history = processor.get_complete_prediction_history()
        history_df = pd.DataFrame(complete_history)
        history_df.to_csv(
            output_dir / f"streaming_complete_history_{sensor_type}.csv", index=False
        )
        print(
            f"Complete prediction history saved: {output_dir / f'streaming_complete_history_{sensor_type}.csv'}"
        )

        # Save constraint violation log
        if len(processor.constraint_violation_log) > 0:
            violation_df = pd.DataFrame(processor.constraint_violation_log)
            violation_df.to_csv(
                output_dir / f"streaming_constraint_violations_{sensor_type}.csv",
                index=False,
            )
            print(
                f"Constraint violations saved: {output_dir / f'streaming_constraint_violations_{sensor_type}.csv'}"
            )

        # Save gesture history summary
        history_summary = processor.get_gesture_history_summary()
        summary_data = []
        for key, summary in history_summary.items():
            summary_data.append(
                {
                    "subject": summary["subject"],
                    "orientation": summary["orientation"],
                    "gesture": summary["gesture"],
                    "total_attempts": summary["total_attempts"],
                    "accepted_count": summary["accepted_count"],
                    "rejected_count": summary["rejected_count"],
                    "acceptance_rate": summary["acceptance_rate"],
                    "min_confidence": summary["min_confidence"],
                    "max_confidence": summary["max_confidence"],
                    "mean_confidence": summary["mean_confidence"],
                }
            )

        summary_df = pd.DataFrame(summary_data)
        summary_df.to_csv(
            output_dir / f"streaming_gesture_summary_{sensor_type}.csv", index=False
        )
        print(
            f"Gesture history summary saved: {output_dir / f'streaming_gesture_summary_{sensor_type}.csv'}"
        )

    print(f"\nStreaming postprocessor results saved to {output_dir}")
    print(f"\nFILES CREATED:")
    print(f"- gesture_orientation_combinations.csv (valid combinations analysis)")
    print(f"- threshold_sweep_results.csv (detailed results for all thresholds)")
    print(f"- threshold_sweep_analysis.png (comprehensive visualization)")
    print(f"- streaming_predictions_full_sensors.csv (final predictions)")
    print(f"- streaming_predictions_imu_only.csv (final predictions)")
    print(f"- streaming_complete_history_*.csv (ALL prediction attempts)")
    print(f"- streaming_constraint_violations_*.csv (constraint violations)")
    print(f"- streaming_gesture_summary_*.csv (gesture attempt summaries)")


if __name__ == "__main__":
    main()