Support FHIR import using fhir.resources

[Zethson]

Description of feature

https://github.com/nazrulworld/fhir.resources

That is the way to go. The nice thing is that this validates external FHIR for us. We need to find ways to generate EHRData objects from that. Should be possible. We might need some code that takes FHIR and create long or wide Pandas DataFrames and then EHRData objects or we go directly to EHRData.

[Zethson]

A LLM generated MVP:

import numpy as np
import pandas as pd
from fhir.resources.bundle import Bundle
from fhir.resources.patient import Patient
from fhir.resources.observation import Observation

def parse_fhir(fhir_data):
    """FHIR to EHRData parser with 3D temporal support."""
    
    if isinstance(fhir_data, dict) and fhir_data.get('resourceType') == 'Bundle':
        bundle = Bundle(**fhir_data)
        resources = [entry.resource for entry in bundle.entry]
    else:
        resources = [fhir_data] if not isinstance(fhir_data, list) else fhir_data
    
    patients = {}
    observations = []
    all_timepoints = set()
    
    for resource in resources:
        resource_type = resource.get_resource_type()
        print(f"Processing: {resource_type}")
        
        if resource_type == 'Patient':
            patients[resource.id] = {
                'gender': resource.gender,
                'birth_date': getattr(resource, 'birthDate', None)
            }
            print(f"  Patient {resource.id}: {patients[resource.id]}")
            
        elif resource_type == 'Observation':
            patient_id = resource.subject.reference.split('/')[-1]
            code = resource.code.coding[0].code
            value = resource.valueQuantity.value
            
            # Extract timepoint
            timepoint = None
            if hasattr(resource, 'effectiveDateTime') and resource.effectiveDateTime:
                timepoint = resource.effectiveDateTime.date().isoformat()
                all_timepoints.add(timepoint)
            
            observations.append({
                'patient_id': patient_id,
                'variable': code,
                'value': float(value),
                'timepoint': timepoint
            })
            print(f"  Observation: {patient_id} -> {code} = {value} at {timepoint}")
    
    print(f"Found {len(patients)} patients, {len(observations)} observations")
    print(f"Timepoints: {sorted(all_timepoints) if all_timepoints else ['No timepoints']}")
    
    if not observations:
        print("No observations found!")
        return None
    
    # Build data structures
    patient_ids = sorted(patients.keys())
    variables = sorted(set(obs['variable'] for obs in observations))
    timepoints = sorted(all_timepoints) if all_timepoints else [None]
    
    print(f"Patients: {patient_ids}")
    print(f"Variables: {variables}")
    print(f"Timepoints: {timepoints}")
    
    # Create 3D array if we have timepoints, otherwise 2D
    if all_timepoints and len(timepoints) > 0 and timepoints[0] is not None:
        # 3D: patients x variables x timepoints
        R = np.full((len(patient_ids), len(variables), len(timepoints)), np.nan)
        
        for obs in observations:
            if obs['patient_id'] in patient_ids:
                p_idx = patient_ids.index(obs['patient_id'])
                v_idx = variables.index(obs['variable'])
                if obs['timepoint'] and obs['timepoint'] in timepoints:
                    t_idx = timepoints.index(obs['timepoint'])
                    R[p_idx, v_idx, t_idx] = obs['value']
        
        # Create X as mean over time
        X = np.nanmean(R, axis=2)
        
        print(f"R tensor shape: {R.shape}")
        print(f"R tensor:\n{R}")
        print(f"X matrix shape: {X.shape}")
        print(f"X matrix:\n{X}")
        
        # Create DataFrames
        obs_df = pd.DataFrame(index=patient_ids)
        for pid in patient_ids:
            if pid in patients:
                for key, value in patients[pid].items():
                    obs_df.loc[pid, key] = value
        
        var_df = pd.DataFrame(index=variables)
        tem_df = pd.DataFrame(index=timepoints)  # Time DataFrame
        
        try:
            from ehrdata import EHRData
            edata = EHRData(X=X, R=R, obs=obs_df, var=var_df, tem=tem_df)
            print(f"Created 3D EHRData: {edata.shape}")
            return edata
        except ImportError:
            print("EHRData not available")
            return {'X': X, 'R': R, 'obs': obs_df, 'var': var_df, 'tem': tem_df}
    
    else:
        # 2D: patients x variables (no time dimension)
        X = np.full((len(patient_ids), len(variables)), np.nan)
        
        for obs in observations:
            if obs['patient_id'] in patient_ids:
                p_idx = patient_ids.index(obs['patient_id'])
                v_idx = variables.index(obs['variable'])
                X[p_idx, v_idx] = obs['value']
        
        print(f"X matrix shape: {X.shape}")
        print(f"X matrix:\n{X}")
        
        # Create DataFrames
        obs_df = pd.DataFrame(index=patient_ids)
        for pid in patient_ids:
            if pid in patients:
                for key, value in patients[pid].items():
                    obs_df.loc[pid, key] = value
        
        var_df = pd.DataFrame(index=variables)
        
        try:
            from ehrdata import EHRData
            edata = EHRData(X=X, obs=obs_df, var=var_df)
            print(f"Created 2D EHRData: {edata.shape}")
            return edata
        except ImportError:
            print("EHRData not available")
            return {'X': X, 'obs': obs_df, 'var': var_df}

# Test with 3x3 temporal data
test_bundle_3d = {
    "resourceType": "Bundle",
    "type": "collection",
    "entry": [
        # 3 Patients
        {"resource": {"resourceType": "Patient", "id": "p1", "gender": "male", "birthDate": "1980-01-01"}},
        {"resource": {"resourceType": "Patient", "id": "p2", "gender": "female", "birthDate": "1975-05-15"}},
        {"resource": {"resourceType": "Patient", "id": "p3", "gender": "male", "birthDate": "1990-12-20"}},
        
        # Blood pressure observations for all patients across 3 timepoints
        {"resource": {"resourceType": "Observation", "status": "final", "subject": {"reference": "Patient/p1"}, 
                     "code": {"coding": [{"code": "blood_pressure"}]}, "valueQuantity": {"value": 120}, 
                     "effectiveDateTime": "2024-01-01T10:00:00Z"}},
        {"resource": {"resourceType": "Observation", "status": "final", "subject": {"reference": "Patient/p1"}, 
                     "code": {"coding": [{"code": "blood_pressure"}]}, "valueQuantity": {"value": 125}, 
                     "effectiveDateTime": "2024-02-01T10:00:00Z"}},
        {"resource": {"resourceType": "Observation", "status": "final", "subject": {"reference": "Patient/p1"}, 
                     "code": {"coding": [{"code": "blood_pressure"}]}, "valueQuantity": {"value": 130}, 
                     "effectiveDateTime": "2024-03-01T10:00:00Z"}},
        
        {"resource": {"resourceType": "Observation", "status": "final", "subject": {"reference": "Patient/p2"}, 
                     "code": {"coding": [{"code": "blood_pressure"}]}, "valueQuantity": {"value": 110}, 
                     "effectiveDateTime": "2024-01-01T10:00:00Z"}},
        {"resource": {"resourceType": "Observation", "status": "final", "subject": {"reference": "Patient/p2"}, 
                     "code": {"coding": [{"code": "blood_pressure"}]}, "valueQuantity": {"value": 115}, 
                     "effectiveDateTime": "2024-02-01T10:00:00Z"}},
        
        {"resource": {"resourceType": "Observation", "status": "final", "subject": {"reference": "Patient/p3"}, 
                     "code": {"coding": [{"code": "blood_pressure"}]}, "valueQuantity": {"value": 140}, 
                     "effectiveDateTime": "2024-03-01T10:00:00Z"}},
        
        # Heart rate observations
        {"resource": {"resourceType": "Observation", "status": "final", "subject": {"reference": "Patient/p1"}, 
                     "code": {"coding": [{"code": "heart_rate"}]}, "valueQuantity": {"value": 72}, 
                     "effectiveDateTime": "2024-01-01T10:00:00Z"}},
        {"resource": {"resourceType": "Observation", "status": "final", "subject": {"reference": "Patient/p2"}, 
                     "code": {"coding": [{"code": "heart_rate"}]}, "valueQuantity": {"value": 68}, 
                     "effectiveDateTime": "2024-02-01T10:00:00Z"}},
        {"resource": {"resourceType": "Observation", "status": "final", "subject": {"reference": "Patient/p3"}, 
                     "code": {"coding": [{"code": "heart_rate"}]}, "valueQuantity": {"value": 80}, 
                     "effectiveDateTime": "2024-03-01T10:00:00Z"}},
        
        # Temperature observations
        {"resource": {"resourceType": "Observation", "status": "final", "subject": {"reference": "Patient/p1"}, 
                     "code": {"coding": [{"code": "temperature"}]}, "valueQuantity": {"value": 98.6}, 
                     "effectiveDateTime": "2024-02-01T10:00:00Z"}},
        {"resource": {"resourceType": "Observation", "status": "final", "subject": {"reference": "Patient/p2"}, 
                     "code": {"coding": [{"code": "temperature"}]}, "valueQuantity": {"value": 99.1}, 
                     "effectiveDateTime": "2024-01-01T10:00:00Z"}},
        {"resource": {"resourceType": "Observation", "status": "final", "subject": {"reference": "Patient/p3"}, 
                     "code": {"coding": [{"code": "temperature"}]}, "valueQuantity": {"value": 97.8}, 
                     "effectiveDateTime": "2024-03-01T10:00:00Z"}},
    ]
}

print("=== 3D FHIR to EHRData Parser (3 patients x 3 variables x 3 timepoints) ===")
result = parse_fhir(test_bundle_3d)