Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/components/DatasetMetadataModal.module.css
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,18 @@
background: var(--agml-surface-soft);
}

.snippetLabel {
font-family: var(--ifm-code-font-family);
font-size: 0.68rem;
font-weight: 600;
color: var(--agml-muted);
margin: 0.9rem 0 0;
}

.snippetLabel:first-of-type {
margin-top: 0;
}

.snippetRow {
display: flex;
align-items: flex-start;
Expand Down
61 changes: 46 additions & 15 deletions src/components/DatasetMetadataModal.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,12 @@ function formatLoaderInstructions(dataset: Dataset) {
code: `from datasets import load_dataset\nloader = load_dataset("Project-AgML/${dataset.name}")`,
};
}
if (dataset.sensor_modality?.toLowerCase().includes('multispectral')) {
return {
title: 'Load from Hugging Face',
code: `from agml.data.multispectral_hf_loader import MultispectralDataLoader\nloader = MultispectralDataLoader("Project-AgML/${dataset.name}")`,
};
}
return {
title: 'Load from Hugging Face',
code: `from agml.data.hf_loader import HuggingFaceDataLoader\nloader = HuggingFaceDataLoader("Project-AgML/${dataset.name}")`,
Expand Down Expand Up @@ -1166,6 +1172,7 @@ export function DatasetMetadataModal({
const [classesExpanded, setClassesExpanded] = useState(false);
const [copied, setCopied] = useState(false);
const [citationCopied, setCitationCopied] = useState(false);
const [bibtexCopied, setBibtexCopied] = useState(false);

const [expandedRowKeys, setExpandedRowKeys] = useState<Set<string>>(new Set());
const toggleExpandedRow = (key: string) => {
Expand Down Expand Up @@ -1595,23 +1602,47 @@ export function DatasetMetadataModal({
)}
</section>

{dataset.citation && (
{(dataset.citation || dataset.bibtex) && (
<section className={styles.section}>
<h3 className={styles.sectionTitle}>Citation</h3>
<div className={styles.snippetRow}>
<pre className={styles.citationCode}>{dataset.citation}</pre>
<button
type="button"
className={styles.snippetCopyButton}
onClick={() => {
navigator.clipboard.writeText(dataset.citation ?? '');
setCitationCopied(true);
setTimeout(() => setCitationCopied(false), 1500);
}}
>
{citationCopied ? 'Copied!' : 'Copy'}
</button>
</div>
{dataset.citation && (
<>
{dataset.bibtex && <p className={styles.snippetLabel}>Citation</p>}
<div className={styles.snippetRow}>
<pre className={styles.citationCode}>{dataset.citation}</pre>
<button
type="button"
className={styles.snippetCopyButton}
onClick={() => {
navigator.clipboard.writeText(dataset.citation ?? '');
setCitationCopied(true);
setTimeout(() => setCitationCopied(false), 1500);
}}
>
{citationCopied ? 'Copied!' : 'Copy'}
</button>
</div>
</>
)}
{dataset.bibtex && (
<>
{dataset.citation && <p className={styles.snippetLabel}>BibTeX</p>}
<div className={styles.snippetRow}>
<pre className={styles.citationCode}>{dataset.bibtex}</pre>
<button
type="button"
className={styles.snippetCopyButton}
onClick={() => {
navigator.clipboard.writeText(dataset.bibtex ?? '');
setBibtexCopied(true);
setTimeout(() => setBibtexCopied(false), 1500);
}}
>
{bibtexCopied ? 'Copied!' : 'Copy'}
</button>
</div>
</>
)}
</section>
)}
</div>
Expand Down
30 changes: 26 additions & 4 deletions src/lib/datasets.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ export interface Dataset {
augmented_counterpart: string | null;
crop_types: string[] | null;
sensor_modality: string | null;
// Derived: `sensor_modality` split into individual lowercase values (it's stored as a single
// comma-joined string, e.g. "rgb, thermal") so the search page can filter by modality even
// when a dataset has more than one.
display_modality: string[] | null;
real_or_synthetic: string | null;
platform: string[] | null;
input_data_format: string | null;
Expand All @@ -34,6 +38,10 @@ export interface Dataset {
point_cloud_sample_url: string | null;
license: string | null;
citation: string | null;
// BibTeX entry for the article the dataset was sourced from, scraped from the dataset's
// Hugging Face README (or resolved from a DOI found there) — distinct from `citation`,
// which is free-form and not always valid BibTeX.
bibtex: string | null;
parent_dataset?: string | null;
// Derived (not present in the source manifests): for a dataset whose child variants are
// hidden from search results (see isChildDataset/includeChildren), the crop/species names
Expand Down Expand Up @@ -158,6 +166,10 @@ function normalizeDataset(raw: unknown): Dataset | null {
raw.ml_task,
raw.task,
);
const sensorModality =
firstString(raw.sensor_modality, raw.sensor, raw.modality) ??
toStringArray(raw.sensor_modality)?.join(", ") ??
null;
const qaType = toStringArray(raw.qa_type);
const taskDimensions = toStringArray(raw.task_dimensions);
const isVlm =
Expand Down Expand Up @@ -188,10 +200,17 @@ function normalizeDataset(raw: unknown): Dataset | null {
?.map((c) => c.trim().toLowerCase())
.filter((c) => c && !/^\+.*\b(more|others?)\b/i.test(c)) ??
null,
sensor_modality:
firstString(raw.sensor_modality, raw.sensor, raw.modality) ??
toStringArray(raw.sensor_modality)?.join(", ") ??
null,
sensor_modality: sensorModality,
display_modality: sensorModality
? Array.from(
new Set(
sensorModality
.split(",")
.map((entry) => entry.trim().toLowerCase())
.filter(Boolean),
),
)
: null,
real_or_synthetic: firstString(
raw.real_or_synthetic,
raw.real_synthetic,
Expand Down Expand Up @@ -225,6 +244,7 @@ function normalizeDataset(raw: unknown): Dataset | null {
),
license: firstString(raw.license),
citation: firstString(raw.citation),
bibtex: firstString(raw.bibtex),
parent_dataset: firstString(raw.parent_dataset, raw.parentDataset),
child_crop_types: null,
zip_size_bytes: toNumber(raw.zip_size_bytes ?? raw.zipSizeBytes),
Expand Down Expand Up @@ -261,6 +281,7 @@ function mergeDataset(current: Dataset, incoming: Dataset): Dataset {
current.augmented_counterpart ?? incoming.augmented_counterpart,
crop_types: current.crop_types ?? incoming.crop_types,
sensor_modality: current.sensor_modality ?? incoming.sensor_modality,
display_modality: current.display_modality ?? incoming.display_modality,
real_or_synthetic:
current.real_or_synthetic ?? incoming.real_or_synthetic,
platform:
Expand Down Expand Up @@ -292,6 +313,7 @@ function mergeDataset(current: Dataset, incoming: Dataset): Dataset {
current.point_cloud_sample_url ?? incoming.point_cloud_sample_url,
license: current.license ?? incoming.license,
citation: current.citation ?? incoming.citation,
bibtex: current.bibtex ?? incoming.bibtex,
parent_dataset: current.parent_dataset ?? incoming.parent_dataset,
child_crop_types: current.child_crop_types ?? incoming.child_crop_types,
zip_size_bytes: current.zip_size_bytes ?? incoming.zip_size_bytes,
Expand Down
8 changes: 8 additions & 0 deletions src/pages/datasets/index.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,14 @@ const DATASET_FILTERS: DatasetFilterConfig[] = [
mode: 'containsAny',
formatOption: (value) => toDisplayLabel(value),
},
{
key: 'modality',
label: 'Modality',
field: 'display_modality',
kind: 'dropdown',
mode: 'containsAny',
formatOption: (value) => toDisplayLabel(value),
},
{
key: 'environment',
label: 'Environment',
Expand Down
2 changes: 1 addition & 1 deletion static/data/embeddings/meta.json

Large diffs are not rendered by default.

Binary file modified static/data/embeddings/vectors.bin
Binary file not shown.
Loading
Loading