External sources

scripts/generate-datasets.mjs

@@ -323,7 +323,12 @@ async function generateDatasets() {
   const datasets = normalizeManifest(readJson(datasetsPath));
   const hfDatasetsRaw = readJson(hfDatasetsPath);
   const hfDatasets = Array.isArray(hfDatasetsRaw)
-    ? hfDatasetsRaw.map((entry) => ({ ...entry, source: 'huggingface' }))
+    ? hfDatasetsRaw.map((entry) => ({
+        ...entry,
+        // Only default to 'huggingface' — externally hosted entries (e.g. source: 'doi') must
+        // keep their own source so the site doesn't present them as rehosted AgML data.
+        source: typeof entry.source === 'string' && entry.source.trim() ? entry.source : 'huggingface',
+      }))
     : hfDatasetsRaw;
 
   writeJson(datasetsPath, datasets);

src/components/DatasetMetadataModal.module.css

@@ -1503,3 +1503,14 @@
 		grid-template-columns: 1fr !important;
 	}
 }
+
+.externalDisclaimer {
+	margin: 0.5rem 0 0;
+	padding: 0.6rem 0.8rem;
+	border-left: 3px solid var(--agml-border-strong);
+	border-radius: 4px;
+	background: var(--agml-tag-bg);
+	color: var(--agml-tag-fg);
+	font-size: 0.82rem;
+	line-height: 1.45;
+}

src/components/DatasetMetadataModal.tsx

@@ -105,6 +105,11 @@ function formatMetricScore(entry: { metrics: { key: string; label: string; value
 	return entry.score != null ? entry.score.toLocaleString() : '—';
 }
 
+// Datasets AgML links to but does not rehost (e.g. source "doi"): no AgML loader exists for them.
+function isExternalDataset(dataset: Dataset) {
+	return dataset.source != null && dataset.source !== 'agml' && dataset.source !== 'huggingface';
+}
+
 function formatLoaderInstructions(dataset: Dataset) {
 	if (dataset.source === 'huggingface') {
 		if(dataset.dataset_type === 'vlm') {
@@ -1199,6 +1204,7 @@ export function DatasetMetadataModal({
 				] as const)
 			: ([['Number of images', formatImageCount(dataset.num_images)]] as const)),
 		['Size', formatBytesDecimal(dataset.zip_size_bytes)],
+		...(dataset.num_files != null ? ([['Number of files', dataset.num_files.toLocaleString()]] as const) : []),
 		...(dataset.augmented_num_images != null
 			? ([['Augmented images', formatImageCount(dataset.augmented_num_images)]] as const)
 			: []),
@@ -1423,6 +1429,16 @@ export function DatasetMetadataModal({
 					)}
 				</section>
 
+				{isExternalDataset(dataset) ? (
+					<section className={styles.section}>
+						<h3 className={styles.sectionTitle}>Get the data</h3>
+						<p className={styles.externalDisclaimer}>
+							This dataset is hosted externally and is not rehosted by AgML. AgML makes no claims about its
+							quality, accuracy, or availability — please review the original source
+							documentation before use.
+						</p>
+					</section>
+				) : (
 				<section className={styles.section}>
 					<h3 className={styles.sectionTitle}>{loader.title}</h3>
 					<div className={styles.snippetRow}>
@@ -1440,14 +1456,20 @@ export function DatasetMetadataModal({
 						</button>
 					</div>
 				</section>
+				)}
 
-				{(dataset.documentation || dataset.hf_link) && (
+				{(dataset.documentation || dataset.hf_link || (isExternalDataset(dataset) && dataset.data_link)) && (
 					<div className={styles.linkRow}>
 						{dataset.documentation && (
 							<a className={styles.externalLink} href={dataset.documentation} target="_blank" rel="noreferrer">
 								Open source documentation
 							</a>
 						)}
+						{isExternalDataset(dataset) && dataset.data_link && (
+							<a className={styles.externalLink} href={dataset.data_link} target="_blank" rel="noreferrer">
+								Download from original source
+							</a>
+						)}
 						{dataset.hf_link && (
 							<a className={styles.hfLink} href={dataset.hf_link} target="_blank" rel="noreferrer">
 								View on Hugging Face

src/lib/datasets.ts

@@ -50,6 +50,11 @@ export interface Dataset {
     child_crop_types?: string[] | null;
     zip_size_bytes?: number | null;
     hf_link?: string | null;
+    // Externally hosted datasets (source "doi"/external): where the data actually lives, and
+    // whether AgML mirrors it unmodified. AgML does not rehost these.
+    data_link?: string | null;
+    num_files?: number | null;
+    hosted_in_original_form?: boolean | null;
     // VLM (vision-language) dataset fields — absent/null on plain vision datasets.
     dataset_type: "vision" | "vlm";
     qa_type: string[] | null;
@@ -247,7 +252,11 @@ function normalizeDataset(raw: unknown): Dataset | null {
         bibtex: firstString(raw.bibtex),
         parent_dataset: firstString(raw.parent_dataset, raw.parentDataset),
         child_crop_types: null,
-        zip_size_bytes: toNumber(raw.zip_size_bytes ?? raw.zipSizeBytes),
+        zip_size_bytes: toNumber(raw.zip_size_bytes ?? raw.zipSizeBytes ?? raw.data_size_bytes),
+        data_link: firstString(raw.data_link),
+        num_files: toNumber(raw.num_files),
+        hosted_in_original_form:
+            typeof raw.hosted_in_original_form === "boolean" ? raw.hosted_in_original_form : null,
         hf_link: firstString(raw.hf_link, raw.huggingface_link, raw.hf_url),
         dataset_type: isVlm ? "vlm" : "vision",
         qa_type: qaType,
@@ -318,6 +327,9 @@ function mergeDataset(current: Dataset, incoming: Dataset): Dataset {
         child_crop_types: current.child_crop_types ?? incoming.child_crop_types,
         zip_size_bytes: current.zip_size_bytes ?? incoming.zip_size_bytes,
         hf_link: current.hf_link ?? incoming.hf_link,
+        data_link: current.data_link ?? incoming.data_link,
+        num_files: current.num_files ?? incoming.num_files,
+        hosted_in_original_form: current.hosted_in_original_form ?? incoming.hosted_in_original_form,
         dataset_type:
             current.dataset_type === "vlm" || incoming.dataset_type === "vlm"
                 ? "vlm"

static/data/dataset_history.json

@@ -1,6 +1,6 @@
 {
   "_readme": "Hand-maintained going forward — do not regenerate. To add a snapshot, run `python3 scripts/append_dataset_snapshot.py` (counts the live static/data/{datasets,hf_datasets}.json the same way src/lib/datasets.ts computeDatasetStats does: top-level datasets only, iNatAg-mini's images excluded) or append to `points` by hand: { period: \"<short x-axis label>\", date: \"<YYYY-MM-DD>\", datasetCount: <n>, imageCount: <n> }.",
-  "generatedAt": "2026-09-24",
+  "generatedAt": "2026-09-25",
   "annotations": [
     {
       "atPeriod": "May 30",
@@ -195,6 +195,12 @@
       "date": "2026-09-24",
       "datasetCount": 370,
       "imageCount": 6940769
+    },
+    {
+      "period": "Sep 25",
+      "date": "2026-09-25",
+      "datasetCount": 408,
+      "imageCount": 7011093
     }
   ]
 }