fix: rebuild rustdoc json for different versions of a same crate

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -523,18 +523,33 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
                 vec![]
             }
             CompileMode::Doc => {
-                let path = if bcx.build_config.intent.wants_doc_json_output() {
-                    self.output_dir(unit)
-                        .join(format!("{}.json", unit.target.crate_name()))
+                let wants_json_doc = bcx.build_config.intent.wants_doc_json_output();
+
+                let path = if wants_json_doc {
+                    // Always use 'new' layout for '--output-format=json'.
+                    let crate_name = unit.target.crate_name();
+                    self.out_dir_new_layout(unit)
+                        .join(format!("{crate_name}.json"))
                 } else {
                     self.output_dir(unit)
                         .join(unit.target.crate_name())
                         .join("index.html")
                 };
 
+                // Uplift if output is json, from 'new' layout location for backward compatibility
+                // See #16773.
+                let hardlink = if wants_json_doc {
+                    Some(
+                        self.output_dir(unit)
+                            .join(format!("{}.json", unit.target.crate_name())),
+                    )
+                } else {
+                    None
+                };
+
                 let mut outputs = vec![OutputFile {
                     path,
-                    hardlink: None,
+                    hardlink,
                     export_path: None,
                     flavor: FileFlavor::Normal,
                 }];

src/cargo/core/compiler/mod.rs

@@ -868,7 +868,16 @@ fn prepare_rustdoc(build_runner: &BuildRunner<'_, '_>, unit: &Unit) -> CargoResu
     add_cap_lints(bcx, unit, &mut rustdoc);
 
     unit.kind.add_target_arg(&mut rustdoc);
-    let doc_dir = build_runner.files().output_dir(unit);
+
+    let doc_dir = if build_runner.bcx.build_config.intent.wants_doc_json_output() {
+        // Always use new layout for '--output-format=json'.
+        // In fix for https://github.com/rust-lang/cargo/issues/16291
+
+        build_runner.files().out_dir_new_layout(unit)
+    } else {
+        build_runner.files().output_dir(unit)
+    };
+
     rustdoc.arg("-o").arg(&doc_dir);
     rustdoc.args(&features_args(unit));
     rustdoc.args(&check_cfg_args(unit));
@@ -972,6 +981,7 @@ fn rustdoc(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> CargoResult<W
     let mut rustdoc = prepare_rustdoc(build_runner, unit)?;
 
     let crate_name = unit.target.crate_name();
+    let is_json_output = build_runner.bcx.build_config.intent.wants_doc_json_output();
     let doc_dir = build_runner.files().output_dir(unit);
     // Create the documentation directory ahead of time as rustdoc currently has
     // a bug where concurrent invocations will race to create this directory if
@@ -1055,13 +1065,15 @@ fn rustdoc(build_runner: &mut BuildRunner<'_, '_>, unit: &Unit) -> CargoResult<W
             }
         }
 
-        let crate_dir = doc_dir.join(&crate_name);
-        if crate_dir.exists() {
-            // Remove output from a previous build. This ensures that stale
-            // files for removed items are removed.
-            debug!("removing pre-existing doc directory {:?}", crate_dir);
-            paths::remove_dir_all(crate_dir)?;
-        }
+        if !is_json_output {
+            let crate_dir = doc_dir.join(&crate_name);
+            if crate_dir.exists() {
+                // Remove output from a previous build. This ensures that stale
+                // files for removed items are removed.
+                debug!("removing pre-existing doc directory {:?}", crate_dir);
+                paths::remove_dir_all(&crate_dir)?;
+            }
+        };
         state.running(&rustdoc);
         let timestamp = paths::set_invocation_time(&fingerprint_dir)?;
 

src/cargo/core/compiler/rustdoc.rs

@@ -250,12 +250,6 @@ pub fn add_output_format(
     build_runner: &BuildRunner<'_, '_>,
     rustdoc: &mut ProcessBuilder,
 ) -> CargoResult<()> {
-    let gctx = build_runner.bcx.gctx;
-    if !gctx.cli_unstable().unstable_options {
-        tracing::debug!("`unstable-options` is ignored, required -Zunstable-options flag");
-        return Ok(());
-    }
-
     if build_runner.bcx.build_config.intent.wants_doc_json_output() {
         rustdoc.arg("-Zunstable-options");
         rustdoc.arg("--output-format=json");

tests/testsuite/build_dir.rs

@@ -501,6 +501,45 @@ fn cargo_doc_should_output_to_target_dir() {
     assert_exists(&docs_dir.join("foo/index.html"));
 }
 
+#[cargo_test(nightly, reason = "--output-format is unstable")]
+fn cargo_rustdoc_json_should_output_to_target_dir() {
+    let p = project()
+        .file("src/lib.rs", "")
+        .file(
+            ".cargo/config.toml",
+            r#"
+            [build]
+            target-dir = "target-dir"
+            build-dir = "build-dir"
+            "#,
+        )
+        .build();
+
+    p.cargo("-Zbuild-dir-new-layout rustdoc -Zunstable-options --output-format json")
+        .masquerade_as_nightly_cargo(&["new build-dir layout", "rustdoc-output-format"])
+        .enable_mac_dsym()
+        .run();
+
+    let docs_dir = p.root().join("target-dir/doc");
+
+    assert_exists(&docs_dir);
+    assert_exists(&docs_dir.join("foo.json"));
+
+    p.root().join("build-dir").assert_build_dir_layout(str![
+        r#"
+[ROOT]/foo/build-dir/.rustc_info.json
+[ROOT]/foo/build-dir/.rustdoc_fingerprint.json
+[ROOT]/foo/build-dir/CACHEDIR.TAG
+[ROOT]/foo/build-dir/debug/.cargo-build-lock
+[ROOT]/foo/build-dir/debug/.fingerprint/foo-[HASH]/doc-lib-foo
+[ROOT]/foo/build-dir/debug/.fingerprint/foo-[HASH]/doc-lib-foo.json
+[ROOT]/foo/build-dir/debug/.fingerprint/foo-[HASH]/invoked.timestamp
+[ROOT]/foo/build-dir/debug/build/foo-[HASH]/out/foo.json
+
+"#
+    ]);
+}
+
 #[cargo_test]
 fn cargo_package_should_build_in_build_dir_and_output_to_target_dir() {
     let p = project()

tests/testsuite/build_dir_legacy.rs

@@ -459,6 +459,45 @@ fn cargo_doc_should_output_to_target_dir() {
     assert_exists(&docs_dir.join("foo/index.html"));
 }
 
+#[cargo_test(nightly, reason = "--output-format is unstable")]
+fn cargo_rustdoc_json_should_output_to_target_dir() {
+    let p = project()
+        .file("src/lib.rs", "")
+        .file(
+            ".cargo/config.toml",
+            r#"
+            [build]
+            target-dir = "target-dir"
+            build-dir = "build-dir"
+            "#,
+        )
+        .build();
+
+    p.cargo("rustdoc -Zunstable-options --output-format json")
+        .masquerade_as_nightly_cargo(&["rustdoc-output-format"])
+        .enable_mac_dsym()
+        .run();
+
+    let docs_dir = p.root().join("target-dir/doc");
+
+    assert_exists(&docs_dir);
+    assert_exists(&docs_dir.join("foo.json"));
+
+    p.root().join("build-dir").assert_build_dir_layout(str![
+        r#"
+[ROOT]/foo/build-dir/.rustc_info.json
+[ROOT]/foo/build-dir/.rustdoc_fingerprint.json
+[ROOT]/foo/build-dir/CACHEDIR.TAG
+[ROOT]/foo/build-dir/debug/.cargo-build-lock
+[ROOT]/foo/build-dir/debug/.fingerprint/foo-[HASH]/doc-lib-foo
+[ROOT]/foo/build-dir/debug/.fingerprint/foo-[HASH]/doc-lib-foo.json
+[ROOT]/foo/build-dir/debug/.fingerprint/foo-[HASH]/invoked.timestamp
+[ROOT]/foo/build-dir/debug/build/foo-[HASH]/out/foo.json
+
+"#
+    ]);
+}
+
 #[cargo_test]
 fn cargo_package_should_build_in_build_dir_and_output_to_target_dir() {
     let p = project()

tests/testsuite/rustdoc.rs

@@ -1,5 +1,7 @@
 //! Tests for the `cargo rustdoc` command.
 
+use std::fs;
+
 use crate::prelude::*;
 use cargo_test_support::str;
 use cargo_test_support::{basic_manifest, cross_compile, project};
@@ -44,7 +46,24 @@ fn rustdoc_simple_json() {
         .masquerade_as_nightly_cargo(&["rustdoc-output-format"])
         .with_stderr_data(str![[r#"
 [DOCUMENTING] foo v0.0.1 ([ROOT]/foo)
-[RUNNING] `rustdoc [..] --crate-name foo [..]-o [ROOT]/foo/target/doc [..] --output-format=json[..]
+[RUNNING] `rustdoc [..] --crate-name foo [..]-o [ROOT]/foo/target/debug/build/foo-[HASH]/out [..] --output-format=json[..]
+[FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
+[GENERATED] [ROOT]/foo/target/doc/foo.json
+
+"#]])
+        .run();
+    assert!(p.root().join("target/doc/foo.json").is_file());
+}
+
+#[cargo_test(nightly, reason = "--output-format is unstable")]
+fn rustdoc_json_with_new_layout() {
+    let p = project().file("src/lib.rs", "").build();
+
+    p.cargo("rustdoc -Z unstable-options -Z build-dir-new-layout  --output-format json -v")
+        .masquerade_as_nightly_cargo(&["rustdoc-output-format"])
+        .with_stderr_data(str![[r#"
+[DOCUMENTING] foo v0.0.1 ([ROOT]/foo)
+[RUNNING] `rustdoc [..] --crate-name foo [..]-o [ROOT]/foo/target/debug/build/foo/[HASH]/out [..] --output-format=json[..]
 [FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
 [GENERATED] [ROOT]/foo/target/doc/foo.json
 
@@ -322,3 +341,99 @@ fn fail_with_glob() {
 "#]])
         .run();
 }
+
+#[cargo_test(nightly, reason = "--output-format is unstable")]
+fn rustdoc_json_same_crate_different_version() {
+    let entry = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "entry"
+                version = "0.1.0"
+                edition = "2021"
+
+                [dependencies]
+                dep_v1 = { path = "../dep_v1", package = "dep" }
+                dep_v2 = { path = "../dep_v2", package = "dep" }
+            "#,
+        )
+        .file("src/lib.rs", "pub fn entry() {}")
+        .build();
+
+    let _dep_v1 = project()
+        .at("dep_v1")
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "dep"
+                version = "1.0.0"
+                edition = "2021"
+            "#,
+        )
+        .file("src/lib.rs", "pub fn dep_v1_fn() {}")
+        .build();
+
+    let _dep_v2 = project()
+        .at("dep_v2")
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "dep"
+                version = "2.0.0"
+                edition = "2021"
+            "#,
+        )
+        .file("src/lib.rs", "pub fn dep_v2_fn() {}")
+        .build();
+
+    entry
+        .cargo("rustdoc -v -Z unstable-options --output-format json -p dep@1.0.0")
+        .masquerade_as_nightly_cargo(&["rustdoc-output-format"])
+        .with_stderr_data(str![[r#"
+[LOCKING] 2 packages to latest compatible versions
+[DOCUMENTING] dep v1.0.0 ([ROOT]/dep_v1)
+[RUNNING] `rustdoc [..] --crate-name dep [ROOT]/dep_v1/src/lib.rs [..] --output-format=json[..]`
+[FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
+[GENERATED] [ROOT]/foo/target/doc/dep.json
+
+"#]])
+        .run();
+
+    let dep_json = fs::read_to_string(entry.root().join("target/doc/dep.json")).unwrap();
+    assert!(dep_json.contains("dep_v1_fn"));
+    assert!(!dep_json.contains("dep_v2_fn"));
+
+    entry
+        .cargo("rustdoc -v -Z unstable-options --output-format json -p dep@2.0.0")
+        .masquerade_as_nightly_cargo(&["rustdoc-output-format"])
+        .with_stderr_data(str![[r#"
+[DOCUMENTING] dep v2.0.0 ([ROOT]/dep_v2)
+[RUNNING] `rustdoc [..] --crate-name dep [ROOT]/dep_v2/src/lib.rs [..] --output-format=json[..]`
+[FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
+[GENERATED] [ROOT]/foo/target/doc/dep.json
+
+"#]])
+        .run();
+
+    let dep_json = fs::read_to_string(entry.root().join("target/doc/dep.json")).unwrap();
+    assert!(!dep_json.contains("dep_v1_fn"));
+    assert!(dep_json.contains("dep_v2_fn"));
+
+    entry
+        .cargo("rustdoc -v -Z unstable-options --output-format json -p dep@1.0.0")
+        .masquerade_as_nightly_cargo(&["rustdoc-output-format"])
+        .with_stderr_data(str![[r#"
+[FRESH] dep v1.0.0 ([ROOT]/dep_v1)
+[FINISHED] `dev` profile [unoptimized + debuginfo] target(s) in [ELAPSED]s
+[GENERATED] [ROOT]/foo/target/doc/dep.json
+
+"#]])
+        .run();
+
+    let dep_json = fs::read_to_string(entry.root().join("target/doc/dep.json")).unwrap();
+    assert!(dep_json.contains("dep_v1_fn"));
+    assert!(!dep_json.contains("dep_v2_fn"));
+}