diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java index ff07100eed..91b55790a9 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CliGitAPIImpl.java @@ -522,6 +522,7 @@ public FetchCommand fetch_() { private Integer timeout; private boolean tags = true; private Integer depth = 1; + private String filterSpec = null; @Override public FetchCommand from(URIish remote, List refspecs) { @@ -566,6 +567,12 @@ public FetchCommand depth(Integer depth) { return this; } + @Override + public FetchCommand filter(String filterSpec) { + this.filterSpec = filterSpec; + return this; + } + @Override public void execute() throws GitException, InterruptedException { listener.getLogger().println("Fetching upstream changes from " + url); @@ -592,6 +599,37 @@ public void execute() throws GitException, InterruptedException { args.add("--depth=" + depth); } + if (filterSpec != null) { + // try to get the current filterspec + String currentFilterSpec = null; + String defaultRemote = null; + try { + defaultRemote = getDefaultRemote(); + if (defaultRemote != null && !defaultRemote.isEmpty()) { + currentFilterSpec = + launchCommand("config", "remote." + defaultRemote + ".partialclonefilter"); + } + // we might fail if we have no promisor configured, catch the exception and just continue + } catch (GitException e) { + // leave currentFilterSpec null + } + + if (isAtLeastVersion(2, 22, 0, 0)) { + args.add("--filter=" + filterSpec); + } + + // if the filterspec has changed + if (defaultRemote != null && !filterSpec.equals(currentFilterSpec)) { + launchCommand("config", "remote." + defaultRemote + ".promisor", "true"); + launchCommand("config", "remote." + defaultRemote + ".partialclonefilter", filterSpec); + + if (isAtLeastVersion(2, 36, 0, 0)) { + // reapply filter and trigger maintenance + args.add("--refetch"); + } + } + } + warnIfWindowsTemporaryDirNameHasSpaces(); StandardCredentials cred = credentials.get(url.toPrivateString()); @@ -727,6 +765,7 @@ public CloneCommand clone_() { private boolean tags = true; private List refspecs; private Integer depth = 1; + private String filterSpec = null; @Override public CloneCommand url(String url) { @@ -802,6 +841,12 @@ public CloneCommand refspecs(List refspecs) { return this; } + @Override + public CloneCommand filter(String filterSpec) { + this.filterSpec = filterSpec; + return this; + } + @Override public void execute() throws GitException, InterruptedException { @@ -872,9 +917,14 @@ public void execute() throws GitException, InterruptedException { if (refspecs == null) { refspecs = Collections.singletonList(new RefSpec("+refs/heads/*:refs/remotes/" + origin + "/*")); } + if (filterSpec != null) { + launchCommand("config", "--add", "remote." + origin + ".promisor", "true"); + launchCommand("config", "--add", "remote." + origin + ".partialclonefilter", filterSpec); + } fetch_().from(urIish, refspecs) .shallow(shallow) .depth(depth) + .filter(filterSpec) .timeout(timeout) .tags(tags) .execute(); @@ -1761,6 +1811,15 @@ public boolean isShallowRepository() { return new File(workspace, pathJoin(".git", "shallow")).exists(); } + /** Returns true if the remote has a promisor configured for missing blobs. */ + boolean hasPromisor(String name) throws GitException, InterruptedException { + try { + return launchCommand("config", "remote." + name + ".promisor").contains("true"); + } catch (GitException ge) { + return false; + } + } + private String pathJoin(String a, String b) { return new File(a, b).toString(); } @@ -2090,6 +2149,17 @@ private String launchCommandWithCredentials( @NonNull URIish url, Integer timeout) throws GitException, InterruptedException { + return launchCommandWithCredentials(args, workDir, environment, credentials, url, timeout); + } + + private String launchCommandWithCredentials( + ArgumentListBuilder args, + File workDir, + EnvVars env, + StandardCredentials credentials, + @NonNull URIish url, + Integer timeout) + throws GitException, InterruptedException { Path key = null; Path ssh = null; @@ -2098,7 +2168,6 @@ private String launchCommandWithCredentials( Path passwordFile = null; Path passphrase = null; Path knownHostsTemp = null; - EnvVars env = environment; if (!PROMPT_FOR_AUTHENTICATION && isAtLeastVersion(2, 3, 0, 0)) { env = new EnvVars(env); env.put("GIT_TERMINAL_PROMPT", "false"); // Don't prompt for auth from command line git @@ -3179,7 +3248,32 @@ public void execute() throws GitException, InterruptedException { args.add("-f"); } args.add(ref); - launchCommandIn(args, workspace, checkoutEnv, timeout); + + String repoUrl = null; + try { + String defaultRemote = getDefaultRemote(); + if (defaultRemote != null && !defaultRemote.isEmpty() && hasPromisor(defaultRemote)) { + repoUrl = getRemoteUrl(defaultRemote); + } + } catch (GitException e) { + /* Nothing to do, just keeping repoUrl = null */ + } + + if (repoUrl != null) { + StandardCredentials cred = credentials.get(repoUrl); + if (cred == null) { + cred = defaultCredentials; + } + + try { + launchCommandWithCredentials( + args, workspace, checkoutEnv, cred, new URIish(repoUrl), timeout); + } catch (URISyntaxException e) { + throw new GitException("Invalid URL " + repoUrl, e); + } + } else { + launchCommandIn(args, workspace, checkoutEnv, timeout); + } if (lfsRemote != null) { final String url = getRemoteUrl(lfsRemote); diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/CloneCommand.java b/src/main/java/org/jenkinsci/plugins/gitclient/CloneCommand.java index 92a23bbbf6..e085331d6b 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/CloneCommand.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/CloneCommand.java @@ -114,4 +114,13 @@ public interface CloneCommand extends GitCommand { * @return a {@link org.jenkinsci.plugins.gitclient.CloneCommand} object. */ CloneCommand depth(Integer depth); + + /** + * Apply an object filter to a partial clone. If unset, a full clone is performed. + * + * @param filterSpec filter of objects to be sent by the server + * @return a {@link org.jenkinsci.plugins.gitclient.CloneCommand} object. + * @since 6.7.0 + */ + CloneCommand filter(String filterSpec); } diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/FetchCommand.java b/src/main/java/org/jenkinsci/plugins/gitclient/FetchCommand.java index 01320584f1..ff8884eb2c 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/FetchCommand.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/FetchCommand.java @@ -64,4 +64,13 @@ public interface FetchCommand extends GitCommand { * @return a {@link org.jenkinsci.plugins.gitclient.CloneCommand} object. */ FetchCommand depth(Integer depth); + + /** + * Apply an object filter to a partial clone. If unset, a full clone is performed. + * + * @param filterSpec filter of objects to be sent by the server + * @return a {@link org.jenkinsci.plugins.gitclient.CloneCommand} object. + * @since 6.7.0 + */ + FetchCommand filter(String filterSpec); } diff --git a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java index 94e63ed330..4fbcc126ee 100644 --- a/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java +++ b/src/main/java/org/jenkinsci/plugins/gitclient/JGitAPIImpl.java @@ -786,6 +786,15 @@ public org.jenkinsci.plugins.gitclient.FetchCommand depth(Integer depth) { return this; } + @Override + public org.jenkinsci.plugins.gitclient.FetchCommand filter(String filterSpec) { + if (filterSpec != null) { + listener.getLogger() + .println("[WARNING] JGit does not support partial clone filters; reverting to full clone"); + } + return this; + } + @Override public void execute() throws GitException { try (Repository repo = getRepository()) { @@ -1638,6 +1647,15 @@ public CloneCommand depth(Integer depth) { return this; } + @Override + public CloneCommand filter(String filterSpec) { + if (filterSpec != null) { + listener.getLogger() + .println("[WARNING] JGit does not support partial clone filters; reverting to full clone"); + } + return this; + } + private RepositoryBuilder newRepositoryBuilder() { RepositoryBuilder builder = new RepositoryBuilder(); builder.setGitDir(new File(workspace, Constants.DOT_GIT)).readEnvironment(); diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/CredentialsTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/CredentialsTest.java index 07b65878ad..25827d11e3 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/CredentialsTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/CredentialsTest.java @@ -516,6 +516,38 @@ void testLfsMergeWithCredentials() throws Exception { "Fast-forward merge failed. master and modified_lfs should be the same."); } + @Test + @Issue("JENKINS-70094") + void testCheckoutPartialCloneWithCredentials() throws Exception { + if (testPeriodExpired() || lfsSpecificTest) { + return; + } + File clonedFile = new File(repo, fileToCheck); + git.init_().workspace(repo.getAbsolutePath()).execute(); + assertFalse(clonedFile.exists(), "file " + fileToCheck + " in " + repo + ", has " + listDir(repo)); + addCredential(); + // clone with remote url + git.clone_() + .url(gitRepoURL) + .repositoryName("origin") + .filter("blob:none") + .execute(); + ObjectId master = git.getHeadRev(gitRepoURL, "master"); + git.checkout() + .branch("master") + .ref(master.getName()) + .deleteBranchIfExist(true) + .execute(); + assertTrue(git.isCommitInRepo(master), "master: " + master + " not in repo"); + assertEquals( + master, + git.withRepository( + (gitRepo, unusedChannel) -> gitRepo.findRef("master").getObjectId()), + "Master != HEAD"); + assertEquals("master", git.withRepository((gitRepo, unusedChanel) -> gitRepo.getBranch()), "Wrong branch"); + assertTrue(clonedFile.exists(), "No file " + fileToCheck + ", has " + listDir(repo)); + } + private static boolean isWindows() { return File.pathSeparatorChar == ';'; } diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java index b1a25c23ae..1daef7b7e3 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientCloneTest.java @@ -40,6 +40,7 @@ import org.junit.jupiter.params.ParameterizedClass; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import org.jvnet.hudson.test.Issue; @ParameterizedClass(name = "{0}") @MethodSource("gitObjects") @@ -427,12 +428,42 @@ void test_clone_huge_timeout_logging() throws Exception { assertTimeout(testGitClient, "fetch", expectedValue); } + @Test + @Issue("JENKINS-70094") + void test_clone_filter() throws Exception { + WorkspaceWithRepo anotherWorkspace = new WorkspaceWithRepo(secondRepo.getRoot(), "git", listener); + GitClient anotherGitClient = anotherWorkspace.getGitClient(); + CloneCommand cmd = anotherGitClient + .clone_() + .url(anotherWorkspace.localMirror()) + .repositoryName("origin") + .filter("blob:none"); + cmd.execute(); + anotherGitClient.checkout().ref("origin/master").branch("master").execute(); + check_remote_url(anotherWorkspace, anotherGitClient, "origin"); + assertBranchesExist(anotherGitClient.getBranches(), "master"); + if (gitImplName.equals("git")) { + assertThat("hasPromisor?", anotherWorkspace.cgit().hasPromisor("origin"), is(true)); + String filterSpec = anotherWorkspace + .launchCommand("git", "config", "remote." + "origin" + ".partialclonefilter") + .trim(); + assertThat("filterSpec", filterSpec, is("blob:none")); + assertPromisorFilesExist(anotherWorkspace.getGitFileDir()); + } + } + private void assertAlternatesFileExists() { final String alternates = ".git" + File.separator + "objects" + File.separator + "info" + File.separator + "alternates"; assertThat(new File(testGitDir, alternates), is(anExistingFile())); } + private void assertPromisorFilesExist(File anotherTestGitDir) { + File pack = new File(anotherTestGitDir, ".git" + File.separator + "objects" + File.separator + "pack"); + File[] promisors = pack.listFiles((dir, name) -> name.endsWith(".promisor")); + assertThat(promisors, is(not(emptyArray()))); + } + private void assertAlternatesFileNotFound() { final String alternates = ".git" + File.separator + "objects" + File.separator + "info" + File.separator + "alternates"; diff --git a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientFetchTest.java b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientFetchTest.java index f2764ff8ea..70416e3866 100644 --- a/src/test/java/org/jenkinsci/plugins/gitclient/GitClientFetchTest.java +++ b/src/test/java/org/jenkinsci/plugins/gitclient/GitClientFetchTest.java @@ -620,6 +620,63 @@ void test_fetch_default_timeout_logging() throws Exception { assertRevParseNotCalled(testGitClient, randomBranchName); } + @Test + @Issue("JENKINS-70094") + void test_fetch_filter() throws Exception { + testGitClient + .clone_() + .url(workspace.localMirror()) + .repositoryName("origin") + .filter("blob:none") + .execute(); + checkoutRandomBranch(); + testGitClient + .fetch_() + .from(new URIish("origin"), null) + .filter("blob:limit=1k") + .execute(); + boolean expectedPromisorValue = gitImplName.equals("git"); + assertThat("hasPromisor?", workspace.cgit().hasPromisor("origin"), is(expectedPromisorValue)); + if (gitImplName.equals("git")) { + String filterSpec = workspace + .launchCommand("git", "config", "remote." + "origin" + ".partialclonefilter") + .trim(); + assertThat("filterSpec", filterSpec, is("blob:limit=1k")); + } + } + + @Test + @Issue("JENKINS-70094") + void test_fetch_filter_changed() throws Exception { + testGitClient + .clone_() + .url(workspace.localMirror()) + .repositoryName("origin") + .filter("blob:none") + .execute(); + checkoutRandomBranch(); + testGitClient.fetch_().from(new URIish("origin"), null).filter("tree:0").execute(); + boolean expectedPromisorValue = gitImplName.equals("git"); + assertThat("hasPromisor?", workspace.cgit().hasPromisor("origin"), is(expectedPromisorValue)); + if (gitImplName.equals("git")) { + String filterSpec = workspace + .launchCommand("git", "config", "remote." + "origin" + ".partialclonefilter") + .trim(); + assertThat("filterSpec", filterSpec, is("tree:0")); + } + testGitClient + .fetch_() + .from(new URIish("origin"), null) + .filter("blob:limit=1k") + .execute(); + if (gitImplName.equals("git")) { + String filterSpec = workspace + .launchCommand("git", "config", "remote." + "origin" + ".partialclonefilter") + .trim(); + assertThat("filterSpec", filterSpec, is("blob:limit=1k")); + } + } + private void check_remote_url(WorkspaceWithRepo workspace, GitClient gitClient, final String repositoryName) throws Exception { assertThat("Wrong remote URL", gitClient.getRemoteUrl(repositoryName), is(workspace.localMirror()));