View Javadoc
1   /*
2    * Copyright (C) 2014, Google Inc. and others
3    *
4    * This program and the accompanying materials are made available under the
5    * terms of the Eclipse Distribution License v. 1.0 which is available at
6    * https://www.eclipse.org/org/documents/edl-v10.php.
7    *
8    * SPDX-License-Identifier: BSD-3-Clause
9    */
10  package org.eclipse.jgit.gitrepo;
11  
12  import java.io.File;
13  import java.io.FileInputStream;
14  import java.io.IOException;
15  import java.io.InputStream;
16  import java.net.URI;
17  import java.text.MessageFormat;
18  import java.util.ArrayList;
19  import java.util.List;
20  import java.util.Map;
21  import java.util.Objects;
22  import java.util.StringJoiner;
23  import java.util.TreeMap;
24  
25  import org.eclipse.jgit.annotations.NonNull;
26  import org.eclipse.jgit.annotations.Nullable;
27  import org.eclipse.jgit.api.Git;
28  import org.eclipse.jgit.api.GitCommand;
29  import org.eclipse.jgit.api.errors.GitAPIException;
30  import org.eclipse.jgit.api.errors.InvalidRefNameException;
31  import org.eclipse.jgit.gitrepo.BareSuperprojectWriter.ExtraContent;
32  import org.eclipse.jgit.gitrepo.ManifestParser.IncludedFileReader;
33  import org.eclipse.jgit.gitrepo.internal.RepoText;
34  import org.eclipse.jgit.internal.JGitText;
35  import org.eclipse.jgit.lib.Constants;
36  import org.eclipse.jgit.lib.FileMode;
37  import org.eclipse.jgit.lib.ObjectId;
38  import org.eclipse.jgit.lib.PersonIdent;
39  import org.eclipse.jgit.lib.ProgressMonitor;
40  import org.eclipse.jgit.lib.Ref;
41  import org.eclipse.jgit.lib.RefDatabase;
42  import org.eclipse.jgit.lib.Repository;
43  import org.eclipse.jgit.revwalk.RevCommit;
44  import org.eclipse.jgit.treewalk.TreeWalk;
45  import org.eclipse.jgit.util.FileUtils;
46  
47  /**
48   * A class used to execute a repo command.
49   *
50   * This will parse a repo XML manifest, convert it into .gitmodules file and the
51   * repository config file.
52   *
53   * If called against a bare repository, it will replace all the existing content
54   * of the repository with the contents populated from the manifest.
55   *
56   * repo manifest allows projects overlapping, e.g. one project's manifestPath is
57   * "foo" and another project's manifestPath is "foo/bar". This won't
58   * work in git submodule, so we'll skip all the sub projects
59   * ("foo/bar" in the example) while converting.
60   *
61   * @see <a href="https://code.google.com/p/git-repo/">git-repo project page</a>
62   * @since 3.4
63   */
64  public class RepoCommand extends GitCommand<RevCommit> {
65  
66  
67  	private String manifestPath;
68  	private String baseUri;
69  	private URI targetUri;
70  	private String groupsParam;
71  	private String branch;
72  	private String targetBranch = Constants.HEAD;
73  	private PersonIdent author;
74  	private RemoteReader callback;
75  	private InputStream inputStream;
76  	private IncludedFileReader includedReader;
77  
78  	private BareSuperprojectWriter.BareWriterConfig bareWriterConfig = BareSuperprojectWriter.BareWriterConfig
79  			.getDefault();
80  
81  	private ProgressMonitor monitor;
82  
83  	private final List<ExtraContent> extraContents = new ArrayList<>();
84  
85  	/**
86  	 * A callback to get ref sha1 of a repository from its uri.
87  	 *
88  	 * We provided a default implementation {@link DefaultRemoteReader} to
89  	 * use ls-remote command to read the sha1 from the repository and clone the
90  	 * repository to read the file. Callers may have their own quicker
91  	 * implementation.
92  	 *
93  	 * @since 3.4
94  	 */
95  	public interface RemoteReader {
96  		/**
97  		 * Read a remote ref sha1.
98  		 *
99  		 * @param uri
100 		 *            The URI of the remote repository
101 		 * @param ref
102 		 *            Name of the ref to lookup. May be a short-hand form, e.g.
103 		 *            "master" which is automatically expanded to
104 		 *            "refs/heads/master" if "refs/heads/master" already exists.
105 		 * @return the sha1 of the remote repository, or null if the ref does
106 		 *         not exist.
107 		 * @throws GitAPIException
108 		 */
109 		@Nullable
110 		public ObjectId sha1(String uri, String ref) throws GitAPIException;
111 
112 		/**
113 		 * Read a file from a remote repository.
114 		 *
115 		 * @param uri
116 		 *            The URI of the remote repository
117 		 * @param ref
118 		 *            The ref (branch/tag/etc.) to read
119 		 * @param path
120 		 *            The relative path (inside the repo) to the file to read
121 		 * @return the file content.
122 		 * @throws GitAPIException
123 		 * @throws IOException
124 		 * @since 3.5
125 		 *
126 		 * @deprecated Use {@link #readFileWithMode(String, String, String)}
127 		 *             instead
128 		 */
129 		@Deprecated
130 		public default byte[] readFile(String uri, String ref, String path)
131 				throws GitAPIException, IOException {
132 			return readFileWithMode(uri, ref, path).getContents();
133 		}
134 
135 		/**
136 		 * Read contents and mode (i.e. permissions) of the file from a remote
137 		 * repository.
138 		 *
139 		 * @param uri
140 		 *            The URI of the remote repository
141 		 * @param ref
142 		 *            Name of the ref to lookup. May be a short-hand form, e.g.
143 		 *            "master" which is automatically expanded to
144 		 *            "refs/heads/master" if "refs/heads/master" already exists.
145 		 * @param path
146 		 *            The relative path (inside the repo) to the file to read
147 		 * @return The contents and file mode of the file in the given
148 		 *         repository and branch. Never null.
149 		 * @throws GitAPIException
150 		 *             If the ref have an invalid or ambiguous name, or it does
151 		 *             not exist in the repository,
152 		 * @throws IOException
153 		 *             If the object does not exist or is too large
154 		 * @since 5.2
155 		 */
156 		@NonNull
157 		public RemoteFile readFileWithMode(String uri, String ref, String path)
158 				throws GitAPIException, IOException;
159 	}
160 
161 	/**
162 	 * Read-only view of contents and file mode (i.e. permissions) for a file in
163 	 * a remote repository.
164 	 *
165 	 * @since 5.2
166 	 */
167 	public static final class RemoteFile {
168 		@NonNull
169 		private final byte[] contents;
170 
171 		@NonNull
172 		private final FileMode fileMode;
173 
174 		/**
175 		 * @param contents
176 		 *            Raw contents of the file.
177 		 * @param fileMode
178 		 *            Git file mode for this file (e.g. executable or regular)
179 		 */
180 		public RemoteFile(@NonNull byte[] contents,
181 				@NonNull FileMode fileMode) {
182 			this.contents = Objects.requireNonNull(contents);
183 			this.fileMode = Objects.requireNonNull(fileMode);
184 		}
185 
186 		/**
187 		 * Contents of the file.
188 		 * <p>
189 		 * Callers who receive this reference must not modify its contents (as
190 		 * it can point to internal cached data).
191 		 *
192 		 * @return Raw contents of the file. Do not modify it.
193 		 */
194 		@NonNull
195 		public byte[] getContents() {
196 			return contents;
197 		}
198 
199 		/**
200 		 * @return Git file mode for this file (e.g. executable or regular)
201 		 */
202 		@NonNull
203 		public FileMode getFileMode() {
204 			return fileMode;
205 		}
206 
207 	}
208 
209 	/** A default implementation of {@link RemoteReader} callback. */
210 	public static class DefaultRemoteReader implements RemoteReader {
211 
212 		@Override
213 		public ObjectId sha1(String uri, String ref) throws GitAPIException {
214 			Map<String, Ref> map = Git
215 					.lsRemoteRepository()
216 					.setRemote(uri)
217 					.callAsMap();
218 			Ref r = RefDatabase.findRef(map, ref);
219 			return r != null ? r.getObjectId() : null;
220 		}
221 
222 		@Override
223 		public RemoteFile readFileWithMode(String uri, String ref, String path)
224 				throws GitAPIException, IOException {
225 			File dir = FileUtils.createTempDir("jgit_", ".git", null); //$NON-NLS-1$ //$NON-NLS-2$
226 			try (Git git = Git.cloneRepository().setBare(true).setDirectory(dir)
227 					.setURI(uri).call()) {
228 				Repository repo = git.getRepository();
229 				ObjectId refCommitId = sha1(uri, ref);
230 				if (refCommitId == null) {
231 					throw new InvalidRefNameException(MessageFormat
232 							.format(JGitText.get().refNotResolved, ref));
233 				}
234 				RevCommit commit = repo.parseCommit(refCommitId);
235 				TreeWalk tw = TreeWalk.forPath(repo, path, commit.getTree());
236 
237 				// TODO(ifrade): Cope better with big files (e.g. using
238 				// InputStream instead of byte[])
239 				return new RemoteFile(
240 						tw.getObjectReader().open(tw.getObjectId(0))
241 								.getCachedBytes(Integer.MAX_VALUE),
242 						tw.getFileMode(0));
243 			} finally {
244 				FileUtils.delete(dir, FileUtils.RECURSIVE);
245 			}
246 		}
247 	}
248 
249 	@SuppressWarnings("serial")
250 	static class ManifestErrorException extends GitAPIException {
251 		ManifestErrorException(Throwable cause) {
252 			super(RepoText.get().invalidManifest, cause);
253 		}
254 	}
255 
256 	@SuppressWarnings("serial")
257 	static class RemoteUnavailableException extends GitAPIException {
258 		RemoteUnavailableException(String uri) {
259 			super(MessageFormat.format(RepoText.get().errorRemoteUnavailable, uri));
260 		}
261 	}
262 
263 	/**
264 	 * Constructor for RepoCommand
265 	 *
266 	 * @param repo
267 	 *            the {@link org.eclipse.jgit.lib.Repository}
268 	 */
269 	public RepoCommand(Repository repo) {
270 		super(repo);
271 	}
272 
273 	/**
274 	 * Set path to the manifest XML file.
275 	 * <p>
276 	 * Calling {@link #setInputStream} will ignore the path set here.
277 	 *
278 	 * @param path
279 	 *            (with <code>/</code> as separator)
280 	 * @return this command
281 	 */
282 	public RepoCommand setPath(String path) {
283 		this.manifestPath = path;
284 		return this;
285 	}
286 
287 	/**
288 	 * Set the input stream to the manifest XML.
289 	 * <p>
290 	 * Setting inputStream will ignore the path set. It will be closed in
291 	 * {@link #call}.
292 	 *
293 	 * @param inputStream a {@link java.io.InputStream} object.
294 	 * @return this command
295 	 * @since 3.5
296 	 */
297 	public RepoCommand setInputStream(InputStream inputStream) {
298 		this.inputStream = inputStream;
299 		return this;
300 	}
301 
302 	/**
303 	 * Set base URI of the paths inside the XML. This is typically the name of
304 	 * the directory holding the manifest repository, eg. for
305 	 * https://android.googlesource.com/platform/manifest, this should be
306 	 * /platform (if you would run this on android.googlesource.com) or
307 	 * https://android.googlesource.com/platform elsewhere.
308 	 *
309 	 * @param uri
310 	 *            the base URI
311 	 * @return this command
312 	 */
313 	public RepoCommand setURI(String uri) {
314 		this.baseUri = uri;
315 		return this;
316 	}
317 
318 	/**
319 	 * Set the URI of the superproject (this repository), so the .gitmodules
320 	 * file can specify the submodule URLs relative to the superproject.
321 	 *
322 	 * @param uri
323 	 *            the URI of the repository holding the superproject.
324 	 * @return this command
325 	 * @since 4.8
326 	 */
327 	public RepoCommand setTargetURI(String uri) {
328 		// The repo name is interpreted as a directory, for example
329 		// Gerrit (http://gerrit.googlesource.com/gerrit) has a
330 		// .gitmodules referencing ../plugins/hooks, which is
331 		// on http://gerrit.googlesource.com/plugins/hooks,
332 		this.targetUri = URI.create(uri + "/"); //$NON-NLS-1$
333 		return this;
334 	}
335 
336 	/**
337 	 * Set groups to sync
338 	 *
339 	 * @param groups groups separated by comma, examples: default|all|G1,-G2,-G3
340 	 * @return this command
341 	 */
342 	public RepoCommand setGroups(String groups) {
343 		this.groupsParam = groups;
344 		return this;
345 	}
346 
347 	/**
348 	 * Set default branch.
349 	 * <p>
350 	 * This is generally the name of the branch the manifest file was in. If
351 	 * there's no default revision (branch) specified in manifest and no
352 	 * revision specified in project, this branch will be used.
353 	 *
354 	 * @param branch
355 	 *            a branch name
356 	 * @return this command
357 	 */
358 	public RepoCommand setBranch(String branch) {
359 		this.branch = branch;
360 		return this;
361 	}
362 
363 	/**
364 	 * Set target branch.
365 	 * <p>
366 	 * This is the target branch of the super project to be updated. If not set,
367 	 * default is HEAD.
368 	 * <p>
369 	 * For non-bare repositories, HEAD will always be used and this will be
370 	 * ignored.
371 	 *
372 	 * @param branch
373 	 *            branch name
374 	 * @return this command
375 	 * @since 4.1
376 	 */
377 	public RepoCommand setTargetBranch(String branch) {
378 		this.targetBranch = Constants.R_HEADS + branch;
379 		return this;
380 	}
381 
382 	/**
383 	 * Set whether the branch name should be recorded in .gitmodules.
384 	 * <p>
385 	 * Submodule entries in .gitmodules can include a "branch" field
386 	 * to indicate what remote branch each submodule tracks.
387 	 * <p>
388 	 * That field is used by "git submodule update --remote" to update
389 	 * to the tip of the tracked branch when asked and by Gerrit to
390 	 * update the superproject when a change on that branch is merged.
391 	 * <p>
392 	 * Subprojects that request a specific commit or tag will not have
393 	 * a branch name recorded.
394 	 * <p>
395 	 * Not implemented for non-bare repositories.
396 	 *
397 	 * @param enable Whether to record the branch name
398 	 * @return this command
399 	 * @since 4.2
400 	 */
401 	public RepoCommand setRecordRemoteBranch(boolean enable) {
402 		this.bareWriterConfig.recordRemoteBranch = enable;
403 		return this;
404 	}
405 
406 	/**
407 	 * Set whether the labels field should be recorded as a label in
408 	 * .gitattributes.
409 	 * <p>
410 	 * Not implemented for non-bare repositories.
411 	 *
412 	 * @param enable Whether to record the labels in the .gitattributes
413 	 * @return this command
414 	 * @since 4.4
415 	 */
416 	public RepoCommand setRecordSubmoduleLabels(boolean enable) {
417 		this.bareWriterConfig.recordSubmoduleLabels = enable;
418 		return this;
419 	}
420 
421 	/**
422 	 * Set whether the clone-depth field should be recorded as a shallow
423 	 * recommendation in .gitmodules.
424 	 * <p>
425 	 * Not implemented for non-bare repositories.
426 	 *
427 	 * @param enable Whether to record the shallow recommendation.
428 	 * @return this command
429 	 * @since 4.4
430 	 */
431 	public RepoCommand setRecommendShallow(boolean enable) {
432 		this.bareWriterConfig.recordShallowSubmodules = enable;
433 		return this;
434 	}
435 
436 	/**
437 	 * The progress monitor associated with the clone operation. By default,
438 	 * this is set to <code>NullProgressMonitor</code>
439 	 *
440 	 * @see org.eclipse.jgit.lib.NullProgressMonitor
441 	 * @param monitor
442 	 *            a {@link org.eclipse.jgit.lib.ProgressMonitor}
443 	 * @return this command
444 	 */
445 	public RepoCommand setProgressMonitor(ProgressMonitor monitor) {
446 		this.monitor = monitor;
447 		return this;
448 	}
449 
450 	/**
451 	 * Set whether to skip projects whose commits don't exist remotely.
452 	 * <p>
453 	 * When set to true, we'll just skip the manifest entry and continue
454 	 * on to the next one.
455 	 * <p>
456 	 * When set to false (default), we'll throw an error when remote
457 	 * failures occur.
458 	 * <p>
459 	 * Not implemented for non-bare repositories.
460 	 *
461 	 * @param ignore Whether to ignore the remote failures.
462 	 * @return this command
463 	 * @since 4.3
464 	 */
465 	public RepoCommand setIgnoreRemoteFailures(boolean ignore) {
466 		this.bareWriterConfig.ignoreRemoteFailures = ignore;
467 		return this;
468 	}
469 
470 	/**
471 	 * Set the author/committer for the bare repository commit.
472 	 * <p>
473 	 * For non-bare repositories, the current user will be used and this will be
474 	 * ignored.
475 	 *
476 	 * @param author
477 	 *            the author's {@link org.eclipse.jgit.lib.PersonIdent}
478 	 * @return this command
479 	 */
480 	public RepoCommand setAuthor(PersonIdent author) {
481 		this.author = author;
482 		return this;
483 	}
484 
485 	/**
486 	 * Set the GetHeadFromUri callback.
487 	 *
488 	 * This is only used in bare repositories.
489 	 *
490 	 * @param callback
491 	 *            a {@link org.eclipse.jgit.gitrepo.RepoCommand.RemoteReader}
492 	 *            object.
493 	 * @return this command
494 	 */
495 	public RepoCommand setRemoteReader(RemoteReader callback) {
496 		this.callback = callback;
497 		return this;
498 	}
499 
500 	/**
501 	 * Set the IncludedFileReader callback.
502 	 *
503 	 * @param reader
504 	 *            a
505 	 *            {@link org.eclipse.jgit.gitrepo.ManifestParser.IncludedFileReader}
506 	 *            object.
507 	 * @return this command
508 	 * @since 4.0
509 	 */
510 	public RepoCommand setIncludedFileReader(IncludedFileReader reader) {
511 		this.includedReader = reader;
512 		return this;
513 	}
514 
515 	/**
516 	 * Create a file with the given content in the destination repository
517 	 *
518 	 * @param path
519 	 *            where to create the file in the destination repository
520 	 * @param contents
521 	 *            content for the create file
522 	 * @return this command
523 	 *
524 	 * @since 6.1
525 	 */
526 	public RepoCommand addToDestination(String path, String contents) {
527 		this.extraContents.add(new ExtraContent(path, contents));
528 		return this;
529 	}
530 
531 	/** {@inheritDoc} */
532 	@Override
533 	public RevCommit call() throws GitAPIException {
534 		checkCallable();
535 		if (baseUri == null) {
536 			baseUri = ""; //$NON-NLS-1$
537 		}
538 		if (inputStream == null) {
539 			if (manifestPath == null || manifestPath.length() == 0)
540 				throw new IllegalArgumentException(
541 						JGitText.get().pathNotConfigured);
542 			try {
543 				inputStream = new FileInputStream(manifestPath);
544 			} catch (IOException e) {
545 				throw new IllegalArgumentException(
546 						JGitText.get().pathNotConfigured, e);
547 			}
548 		}
549 
550 		List<RepoProject> filteredProjects;
551 		try {
552 			ManifestParser parser = new ManifestParser(includedReader,
553 					manifestPath, branch, baseUri, groupsParam, repo);
554 			parser.read(inputStream);
555 			filteredProjects = parser.getFilteredProjects();
556 		} catch (IOException e) {
557 			throw new ManifestErrorException(e);
558 		} finally {
559 			try {
560 				inputStream.close();
561 			} catch (IOException e) {
562 				// Just ignore it, it's not important.
563 			}
564 		}
565 
566 		if (repo.isBare()) {
567 			List<RepoProject> renamedProjects = renameProjects(filteredProjects);
568 			BareSuperprojectWriter writer = new BareSuperprojectWriter(repo, targetUri,
569 					targetBranch,
570 					author == null ? new PersonIdent(repo) : author,
571 					callback == null ? new DefaultRemoteReader() : callback,
572 					bareWriterConfig, extraContents);
573 			return writer.write(renamedProjects);
574 		}
575 
576 
577 		RegularSuperprojectWriter writer = new RegularSuperprojectWriter(repo, monitor);
578 		return writer.write(filteredProjects);
579 	}
580 
581 	/**
582 	 * Rename the projects if there's a conflict when converted to submodules.
583 	 *
584 	 * @param projects
585 	 *            parsed projects
586 	 * @return projects that are renamed if necessary
587 	 */
588 	private List<RepoProject> renameProjects(List<RepoProject> projects) {
589 		Map<String, List<RepoProject>> m = new TreeMap<>();
590 		for (RepoProject proj : projects) {
591 			List<RepoProject> l = m.get(proj.getName());
592 			if (l == null) {
593 				l = new ArrayList<>();
594 				m.put(proj.getName(), l);
595 			}
596 			l.add(proj);
597 		}
598 
599 		List<RepoProject> ret = new ArrayList<>();
600 		for (List<RepoProject> ps : m.values()) {
601 			boolean nameConflict = ps.size() != 1;
602 			for (RepoProject proj : ps) {
603 				String name = proj.getName();
604 				if (nameConflict) {
605 					name += SLASH + proj.getPath();
606 				}
607 				RepoProject p = new RepoProject(name,
608 						proj.getPath(), proj.getRevision(), null,
609 						proj.getGroups(), proj.getRecommendShallow());
610 				p.setUrl(proj.getUrl());
611 				p.addCopyFiles(proj.getCopyFiles());
612 				p.addLinkFiles(proj.getLinkFiles());
613 				ret.add(p);
614 			}
615 		}
616 		return ret;
617 	}
618 
619 	/*
620 	 * Assume we are document "a/b/index.html", what should we put in a href to get to "a/" ?
621 	 * Returns the child if either base or child is not a bare path. This provides a missing feature in
622 	 * java.net.URI (see http://bugs.java.com/view_bug.do?bug_id=6226081).
623 	 */
624 	private static final String SLASH = "/"; //$NON-NLS-1$
625 	static URI relativize(URI current, URI target) {
626 		if (!Objects.equals(current.getHost(), target.getHost())) {
627 			return target;
628 		}
629 
630 		String cur = current.normalize().getPath();
631 		String dest = target.normalize().getPath();
632 
633 		// TODO(hanwen): maybe (absolute, relative) should throw an exception.
634 		if (cur.startsWith(SLASH) != dest.startsWith(SLASH)) {
635 			return target;
636 		}
637 
638 		while (cur.startsWith(SLASH)) {
639 			cur = cur.substring(1);
640 		}
641 		while (dest.startsWith(SLASH)) {
642 			dest = dest.substring(1);
643 		}
644 
645 		if (cur.indexOf('/') == -1 || dest.indexOf('/') == -1) {
646 			// Avoid having to special-casing in the next two ifs.
647 			String prefix = "prefix/"; //$NON-NLS-1$
648 			cur = prefix + cur;
649 			dest = prefix + dest;
650 		}
651 
652 		if (!cur.endsWith(SLASH)) {
653 			// The current file doesn't matter.
654 			int lastSlash = cur.lastIndexOf('/');
655 			cur = cur.substring(0, lastSlash);
656 		}
657 		String destFile = ""; //$NON-NLS-1$
658 		if (!dest.endsWith(SLASH)) {
659 			// We always have to provide the destination file.
660 			int lastSlash = dest.lastIndexOf('/');
661 			destFile = dest.substring(lastSlash + 1, dest.length());
662 			dest = dest.substring(0, dest.lastIndexOf('/'));
663 		}
664 
665 		String[] cs = cur.split(SLASH);
666 		String[] ds = dest.split(SLASH);
667 
668 		int common = 0;
669 		while (common < cs.length && common < ds.length && cs[common].equals(ds[common])) {
670 			common++;
671 		}
672 
673 		StringJoiner j = new StringJoiner(SLASH);
674 		for (int i = common; i < cs.length; i++) {
675 			j.add(".."); //$NON-NLS-1$
676 		}
677 		for (int i = common; i < ds.length; i++) {
678 			j.add(ds[i]);
679 		}
680 
681 		j.add(destFile);
682 		return URI.create(j.toString());
683 	}
684 
685 }