View Javadoc
1   /*
2    * Copyright (C) 2014, Google Inc.
3    * and other copyright owners as documented in the project's IP log.
4    *
5    * This program and the accompanying materials are made available
6    * under the terms of the Eclipse Distribution License v1.0 which
7    * accompanies this distribution, is reproduced below, and is
8    * available at http://www.eclipse.org/org/documents/edl-v10.php
9    *
10   * All rights reserved.
11   *
12   * Redistribution and use in source and binary forms, with or
13   * without modification, are permitted provided that the following
14   * conditions are met:
15   *
16   * - Redistributions of source code must retain the above copyright
17   *   notice, this list of conditions and the following disclaimer.
18   *
19   * - Redistributions in binary form must reproduce the above
20   *   copyright notice, this list of conditions and the following
21   *   disclaimer in the documentation and/or other materials provided
22   *   with the distribution.
23   *
24   * - Neither the name of the Eclipse Foundation, Inc. nor the
25   *   names of its contributors may be used to endorse or promote
26   *   products derived from this software without specific prior
27   *   written permission.
28   *
29   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
30   * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
31   * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
32   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
34   * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
35   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
36   * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
37   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
38   * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
39   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
40   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
41   * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42   */
43  package org.eclipse.jgit.gitrepo;
44  
45  import java.io.File;
46  import java.io.FileInputStream;
47  import java.io.IOException;
48  import java.io.InputStream;
49  import java.text.MessageFormat;
50  import java.util.ArrayList;
51  import java.util.List;
52  import java.util.Map;
53  
54  import org.eclipse.jgit.api.Git;
55  import org.eclipse.jgit.api.GitCommand;
56  import org.eclipse.jgit.api.SubmoduleAddCommand;
57  import org.eclipse.jgit.api.errors.ConcurrentRefUpdateException;
58  import org.eclipse.jgit.api.errors.GitAPIException;
59  import org.eclipse.jgit.api.errors.JGitInternalException;
60  import org.eclipse.jgit.dircache.DirCache;
61  import org.eclipse.jgit.dircache.DirCacheBuilder;
62  import org.eclipse.jgit.dircache.DirCacheEntry;
63  import org.eclipse.jgit.gitrepo.ManifestParser.IncludedFileReader;
64  import org.eclipse.jgit.gitrepo.RepoProject.CopyFile;
65  import org.eclipse.jgit.gitrepo.internal.RepoText;
66  import org.eclipse.jgit.internal.JGitText;
67  import org.eclipse.jgit.lib.CommitBuilder;
68  import org.eclipse.jgit.lib.Config;
69  import org.eclipse.jgit.lib.Constants;
70  import org.eclipse.jgit.lib.FileMode;
71  import org.eclipse.jgit.lib.ObjectId;
72  import org.eclipse.jgit.lib.ObjectInserter;
73  import org.eclipse.jgit.lib.ObjectReader;
74  import org.eclipse.jgit.lib.PersonIdent;
75  import org.eclipse.jgit.lib.ProgressMonitor;
76  import org.eclipse.jgit.lib.Ref;
77  import org.eclipse.jgit.lib.RefDatabase;
78  import org.eclipse.jgit.lib.RefUpdate;
79  import org.eclipse.jgit.lib.RefUpdate.Result;
80  import org.eclipse.jgit.lib.Repository;
81  import org.eclipse.jgit.revwalk.RevCommit;
82  import org.eclipse.jgit.revwalk.RevWalk;
83  import org.eclipse.jgit.util.FileUtils;
84  
85  /**
86   * A class used to execute a repo command.
87   *
88   * This will parse a repo XML manifest, convert it into .gitmodules file and the
89   * repository config file.
90   *
91   * If called against a bare repository, it will replace all the existing content
92   * of the repository with the contents populated from the manifest.
93   *
94   * repo manifest allows projects overlapping, e.g. one project's path is
95   * "foo" and another project's path is "foo/bar". This won't
96   * work in git submodule, so we'll skip all the sub projects
97   * ("foo/bar" in the example) while converting.
98   *
99   * @see <a href="https://code.google.com/p/git-repo/">git-repo project page</a>
100  * @since 3.4
101  */
102 public class RepoCommand extends GitCommand<RevCommit> {
103 
104 	private String path;
105 	private String uri;
106 	private String groups;
107 	private String branch;
108 	private String targetBranch = Constants.HEAD;
109 	private PersonIdent author;
110 	private RemoteReader callback;
111 	private InputStream inputStream;
112 	private IncludedFileReader includedReader;
113 
114 	private List<RepoProject> bareProjects;
115 	private Git git;
116 	private ProgressMonitor monitor;
117 
118 	/**
119 	 * A callback to get ref sha1 of a repository from its uri.
120 	 *
121 	 * We provided a default implementation {@link DefaultRemoteReader} to
122 	 * use ls-remote command to read the sha1 from the repository and clone the
123 	 * repository to read the file. Callers may have their own quicker
124 	 * implementation.
125 	 *
126 	 * @since 3.4
127 	 */
128 	public interface RemoteReader {
129 		/**
130 		 * Read a remote ref sha1.
131 		 *
132 		 * @param uri
133 		 *            The URI of the remote repository
134 		 * @param ref
135 		 *            The ref (branch/tag/etc.) to read
136 		 * @return the sha1 of the remote repository
137 		 * @throws GitAPIException
138 		 */
139 		public ObjectId sha1(String uri, String ref) throws GitAPIException;
140 
141 		/**
142 		 * Read a file from a remote repository.
143 		 *
144 		 * @param uri
145 		 *            The URI of the remote repository
146 		 * @param ref
147 		 *            The ref (branch/tag/etc.) to read
148 		 * @param path
149 		 *            The relative path (inside the repo) to the file to read
150 		 * @return the file content.
151 		 * @throws GitAPIException
152 		 * @throws IOException
153 		 * @since 3.5
154 		 */
155 		public byte[] readFile(String uri, String ref, String path)
156 				throws GitAPIException, IOException;
157 	}
158 
159 	/** A default implementation of {@link RemoteReader} callback. */
160 	public static class DefaultRemoteReader implements RemoteReader {
161 		public ObjectId sha1(String uri, String ref) throws GitAPIException {
162 			Map<String, Ref> map = Git
163 					.lsRemoteRepository()
164 					.setRemote(uri)
165 					.callAsMap();
166 			Ref r = RefDatabase.findRef(map, ref);
167 			return r != null ? r.getObjectId() : null;
168 		}
169 
170 		public byte[] readFile(String uri, String ref, String path)
171 				throws GitAPIException, IOException {
172 			File dir = FileUtils.createTempDir("jgit_", ".git", null); //$NON-NLS-1$ //$NON-NLS-2$
173 			Repository repo = Git
174 					.cloneRepository()
175 					.setBare(true)
176 					.setDirectory(dir)
177 					.setURI(uri)
178 					.call()
179 					.getRepository();
180 			try {
181 				return readFileFromRepo(repo, ref, path);
182 			} finally {
183 				repo.close();
184 				FileUtils.delete(dir, FileUtils.RECURSIVE);
185 			}
186 		}
187 
188 		/**
189 		 * Read a file from the repository
190 		 *
191 		 * @param repo
192 		 *            The repository containing the file
193 		 * @param ref
194 		 *            The ref (branch/tag/etc.) to read
195 		 * @param path
196 		 *            The relative path (inside the repo) to the file to read
197 		 * @return the file's content
198 		 * @throws GitAPIException
199 		 * @throws IOException
200 		 * @since 3.5
201 		 */
202 		protected byte[] readFileFromRepo(Repository repo,
203 				String ref, String path) throws GitAPIException, IOException {
204 			try (ObjectReader reader = repo.newObjectReader()) {
205 				ObjectId oid = repo.resolve(ref + ":" + path); //$NON-NLS-1$
206 				return reader.open(oid).getBytes(Integer.MAX_VALUE);
207 			}
208 		}
209 	}
210 
211 	@SuppressWarnings("serial")
212 	private static class ManifestErrorException extends GitAPIException {
213 		ManifestErrorException(Throwable cause) {
214 			super(RepoText.get().invalidManifest, cause);
215 		}
216 	}
217 
218 	@SuppressWarnings("serial")
219 	private static class RemoteUnavailableException extends GitAPIException {
220 		RemoteUnavailableException(String uri) {
221 			super(MessageFormat.format(RepoText.get().errorRemoteUnavailable, uri));
222 		}
223 	}
224 
225 	/**
226 	 * @param repo
227 	 */
228 	public RepoCommand(Repository repo) {
229 		super(repo);
230 	}
231 
232 	/**
233 	 * Set path to the manifest XML file.
234 	 * <p>
235 	 * Calling {@link #setInputStream} will ignore the path set here.
236 	 *
237 	 * @param path
238 	 *            (with <code>/</code> as separator)
239 	 * @return this command
240 	 */
241 	public RepoCommand setPath(String path) {
242 		this.path = path;
243 		return this;
244 	}
245 
246 	/**
247 	 * Set the input stream to the manifest XML.
248 	 * <p>
249 	 * Setting inputStream will ignore the path set. It will be closed in
250 	 * {@link #call}.
251 	 *
252 	 * @param inputStream
253 	 * @return this command
254 	 * @since 3.5
255 	 */
256 	public RepoCommand setInputStream(InputStream inputStream) {
257 		this.inputStream = inputStream;
258 		return this;
259 	}
260 
261 	/**
262 	 * Set base URI of the pathes inside the XML
263 	 *
264 	 * @param uri
265 	 * @return this command
266 	 */
267 	public RepoCommand setURI(String uri) {
268 		this.uri = uri;
269 		return this;
270 	}
271 
272 	/**
273 	 * Set groups to sync
274 	 *
275 	 * @param groups groups separated by comma, examples: default|all|G1,-G2,-G3
276 	 * @return this command
277 	 */
278 	public RepoCommand setGroups(String groups) {
279 		this.groups = groups;
280 		return this;
281 	}
282 
283 	/**
284 	 * Set default branch.
285 	 * <p>
286 	 * This is generally the name of the branch the manifest file was in. If
287 	 * there's no default revision (branch) specified in manifest and no
288 	 * revision specified in project, this branch will be used.
289 	 *
290 	 * @param branch
291 	 * @return this command
292 	 */
293 	public RepoCommand setBranch(String branch) {
294 		this.branch = branch;
295 		return this;
296 	}
297 
298 	/**
299 	 * Set target branch.
300 	 * <p>
301 	 * This is the target branch of the super project to be updated. If not set,
302 	 * default is HEAD.
303 	 * <p>
304 	 * For non-bare repositories, HEAD will always be used and this will be
305 	 * ignored.
306 	 *
307 	 * @param branch
308 	 * @return this command
309 	 * @since 4.1
310 	 */
311 	public RepoCommand setTargetBranch(String branch) {
312 		this.targetBranch = Constants.R_HEADS + branch;
313 		return this;
314 	}
315 
316 	/**
317 	 * The progress monitor associated with the clone operation. By default,
318 	 * this is set to <code>NullProgressMonitor</code>
319 	 *
320 	 * @see org.eclipse.jgit.lib.NullProgressMonitor
321 	 * @param monitor
322 	 * @return this command
323 	 */
324 	public RepoCommand setProgressMonitor(final ProgressMonitor monitor) {
325 		this.monitor = monitor;
326 		return this;
327 	}
328 
329 	/**
330 	 * Set the author/committer for the bare repository commit.
331 	 * <p>
332 	 * For non-bare repositories, the current user will be used and this will be
333 	 * ignored.
334 	 *
335 	 * @param author
336 	 * @return this command
337 	 */
338 	public RepoCommand setAuthor(final PersonIdent author) {
339 		this.author = author;
340 		return this;
341 	}
342 
343 	/**
344 	 * Set the GetHeadFromUri callback.
345 	 *
346 	 * This is only used in bare repositories.
347 	 *
348 	 * @param callback
349 	 * @return this command
350 	 */
351 	public RepoCommand setRemoteReader(final RemoteReader callback) {
352 		this.callback = callback;
353 		return this;
354 	}
355 
356 	/**
357 	 * Set the IncludedFileReader callback.
358 	 *
359 	 * @param reader
360 	 * @return this command
361 	 * @since 4.0
362 	 */
363 	public RepoCommand setIncludedFileReader(IncludedFileReader reader) {
364 		this.includedReader = reader;
365 		return this;
366 	}
367 
368 	@Override
369 	public RevCommit call() throws GitAPIException {
370 		try {
371 			checkCallable();
372 			if (uri == null || uri.length() == 0)
373 				throw new IllegalArgumentException(
374 						JGitText.get().uriNotConfigured);
375 			if (inputStream == null) {
376 				if (path == null || path.length() == 0)
377 					throw new IllegalArgumentException(
378 							JGitText.get().pathNotConfigured);
379 				try {
380 					inputStream = new FileInputStream(path);
381 				} catch (IOException e) {
382 					throw new IllegalArgumentException(
383 							JGitText.get().pathNotConfigured);
384 				}
385 			}
386 
387 			if (repo.isBare()) {
388 				bareProjects = new ArrayList<RepoProject>();
389 				if (author == null)
390 					author = new PersonIdent(repo);
391 				if (callback == null)
392 					callback = new DefaultRemoteReader();
393 			} else
394 				git = new Git(repo);
395 
396 			ManifestParser parser = new ManifestParser(
397 					includedReader, path, branch, uri, groups, repo);
398 			try {
399 				parser.read(inputStream);
400 				for (RepoProject proj : parser.getFilteredProjects()) {
401 					addSubmodule(proj.getUrl(),
402 							proj.getPath(),
403 							proj.getRevision(),
404 							proj.getCopyFiles());
405 				}
406 			} catch (GitAPIException | IOException e) {
407 				throw new ManifestErrorException(e);
408 			}
409 		} finally {
410 			try {
411 				if (inputStream != null)
412 					inputStream.close();
413 			} catch (IOException e) {
414 				// Just ignore it, it's not important.
415 			}
416 		}
417 
418 		if (repo.isBare()) {
419 			DirCache index = DirCache.newInCore();
420 			DirCacheBuilder builder = index.builder();
421 			ObjectInserter inserter = repo.newObjectInserter();
422 			try (RevWalk rw = new RevWalk(repo)) {
423 				Config cfg = new Config();
424 				for (RepoProject proj : bareProjects) {
425 					String name = proj.getPath();
426 					String nameUri = proj.getName();
427 					cfg.setString("submodule", name, "path", name); //$NON-NLS-1$ //$NON-NLS-2$
428 					cfg.setString("submodule", name, "url", nameUri); //$NON-NLS-1$ //$NON-NLS-2$
429 					// create gitlink
430 					DirCacheEntry dcEntry = new DirCacheEntry(name);
431 					ObjectId objectId;
432 					if (ObjectId.isId(proj.getRevision()))
433 						objectId = ObjectId.fromString(proj.getRevision());
434 					else {
435 						objectId = callback.sha1(nameUri, proj.getRevision());
436 					}
437 					if (objectId == null)
438 						throw new RemoteUnavailableException(nameUri);
439 					dcEntry.setObjectId(objectId);
440 					dcEntry.setFileMode(FileMode.GITLINK);
441 					builder.add(dcEntry);
442 
443 					for (CopyFile copyfile : proj.getCopyFiles()) {
444 						byte[] src = callback.readFile(
445 								nameUri, proj.getRevision(), copyfile.src);
446 						objectId = inserter.insert(Constants.OBJ_BLOB, src);
447 						dcEntry = new DirCacheEntry(copyfile.dest);
448 						dcEntry.setObjectId(objectId);
449 						dcEntry.setFileMode(FileMode.REGULAR_FILE);
450 						builder.add(dcEntry);
451 					}
452 				}
453 				String content = cfg.toText();
454 
455 				// create a new DirCacheEntry for .gitmodules file.
456 				final DirCacheEntry dcEntry = new DirCacheEntry(Constants.DOT_GIT_MODULES);
457 				ObjectId objectId = inserter.insert(Constants.OBJ_BLOB,
458 						content.getBytes(Constants.CHARACTER_ENCODING));
459 				dcEntry.setObjectId(objectId);
460 				dcEntry.setFileMode(FileMode.REGULAR_FILE);
461 				builder.add(dcEntry);
462 
463 				builder.finish();
464 				ObjectId treeId = index.writeTree(inserter);
465 
466 				// Create a Commit object, populate it and write it
467 				ObjectId headId = repo.resolve(targetBranch + "^{commit}"); //$NON-NLS-1$
468 				CommitBuilder commit = new CommitBuilder();
469 				commit.setTreeId(treeId);
470 				if (headId != null)
471 					commit.setParentIds(headId);
472 				commit.setAuthor(author);
473 				commit.setCommitter(author);
474 				commit.setMessage(RepoText.get().repoCommitMessage);
475 
476 				ObjectId commitId = inserter.insert(commit);
477 				inserter.flush();
478 
479 				RefUpdate ru = repo.updateRef(targetBranch);
480 				ru.setNewObjectId(commitId);
481 				ru.setExpectedOldObjectId(headId != null ? headId : ObjectId.zeroId());
482 				Result rc = ru.update(rw);
483 
484 				switch (rc) {
485 					case NEW:
486 					case FORCED:
487 					case FAST_FORWARD:
488 						// Successful. Do nothing.
489 						break;
490 					case REJECTED:
491 					case LOCK_FAILURE:
492 						throw new ConcurrentRefUpdateException(
493 								MessageFormat.format(
494 										JGitText.get().cannotLock, targetBranch),
495 								ru.getRef(),
496 								rc);
497 					default:
498 						throw new JGitInternalException(MessageFormat.format(
499 								JGitText.get().updatingRefFailed,
500 								targetBranch, commitId.name(), rc));
501 				}
502 
503 				return rw.parseCommit(commitId);
504 			} catch (IOException e) {
505 				throw new ManifestErrorException(e);
506 			}
507 		} else {
508 			return git
509 				.commit()
510 				.setMessage(RepoText.get().repoCommitMessage)
511 				.call();
512 		}
513 	}
514 
515 	private void addSubmodule(String url, String name, String revision,
516 			List<CopyFile> copyfiles) throws GitAPIException, IOException {
517 		if (repo.isBare()) {
518 			RepoProject proj = new RepoProject(url, name, revision, null, null);
519 			proj.addCopyFiles(copyfiles);
520 			bareProjects.add(proj);
521 		} else {
522 			SubmoduleAddCommand add = git
523 				.submoduleAdd()
524 				.setPath(name)
525 				.setURI(url);
526 			if (monitor != null)
527 				add.setProgressMonitor(monitor);
528 
529 			Repository subRepo = add.call();
530 			if (revision != null) {
531 				try (Git sub = new Git(subRepo)) {
532 					sub.checkout().setName(findRef(revision, subRepo))
533 							.call();
534 				}
535 				subRepo.close();
536 				git.add().addFilepattern(name).call();
537 			}
538 			for (CopyFile copyfile : copyfiles) {
539 				copyfile.copy();
540 				git.add().addFilepattern(copyfile.dest).call();
541 			}
542 		}
543 	}
544 
545 	private static String findRef(String ref, Repository repo)
546 			throws IOException {
547 		if (!ObjectId.isId(ref)) {
548 			Ref r = repo.getRef(Constants.DEFAULT_REMOTE_NAME + "/" + ref); //$NON-NLS-1$
549 			if (r != null)
550 				return r.getName();
551 		}
552 		return ref;
553 	}
554 }