View Javadoc
1   /*
2    * Copyright (C) 2021, Google Inc. and others
3    *
4    * This program and the accompanying materials are made available under the
5    * terms of the Eclipse Distribution License v. 1.0 which is available at
6    * https://www.eclipse.org/org/documents/edl-v10.php.
7    *
8    * SPDX-License-Identifier: BSD-3-Clause
9    */
10  package org.eclipse.jgit.gitrepo;
11  
12  import static java.nio.charset.StandardCharsets.UTF_8;
13  import static org.eclipse.jgit.lib.Constants.R_TAGS;
14  
15  import java.io.IOException;
16  import java.net.URI;
17  import java.text.MessageFormat;
18  import java.util.List;
19  
20  import org.eclipse.jgit.api.errors.ConcurrentRefUpdateException;
21  import org.eclipse.jgit.api.errors.GitAPIException;
22  import org.eclipse.jgit.api.errors.JGitInternalException;
23  import org.eclipse.jgit.dircache.DirCache;
24  import org.eclipse.jgit.dircache.DirCacheBuilder;
25  import org.eclipse.jgit.dircache.DirCacheEntry;
26  import org.eclipse.jgit.gitrepo.RepoCommand.ManifestErrorException;
27  import org.eclipse.jgit.gitrepo.RepoCommand.RemoteFile;
28  import org.eclipse.jgit.gitrepo.RepoCommand.RemoteReader;
29  import org.eclipse.jgit.gitrepo.RepoCommand.RemoteUnavailableException;
30  import org.eclipse.jgit.gitrepo.RepoProject.CopyFile;
31  import org.eclipse.jgit.gitrepo.RepoProject.LinkFile;
32  import org.eclipse.jgit.gitrepo.internal.RepoText;
33  import org.eclipse.jgit.internal.JGitText;
34  import org.eclipse.jgit.lib.CommitBuilder;
35  import org.eclipse.jgit.lib.Config;
36  import org.eclipse.jgit.lib.Constants;
37  import org.eclipse.jgit.lib.FileMode;
38  import org.eclipse.jgit.lib.ObjectId;
39  import org.eclipse.jgit.lib.ObjectInserter;
40  import org.eclipse.jgit.lib.PersonIdent;
41  import org.eclipse.jgit.lib.RefUpdate;
42  import org.eclipse.jgit.lib.RefUpdate.Result;
43  import org.eclipse.jgit.lib.Repository;
44  import org.eclipse.jgit.revwalk.RevCommit;
45  import org.eclipse.jgit.revwalk.RevWalk;
46  import org.eclipse.jgit.util.FileUtils;
47  
48  /**
49   * Writes .gitmodules and gitlinks of parsed manifest projects into a bare
50   * repository.
51   *
52   * To write on a regular repository, see {@link RegularSuperprojectWriter}.
53   */
54  class BareSuperprojectWriter {
55  	private static final int LOCK_FAILURE_MAX_RETRIES = 5;
56  
57  	// Retry exponentially with delays in this range
58  	private static final int LOCK_FAILURE_MIN_RETRY_DELAY_MILLIS = 50;
59  
60  	private static final int LOCK_FAILURE_MAX_RETRY_DELAY_MILLIS = 5000;
61  
62  	private final Repository repo;
63  
64  	private final URI targetUri;
65  
66  	private final String targetBranch;
67  
68  	private final RemoteReader callback;
69  
70  	private final BareWriterConfig config;
71  
72  	private final PersonIdent author;
73  
74  	private List<ExtraContent> extraContents;
75  
76  	static class BareWriterConfig {
77  		boolean ignoreRemoteFailures = false;
78  
79  		boolean recordRemoteBranch = true;
80  
81  		boolean recordSubmoduleLabels = true;
82  
83  		boolean recordShallowSubmodules = true;
84  
85  		static BareWriterConfig getDefault() {
86  			return new BareWriterConfig();
87  		}
88  
89  		private BareWriterConfig() {
90  		}
91  	}
92  
93  	static class ExtraContent {
94  		final String path;
95  
96  		final String content;
97  
98  		ExtraContent(String path, String content) {
99  			this.path = path;
100 			this.content = content;
101 		}
102 	}
103 
104 	BareSuperprojectWriter(Repository repo, URI targetUri,
105 			String targetBranch,
106 			PersonIdent author, RemoteReader callback,
107 			BareWriterConfig config,
108 			List<ExtraContent> extraContents) {
109 		assert (repo.isBare());
110 		this.repo = repo;
111 		this.targetUri = targetUri;
112 		this.targetBranch = targetBranch;
113 		this.author = author;
114 		this.callback = callback;
115 		this.config = config;
116 		this.extraContents = extraContents;
117 	}
118 
119 	RevCommit write(List<RepoProject> repoProjects)
120 			throws GitAPIException {
121 		DirCache index = DirCache.newInCore();
122 		ObjectInserter inserter = repo.newObjectInserter();
123 
124 		try (RevWalk rw = new RevWalk(repo)) {
125 			prepareIndex(repoProjects, index, inserter);
126 			ObjectId treeId = index.writeTree(inserter);
127 			long prevDelay = 0;
128 			for (int i = 0; i < LOCK_FAILURE_MAX_RETRIES - 1; i++) {
129 				try {
130 					return commitTreeOnCurrentTip(inserter, rw, treeId);
131 				} catch (ConcurrentRefUpdateException e) {
132 					prevDelay = FileUtils.delay(prevDelay,
133 							LOCK_FAILURE_MIN_RETRY_DELAY_MILLIS,
134 							LOCK_FAILURE_MAX_RETRY_DELAY_MILLIS);
135 					Thread.sleep(prevDelay);
136 					repo.getRefDatabase().refresh();
137 				}
138 			}
139 			// In the last try, just propagate the exceptions
140 			return commitTreeOnCurrentTip(inserter, rw, treeId);
141 		} catch (IOException | InterruptedException e) {
142 			throw new ManifestErrorException(e);
143 		}
144 	}
145 
146 	private void prepareIndex(List<RepoProject> projects, DirCache index,
147 			ObjectInserter inserter) throws IOException, GitAPIException {
148 		Config cfg = new Config();
149 		StringBuilder attributes = new StringBuilder();
150 		DirCacheBuilder builder = index.builder();
151 		for (RepoProject proj : projects) {
152 			String name = proj.getName();
153 			String path = proj.getPath();
154 			String url = proj.getUrl();
155 			ObjectId objectId;
156 			if (ObjectId.isId(proj.getRevision())) {
157 				objectId = ObjectId.fromString(proj.getRevision());
158 			} else {
159 				objectId = callback.sha1(url, proj.getRevision());
160 				if (objectId == null && !config.ignoreRemoteFailures) {
161 					throw new RemoteUnavailableException(url);
162 				}
163 				if (config.recordRemoteBranch) {
164 					// "branch" field is only for non-tag references.
165 					// Keep tags in "ref" field as hint for other tools.
166 					String field = proj.getRevision().startsWith(R_TAGS) ? "ref" //$NON-NLS-1$
167 							: "branch"; //$NON-NLS-1$
168 					cfg.setString("submodule", name, field, //$NON-NLS-1$
169 							proj.getRevision());
170 				}
171 
172 				if (config.recordShallowSubmodules
173 						&& proj.getRecommendShallow() != null) {
174 					// The shallow recommendation is losing information.
175 					// As the repo manifests stores the recommended
176 					// depth in the 'clone-depth' field, while
177 					// git core only uses a binary 'shallow = true/false'
178 					// hint, we'll map any depth to 'shallow = true'
179 					cfg.setBoolean("submodule", name, "shallow", //$NON-NLS-1$ //$NON-NLS-2$
180 							true);
181 				}
182 			}
183 			if (config.recordSubmoduleLabels) {
184 				StringBuilder rec = new StringBuilder();
185 				rec.append("/"); //$NON-NLS-1$
186 				rec.append(path);
187 				for (String group : proj.getGroups()) {
188 					rec.append(" "); //$NON-NLS-1$
189 					rec.append(group);
190 				}
191 				rec.append("\n"); //$NON-NLS-1$
192 				attributes.append(rec.toString());
193 			}
194 
195 			URI submodUrl = URI.create(url);
196 			if (targetUri != null) {
197 				submodUrl = RepoCommand.relativize(targetUri, submodUrl);
198 			}
199 			cfg.setString("submodule", name, "path", path); //$NON-NLS-1$ //$NON-NLS-2$
200 			cfg.setString("submodule", name, "url", //$NON-NLS-1$ //$NON-NLS-2$
201 					submodUrl.toString());
202 
203 			// create gitlink
204 			if (objectId != null) {
205 				DirCacheEntry dcEntry = new DirCacheEntry(path);
206 				dcEntry.setObjectId(objectId);
207 				dcEntry.setFileMode(FileMode.GITLINK);
208 				builder.add(dcEntry);
209 
210 				for (CopyFile copyfile : proj.getCopyFiles()) {
211 					RemoteFile rf = callback.readFileWithMode(url,
212 							proj.getRevision(), copyfile.src);
213 					objectId = inserter.insert(Constants.OBJ_BLOB,
214 							rf.getContents());
215 					dcEntry = new DirCacheEntry(copyfile.dest);
216 					dcEntry.setObjectId(objectId);
217 					dcEntry.setFileMode(rf.getFileMode());
218 					builder.add(dcEntry);
219 				}
220 				for (LinkFile linkfile : proj.getLinkFiles()) {
221 					String link;
222 					if (linkfile.dest.contains("/")) { //$NON-NLS-1$
223 						link = FileUtils.relativizeGitPath(
224 								linkfile.dest.substring(0,
225 										linkfile.dest.lastIndexOf('/')),
226 								proj.getPath() + "/" + linkfile.src); //$NON-NLS-1$
227 					} else {
228 						link = proj.getPath() + "/" + linkfile.src; //$NON-NLS-1$
229 					}
230 
231 					objectId = inserter.insert(Constants.OBJ_BLOB,
232 							link.getBytes(UTF_8));
233 					dcEntry = new DirCacheEntry(linkfile.dest);
234 					dcEntry.setObjectId(objectId);
235 					dcEntry.setFileMode(FileMode.SYMLINK);
236 					builder.add(dcEntry);
237 				}
238 			}
239 		}
240 		String content = cfg.toText();
241 
242 		// create a new DirCacheEntry for .gitmodules file.
243 		DirCacheEntry dcEntry = new DirCacheEntry(
244 				Constants.DOT_GIT_MODULES);
245 		ObjectId objectId = inserter.insert(Constants.OBJ_BLOB,
246 				content.getBytes(UTF_8));
247 		dcEntry.setObjectId(objectId);
248 		dcEntry.setFileMode(FileMode.REGULAR_FILE);
249 		builder.add(dcEntry);
250 
251 		if (config.recordSubmoduleLabels) {
252 			// create a new DirCacheEntry for .gitattributes file.
253 			DirCacheEntry dcEntryAttr = new DirCacheEntry(
254 					Constants.DOT_GIT_ATTRIBUTES);
255 			ObjectId attrId = inserter.insert(Constants.OBJ_BLOB,
256 					attributes.toString().getBytes(UTF_8));
257 			dcEntryAttr.setObjectId(attrId);
258 			dcEntryAttr.setFileMode(FileMode.REGULAR_FILE);
259 			builder.add(dcEntryAttr);
260 		}
261 
262 		for (ExtraContent ec : extraContents) {
263 			DirCacheEntry extraDcEntry = new DirCacheEntry(ec.path);
264 
265 			ObjectId oid = inserter.insert(Constants.OBJ_BLOB,
266 					ec.content.getBytes(UTF_8));
267 			extraDcEntry.setObjectId(oid);
268 			extraDcEntry.setFileMode(FileMode.REGULAR_FILE);
269 			builder.add(extraDcEntry);
270 		}
271 
272 		builder.finish();
273 	}
274 
275 	private RevCommit commitTreeOnCurrentTip(ObjectInserter inserter,
276 			RevWalk rw, ObjectId treeId)
277 			throws IOException, ConcurrentRefUpdateException {
278 		ObjectId headId = repo.resolve(targetBranch + "^{commit}"); //$NON-NLS-1$
279 		if (headId != null
280 				&& rw.parseCommit(headId).getTree().getId().equals(treeId)) {
281 			// No change. Do nothing.
282 			return rw.parseCommit(headId);
283 		}
284 
285 		CommitBuilder commit = new CommitBuilder();
286 		commit.setTreeId(treeId);
287 		if (headId != null) {
288 			commit.setParentIds(headId);
289 		}
290 		commit.setAuthor(author);
291 		commit.setCommitter(author);
292 		commit.setMessage(RepoText.get().repoCommitMessage);
293 
294 		ObjectId commitId = inserter.insert(commit);
295 		inserter.flush();
296 
297 		RefUpdate ru = repo.updateRef(targetBranch);
298 		ru.setNewObjectId(commitId);
299 		ru.setExpectedOldObjectId(headId != null ? headId : ObjectId.zeroId());
300 		Result rc = ru.update(rw);
301 		switch (rc) {
302 		case NEW:
303 		case FORCED:
304 		case FAST_FORWARD:
305 			// Successful. Do nothing.
306 			break;
307 		case REJECTED:
308 		case LOCK_FAILURE:
309 			throw new ConcurrentRefUpdateException(MessageFormat.format(
310 					JGitText.get().cannotLock, targetBranch), ru.getRef(), rc);
311 		default:
312 			throw new JGitInternalException(
313 					MessageFormat.format(JGitText.get().updatingRefFailed,
314 							targetBranch, commitId.name(), rc));
315 		}
316 
317 		return rw.parseCommit(commitId);
318 	}
319 }