View Javadoc
1   /*
2    * Copyright (C) 2019 Google LLC and others
3    *
4    * This program and the accompanying materials are made available under the
5    * terms of the Eclipse Distribution License v. 1.0 which is available at
6    * https://www.eclipse.org/org/documents/edl-v10.php.
7    *
8    * SPDX-License-Identifier: BSD-3-Clause
9    */
10  
11  package org.eclipse.jgit.internal.storage.file;
12  
13  import static org.eclipse.jgit.lib.Ref.UNDEFINED_UPDATE_INDEX;
14  import static org.eclipse.jgit.lib.Ref.Storage.NEW;
15  import static org.eclipse.jgit.lib.Ref.Storage.PACKED;
16  
17  import java.io.File;
18  import java.io.IOException;
19  import java.util.ArrayList;
20  import java.util.Collections;
21  import java.util.HashSet;
22  import java.util.List;
23  import java.util.Map;
24  import java.util.TreeSet;
25  import java.util.concurrent.locks.ReentrantLock;
26  import java.util.stream.Collectors;
27  
28  import org.eclipse.jgit.annotations.NonNull;
29  import org.eclipse.jgit.events.RefsChangedEvent;
30  import org.eclipse.jgit.internal.storage.reftable.MergedReftable;
31  import org.eclipse.jgit.internal.storage.reftable.ReftableBatchRefUpdate;
32  import org.eclipse.jgit.internal.storage.reftable.ReftableDatabase;
33  import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
34  import org.eclipse.jgit.lib.BatchRefUpdate;
35  import org.eclipse.jgit.lib.Constants;
36  import org.eclipse.jgit.lib.ObjectId;
37  import org.eclipse.jgit.lib.ObjectIdRef;
38  import org.eclipse.jgit.lib.PersonIdent;
39  import org.eclipse.jgit.lib.Ref;
40  import org.eclipse.jgit.lib.RefDatabase;
41  import org.eclipse.jgit.lib.RefRename;
42  import org.eclipse.jgit.lib.RefUpdate;
43  import org.eclipse.jgit.lib.ReflogEntry;
44  import org.eclipse.jgit.lib.ReflogReader;
45  import org.eclipse.jgit.lib.Repository;
46  import org.eclipse.jgit.lib.SymbolicRef;
47  import org.eclipse.jgit.revwalk.RevObject;
48  import org.eclipse.jgit.revwalk.RevTag;
49  import org.eclipse.jgit.revwalk.RevWalk;
50  import org.eclipse.jgit.transport.ReceiveCommand;
51  import org.eclipse.jgit.util.FileUtils;
52  import org.eclipse.jgit.util.RefList;
53  import org.eclipse.jgit.util.RefMap;
54  
55  /**
56   * Implements RefDatabase using reftable for storage.
57   *
58   * This class is threadsafe.
59   */
60  public class FileReftableDatabase extends RefDatabase {
61  	private final ReftableDatabase reftableDatabase;
62  
63  	private final FileRepository fileRepository;
64  
65  	private final FileReftableStack reftableStack;
66  
67  	FileReftableDatabase(FileRepository repo) throws IOException {
68  		this(repo, new File(new File(repo.getDirectory(), Constants.REFTABLE),
69  				Constants.TABLES_LIST));
70  	}
71  
72  	FileReftableDatabase(FileRepository repo, File refstackName) throws IOException {
73  		this.fileRepository = repo;
74  		this.reftableStack = new FileReftableStack(refstackName,
75  			new File(fileRepository.getDirectory(), Constants.REFTABLE),
76  			() -> fileRepository.fireEvent(new RefsChangedEvent()),
77  			() -> fileRepository.getConfig());
78  		this.reftableDatabase = new ReftableDatabase() {
79  
80  			@Override
81  			public MergedReftable openMergedReftable() throws IOException {
82  				return reftableStack.getMergedReftable();
83  			}
84  		};
85  	}
86  
87  	ReflogReader getReflogReader(String refname) throws IOException {
88  		return reftableDatabase.getReflogReader(refname);
89  	}
90  
91  	/**
92  	 * @param repoDir
93  	 * @return whether the given repo uses reftable for refdb storage.
94  	 */
95  	public static boolean isReftable(File repoDir) {
96  		return new File(repoDir, Constants.REFTABLE).isDirectory();
97  	}
98  
99  	/** {@inheritDoc} */
100 	@Override
101 	public boolean hasFastTipsWithSha1() throws IOException {
102 		return reftableDatabase.hasFastTipsWithSha1();
103 	}
104 
105 	/**
106 	 * Runs a full compaction for GC purposes.
107 	 * @throws IOException on I/O errors
108 	 */
109 	public void compactFully() throws IOException {
110 		reftableDatabase.getLock().lock();
111 		try {
112 			reftableStack.compactFully();
113 			reftableDatabase.clearCache();
114 		} finally {
115 			reftableDatabase.getLock().unlock();
116 		}
117 	}
118 
119 	private ReentrantLock getLock() {
120 		return reftableDatabase.getLock();
121 	}
122 
123 	/** {@inheritDoc} */
124 	@Override
125 	public boolean performsAtomicTransactions() {
126 		return true;
127 	}
128 
129 	/** {@inheritDoc} */
130 	@NonNull
131 	@Override
132 	public BatchRefUpdate newBatchUpdate() {
133 		return new FileReftableBatchRefUpdate(this, fileRepository);
134 	}
135 
136 	/** {@inheritDoc} */
137 	@Override
138 	public RefUpdate newUpdate(String refName, boolean detach)
139 			throws IOException {
140 		boolean detachingSymbolicRef = false;
141 		Ref ref = exactRef(refName);
142 
143 		if (ref == null) {
144 			ref = new ObjectIdRef.Unpeeled(NEW, refName, null);
145 		} else {
146 			detachingSymbolicRef = detach && ref.isSymbolic();
147 		}
148 
149 		RefUpdate update = new FileReftableRefUpdate(ref);
150 		if (detachingSymbolicRef) {
151 			update.setDetachingSymbolicRef();
152 		}
153 		return update;
154 	}
155 
156 	/** {@inheritDoc} */
157 	@Override
158 	public Ref exactRef(String name) throws IOException {
159 		return reftableDatabase.exactRef(name);
160 	}
161 
162 	/** {@inheritDoc} */
163 	@Override
164 	public List<Ref> getRefs() throws IOException {
165 		return super.getRefs();
166 	}
167 
168 	/** {@inheritDoc} */
169 	@Override
170 	public Map<String, Ref> getRefs(String prefix) throws IOException {
171 		List<Ref> refs = reftableDatabase.getRefsByPrefix(prefix);
172 		RefList.Builder<Ref> builder = new RefList.Builder<>(refs.size());
173 		for (Ref r : refs) {
174 			builder.add(r);
175 		}
176 		return new RefMap(prefix, builder.toRefList(), RefList.emptyList(),
177 				RefList.emptyList());
178 	}
179 
180 	/** {@inheritDoc} */
181 	@Override
182 	public List<Ref> getAdditionalRefs() throws IOException {
183 		return Collections.emptyList();
184 	}
185 
186 	/** {@inheritDoc} */
187 	@Override
188 	public Ref" href="../../../../../../org/eclipse/jgit/lib/Ref.html#Ref">Ref peel(Ref ref) throws IOException {
189 		Ref oldLeaf = ref.getLeaf();
190 		if (oldLeaf.isPeeled() || oldLeaf.getObjectId() == null) {
191 			return ref;
192 		}
193 		return recreate(ref, doPeel(oldLeaf), hasVersioning());
194 
195 	}
196 
197 	private Refhref="../../../../../../org/eclipse/jgit/lib/Ref.html#Ref">Ref doPeel(Ref leaf) throws IOException {
198 		try (RevWalkvwalk/RevWalk.html#RevWalk">RevWalk rw = new RevWalk(fileRepository)) {
199 			RevObject obj = rw.parseAny(leaf.getObjectId());
200 			if (obj instanceof RevTag) {
201 				return new ObjectIdRef.PeeledTag(leaf.getStorage(),
202 						leaf.getName(), leaf.getObjectId(), rw.peel(obj).copy(),
203 						hasVersioning() ? leaf.getUpdateIndex()
204 								: UNDEFINED_UPDATE_INDEX);
205 			}
206 			return new ObjectIdRef.PeeledNonTag(leaf.getStorage(),
207 					leaf.getName(), leaf.getObjectId(),
208 					hasVersioning() ? leaf.getUpdateIndex()
209 							: UNDEFINED_UPDATE_INDEX);
210 
211 		}
212 	}
213 
214 	private static Ref" href="../../../../../../org/eclipse/jgit/lib/Ref.html#Ref">Refef="../../../../../../org/eclipse/jgit/lib/Ref.html#Ref">Ref recreate(Ref" href="../../../../../../org/eclipse/jgit/lib/Ref.html#Ref">Ref old, Ref leaf, boolean hasVersioning) {
215 		if (old.isSymbolic()) {
216 			Ref dst = recreate(old.getTarget(), leaf, hasVersioning);
217 			return new SymbolicRef(old.getName(), dst,
218 					hasVersioning ? old.getUpdateIndex()
219 							: UNDEFINED_UPDATE_INDEX);
220 		}
221 		return leaf;
222 	}
223 
224 	private class FileRefRename extends RefRename {
225 		FileRefRename(RefUpdate="../../../../../../org/eclipse/jgit/lib/RefUpdate.html#RefUpdate">RefUpdate src, RefUpdate dst) {
226 			super(src, dst);
227 		}
228 
229 		void writeRename(ReftableWriter w) throws IOException {
230 			long idx = reftableDatabase.nextUpdateIndex();
231 			w.setMinUpdateIndex(idx).setMaxUpdateIndex(idx).begin();
232 			List<Ref> refs = new ArrayList<>(3);
233 
234 			Ref dest = destination.getRef();
235 			Ref head = exactRef(Constants.HEAD);
236 			if (head != null && head.isSymbolic()
237 					&& head.getLeaf().getName().equals(source.getName())) {
238 				head = new SymbolicRef(Constants.HEAD, dest, idx);
239 				refs.add(head);
240 			}
241 
242 			ObjectId objId = source.getRef().getObjectId();
243 
244 			// XXX should we check if the source is a Tag vs. NonTag?
245 			refs.add(new ObjectIdRef.PeeledNonTag(Ref.Storage.NEW,
246 					destination.getName(), objId));
247 			refs.add(new ObjectIdRef.Unpeeled(Ref.Storage.NEW, source.getName(),
248 					null));
249 
250 			w.sortAndWriteRefs(refs);
251 			PersonIdent who = destination.getRefLogIdent();
252 			if (who == null) {
253 				who = new PersonIdent(fileRepository);
254 			}
255 
256 			if (!destination.getRefLogMessage().isEmpty()) {
257 				List<String> refnames = refs.stream().map(r -> r.getName())
258 						.collect(Collectors.toList());
259 				Collections.sort(refnames);
260 				for (String s : refnames) {
261 					ObjectId old = (Constants.HEAD.equals(s)
262 							|| s.equals(source.getName())) ? objId
263 									: ObjectId.zeroId();
264 					ObjectId newId = (Constants.HEAD.equals(s)
265 							|| s.equals(destination.getName())) ? objId
266 									: ObjectId.zeroId();
267 
268 					w.writeLog(s, idx, who, old, newId,
269 							destination.getRefLogMessage());
270 				}
271 			}
272 		}
273 
274 		@Override
275 		protected RefUpdate.Result doRename() throws IOException {
276 			Ref src = exactRef(source.getName());
277 			if (exactRef(destination.getName()) != null || src == null
278 					|| !source.getOldObjectId().equals(src.getObjectId())) {
279 				return RefUpdate.Result.LOCK_FAILURE;
280 			}
281 
282 			if (src.isSymbolic()) {
283 				// We could support this, but this is easier and compatible.
284 				return RefUpdate.Result.IO_FAILURE;
285 			}
286 
287 			if (!addReftable(this::writeRename)) {
288 				return RefUpdate.Result.LOCK_FAILURE;
289 			}
290 
291 			return RefUpdate.Result.RENAMED;
292 		}
293 	}
294 
295 	/** {@inheritDoc} */
296 	@Override
297 	public RefRename newRename(String fromName, String toName)
298 			throws IOException {
299 		RefUpdate src = newUpdate(fromName, true);
300 		RefUpdate dst = newUpdate(toName, true);
301 		return new FileRefRename(src, dst);
302 	}
303 
304 	/** {@inheritDoc} */
305 	@Override
306 	public boolean isNameConflicting(String name) throws IOException {
307 		return reftableDatabase.isNameConflicting(name, new TreeSet<>(),
308 				new HashSet<>());
309 	}
310 
311 	/** {@inheritDoc} */
312 	@Override
313 	public void close() {
314 		reftableStack.close();
315 	}
316 
317 	/** {@inheritDoc} */
318 	@Override
319 	public void create() throws IOException {
320 		FileUtils.mkdir(
321 				new File(fileRepository.getDirectory(), Constants.REFTABLE),
322 				true);
323 	}
324 
325 	private boolean addReftable(FileReftableStack.Writer w) throws IOException {
326 		if (!reftableStack.addReftable(w)) {
327 			reftableStack.reload();
328 			reftableDatabase.clearCache();
329 			return false;
330 		}
331 		reftableDatabase.clearCache();
332 
333 		return true;
334 	}
335 
336 	private class FileReftableBatchRefUpdate extends ReftableBatchRefUpdate {
337 		FileReftableBatchRefUpdate(FileReftableDatabase db,
338 				Repository repository) {
339 			super(db, db.reftableDatabase, db.getLock(), repository);
340 		}
341 
342 		@Override
343 		protected void applyUpdates(List<Ref> newRefs,
344 				List<ReceiveCommand> pending) throws IOException {
345 			if (!addReftable(rw -> write(rw, newRefs, pending))) {
346 				for (ReceiveCommand c : pending) {
347 					if (c.getResult() == ReceiveCommand.Result.NOT_ATTEMPTED) {
348 						c.setResult(RefUpdate.Result.LOCK_FAILURE);
349 					}
350 				}
351 			}
352 		}
353 	}
354 
355 	private class FileReftableRefUpdate extends RefUpdate {
356 		FileReftableRefUpdate(Ref ref) {
357 			super(ref);
358 		}
359 
360 		@Override
361 		protected RefDatabase getRefDatabase() {
362 			return FileReftableDatabase.this;
363 		}
364 
365 		@Override
366 		protected Repository getRepository() {
367 			return FileReftableDatabase.this.fileRepository;
368 		}
369 
370 		@Override
371 		protected void unlock() {
372 			// nop.
373 		}
374 
375 		private RevWalk rw;
376 
377 		private Ref dstRef;
378 
379 		@Override
380 		public Result update(RevWalk walk) throws IOException {
381 			try {
382 				rw = walk;
383 				return super.update(walk);
384 			} finally {
385 				rw = null;
386 			}
387 		}
388 
389 		@Override
390 		protected boolean tryLock(boolean deref) throws IOException {
391 			dstRef = getRef();
392 			if (deref) {
393 				dstRef = dstRef.getLeaf();
394 			}
395 
396 			Ref derefed = exactRef(dstRef.getName());
397 			if (derefed != null) {
398 				setOldObjectId(derefed.getObjectId());
399 			}
400 
401 			return true;
402 		}
403 
404 		void writeUpdate(ReftableWriter w) throws IOException {
405 			Ref newRef = null;
406 			if (rw != null && !ObjectId.zeroId().equals(getNewObjectId())) {
407 				RevObject obj = rw.parseAny(getNewObjectId());
408 				if (obj instanceof RevTag) {
409 					newRef = new ObjectIdRef.PeeledTag(Ref.Storage.PACKED,
410 							dstRef.getName(), getNewObjectId(),
411 							rw.peel(obj).copy());
412 				}
413 			}
414 			if (newRef == null) {
415 				newRef = new ObjectIdRef.PeeledNonTag(Ref.Storage.PACKED,
416 						dstRef.getName(), getNewObjectId());
417 			}
418 
419 			long idx = reftableDatabase.nextUpdateIndex();
420 			w.setMinUpdateIndex(idx).setMaxUpdateIndex(idx).begin()
421 					.writeRef(newRef);
422 
423 			ObjectId oldId = getOldObjectId();
424 			if (oldId == null) {
425 				oldId = ObjectId.zeroId();
426 			}
427 			w.writeLog(dstRef.getName(), idx, getRefLogIdent(), oldId,
428 					getNewObjectId(), getRefLogMessage());
429 		}
430 
431 		@Override
432 		public PersonIdent getRefLogIdent() {
433 			PersonIdent who = super.getRefLogIdent();
434 			if (who == null) {
435 				who = new PersonIdent(getRepository());
436 			}
437 			return who;
438 		}
439 
440 		void writeDelete(ReftableWriter w) throws IOException {
441 			Ref newRef = new ObjectIdRef.Unpeeled(Ref.Storage.NEW,
442 					dstRef.getName(), null);
443 			long idx = reftableDatabase.nextUpdateIndex();
444 			w.setMinUpdateIndex(idx).setMaxUpdateIndex(idx).begin()
445 					.writeRef(newRef);
446 
447 			ObjectId oldId = ObjectId.zeroId();
448 			Ref old = exactRef(dstRef.getName());
449 			if (old != null) {
450 				old = old.getLeaf();
451 				if (old.getObjectId() != null) {
452 					oldId = old.getObjectId();
453 				}
454 			}
455 
456 			w.writeLog(dstRef.getName(), idx, getRefLogIdent(), oldId,
457 					ObjectId.zeroId(), getRefLogMessage());
458 		}
459 
460 		@Override
461 		protected Result doUpdate(Result desiredResult) throws IOException {
462 			if (isRefLogIncludingResult()) {
463 				setRefLogMessage(
464 						getRefLogMessage() + ": " + desiredResult.toString(), //$NON-NLS-1$
465 						false);
466 			}
467 
468 			if (!addReftable(this::writeUpdate)) {
469 				return Result.LOCK_FAILURE;
470 			}
471 
472 			return desiredResult;
473 		}
474 
475 		@Override
476 		protected Result doDelete(Result desiredResult) throws IOException {
477 
478 			if (isRefLogIncludingResult()) {
479 				setRefLogMessage(
480 						getRefLogMessage() + ": " + desiredResult.toString(), //$NON-NLS-1$
481 						false);
482 			}
483 
484 			if (!addReftable(this::writeDelete)) {
485 				return Result.LOCK_FAILURE;
486 			}
487 
488 			return desiredResult;
489 		}
490 
491 		void writeLink(ReftableWriter w) throws IOException {
492 			long idx = reftableDatabase.nextUpdateIndex();
493 			w.setMinUpdateIndex(idx).setMaxUpdateIndex(idx).begin()
494 					.writeRef(dstRef);
495 
496 			ObjectId beforeId = ObjectId.zeroId();
497 			Ref before = exactRef(dstRef.getName());
498 			if (before != null) {
499 				before = before.getLeaf();
500 				if (before.getObjectId() != null) {
501 					beforeId = before.getObjectId();
502 				}
503 			}
504 
505 			Ref after = dstRef.getLeaf();
506 			ObjectId afterId = ObjectId.zeroId();
507 			if (after.getObjectId() != null) {
508 				afterId = after.getObjectId();
509 			}
510 
511 			w.writeLog(dstRef.getName(), idx, getRefLogIdent(), beforeId,
512 					afterId, getRefLogMessage());
513 		}
514 
515 		@Override
516 		protected Result doLink(String target) throws IOException {
517 			if (isRefLogIncludingResult()) {
518 				setRefLogMessage(
519 						getRefLogMessage() + ": " + Result.FORCED.toString(), //$NON-NLS-1$
520 						false);
521 			}
522 
523 			boolean exists = exactRef(getName()) != null;
524 			dstRef = new SymbolicRef(getName(),
525 					new ObjectIdRef.Unpeeled(Ref.Storage.NEW, target, null),
526 					reftableDatabase.nextUpdateIndex());
527 
528 			if (!addReftable(this::writeLink)) {
529 				return Result.LOCK_FAILURE;
530 			}
531 			// XXX unclear if we should support FORCED here. Baseclass says
532 			// NEW is OK ?
533 			return exists ? Result.FORCED : Result.NEW;
534 		}
535 	}
536 
537 	private static void writeConvertTable(Repository repo, ReftableWriter w,
538 			boolean writeLogs) throws IOException {
539 		int size = 0;
540 		List<Ref> refs = repo.getRefDatabase().getRefs();
541 		if (writeLogs) {
542 			for (Ref r : refs) {
543 				ReflogReader rlr = repo.getReflogReader(r.getName());
544 				if (rlr != null) {
545 					size = Math.max(rlr.getReverseEntries().size(), size);
546 				}
547 			}
548 		}
549 		// We must use 1 here, nextUpdateIndex() on the empty stack is 1.
550 		w.setMinUpdateIndex(1).setMaxUpdateIndex(size + 1).begin();
551 
552 		// The spec says to write the logs in the first table, and put refs in a
553 		// separate table, but this complicates the compaction (when we can we drop
554 		// deletions? Can we compact the .log table and the .ref table together?)
555 		try (RevWalkvwalk/RevWalk.html#RevWalk">RevWalk rw = new RevWalk(repo)) {
556 			List<Ref> toWrite = new ArrayList<>(refs.size());
557 			for (Ref r : refs) {
558 				toWrite.add(refForWrite(rw, r));
559 			}
560 			w.sortAndWriteRefs(toWrite);
561 		}
562 
563 		if (writeLogs) {
564 			for (Ref r : refs) {
565 				long idx = size;
566 				ReflogReader reader = repo.getReflogReader(r.getName());
567 				if (reader == null) {
568 					continue;
569 				}
570 				for (ReflogEntry e : reader.getReverseEntries()) {
571 					w.writeLog(r.getName(), idx, e.getWho(), e.getOldId(),
572 							e.getNewId(), e.getComment());
573 					idx--;
574 				}
575 			}
576 		}
577 	}
578 
579 	private static Ref/../../org/eclipse/jgit/lib/Ref.html#Ref">Ref refForWrite(RevWalk rw, Ref r) throws IOException {
580 		if (r.isSymbolic()) {
581 			return new SymbolicRef(r.getName(), new ObjectIdRef.Unpeeled(NEW,
582 					r.getTarget().getName(), null));
583 		}
584 		ObjectId newId = r.getObjectId();
585 		RevObject obj = rw.parseAny(newId);
586 		RevObject peel = null;
587 		if (obj instanceof RevTag) {
588 			peel = rw.peel(obj);
589 		}
590 		if (peel != null) {
591 			return new ObjectIdRef.PeeledTag(PACKED, r.getName(), newId,
592 					peel.copy());
593 		}
594 		return new ObjectIdRef.PeeledNonTag(PACKED, r.getName(), newId);
595 	}
596 
597 	/**
598 	 * @param repo
599 	 *            the repository
600 	 * @param writeLogs
601 	 *            whether to write reflogs
602 	 * @return a reftable based RefDB from an existing repository.
603 	 * @throws IOException
604 	 *             on IO error
605 	 */
606 	public static FileReftableDatabase convertFrom(FileRepository repo,
607 			boolean writeLogs) throws IOException {
608 		FileReftableDatabase newDb = null;
609 		File reftableList = null;
610 		try {
611 			File reftableDir = new File(repo.getDirectory(),
612 					Constants.REFTABLE);
613 			reftableList = new File(reftableDir, Constants.TABLES_LIST);
614 			if (!reftableDir.isDirectory()) {
615 				reftableDir.mkdir();
616 			}
617 
618 			try (FileReftableStacke/file/FileReftableStack.html#FileReftableStack">FileReftableStack stack = new FileReftableStack(reftableList,
619 					reftableDir, null, () -> repo.getConfig())) {
620 				stack.addReftable(rw -> writeConvertTable(repo, rw, writeLogs));
621 			}
622 			reftableList = null;
623 		} finally {
624 			if (reftableList != null) {
625 				reftableList.delete();
626 			}
627 		}
628 		return newDb;
629 	}
630 }