View Javadoc
1   /*
2    * Copyright (C) 2012, Christian Halstrick <christian.halstrick@sap.com>
3    * Copyright (C) 2011, Shawn O. Pearce <spearce@spearce.org>
4    * and other copyright owners as documented in the project's IP log.
5    *
6    * This program and the accompanying materials are made available
7    * under the terms of the Eclipse Distribution License v1.0 which
8    * accompanies this distribution, is reproduced below, and is
9    * available at http://www.eclipse.org/org/documents/edl-v10.php
10   *
11   * All rights reserved.
12   *
13   * Redistribution and use in source and binary forms, with or
14   * without modification, are permitted provided that the following
15   * conditions are met:
16   *
17   * - Redistributions of source code must retain the above copyright
18   *   notice, this list of conditions and the following disclaimer.
19   *
20   * - Redistributions in binary form must reproduce the above
21   *   copyright notice, this list of conditions and the following
22   *   disclaimer in the documentation and/or other materials provided
23   *   with the distribution.
24   *
25   * - Neither the name of the Eclipse Foundation, Inc. nor the
26   *   names of its contributors may be used to endorse or promote
27   *   products derived from this software without specific prior
28   *   written permission.
29   *
30   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
31   * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
32   * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
33   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
34   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
35   * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
36   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
37   * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
38   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
39   * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
40   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
41   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
42   * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43   */
44  package org.eclipse.jgit.internal.storage.file;
45  
46  import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
47  import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
48  
49  import java.io.File;
50  import java.io.FileOutputStream;
51  import java.io.IOException;
52  import java.io.OutputStream;
53  import java.io.PrintWriter;
54  import java.io.StringWriter;
55  import java.nio.channels.Channels;
56  import java.nio.channels.FileChannel;
57  import java.nio.file.DirectoryNotEmptyException;
58  import java.nio.file.DirectoryStream;
59  import java.nio.file.Files;
60  import java.nio.file.Path;
61  import java.nio.file.StandardCopyOption;
62  import java.text.MessageFormat;
63  import java.text.ParseException;
64  import java.time.Instant;
65  import java.time.temporal.ChronoUnit;
66  import java.util.ArrayList;
67  import java.util.Collection;
68  import java.util.Collections;
69  import java.util.Comparator;
70  import java.util.Date;
71  import java.util.HashMap;
72  import java.util.HashSet;
73  import java.util.Iterator;
74  import java.util.LinkedList;
75  import java.util.List;
76  import java.util.Map;
77  import java.util.Objects;
78  import java.util.Set;
79  import java.util.TreeMap;
80  import java.util.concurrent.Callable;
81  import java.util.concurrent.ExecutorService;
82  import java.util.regex.Pattern;
83  import java.util.stream.Collectors;
84  import java.util.stream.Stream;
85  
86  import org.eclipse.jgit.annotations.NonNull;
87  import org.eclipse.jgit.dircache.DirCacheIterator;
88  import org.eclipse.jgit.errors.CancelledException;
89  import org.eclipse.jgit.errors.CorruptObjectException;
90  import org.eclipse.jgit.errors.IncorrectObjectTypeException;
91  import org.eclipse.jgit.errors.MissingObjectException;
92  import org.eclipse.jgit.errors.NoWorkTreeException;
93  import org.eclipse.jgit.internal.JGitText;
94  import org.eclipse.jgit.internal.storage.pack.PackExt;
95  import org.eclipse.jgit.internal.storage.pack.PackWriter;
96  import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
97  import org.eclipse.jgit.lib.ConfigConstants;
98  import org.eclipse.jgit.lib.Constants;
99  import org.eclipse.jgit.lib.FileMode;
100 import org.eclipse.jgit.lib.NullProgressMonitor;
101 import org.eclipse.jgit.lib.ObjectId;
102 import org.eclipse.jgit.lib.ObjectIdSet;
103 import org.eclipse.jgit.lib.ObjectLoader;
104 import org.eclipse.jgit.lib.ObjectReader;
105 import org.eclipse.jgit.lib.ProgressMonitor;
106 import org.eclipse.jgit.lib.Ref;
107 import org.eclipse.jgit.lib.Ref.Storage;
108 import org.eclipse.jgit.lib.RefDatabase;
109 import org.eclipse.jgit.lib.ReflogEntry;
110 import org.eclipse.jgit.lib.ReflogReader;
111 import org.eclipse.jgit.lib.internal.WorkQueue;
112 import org.eclipse.jgit.revwalk.ObjectWalk;
113 import org.eclipse.jgit.revwalk.RevObject;
114 import org.eclipse.jgit.revwalk.RevWalk;
115 import org.eclipse.jgit.storage.pack.PackConfig;
116 import org.eclipse.jgit.treewalk.TreeWalk;
117 import org.eclipse.jgit.treewalk.filter.TreeFilter;
118 import org.eclipse.jgit.util.FileUtils;
119 import org.eclipse.jgit.util.GitDateParser;
120 import org.eclipse.jgit.util.SystemReader;
121 import org.slf4j.Logger;
122 import org.slf4j.LoggerFactory;
123 
124 /**
125  * A garbage collector for git
126  * {@link org.eclipse.jgit.internal.storage.file.FileRepository}. Instances of
127  * this class are not thread-safe. Don't use the same instance from multiple
128  * threads.
129  *
130  * This class started as a copy of DfsGarbageCollector from Shawn O. Pearce
131  * adapted to FileRepositories.
132  */
133 public class GC {
134 	private final static Logger LOG = LoggerFactory
135 			.getLogger(GC.class);
136 
137 	private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago"; //$NON-NLS-1$
138 
139 	private static final String PRUNE_PACK_EXPIRE_DEFAULT = "1.hour.ago"; //$NON-NLS-1$
140 
141 	private static final Pattern PATTERN_LOOSE_OBJECT = Pattern
142 			.compile("[0-9a-fA-F]{38}"); //$NON-NLS-1$
143 
144 	private static final String PACK_EXT = "." + PackExt.PACK.getExtension();//$NON-NLS-1$
145 
146 	private static final String BITMAP_EXT = "." //$NON-NLS-1$
147 			+ PackExt.BITMAP_INDEX.getExtension();
148 
149 	private static final String INDEX_EXT = "." + PackExt.INDEX.getExtension(); //$NON-NLS-1$
150 
151 	private static final int DEFAULT_AUTOPACKLIMIT = 50;
152 
153 	private static final int DEFAULT_AUTOLIMIT = 6700;
154 
155 	private static volatile ExecutorService executor;
156 
157 	/**
158 	 * Set the executor for running auto-gc in the background. If no executor is
159 	 * set JGit's own WorkQueue will be used.
160 	 *
161 	 * @param e
162 	 *            the executor to be used for running auto-gc
163 	 */
164 	public static void setExecutor(ExecutorService e) {
165 		executor = e;
166 	}
167 
168 	private final FileRepository repo;
169 
170 	private ProgressMonitor pm;
171 
172 	private long expireAgeMillis = -1;
173 
174 	private Date expire;
175 
176 	private long packExpireAgeMillis = -1;
177 
178 	private Date packExpire;
179 
180 	private PackConfig pconfig;
181 
182 	/**
183 	 * the refs which existed during the last call to {@link #repack()}. This is
184 	 * needed during {@link #prune(Set)} where we can optimize by looking at the
185 	 * difference between the current refs and the refs which existed during
186 	 * last {@link #repack()}.
187 	 */
188 	private Collection<Ref> lastPackedRefs;
189 
190 	/**
191 	 * Holds the starting time of the last repack() execution. This is needed in
192 	 * prune() to inspect only those reflog entries which have been added since
193 	 * last repack().
194 	 */
195 	private long lastRepackTime;
196 
197 	/**
198 	 * Whether gc should do automatic housekeeping
199 	 */
200 	private boolean automatic;
201 
202 	/**
203 	 * Whether to run gc in a background thread
204 	 */
205 	private boolean background;
206 
207 	/**
208 	 * Creates a new garbage collector with default values. An expirationTime of
209 	 * two weeks and <code>null</code> as progress monitor will be used.
210 	 *
211 	 * @param repo
212 	 *            the repo to work on
213 	 */
214 	public GC(FileRepository repo) {
215 		this.repo = repo;
216 		this.pconfig = new PackConfig(repo);
217 		this.pm = NullProgressMonitor.INSTANCE;
218 	}
219 
220 	/**
221 	 * Runs a garbage collector on a
222 	 * {@link org.eclipse.jgit.internal.storage.file.FileRepository}. It will
223 	 * <ul>
224 	 * <li>pack loose references into packed-refs</li>
225 	 * <li>repack all reachable objects into new pack files and delete the old
226 	 * pack files</li>
227 	 * <li>prune all loose objects which are now reachable by packs</li>
228 	 * </ul>
229 	 *
230 	 * If {@link #setAuto(boolean)} was set to {@code true} {@code gc} will
231 	 * first check whether any housekeeping is required; if not, it exits
232 	 * without performing any work.
233 	 *
234 	 * If {@link #setBackground(boolean)} was set to {@code true}
235 	 * {@code collectGarbage} will start the gc in the background, and then
236 	 * return immediately. In this case, errors will not be reported except in
237 	 * gc.log.
238 	 *
239 	 * @return the collection of
240 	 *         {@link org.eclipse.jgit.internal.storage.file.PackFile}'s which
241 	 *         are newly created
242 	 * @throws java.io.IOException
243 	 * @throws java.text.ParseException
244 	 *             If the configuration parameter "gc.pruneexpire" couldn't be
245 	 *             parsed
246 	 */
247 	// TODO(ms): change signature and return Future<Collection<PackFile>>
248 	@SuppressWarnings("FutureReturnValueIgnored")
249 	public Collection<PackFile> gc() throws IOException, ParseException {
250 		if (!background) {
251 			return doGc();
252 		}
253 		final GcLogernal/storage/file/GcLog.html#GcLog">GcLog gcLog = new GcLog(repo);
254 		if (!gcLog.lock()) {
255 			// there is already a background gc running
256 			return Collections.emptyList();
257 		}
258 
259 		Callable<Collection<PackFile>> gcTask = () -> {
260 			try {
261 				Collection<PackFile> newPacks = doGc();
262 				if (automatic && tooManyLooseObjects()) {
263 					String message = JGitText.get().gcTooManyUnpruned;
264 					gcLog.write(message);
265 					gcLog.commit();
266 				}
267 				return newPacks;
268 			} catch (IOException | ParseException e) {
269 				try {
270 					gcLog.write(e.getMessage());
271 					StringWriter sw = new StringWriter();
272 					e.printStackTrace(new PrintWriter(sw));
273 					gcLog.write(sw.toString());
274 					gcLog.commit();
275 				} catch (IOException e2) {
276 					e2.addSuppressed(e);
277 					LOG.error(e2.getMessage(), e2);
278 				}
279 			} finally {
280 				gcLog.unlock();
281 			}
282 			return Collections.emptyList();
283 		};
284 		// TODO(ms): change signature and return the Future
285 		executor().submit(gcTask);
286 		return Collections.emptyList();
287 	}
288 
289 	private ExecutorService executor() {
290 		return (executor != null) ? executor : WorkQueue.getExecutor();
291 	}
292 
293 	private Collection<PackFile> doGc() throws IOException, ParseException {
294 		if (automatic && !needGc()) {
295 			return Collections.emptyList();
296 		}
297 		pm.start(6 /* tasks */);
298 		packRefs();
299 		// TODO: implement reflog_expire(pm, repo);
300 		Collection<PackFile> newPacks = repack();
301 		prune(Collections.emptySet());
302 		// TODO: implement rerere_gc(pm);
303 		return newPacks;
304 	}
305 
306 	/**
307 	 * Loosen objects in a pack file which are not also in the newly-created
308 	 * pack files.
309 	 *
310 	 * @param inserter
311 	 * @param reader
312 	 * @param pack
313 	 * @param existing
314 	 * @throws IOException
315 	 */
316 	private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, PackFile pack, HashSet<ObjectId> existing)
317 			throws IOException {
318 		for (PackIndex.MutableEntry entry : pack) {
319 			ObjectId oid = entry.toObjectId();
320 			if (existing.contains(oid)) {
321 				continue;
322 			}
323 			existing.add(oid);
324 			ObjectLoader loader = reader.open(oid);
325 			inserter.insert(loader.getType(),
326 					loader.getSize(),
327 					loader.openStream(),
328 					true /* create this object even though it's a duplicate */);
329 		}
330 	}
331 
332 	/**
333 	 * Delete old pack files. What is 'old' is defined by specifying a set of
334 	 * old pack files and a set of new pack files. Each pack file contained in
335 	 * old pack files but not contained in new pack files will be deleted. If
336 	 * preserveOldPacks is set, keep a copy of the pack file in the preserve
337 	 * directory. If an expirationDate is set then pack files which are younger
338 	 * than the expirationDate will not be deleted nor preserved.
339 	 * <p>
340 	 * If we're not immediately expiring loose objects, loosen any objects
341 	 * in the old pack files which aren't in the new pack files.
342 	 *
343 	 * @param oldPacks
344 	 * @param newPacks
345 	 * @throws ParseException
346 	 * @throws IOException
347 	 */
348 	private void deleteOldPacks(Collection<PackFile> oldPacks,
349 			Collection<PackFile> newPacks) throws ParseException, IOException {
350 		HashSet<ObjectId> ids = new HashSet<>();
351 		for (PackFile pack : newPacks) {
352 			for (PackIndex.MutableEntry entry : pack) {
353 				ids.add(entry.toObjectId());
354 			}
355 		}
356 		ObjectReader reader = repo.newObjectReader();
357 		ObjectDirectory dir = repo.getObjectDatabase();
358 		ObjectDirectoryInserter inserter = dir.newInserter();
359 		boolean shouldLoosen = !"now".equals(getPruneExpireStr()) && //$NON-NLS-1$
360 			getExpireDate() < Long.MAX_VALUE;
361 
362 		prunePreserved();
363 		long packExpireDate = getPackExpireDate();
364 		oldPackLoop: for (PackFile oldPack : oldPacks) {
365 			checkCancelled();
366 			String oldName = oldPack.getPackName();
367 			// check whether an old pack file is also among the list of new
368 			// pack files. Then we must not delete it.
369 			for (PackFile newPack : newPacks)
370 				if (oldName.equals(newPack.getPackName()))
371 					continue oldPackLoop;
372 
373 			if (!oldPack.shouldBeKept()
374 					&& repo.getFS()
375 							.lastModifiedInstant(oldPack.getPackFile())
376 							.toEpochMilli() < packExpireDate) {
377 				oldPack.close();
378 				if (shouldLoosen) {
379 					loosen(inserter, reader, oldPack, ids);
380 				}
381 				prunePack(oldName);
382 			}
383 		}
384 
385 		// close the complete object database. That's my only chance to force
386 		// rescanning and to detect that certain pack files are now deleted.
387 		repo.getObjectDatabase().close();
388 	}
389 
390 	/**
391 	 * Deletes old pack file, unless 'preserve-oldpacks' is set, in which case it
392 	 * moves the pack file to the preserved directory
393 	 *
394 	 * @param packFile
395 	 * @param packName
396 	 * @param ext
397 	 * @param deleteOptions
398 	 * @throws IOException
399 	 */
400 	private void removeOldPack(File packFile, String packName, PackExt ext,
401 			int deleteOptions) throws IOException {
402 		if (pconfig.isPreserveOldPacks()) {
403 			File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
404 			FileUtils.mkdir(oldPackDir, true);
405 
406 			String oldPackName = "pack-" + packName + ".old-" + ext.getExtension();  //$NON-NLS-1$ //$NON-NLS-2$
407 			File oldPackFile = new File(oldPackDir, oldPackName);
408 			FileUtils.rename(packFile, oldPackFile);
409 		} else {
410 			FileUtils.delete(packFile, deleteOptions);
411 		}
412 	}
413 
414 	/**
415 	 * Delete the preserved directory including all pack files within
416 	 */
417 	private void prunePreserved() {
418 		if (pconfig.isPrunePreserved()) {
419 			try {
420 				FileUtils.delete(repo.getObjectDatabase().getPreservedDirectory(),
421 						FileUtils.RECURSIVE | FileUtils.RETRY | FileUtils.SKIP_MISSING);
422 			} catch (IOException e) {
423 				// Deletion of the preserved pack files failed. Silently return.
424 			}
425 		}
426 	}
427 
428 	/**
429 	 * Delete files associated with a single pack file. First try to delete the
430 	 * ".pack" file because on some platforms the ".pack" file may be locked and
431 	 * can't be deleted. In such a case it is better to detect this early and
432 	 * give up on deleting files for this packfile. Otherwise we may delete the
433 	 * ".index" file and when failing to delete the ".pack" file we are left
434 	 * with a ".pack" file without a ".index" file.
435 	 *
436 	 * @param packName
437 	 */
438 	private void prunePack(String packName) {
439 		PackExt[] extensions = PackExt.values();
440 		try {
441 			// Delete the .pack file first and if this fails give up on deleting
442 			// the other files
443 			int deleteOptions = FileUtils.RETRY | FileUtils.SKIP_MISSING;
444 			for (PackExt ext : extensions)
445 				if (PackExt.PACK.equals(ext)) {
446 					File f = nameFor(packName, "." + ext.getExtension()); //$NON-NLS-1$
447 					removeOldPack(f, packName, ext, deleteOptions);
448 					break;
449 				}
450 			// The .pack file has been deleted. Delete as many as the other
451 			// files as you can.
452 			deleteOptions |= FileUtils.IGNORE_ERRORS;
453 			for (PackExt ext : extensions) {
454 				if (!PackExt.PACK.equals(ext)) {
455 					File f = nameFor(packName, "." + ext.getExtension()); //$NON-NLS-1$
456 					removeOldPack(f, packName, ext, deleteOptions);
457 				}
458 			}
459 		} catch (IOException e) {
460 			// Deletion of the .pack file failed. Silently return.
461 		}
462 	}
463 
464 	/**
465 	 * Like "git prune-packed" this method tries to prune all loose objects
466 	 * which can be found in packs. If certain objects can't be pruned (e.g.
467 	 * because the filesystem delete operation fails) this is silently ignored.
468 	 *
469 	 * @throws java.io.IOException
470 	 */
471 	public void prunePacked() throws IOException {
472 		ObjectDirectory objdb = repo.getObjectDatabase();
473 		Collection<PackFile> packs = objdb.getPacks();
474 		File objects = repo.getObjectsDirectory();
475 		String[] fanout = objects.list();
476 
477 		if (fanout != null && fanout.length > 0) {
478 			pm.beginTask(JGitText.get().pruneLoosePackedObjects, fanout.length);
479 			try {
480 				for (String d : fanout) {
481 					checkCancelled();
482 					pm.update(1);
483 					if (d.length() != 2)
484 						continue;
485 					String[] entries = new File(objects, d).list();
486 					if (entries == null)
487 						continue;
488 					for (String e : entries) {
489 						checkCancelled();
490 						if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
491 							continue;
492 						ObjectId id;
493 						try {
494 							id = ObjectId.fromString(d + e);
495 						} catch (IllegalArgumentException notAnObject) {
496 							// ignoring the file that does not represent loose
497 							// object
498 							continue;
499 						}
500 						boolean found = false;
501 						for (PackFile p : packs) {
502 							checkCancelled();
503 							if (p.hasObject(id)) {
504 								found = true;
505 								break;
506 							}
507 						}
508 						if (found)
509 							FileUtils.delete(objdb.fileFor(id), FileUtils.RETRY
510 									| FileUtils.SKIP_MISSING
511 									| FileUtils.IGNORE_ERRORS);
512 					}
513 				}
514 			} finally {
515 				pm.endTask();
516 			}
517 		}
518 	}
519 
520 	/**
521 	 * Like "git prune" this method tries to prune all loose objects which are
522 	 * unreferenced. If certain objects can't be pruned (e.g. because the
523 	 * filesystem delete operation fails) this is silently ignored.
524 	 *
525 	 * @param objectsToKeep
526 	 *            a set of objects which should explicitly not be pruned
527 	 * @throws java.io.IOException
528 	 * @throws java.text.ParseException
529 	 *             If the configuration parameter "gc.pruneexpire" couldn't be
530 	 *             parsed
531 	 */
532 	public void prune(Set<ObjectId> objectsToKeep) throws IOException,
533 			ParseException {
534 		long expireDate = getExpireDate();
535 
536 		// Collect all loose objects which are old enough, not referenced from
537 		// the index and not in objectsToKeep
538 		Map<ObjectId, File> deletionCandidates = new HashMap<>();
539 		Set<ObjectId> indexObjects = null;
540 		File objects = repo.getObjectsDirectory();
541 		String[] fanout = objects.list();
542 		if (fanout == null || fanout.length == 0) {
543 			return;
544 		}
545 		pm.beginTask(JGitText.get().pruneLooseUnreferencedObjects,
546 				fanout.length);
547 		try {
548 			for (String d : fanout) {
549 				checkCancelled();
550 				pm.update(1);
551 				if (d.length() != 2)
552 					continue;
553 				File dir = new File(objects, d);
554 				File[] entries = dir.listFiles();
555 				if (entries == null || entries.length == 0) {
556 					FileUtils.delete(dir, FileUtils.IGNORE_ERRORS);
557 					continue;
558 				}
559 				for (File f : entries) {
560 					checkCancelled();
561 					String fName = f.getName();
562 					if (fName.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
563 						continue;
564 					if (repo.getFS().lastModifiedInstant(f)
565 							.toEpochMilli() >= expireDate) {
566 						continue;
567 					}
568 					try {
569 						ObjectId id = ObjectId.fromString(d + fName);
570 						if (objectsToKeep.contains(id))
571 							continue;
572 						if (indexObjects == null)
573 							indexObjects = listNonHEADIndexObjects();
574 						if (indexObjects.contains(id))
575 							continue;
576 						deletionCandidates.put(id, f);
577 					} catch (IllegalArgumentException notAnObject) {
578 						// ignoring the file that does not represent loose
579 						// object
580 					}
581 				}
582 			}
583 		} finally {
584 			pm.endTask();
585 		}
586 
587 		if (deletionCandidates.isEmpty()) {
588 			return;
589 		}
590 
591 		checkCancelled();
592 
593 		// From the set of current refs remove all those which have been handled
594 		// during last repack(). Only those refs will survive which have been
595 		// added or modified since the last repack. Only these can save existing
596 		// loose refs from being pruned.
597 		Collection<Ref> newRefs;
598 		if (lastPackedRefs == null || lastPackedRefs.isEmpty())
599 			newRefs = getAllRefs();
600 		else {
601 			Map<String, Ref> last = new HashMap<>();
602 			for (Ref r : lastPackedRefs) {
603 				last.put(r.getName(), r);
604 			}
605 			newRefs = new ArrayList<>();
606 			for (Ref r : getAllRefs()) {
607 				Ref old = last.get(r.getName());
608 				if (!equals(r, old)) {
609 					newRefs.add(r);
610 				}
611 			}
612 		}
613 
614 		if (!newRefs.isEmpty()) {
615 			// There are new/modified refs! Check which loose objects are now
616 			// referenced by these modified refs (or their reflogentries).
617 			// Remove these loose objects
618 			// from the deletionCandidates. When the last candidate is removed
619 			// leave this method.
620 			ObjectWalk w = new ObjectWalk(repo);
621 			try {
622 				for (Ref cr : newRefs) {
623 					checkCancelled();
624 					w.markStart(w.parseAny(cr.getObjectId()));
625 				}
626 				if (lastPackedRefs != null)
627 					for (Ref lpr : lastPackedRefs) {
628 						w.markUninteresting(w.parseAny(lpr.getObjectId()));
629 					}
630 				removeReferenced(deletionCandidates, w);
631 			} finally {
632 				w.dispose();
633 			}
634 		}
635 
636 		if (deletionCandidates.isEmpty())
637 			return;
638 
639 		// Since we have not left the method yet there are still
640 		// deletionCandidates. Last chance for these objects not to be pruned is
641 		// that they are referenced by reflog entries. Even refs which currently
642 		// point to the same object as during last repack() may have
643 		// additional reflog entries not handled during last repack()
644 		ObjectWalk w = new ObjectWalk(repo);
645 		try {
646 			for (Ref ar : getAllRefs())
647 				for (ObjectId id : listRefLogObjects(ar, lastRepackTime)) {
648 					checkCancelled();
649 					w.markStart(w.parseAny(id));
650 				}
651 			if (lastPackedRefs != null)
652 				for (Ref lpr : lastPackedRefs) {
653 					checkCancelled();
654 					w.markUninteresting(w.parseAny(lpr.getObjectId()));
655 				}
656 			removeReferenced(deletionCandidates, w);
657 		} finally {
658 			w.dispose();
659 		}
660 
661 		if (deletionCandidates.isEmpty())
662 			return;
663 
664 		checkCancelled();
665 
666 		// delete all candidates which have survived: these are unreferenced
667 		// loose objects. Make a last check, though, to avoid deleting objects
668 		// that could have been referenced while the candidates list was being
669 		// built (by an incoming push, for example).
670 		Set<File> touchedFanout = new HashSet<>();
671 		for (File f : deletionCandidates.values()) {
672 			if (f.lastModified() < expireDate) {
673 				f.delete();
674 				touchedFanout.add(f.getParentFile());
675 			}
676 		}
677 
678 		for (File f : touchedFanout) {
679 			FileUtils.delete(f,
680 					FileUtils.EMPTY_DIRECTORIES_ONLY | FileUtils.IGNORE_ERRORS);
681 		}
682 
683 		repo.getObjectDatabase().close();
684 	}
685 
686 	private long getExpireDate() throws ParseException {
687 		long expireDate = Long.MAX_VALUE;
688 
689 		if (expire == null && expireAgeMillis == -1) {
690 			String pruneExpireStr = getPruneExpireStr();
691 			if (pruneExpireStr == null)
692 				pruneExpireStr = PRUNE_EXPIRE_DEFAULT;
693 			expire = GitDateParser.parse(pruneExpireStr, null, SystemReader
694 					.getInstance().getLocale());
695 			expireAgeMillis = -1;
696 		}
697 		if (expire != null)
698 			expireDate = expire.getTime();
699 		if (expireAgeMillis != -1)
700 			expireDate = System.currentTimeMillis() - expireAgeMillis;
701 		return expireDate;
702 	}
703 
704 	private String getPruneExpireStr() {
705 		return repo.getConfig().getString(
706                         ConfigConstants.CONFIG_GC_SECTION, null,
707                         ConfigConstants.CONFIG_KEY_PRUNEEXPIRE);
708 	}
709 
710 	private long getPackExpireDate() throws ParseException {
711 		long packExpireDate = Long.MAX_VALUE;
712 
713 		if (packExpire == null && packExpireAgeMillis == -1) {
714 			String prunePackExpireStr = repo.getConfig().getString(
715 					ConfigConstants.CONFIG_GC_SECTION, null,
716 					ConfigConstants.CONFIG_KEY_PRUNEPACKEXPIRE);
717 			if (prunePackExpireStr == null)
718 				prunePackExpireStr = PRUNE_PACK_EXPIRE_DEFAULT;
719 			packExpire = GitDateParser.parse(prunePackExpireStr, null,
720 					SystemReader.getInstance().getLocale());
721 			packExpireAgeMillis = -1;
722 		}
723 		if (packExpire != null)
724 			packExpireDate = packExpire.getTime();
725 		if (packExpireAgeMillis != -1)
726 			packExpireDate = System.currentTimeMillis() - packExpireAgeMillis;
727 		return packExpireDate;
728 	}
729 
730 	/**
731 	 * Remove all entries from a map which key is the id of an object referenced
732 	 * by the given ObjectWalk
733 	 *
734 	 * @param id2File
735 	 * @param w
736 	 * @throws MissingObjectException
737 	 * @throws IncorrectObjectTypeException
738 	 * @throws IOException
739 	 */
740 	private void removeReferenced(Map<ObjectId, File> id2File,
741 			ObjectWalk w) throws MissingObjectException,
742 			IncorrectObjectTypeException, IOException {
743 		RevObject ro = w.next();
744 		while (ro != null) {
745 			checkCancelled();
746 			if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
747 				return;
748 			}
749 			ro = w.next();
750 		}
751 		ro = w.nextObject();
752 		while (ro != null) {
753 			checkCancelled();
754 			if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
755 				return;
756 			}
757 			ro = w.nextObject();
758 		}
759 	}
760 
761 	private static boolean equals(Reff" href="../../../../../../org/eclipse/jgit/lib/Ref.html#Ref">Ref r1, Ref r2) {
762 		if (r1 == null || r2 == null) {
763 			return false;
764 		}
765 		if (r1.isSymbolic()) {
766 			return r2.isSymbolic() && r1.getTarget().getName()
767 					.equals(r2.getTarget().getName());
768 		}
769 		return !r2.isSymbolic()
770 				&& Objects.equals(r1.getObjectId(), r2.getObjectId());
771 	}
772 
773 	/**
774 	 * Packs all non-symbolic, loose refs into packed-refs.
775 	 *
776 	 * @throws java.io.IOException
777 	 */
778 	public void packRefs() throws IOException {
779 		Collection<Ref> refs = repo.getRefDatabase()
780 				.getRefsByPrefix(Constants.R_REFS);
781 		List<String> refsToBePacked = new ArrayList<>(refs.size());
782 		pm.beginTask(JGitText.get().packRefs, refs.size());
783 		try {
784 			for (Ref ref : refs) {
785 				checkCancelled();
786 				if (!ref.isSymbolic() && ref.getStorage().isLoose())
787 					refsToBePacked.add(ref.getName());
788 				pm.update(1);
789 			}
790 			((RefDirectory) repo.getRefDatabase()).pack(refsToBePacked);
791 		} finally {
792 			pm.endTask();
793 		}
794 	}
795 
796 	/**
797 	 * Packs all objects which reachable from any of the heads into one pack
798 	 * file. Additionally all objects which are not reachable from any head but
799 	 * which are reachable from any of the other refs (e.g. tags), special refs
800 	 * (e.g. FETCH_HEAD) or index are packed into a separate pack file. Objects
801 	 * included in pack files which have a .keep file associated are never
802 	 * repacked. All old pack files which existed before are deleted.
803 	 *
804 	 * @return a collection of the newly created pack files
805 	 * @throws java.io.IOException
806 	 *             when during reading of refs, index, packfiles, objects,
807 	 *             reflog-entries or during writing to the packfiles
808 	 *             {@link java.io.IOException} occurs
809 	 */
810 	public Collection<PackFile> repack() throws IOException {
811 		Collection<PackFile> toBeDeleted = repo.getObjectDatabase().getPacks();
812 
813 		long time = System.currentTimeMillis();
814 		Collection<Ref> refsBefore = getAllRefs();
815 
816 		Set<ObjectId> allHeadsAndTags = new HashSet<>();
817 		Set<ObjectId> allHeads = new HashSet<>();
818 		Set<ObjectId> allTags = new HashSet<>();
819 		Set<ObjectId> nonHeads = new HashSet<>();
820 		Set<ObjectId> txnHeads = new HashSet<>();
821 		Set<ObjectId> tagTargets = new HashSet<>();
822 		Set<ObjectId> indexObjects = listNonHEADIndexObjects();
823 		RefDatabase refdb = repo.getRefDatabase();
824 
825 		for (Ref ref : refsBefore) {
826 			checkCancelled();
827 			nonHeads.addAll(listRefLogObjects(ref, 0));
828 			if (ref.isSymbolic() || ref.getObjectId() == null) {
829 				continue;
830 			}
831 			if (isHead(ref)) {
832 				allHeads.add(ref.getObjectId());
833 			} else if (isTag(ref)) {
834 				allTags.add(ref.getObjectId());
835 			} else if (RefTreeNames.isRefTree(refdb, ref.getName())) {
836 				txnHeads.add(ref.getObjectId());
837 			} else {
838 				nonHeads.add(ref.getObjectId());
839 			}
840 			if (ref.getPeeledObjectId() != null) {
841 				tagTargets.add(ref.getPeeledObjectId());
842 			}
843 		}
844 
845 		List<ObjectIdSet> excluded = new LinkedList<>();
846 		for (PackFile f : repo.getObjectDatabase().getPacks()) {
847 			checkCancelled();
848 			if (f.shouldBeKept())
849 				excluded.add(f.getIndex());
850 		}
851 
852 		// Don't exclude tags that are also branch tips
853 		allTags.removeAll(allHeads);
854 		allHeadsAndTags.addAll(allHeads);
855 		allHeadsAndTags.addAll(allTags);
856 
857 		// Hoist all branch tips and tags earlier in the pack file
858 		tagTargets.addAll(allHeadsAndTags);
859 		nonHeads.addAll(indexObjects);
860 
861 		// Combine the GC_REST objects into the GC pack if requested
862 		if (pconfig.getSinglePack()) {
863 			allHeadsAndTags.addAll(nonHeads);
864 			nonHeads.clear();
865 		}
866 
867 		List<PackFile> ret = new ArrayList<>(2);
868 		PackFile heads = null;
869 		if (!allHeadsAndTags.isEmpty()) {
870 			heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags,
871 					tagTargets, excluded);
872 			if (heads != null) {
873 				ret.add(heads);
874 				excluded.add(0, heads.getIndex());
875 			}
876 		}
877 		if (!nonHeads.isEmpty()) {
878 			PackFile rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE,
879 					tagTargets, excluded);
880 			if (rest != null)
881 				ret.add(rest);
882 		}
883 		if (!txnHeads.isEmpty()) {
884 			PackFile txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE,
885 					null, excluded);
886 			if (txn != null)
887 				ret.add(txn);
888 		}
889 		try {
890 			deleteOldPacks(toBeDeleted, ret);
891 		} catch (ParseException e) {
892 			// TODO: the exception has to be wrapped into an IOException because
893 			// throwing the ParseException directly would break the API, instead
894 			// we should throw a ConfigInvalidException
895 			throw new IOException(e);
896 		}
897 		prunePacked();
898 		deleteEmptyRefsFolders();
899 		deleteOrphans();
900 		deleteTempPacksIdx();
901 
902 		lastPackedRefs = refsBefore;
903 		lastRepackTime = time;
904 		return ret;
905 	}
906 
907 	private static boolean isHead(Ref ref) {
908 		return ref.getName().startsWith(Constants.R_HEADS);
909 	}
910 
911 	private static boolean isTag(Ref ref) {
912 		return ref.getName().startsWith(Constants.R_TAGS);
913 	}
914 
915 	private void deleteEmptyRefsFolders() throws IOException {
916 		Path refs = repo.getDirectory().toPath().resolve(Constants.R_REFS);
917 		// Avoid deleting a folder that was created after the threshold so that concurrent
918 		// operations trying to create a reference are not impacted
919 		Instant threshold = Instant.now().minus(30, ChronoUnit.SECONDS);
920 		try (Stream<Path> entries = Files.list(refs)
921 				.filter(Files::isDirectory)) {
922 			Iterator<Path> iterator = entries.iterator();
923 			while (iterator.hasNext()) {
924 				try (Stream<Path> s = Files.list(iterator.next())) {
925 					s.filter(path -> canBeSafelyDeleted(path, threshold)).forEach(this::deleteDir);
926 				}
927 			}
928 		}
929 	}
930 
931 	private boolean canBeSafelyDeleted(Path path, Instant threshold) {
932 		try {
933 			return Files.getLastModifiedTime(path).toInstant().isBefore(threshold);
934 		}
935 		catch (IOException e) {
936 			LOG.warn(MessageFormat.format(
937 					JGitText.get().cannotAccessLastModifiedForSafeDeletion,
938 					path), e);
939 			return false;
940 		}
941 	}
942 
943 	private void deleteDir(Path dir) {
944 		try (Stream<Path> dirs = Files.walk(dir)) {
945 			dirs.filter(this::isDirectory).sorted(Comparator.reverseOrder())
946 					.forEach(this::delete);
947 		} catch (IOException e) {
948 			LOG.error(e.getMessage(), e);
949 		}
950 	}
951 
952 	private boolean isDirectory(Path p) {
953 		return p.toFile().isDirectory();
954 	}
955 
956 	private void delete(Path d) {
957 		try {
958 			Files.delete(d);
959 		} catch (DirectoryNotEmptyException e) {
960 			// Don't log
961 		} catch (IOException e) {
962 			LOG.error(MessageFormat.format(JGitText.get().cannotDeleteFile, d),
963 					e);
964 		}
965 	}
966 
967 	/**
968 	 * Deletes orphans
969 	 * <p>
970 	 * A file is considered an orphan if it is either a "bitmap" or an index
971 	 * file, and its corresponding pack file is missing in the list.
972 	 * </p>
973 	 */
974 	private void deleteOrphans() {
975 		Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
976 		List<String> fileNames = null;
977 		try (Stream<Path> files = Files.list(packDir)) {
978 			fileNames = files.map(path -> path.getFileName().toString())
979 					.filter(name -> (name.endsWith(PACK_EXT)
980 							|| name.endsWith(BITMAP_EXT)
981 							|| name.endsWith(INDEX_EXT)))
982 					.sorted(Collections.reverseOrder())
983 					.collect(Collectors.toList());
984 		} catch (IOException e1) {
985 			// ignore
986 		}
987 		if (fileNames == null) {
988 			return;
989 		}
990 
991 		String base = null;
992 		for (String n : fileNames) {
993 			if (n.endsWith(PACK_EXT)) {
994 				base = n.substring(0, n.lastIndexOf('.'));
995 			} else {
996 				if (base == null || !n.startsWith(base)) {
997 					try {
998 						Files.delete(packDir.resolve(n));
999 					} catch (IOException e) {
1000 						LOG.error(e.getMessage(), e);
1001 					}
1002 				}
1003 			}
1004 		}
1005 	}
1006 
1007 	private void deleteTempPacksIdx() {
1008 		Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
1009 		Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS);
1010 		if (!Files.exists(packDir)) {
1011 			return;
1012 		}
1013 		try (DirectoryStream<Path> stream =
1014 				Files.newDirectoryStream(packDir, "gc_*_tmp")) { //$NON-NLS-1$
1015 			stream.forEach(t -> {
1016 				try {
1017 					Instant lastModified = Files.getLastModifiedTime(t)
1018 							.toInstant();
1019 					if (lastModified.isBefore(threshold)) {
1020 						Files.deleteIfExists(t);
1021 					}
1022 				} catch (IOException e) {
1023 					LOG.error(e.getMessage(), e);
1024 				}
1025 			});
1026 		} catch (IOException e) {
1027 			LOG.error(e.getMessage(), e);
1028 		}
1029 	}
1030 
1031 	/**
1032 	 * @param ref
1033 	 *            the ref which log should be inspected
1034 	 * @param minTime only reflog entries not older then this time are processed
1035 	 * @return the {@link ObjectId}s contained in the reflog
1036 	 * @throws IOException
1037 	 */
1038 	private Set<ObjectId> listRefLogObjects(Ref ref, long minTime) throws IOException {
1039 		ReflogReader reflogReader = repo.getReflogReader(ref.getName());
1040 		if (reflogReader == null) {
1041 			return Collections.emptySet();
1042 		}
1043 		List<ReflogEntry> rlEntries = reflogReader
1044 				.getReverseEntries();
1045 		if (rlEntries == null || rlEntries.isEmpty())
1046 			return Collections.emptySet();
1047 		Set<ObjectId> ret = new HashSet<>();
1048 		for (ReflogEntry e : rlEntries) {
1049 			if (e.getWho().getWhen().getTime() < minTime)
1050 				break;
1051 			ObjectId newId = e.getNewId();
1052 			if (newId != null && !ObjectId.zeroId().equals(newId))
1053 				ret.add(newId);
1054 			ObjectId oldId = e.getOldId();
1055 			if (oldId != null && !ObjectId.zeroId().equals(oldId))
1056 				ret.add(oldId);
1057 		}
1058 		return ret;
1059 	}
1060 
1061 	/**
1062 	 * Returns a collection of all refs and additional refs.
1063 	 *
1064 	 * Additional refs which don't start with "refs/" are not returned because
1065 	 * they should not save objects from being garbage collected. Examples for
1066 	 * such references are ORIG_HEAD, MERGE_HEAD, FETCH_HEAD and
1067 	 * CHERRY_PICK_HEAD.
1068 	 *
1069 	 * @return a collection of refs pointing to live objects.
1070 	 * @throws IOException
1071 	 */
1072 	private Collection<Ref> getAllRefs() throws IOException {
1073 		RefDatabase refdb = repo.getRefDatabase();
1074 		Collection<Ref> refs = refdb.getRefs();
1075 		List<Ref> addl = refdb.getAdditionalRefs();
1076 		if (!addl.isEmpty()) {
1077 			List<Ref> all = new ArrayList<>(refs.size() + addl.size());
1078 			all.addAll(refs);
1079 			// add additional refs which start with refs/
1080 			for (Ref r : addl) {
1081 				checkCancelled();
1082 				if (r.getName().startsWith(Constants.R_REFS)) {
1083 					all.add(r);
1084 				}
1085 			}
1086 			return all;
1087 		}
1088 		return refs;
1089 	}
1090 
1091 	/**
1092 	 * Return a list of those objects in the index which differ from whats in
1093 	 * HEAD
1094 	 *
1095 	 * @return a set of ObjectIds of changed objects in the index
1096 	 * @throws IOException
1097 	 * @throws CorruptObjectException
1098 	 * @throws NoWorkTreeException
1099 	 */
1100 	private Set<ObjectId> listNonHEADIndexObjects()
1101 			throws CorruptObjectException, IOException {
1102 		if (repo.isBare()) {
1103 			return Collections.emptySet();
1104 		}
1105 		try (TreeWalkTreeWalk.html#TreeWalk">TreeWalk treeWalk = new TreeWalk(repo)) {
1106 			treeWalk.addTree(new DirCacheIterator(repo.readDirCache()));
1107 			ObjectId headID = repo.resolve(Constants.HEAD);
1108 			if (headID != null) {
1109 				try (RevWalk/RevWalk.html#RevWalk">RevWalk revWalk = new RevWalk(repo)) {
1110 					treeWalk.addTree(revWalk.parseTree(headID));
1111 				}
1112 			}
1113 
1114 			treeWalk.setFilter(TreeFilter.ANY_DIFF);
1115 			treeWalk.setRecursive(true);
1116 			Set<ObjectId> ret = new HashSet<>();
1117 
1118 			while (treeWalk.next()) {
1119 				checkCancelled();
1120 				ObjectId objectId = treeWalk.getObjectId(0);
1121 				switch (treeWalk.getRawMode(0) & FileMode.TYPE_MASK) {
1122 				case FileMode.TYPE_MISSING:
1123 				case FileMode.TYPE_GITLINK:
1124 					continue;
1125 				case FileMode.TYPE_TREE:
1126 				case FileMode.TYPE_FILE:
1127 				case FileMode.TYPE_SYMLINK:
1128 					ret.add(objectId);
1129 					continue;
1130 				default:
1131 					throw new IOException(MessageFormat.format(
1132 							JGitText.get().corruptObjectInvalidMode3,
1133 							String.format("%o", //$NON-NLS-1$
1134 									Integer.valueOf(treeWalk.getRawMode(0))),
1135 							(objectId == null) ? "null" : objectId.name(), //$NON-NLS-1$
1136 							treeWalk.getPathString(), //
1137 							repo.getIndexFile()));
1138 				}
1139 			}
1140 			return ret;
1141 		}
1142 	}
1143 
1144 	private PackFile writePack(@NonNull Set<? extends ObjectId> want,
1145 			@NonNull Set<? extends ObjectId> have, @NonNull Set<ObjectId> tags,
1146 			Set<ObjectId> tagTargets, List<ObjectIdSet> excludeObjects)
1147 			throws IOException {
1148 		checkCancelled();
1149 		File tmpPack = null;
1150 		Map<PackExt, File> tmpExts = new TreeMap<>((o1, o2) -> {
1151 			// INDEX entries must be returned last, so the pack
1152 			// scanner does pick up the new pack until all the
1153 			// PackExt entries have been written.
1154 			if (o1 == o2) {
1155 				return 0;
1156 			}
1157 			if (o1 == PackExt.INDEX) {
1158 				return 1;
1159 			}
1160 			if (o2 == PackExt.INDEX) {
1161 				return -1;
1162 			}
1163 			return Integer.signum(o1.hashCode() - o2.hashCode());
1164 		});
1165 		try (PackWriternal/storage/pack/PackWriter.html#PackWriter">PackWriter pw = new PackWriter(
1166 				pconfig,
1167 				repo.newObjectReader())) {
1168 			// prepare the PackWriter
1169 			pw.setDeltaBaseAsOffset(true);
1170 			pw.setReuseDeltaCommits(false);
1171 			if (tagTargets != null) {
1172 				pw.setTagTargets(tagTargets);
1173 			}
1174 			if (excludeObjects != null)
1175 				for (ObjectIdSet idx : excludeObjects)
1176 					pw.excludeObjects(idx);
1177 			pw.preparePack(pm, want, have, PackWriter.NONE, tags);
1178 			if (pw.getObjectCount() == 0)
1179 				return null;
1180 			checkCancelled();
1181 
1182 			// create temporary files
1183 			String id = pw.computeName().getName();
1184 			File packdir = repo.getObjectDatabase().getPackDirectory();
1185 			tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir); //$NON-NLS-1$ //$NON-NLS-2$
1186 			final String tmpBase = tmpPack.getName()
1187 					.substring(0, tmpPack.getName().lastIndexOf('.'));
1188 			File tmpIdx = new File(packdir, tmpBase + ".idx_tmp"); //$NON-NLS-1$
1189 			tmpExts.put(INDEX, tmpIdx);
1190 
1191 			if (!tmpIdx.createNewFile())
1192 				throw new IOException(MessageFormat.format(
1193 						JGitText.get().cannotCreateIndexfile, tmpIdx.getPath()));
1194 
1195 			// write the packfile
1196 			try (FileOutputStream fos = new FileOutputStream(tmpPack);
1197 					FileChannel channel = fos.getChannel();
1198 					OutputStream channelStream = Channels
1199 							.newOutputStream(channel)) {
1200 				pw.writePack(pm, pm, channelStream);
1201 				channel.force(true);
1202 			}
1203 
1204 			// write the packindex
1205 			try (FileOutputStream fos = new FileOutputStream(tmpIdx);
1206 					FileChannel idxChannel = fos.getChannel();
1207 					OutputStream idxStream = Channels
1208 							.newOutputStream(idxChannel)) {
1209 				pw.writeIndex(idxStream);
1210 				idxChannel.force(true);
1211 			}
1212 
1213 			if (pw.prepareBitmapIndex(pm)) {
1214 				File tmpBitmapIdx = new File(packdir, tmpBase + ".bitmap_tmp"); //$NON-NLS-1$
1215 				tmpExts.put(BITMAP_INDEX, tmpBitmapIdx);
1216 
1217 				if (!tmpBitmapIdx.createNewFile())
1218 					throw new IOException(MessageFormat.format(
1219 							JGitText.get().cannotCreateIndexfile,
1220 							tmpBitmapIdx.getPath()));
1221 
1222 				try (FileOutputStream fos = new FileOutputStream(tmpBitmapIdx);
1223 						FileChannel idxChannel = fos.getChannel();
1224 						OutputStream idxStream = Channels
1225 								.newOutputStream(idxChannel)) {
1226 					pw.writeBitmapIndex(idxStream);
1227 					idxChannel.force(true);
1228 				}
1229 			}
1230 
1231 			// rename the temporary files to real files
1232 			File realPack = nameFor(id, ".pack"); //$NON-NLS-1$
1233 
1234 			repo.getObjectDatabase().closeAllPackHandles(realPack);
1235 			tmpPack.setReadOnly();
1236 
1237 			FileUtils.rename(tmpPack, realPack, StandardCopyOption.ATOMIC_MOVE);
1238 			for (Map.Entry<PackExt, File> tmpEntry : tmpExts.entrySet()) {
1239 				File tmpExt = tmpEntry.getValue();
1240 				tmpExt.setReadOnly();
1241 
1242 				File realExt = nameFor(id,
1243 						"." + tmpEntry.getKey().getExtension()); //$NON-NLS-1$
1244 				try {
1245 					FileUtils.rename(tmpExt, realExt,
1246 							StandardCopyOption.ATOMIC_MOVE);
1247 				} catch (IOException e) {
1248 					File newExt = new File(realExt.getParentFile(),
1249 							realExt.getName() + ".new"); //$NON-NLS-1$
1250 					try {
1251 						FileUtils.rename(tmpExt, newExt,
1252 								StandardCopyOption.ATOMIC_MOVE);
1253 					} catch (IOException e2) {
1254 						newExt = tmpExt;
1255 						e = e2;
1256 					}
1257 					throw new IOException(MessageFormat.format(
1258 							JGitText.get().panicCantRenameIndexFile, newExt,
1259 							realExt), e);
1260 				}
1261 			}
1262 			boolean interrupted = false;
1263 			try {
1264 				FileSnapshot snapshot = FileSnapshot.save(realPack);
1265 				if (pconfig.doWaitPreventRacyPack(snapshot.size())) {
1266 					snapshot.waitUntilNotRacy();
1267 				}
1268 			} catch (InterruptedException e) {
1269 				interrupted = true;
1270 			}
1271 			try {
1272 				return repo.getObjectDatabase().openPack(realPack);
1273 			} finally {
1274 				if (interrupted) {
1275 					// Re-set interrupted flag
1276 					Thread.currentThread().interrupt();
1277 				}
1278 			}
1279 		} finally {
1280 			if (tmpPack != null && tmpPack.exists())
1281 				tmpPack.delete();
1282 			for (File tmpExt : tmpExts.values()) {
1283 				if (tmpExt.exists())
1284 					tmpExt.delete();
1285 			}
1286 		}
1287 	}
1288 
1289 	private File nameFor(String name, String ext) {
1290 		File packdir = repo.getObjectDatabase().getPackDirectory();
1291 		return new File(packdir, "pack-" + name + ext); //$NON-NLS-1$
1292 	}
1293 
1294 	private void checkCancelled() throws CancelledException {
1295 		if (pm.isCancelled() || Thread.currentThread().isInterrupted()) {
1296 			throw new CancelledException(JGitText.get().operationCanceled);
1297 		}
1298 	}
1299 
1300 	/**
1301 	 * A class holding statistical data for a FileRepository regarding how many
1302 	 * objects are stored as loose or packed objects
1303 	 */
1304 	public static class RepoStatistics {
1305 		/**
1306 		 * The number of objects stored in pack files. If the same object is
1307 		 * stored in multiple pack files then it is counted as often as it
1308 		 * occurs in pack files.
1309 		 */
1310 		public long numberOfPackedObjects;
1311 
1312 		/**
1313 		 * The number of pack files
1314 		 */
1315 		public long numberOfPackFiles;
1316 
1317 		/**
1318 		 * The number of objects stored as loose objects.
1319 		 */
1320 		public long numberOfLooseObjects;
1321 
1322 		/**
1323 		 * The sum of the sizes of all files used to persist loose objects.
1324 		 */
1325 		public long sizeOfLooseObjects;
1326 
1327 		/**
1328 		 * The sum of the sizes of all pack files.
1329 		 */
1330 		public long sizeOfPackedObjects;
1331 
1332 		/**
1333 		 * The number of loose refs.
1334 		 */
1335 		public long numberOfLooseRefs;
1336 
1337 		/**
1338 		 * The number of refs stored in pack files.
1339 		 */
1340 		public long numberOfPackedRefs;
1341 
1342 		/**
1343 		 * The number of bitmaps in the bitmap indices.
1344 		 */
1345 		public long numberOfBitmaps;
1346 
1347 		@Override
1348 		public String toString() {
1349 			final StringBuilder b = new StringBuilder();
1350 			b.append("numberOfPackedObjects=").append(numberOfPackedObjects); //$NON-NLS-1$
1351 			b.append(", numberOfPackFiles=").append(numberOfPackFiles); //$NON-NLS-1$
1352 			b.append(", numberOfLooseObjects=").append(numberOfLooseObjects); //$NON-NLS-1$
1353 			b.append(", numberOfLooseRefs=").append(numberOfLooseRefs); //$NON-NLS-1$
1354 			b.append(", numberOfPackedRefs=").append(numberOfPackedRefs); //$NON-NLS-1$
1355 			b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects); //$NON-NLS-1$
1356 			b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects); //$NON-NLS-1$
1357 			b.append(", numberOfBitmaps=").append(numberOfBitmaps); //$NON-NLS-1$
1358 			return b.toString();
1359 		}
1360 	}
1361 
1362 	/**
1363 	 * Returns information about objects and pack files for a FileRepository.
1364 	 *
1365 	 * @return information about objects and pack files for a FileRepository
1366 	 * @throws java.io.IOException
1367 	 */
1368 	public RepoStatistics getStatistics() throws IOException {
1369 		RepoStatistics ret = new RepoStatistics();
1370 		Collection<PackFile> packs = repo.getObjectDatabase().getPacks();
1371 		for (PackFile f : packs) {
1372 			ret.numberOfPackedObjects += f.getIndex().getObjectCount();
1373 			ret.numberOfPackFiles++;
1374 			ret.sizeOfPackedObjects += f.getPackFile().length();
1375 			if (f.getBitmapIndex() != null)
1376 				ret.numberOfBitmaps += f.getBitmapIndex().getBitmapCount();
1377 		}
1378 		File objDir = repo.getObjectsDirectory();
1379 		String[] fanout = objDir.list();
1380 		if (fanout != null && fanout.length > 0) {
1381 			for (String d : fanout) {
1382 				if (d.length() != 2)
1383 					continue;
1384 				File[] entries = new File(objDir, d).listFiles();
1385 				if (entries == null)
1386 					continue;
1387 				for (File f : entries) {
1388 					if (f.getName().length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
1389 						continue;
1390 					ret.numberOfLooseObjects++;
1391 					ret.sizeOfLooseObjects += f.length();
1392 				}
1393 			}
1394 		}
1395 
1396 		RefDatabase refDb = repo.getRefDatabase();
1397 		for (Ref r : refDb.getRefs()) {
1398 			Storage storage = r.getStorage();
1399 			if (storage == Storage.LOOSE || storage == Storage.LOOSE_PACKED)
1400 				ret.numberOfLooseRefs++;
1401 			if (storage == Storage.PACKED || storage == Storage.LOOSE_PACKED)
1402 				ret.numberOfPackedRefs++;
1403 		}
1404 
1405 		return ret;
1406 	}
1407 
1408 	/**
1409 	 * Set the progress monitor used for garbage collection methods.
1410 	 *
1411 	 * @param pm a {@link org.eclipse.jgit.lib.ProgressMonitor} object.
1412 	 * @return this
1413 	 */
1414 	public GC setProgressMonitor(ProgressMonitor pm) {
1415 		this.pm = (pm == null) ? NullProgressMonitor.INSTANCE : pm;
1416 		return this;
1417 	}
1418 
1419 	/**
1420 	 * During gc() or prune() each unreferenced, loose object which has been
1421 	 * created or modified in the last <code>expireAgeMillis</code> milliseconds
1422 	 * will not be pruned. Only older objects may be pruned. If set to 0 then
1423 	 * every object is a candidate for pruning.
1424 	 *
1425 	 * @param expireAgeMillis
1426 	 *            minimal age of objects to be pruned in milliseconds.
1427 	 */
1428 	public void setExpireAgeMillis(long expireAgeMillis) {
1429 		this.expireAgeMillis = expireAgeMillis;
1430 		expire = null;
1431 	}
1432 
1433 	/**
1434 	 * During gc() or prune() packfiles which are created or modified in the
1435 	 * last <code>packExpireAgeMillis</code> milliseconds will not be deleted.
1436 	 * Only older packfiles may be deleted. If set to 0 then every packfile is a
1437 	 * candidate for deletion.
1438 	 *
1439 	 * @param packExpireAgeMillis
1440 	 *            minimal age of packfiles to be deleted in milliseconds.
1441 	 */
1442 	public void setPackExpireAgeMillis(long packExpireAgeMillis) {
1443 		this.packExpireAgeMillis = packExpireAgeMillis;
1444 		expire = null;
1445 	}
1446 
1447 	/**
1448 	 * Set the PackConfig used when (re-)writing packfiles. This allows to
1449 	 * influence how packs are written and to implement something similar to
1450 	 * "git gc --aggressive"
1451 	 *
1452 	 * @param pconfig
1453 	 *            the {@link org.eclipse.jgit.storage.pack.PackConfig} used when
1454 	 *            writing packs
1455 	 */
1456 	public void setPackConfig(@NonNull PackConfig pconfig) {
1457 		this.pconfig = pconfig;
1458 	}
1459 
1460 	/**
1461 	 * During gc() or prune() each unreferenced, loose object which has been
1462 	 * created or modified after or at <code>expire</code> will not be pruned.
1463 	 * Only older objects may be pruned. If set to null then every object is a
1464 	 * candidate for pruning.
1465 	 *
1466 	 * @param expire
1467 	 *            instant in time which defines object expiration
1468 	 *            objects with modification time before this instant are expired
1469 	 *            objects with modification time newer or equal to this instant
1470 	 *            are not expired
1471 	 */
1472 	public void setExpire(Date expire) {
1473 		this.expire = expire;
1474 		expireAgeMillis = -1;
1475 	}
1476 
1477 	/**
1478 	 * During gc() or prune() packfiles which are created or modified after or
1479 	 * at <code>packExpire</code> will not be deleted. Only older packfiles may
1480 	 * be deleted. If set to null then every packfile is a candidate for
1481 	 * deletion.
1482 	 *
1483 	 * @param packExpire
1484 	 *            instant in time which defines packfile expiration
1485 	 */
1486 	public void setPackExpire(Date packExpire) {
1487 		this.packExpire = packExpire;
1488 		packExpireAgeMillis = -1;
1489 	}
1490 
1491 	/**
1492 	 * Set the {@code gc --auto} option.
1493 	 *
1494 	 * With this option, gc checks whether any housekeeping is required; if not,
1495 	 * it exits without performing any work. Some JGit commands run
1496 	 * {@code gc --auto} after performing operations that could create many
1497 	 * loose objects.
1498 	 * <p>
1499 	 * Housekeeping is required if there are too many loose objects or too many
1500 	 * packs in the repository. If the number of loose objects exceeds the value
1501 	 * of the gc.auto option JGit GC consolidates all existing packs into a
1502 	 * single pack (equivalent to {@code -A} option), whereas git-core would
1503 	 * combine all loose objects into a single pack using {@code repack -d -l}.
1504 	 * Setting the value of {@code gc.auto} to 0 disables automatic packing of
1505 	 * loose objects.
1506 	 * <p>
1507 	 * If the number of packs exceeds the value of {@code gc.autoPackLimit},
1508 	 * then existing packs (except those marked with a .keep file) are
1509 	 * consolidated into a single pack by using the {@code -A} option of repack.
1510 	 * Setting {@code gc.autoPackLimit} to 0 disables automatic consolidation of
1511 	 * packs.
1512 	 * <p>
1513 	 * Like git the following jgit commands run auto gc:
1514 	 * <ul>
1515 	 * <li>fetch</li>
1516 	 * <li>merge</li>
1517 	 * <li>rebase</li>
1518 	 * <li>receive-pack</li>
1519 	 * </ul>
1520 	 * The auto gc for receive-pack can be suppressed by setting the config
1521 	 * option {@code receive.autogc = false}
1522 	 *
1523 	 * @param auto
1524 	 *            defines whether gc should do automatic housekeeping
1525 	 */
1526 	public void setAuto(boolean auto) {
1527 		this.automatic = auto;
1528 	}
1529 
1530 	/**
1531 	 * @param background
1532 	 *            whether to run the gc in a background thread.
1533 	 */
1534 	void setBackground(boolean background) {
1535 		this.background = background;
1536 	}
1537 
1538 	private boolean needGc() {
1539 		if (tooManyPacks()) {
1540 			addRepackAllOption();
1541 		} else {
1542 			return tooManyLooseObjects();
1543 		}
1544 		// TODO run pre-auto-gc hook, if it fails return false
1545 		return true;
1546 	}
1547 
1548 	private void addRepackAllOption() {
1549 		// TODO: if JGit GC is enhanced to support repack's option -l this
1550 		// method needs to be implemented
1551 	}
1552 
1553 	/**
1554 	 * @return {@code true} if number of packs > gc.autopacklimit (default 50)
1555 	 */
1556 	boolean tooManyPacks() {
1557 		int autopacklimit = repo.getConfig().getInt(
1558 				ConfigConstants.CONFIG_GC_SECTION,
1559 				ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT,
1560 				DEFAULT_AUTOPACKLIMIT);
1561 		if (autopacklimit <= 0) {
1562 			return false;
1563 		}
1564 		// JGit always creates two packfiles, one for the objects reachable from
1565 		// branches, and another one for the rest
1566 		return repo.getObjectDatabase().getPacks().size() > (autopacklimit + 1);
1567 	}
1568 
1569 	/**
1570 	 * Quickly estimate number of loose objects, SHA1 is distributed evenly so
1571 	 * counting objects in one directory (bucket 17) is sufficient
1572 	 *
1573 	 * @return {@code true} if number of loose objects > gc.auto (default 6700)
1574 	 */
1575 	boolean tooManyLooseObjects() {
1576 		int auto = getLooseObjectLimit();
1577 		if (auto <= 0) {
1578 			return false;
1579 		}
1580 		int n = 0;
1581 		int threshold = (auto + 255) / 256;
1582 		Path dir = repo.getObjectsDirectory().toPath().resolve("17"); //$NON-NLS-1$
1583 		if (!dir.toFile().exists()) {
1584 			return false;
1585 		}
1586 		try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, file -> {
1587 					Path fileName = file.getFileName();
1588 					return file.toFile().isFile() && fileName != null
1589 							&& PATTERN_LOOSE_OBJECT.matcher(fileName.toString())
1590 									.matches();
1591 				})) {
1592 			for (Iterator<Path> iter = stream.iterator(); iter.hasNext(); iter
1593 					.next()) {
1594 				if (++n > threshold) {
1595 					return true;
1596 				}
1597 			}
1598 		} catch (IOException e) {
1599 			LOG.error(e.getMessage(), e);
1600 		}
1601 		return false;
1602 	}
1603 
1604 	private int getLooseObjectLimit() {
1605 		return repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION,
1606 				ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT);
1607 	}
1608 }