1
2
3
4
5
6
7
8
9
10
11 package org.eclipse.jgit.internal.storage.file;
12
13 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
14 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
15 import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP;
16 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
17
18 import java.io.File;
19 import java.io.FileOutputStream;
20 import java.io.IOException;
21 import java.io.OutputStream;
22 import java.io.PrintWriter;
23 import java.io.StringWriter;
24 import java.nio.channels.Channels;
25 import java.nio.channels.FileChannel;
26 import java.nio.file.DirectoryNotEmptyException;
27 import java.nio.file.DirectoryStream;
28 import java.nio.file.Files;
29 import java.nio.file.Path;
30 import java.nio.file.StandardCopyOption;
31 import java.text.MessageFormat;
32 import java.text.ParseException;
33 import java.time.Instant;
34 import java.time.temporal.ChronoUnit;
35 import java.util.ArrayList;
36 import java.util.Collection;
37 import java.util.Collections;
38 import java.util.Comparator;
39 import java.util.Date;
40 import java.util.HashMap;
41 import java.util.HashSet;
42 import java.util.Iterator;
43 import java.util.LinkedList;
44 import java.util.List;
45 import java.util.Map;
46 import java.util.Objects;
47 import java.util.Set;
48 import java.util.TreeMap;
49 import java.util.concurrent.CompletableFuture;
50 import java.util.concurrent.ExecutorService;
51 import java.util.function.Supplier;
52 import java.util.regex.Pattern;
53 import java.util.stream.Collectors;
54 import java.util.stream.Stream;
55
56 import org.eclipse.jgit.annotations.NonNull;
57 import org.eclipse.jgit.dircache.DirCacheIterator;
58 import org.eclipse.jgit.errors.CancelledException;
59 import org.eclipse.jgit.errors.CorruptObjectException;
60 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
61 import org.eclipse.jgit.errors.MissingObjectException;
62 import org.eclipse.jgit.errors.NoWorkTreeException;
63 import org.eclipse.jgit.internal.JGitText;
64 import org.eclipse.jgit.internal.storage.pack.PackExt;
65 import org.eclipse.jgit.internal.storage.pack.PackWriter;
66 import org.eclipse.jgit.lib.ConfigConstants;
67 import org.eclipse.jgit.lib.Constants;
68 import org.eclipse.jgit.lib.FileMode;
69 import org.eclipse.jgit.lib.NullProgressMonitor;
70 import org.eclipse.jgit.lib.ObjectId;
71 import org.eclipse.jgit.lib.ObjectIdSet;
72 import org.eclipse.jgit.lib.ObjectLoader;
73 import org.eclipse.jgit.lib.ObjectReader;
74 import org.eclipse.jgit.lib.ProgressMonitor;
75 import org.eclipse.jgit.lib.Ref;
76 import org.eclipse.jgit.lib.Ref.Storage;
77 import org.eclipse.jgit.lib.RefDatabase;
78 import org.eclipse.jgit.lib.ReflogEntry;
79 import org.eclipse.jgit.lib.ReflogReader;
80 import org.eclipse.jgit.lib.internal.WorkQueue;
81 import org.eclipse.jgit.revwalk.ObjectWalk;
82 import org.eclipse.jgit.revwalk.RevObject;
83 import org.eclipse.jgit.revwalk.RevWalk;
84 import org.eclipse.jgit.storage.pack.PackConfig;
85 import org.eclipse.jgit.treewalk.TreeWalk;
86 import org.eclipse.jgit.treewalk.filter.TreeFilter;
87 import org.eclipse.jgit.util.FileUtils;
88 import org.eclipse.jgit.util.GitDateParser;
89 import org.eclipse.jgit.util.SystemReader;
90 import org.slf4j.Logger;
91 import org.slf4j.LoggerFactory;
92
93
94
95
96
97
98
99
100
101
102 public class GC {
103 private static final Logger LOG = LoggerFactory
104 .getLogger(GC.class);
105
106 private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago";
107
108 private static final String PRUNE_PACK_EXPIRE_DEFAULT = "1.hour.ago";
109
110 private static final Pattern PATTERN_LOOSE_OBJECT = Pattern
111 .compile("[0-9a-fA-F]{38}");
112
113 private static final String PACK_EXT = "." + PackExt.PACK.getExtension();
114
115 private static final String BITMAP_EXT = "."
116 + PackExt.BITMAP_INDEX.getExtension();
117
118 private static final String INDEX_EXT = "." + PackExt.INDEX.getExtension();
119
120 private static final String KEEP_EXT = "." + PackExt.KEEP.getExtension();
121
122 private static final int DEFAULT_AUTOPACKLIMIT = 50;
123
124 private static final int DEFAULT_AUTOLIMIT = 6700;
125
126 private static volatile ExecutorService executor;
127
128
129
130
131
132
133
134
135 public static void setExecutor(ExecutorService e) {
136 executor = e;
137 }
138
139 private final FileRepository repo;
140
141 private ProgressMonitor pm;
142
143 private long expireAgeMillis = -1;
144
145 private Date expire;
146
147 private long packExpireAgeMillis = -1;
148
149 private Date packExpire;
150
151 private PackConfig pconfig;
152
153
154
155
156
157
158
159 private Collection<Ref> lastPackedRefs;
160
161
162
163
164
165
166 private long lastRepackTime;
167
168
169
170
171 private boolean automatic;
172
173
174
175
176 private boolean background;
177
178
179
180
181
182
183
184
185 public GC(FileRepository repo) {
186 this.repo = repo;
187 this.pconfig = new PackConfig(repo);
188 this.pm = NullProgressMonitor.INSTANCE;
189 }
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218 public CompletableFuture<Collection<Pack>> gc()
219 throws IOException, ParseException {
220 if (!background) {
221 return CompletableFuture.completedFuture(doGc());
222 }
223 final GcLog gcLog = new GcLog(repo);
224 if (!gcLog.lock()) {
225
226 return CompletableFuture.completedFuture(Collections.emptyList());
227 }
228
229 Supplier<Collection<Pack>> gcTask = () -> {
230 try {
231 Collection<Pack> newPacks = doGc();
232 if (automatic && tooManyLooseObjects()) {
233 String message = JGitText.get().gcTooManyUnpruned;
234 gcLog.write(message);
235 gcLog.commit();
236 }
237 return newPacks;
238 } catch (IOException | ParseException e) {
239 try {
240 gcLog.write(e.getMessage());
241 StringWriter sw = new StringWriter();
242 e.printStackTrace(new PrintWriter(sw));
243 gcLog.write(sw.toString());
244 gcLog.commit();
245 } catch (IOException e2) {
246 e2.addSuppressed(e);
247 LOG.error(e2.getMessage(), e2);
248 }
249 } finally {
250 gcLog.unlock();
251 }
252 return Collections.emptyList();
253 };
254 return CompletableFuture.supplyAsync(gcTask, executor());
255 }
256
257 private ExecutorService executor() {
258 return (executor != null) ? executor : WorkQueue.getExecutor();
259 }
260
261 private Collection<Pack> doGc() throws IOException, ParseException {
262 if (automatic && !needGc()) {
263 return Collections.emptyList();
264 }
265 pm.start(6 );
266 packRefs();
267
268 Collection<Pack> newPacks = repack();
269 prune(Collections.emptySet());
270
271 return newPacks;
272 }
273
274
275
276
277
278
279
280
281
282
283
284 private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, Pack pack, HashSet<ObjectId> existing)
285 throws IOException {
286 for (PackIndex.MutableEntry entry : pack) {
287 ObjectId oid = entry.toObjectId();
288 if (existing.contains(oid)) {
289 continue;
290 }
291 existing.add(oid);
292 ObjectLoader loader = reader.open(oid);
293 inserter.insert(loader.getType(),
294 loader.getSize(),
295 loader.openStream(),
296 true );
297 }
298 }
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316 private void deleteOldPacks(Collection<Pack> oldPacks,
317 Collection<Pack> newPacks) throws ParseException, IOException {
318 HashSet<ObjectId> ids = new HashSet<>();
319 for (Pack pack : newPacks) {
320 for (PackIndex.MutableEntry entry : pack) {
321 ids.add(entry.toObjectId());
322 }
323 }
324 ObjectReader reader = repo.newObjectReader();
325 ObjectDirectory dir = repo.getObjectDatabase();
326 ObjectDirectoryInserter inserter = dir.newInserter();
327 boolean shouldLoosen = !"now".equals(getPruneExpireStr()) &&
328 getExpireDate() < Long.MAX_VALUE;
329
330 prunePreserved();
331 long packExpireDate = getPackExpireDate();
332 oldPackLoop: for (Pack oldPack : oldPacks) {
333 checkCancelled();
334 String oldName = oldPack.getPackName();
335
336
337 for (Pack newPack : newPacks)
338 if (oldName.equals(newPack.getPackName()))
339 continue oldPackLoop;
340
341 if (!oldPack.shouldBeKept()
342 && repo.getFS()
343 .lastModifiedInstant(oldPack.getPackFile())
344 .toEpochMilli() < packExpireDate) {
345 if (shouldLoosen) {
346 loosen(inserter, reader, oldPack, ids);
347 }
348 oldPack.close();
349 prunePack(oldPack.getPackFile());
350 }
351 }
352
353
354
355 repo.getObjectDatabase().close();
356 }
357
358
359
360
361
362
363
364
365
366 private void removeOldPack(PackFile packFile, int deleteOptions)
367 throws IOException {
368 if (pconfig.isPreserveOldPacks()) {
369 File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
370 FileUtils.mkdir(oldPackDir, true);
371
372 PackFile oldPackFile = packFile
373 .createPreservedForDirectory(oldPackDir);
374 FileUtils.rename(packFile, oldPackFile);
375 } else {
376 FileUtils.delete(packFile, deleteOptions);
377 }
378 }
379
380
381
382
383 private void prunePreserved() {
384 if (pconfig.isPrunePreserved()) {
385 try {
386 FileUtils.delete(repo.getObjectDatabase().getPreservedDirectory(),
387 FileUtils.RECURSIVE | FileUtils.RETRY | FileUtils.SKIP_MISSING);
388 } catch (IOException e) {
389
390 }
391 }
392 }
393
394
395
396
397
398
399
400
401
402
403
404 private void prunePack(PackFile packFile) {
405 try {
406
407
408 int deleteOptions = FileUtils.RETRY | FileUtils.SKIP_MISSING;
409 removeOldPack(packFile.create(PackExt.PACK), deleteOptions);
410
411
412
413 deleteOptions |= FileUtils.IGNORE_ERRORS;
414 for (PackExt ext : PackExt.values()) {
415 if (!PackExt.PACK.equals(ext)) {
416 removeOldPack(packFile.create(ext), deleteOptions);
417 }
418 }
419 } catch (IOException e) {
420
421 }
422 }
423
424
425
426
427
428
429
430
431 public void prunePacked() throws IOException {
432 ObjectDirectory objdb = repo.getObjectDatabase();
433 Collection<Pack> packs = objdb.getPacks();
434 File objects = repo.getObjectsDirectory();
435 String[] fanout = objects.list();
436
437 if (fanout != null && fanout.length > 0) {
438 pm.beginTask(JGitText.get().pruneLoosePackedObjects, fanout.length);
439 try {
440 for (String d : fanout) {
441 checkCancelled();
442 pm.update(1);
443 if (d.length() != 2)
444 continue;
445 String[] entries = new File(objects, d).list();
446 if (entries == null)
447 continue;
448 for (String e : entries) {
449 checkCancelled();
450 if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
451 continue;
452 ObjectId id;
453 try {
454 id = ObjectId.fromString(d + e);
455 } catch (IllegalArgumentException notAnObject) {
456
457
458 continue;
459 }
460 boolean found = false;
461 for (Pack p : packs) {
462 checkCancelled();
463 if (p.hasObject(id)) {
464 found = true;
465 break;
466 }
467 }
468 if (found)
469 FileUtils.delete(objdb.fileFor(id), FileUtils.RETRY
470 | FileUtils.SKIP_MISSING
471 | FileUtils.IGNORE_ERRORS);
472 }
473 }
474 } finally {
475 pm.endTask();
476 }
477 }
478 }
479
480
481
482
483
484
485
486
487
488
489
490
491
492 public void prune(Set<ObjectId> objectsToKeep) throws IOException,
493 ParseException {
494 long expireDate = getExpireDate();
495
496
497
498 Map<ObjectId, File> deletionCandidates = new HashMap<>();
499 Set<ObjectId> indexObjects = null;
500 File objects = repo.getObjectsDirectory();
501 String[] fanout = objects.list();
502 if (fanout == null || fanout.length == 0) {
503 return;
504 }
505 pm.beginTask(JGitText.get().pruneLooseUnreferencedObjects,
506 fanout.length);
507 try {
508 for (String d : fanout) {
509 checkCancelled();
510 pm.update(1);
511 if (d.length() != 2)
512 continue;
513 File dir = new File(objects, d);
514 File[] entries = dir.listFiles();
515 if (entries == null || entries.length == 0) {
516 FileUtils.delete(dir, FileUtils.IGNORE_ERRORS);
517 continue;
518 }
519 for (File f : entries) {
520 checkCancelled();
521 String fName = f.getName();
522 if (fName.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
523 continue;
524 if (repo.getFS().lastModifiedInstant(f)
525 .toEpochMilli() >= expireDate) {
526 continue;
527 }
528 try {
529 ObjectId id = ObjectId.fromString(d + fName);
530 if (objectsToKeep.contains(id))
531 continue;
532 if (indexObjects == null)
533 indexObjects = listNonHEADIndexObjects();
534 if (indexObjects.contains(id))
535 continue;
536 deletionCandidates.put(id, f);
537 } catch (IllegalArgumentException notAnObject) {
538
539
540 }
541 }
542 }
543 } finally {
544 pm.endTask();
545 }
546
547 if (deletionCandidates.isEmpty()) {
548 return;
549 }
550
551 checkCancelled();
552
553
554
555
556
557 Collection<Ref> newRefs;
558 if (lastPackedRefs == null || lastPackedRefs.isEmpty())
559 newRefs = getAllRefs();
560 else {
561 Map<String, Ref> last = new HashMap<>();
562 for (Ref r : lastPackedRefs) {
563 last.put(r.getName(), r);
564 }
565 newRefs = new ArrayList<>();
566 for (Ref r : getAllRefs()) {
567 Ref old = last.get(r.getName());
568 if (!equals(r, old)) {
569 newRefs.add(r);
570 }
571 }
572 }
573
574 if (!newRefs.isEmpty()) {
575
576
577
578
579
580 ObjectWalk w = new ObjectWalk(repo);
581 try {
582 for (Ref cr : newRefs) {
583 checkCancelled();
584 w.markStart(w.parseAny(cr.getObjectId()));
585 }
586 if (lastPackedRefs != null)
587 for (Ref lpr : lastPackedRefs) {
588 w.markUninteresting(w.parseAny(lpr.getObjectId()));
589 }
590 removeReferenced(deletionCandidates, w);
591 } finally {
592 w.dispose();
593 }
594 }
595
596 if (deletionCandidates.isEmpty())
597 return;
598
599
600
601
602
603
604 ObjectWalk w = new ObjectWalk(repo);
605 try {
606 for (Ref ar : getAllRefs())
607 for (ObjectId id : listRefLogObjects(ar, lastRepackTime)) {
608 checkCancelled();
609 w.markStart(w.parseAny(id));
610 }
611 if (lastPackedRefs != null)
612 for (Ref lpr : lastPackedRefs) {
613 checkCancelled();
614 w.markUninteresting(w.parseAny(lpr.getObjectId()));
615 }
616 removeReferenced(deletionCandidates, w);
617 } finally {
618 w.dispose();
619 }
620
621 if (deletionCandidates.isEmpty())
622 return;
623
624 checkCancelled();
625
626
627
628
629
630 Set<File> touchedFanout = new HashSet<>();
631 for (File f : deletionCandidates.values()) {
632 if (f.lastModified() < expireDate) {
633 f.delete();
634 touchedFanout.add(f.getParentFile());
635 }
636 }
637
638 for (File f : touchedFanout) {
639 FileUtils.delete(f,
640 FileUtils.EMPTY_DIRECTORIES_ONLY | FileUtils.IGNORE_ERRORS);
641 }
642
643 repo.getObjectDatabase().close();
644 }
645
646 private long getExpireDate() throws ParseException {
647 long expireDate = Long.MAX_VALUE;
648
649 if (expire == null && expireAgeMillis == -1) {
650 String pruneExpireStr = getPruneExpireStr();
651 if (pruneExpireStr == null)
652 pruneExpireStr = PRUNE_EXPIRE_DEFAULT;
653 expire = GitDateParser.parse(pruneExpireStr, null, SystemReader
654 .getInstance().getLocale());
655 expireAgeMillis = -1;
656 }
657 if (expire != null)
658 expireDate = expire.getTime();
659 if (expireAgeMillis != -1)
660 expireDate = System.currentTimeMillis() - expireAgeMillis;
661 return expireDate;
662 }
663
664 private String getPruneExpireStr() {
665 return repo.getConfig().getString(
666 ConfigConstants.CONFIG_GC_SECTION, null,
667 ConfigConstants.CONFIG_KEY_PRUNEEXPIRE);
668 }
669
670 private long getPackExpireDate() throws ParseException {
671 long packExpireDate = Long.MAX_VALUE;
672
673 if (packExpire == null && packExpireAgeMillis == -1) {
674 String prunePackExpireStr = repo.getConfig().getString(
675 ConfigConstants.CONFIG_GC_SECTION, null,
676 ConfigConstants.CONFIG_KEY_PRUNEPACKEXPIRE);
677 if (prunePackExpireStr == null)
678 prunePackExpireStr = PRUNE_PACK_EXPIRE_DEFAULT;
679 packExpire = GitDateParser.parse(prunePackExpireStr, null,
680 SystemReader.getInstance().getLocale());
681 packExpireAgeMillis = -1;
682 }
683 if (packExpire != null)
684 packExpireDate = packExpire.getTime();
685 if (packExpireAgeMillis != -1)
686 packExpireDate = System.currentTimeMillis() - packExpireAgeMillis;
687 return packExpireDate;
688 }
689
690
691
692
693
694
695
696
697
698
699
700 private void removeReferenced(Map<ObjectId, File> id2File,
701 ObjectWalk w) throws MissingObjectException,
702 IncorrectObjectTypeException, IOException {
703 RevObject ro = w.next();
704 while (ro != null) {
705 checkCancelled();
706 if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
707 return;
708 }
709 ro = w.next();
710 }
711 ro = w.nextObject();
712 while (ro != null) {
713 checkCancelled();
714 if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
715 return;
716 }
717 ro = w.nextObject();
718 }
719 }
720
721 private static boolean equals(Ref r1, Ref r2) {
722 if (r1 == null || r2 == null) {
723 return false;
724 }
725 if (r1.isSymbolic()) {
726 return r2.isSymbolic() && r1.getTarget().getName()
727 .equals(r2.getTarget().getName());
728 }
729 return !r2.isSymbolic()
730 && Objects.equals(r1.getObjectId(), r2.getObjectId());
731 }
732
733
734
735
736
737
738
739
740 public void packRefs() throws IOException {
741 RefDatabase refDb = repo.getRefDatabase();
742 if (refDb instanceof FileReftableDatabase) {
743
744 pm.beginTask(JGitText.get().packRefs, 1);
745 try {
746 ((FileReftableDatabase) refDb).compactFully();
747 } finally {
748 pm.endTask();
749 }
750 return;
751 }
752
753 Collection<Ref> refs = refDb.getRefsByPrefix(Constants.R_REFS);
754 List<String> refsToBePacked = new ArrayList<>(refs.size());
755 pm.beginTask(JGitText.get().packRefs, refs.size());
756 try {
757 for (Ref ref : refs) {
758 checkCancelled();
759 if (!ref.isSymbolic() && ref.getStorage().isLoose())
760 refsToBePacked.add(ref.getName());
761 pm.update(1);
762 }
763 ((RefDirectory) repo.getRefDatabase()).pack(refsToBePacked);
764 } finally {
765 pm.endTask();
766 }
767 }
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783 public Collection<Pack> repack() throws IOException {
784 Collection<Pack> toBeDeleted = repo.getObjectDatabase().getPacks();
785
786 long time = System.currentTimeMillis();
787 Collection<Ref> refsBefore = getAllRefs();
788
789 Set<ObjectId> allHeadsAndTags = new HashSet<>();
790 Set<ObjectId> allHeads = new HashSet<>();
791 Set<ObjectId> allTags = new HashSet<>();
792 Set<ObjectId> nonHeads = new HashSet<>();
793 Set<ObjectId> txnHeads = new HashSet<>();
794 Set<ObjectId> tagTargets = new HashSet<>();
795 Set<ObjectId> indexObjects = listNonHEADIndexObjects();
796
797 for (Ref ref : refsBefore) {
798 checkCancelled();
799 nonHeads.addAll(listRefLogObjects(ref, 0));
800 if (ref.isSymbolic() || ref.getObjectId() == null) {
801 continue;
802 }
803 if (isHead(ref)) {
804 allHeads.add(ref.getObjectId());
805 } else if (isTag(ref)) {
806 allTags.add(ref.getObjectId());
807 } else {
808 nonHeads.add(ref.getObjectId());
809 }
810 if (ref.getPeeledObjectId() != null) {
811 tagTargets.add(ref.getPeeledObjectId());
812 }
813 }
814
815 List<ObjectIdSet> excluded = new LinkedList<>();
816 for (Pack p : repo.getObjectDatabase().getPacks()) {
817 checkCancelled();
818 if (p.shouldBeKept())
819 excluded.add(p.getIndex());
820 }
821
822
823 allTags.removeAll(allHeads);
824 allHeadsAndTags.addAll(allHeads);
825 allHeadsAndTags.addAll(allTags);
826
827
828 tagTargets.addAll(allHeadsAndTags);
829 nonHeads.addAll(indexObjects);
830
831
832 if (pconfig.getSinglePack()) {
833 allHeadsAndTags.addAll(nonHeads);
834 nonHeads.clear();
835 }
836
837 List<Pack> ret = new ArrayList<>(2);
838 Pack heads = null;
839 if (!allHeadsAndTags.isEmpty()) {
840 heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags,
841 tagTargets, excluded);
842 if (heads != null) {
843 ret.add(heads);
844 excluded.add(0, heads.getIndex());
845 }
846 }
847 if (!nonHeads.isEmpty()) {
848 Pack rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE,
849 tagTargets, excluded);
850 if (rest != null)
851 ret.add(rest);
852 }
853 if (!txnHeads.isEmpty()) {
854 Pack txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE,
855 null, excluded);
856 if (txn != null)
857 ret.add(txn);
858 }
859 try {
860 deleteOldPacks(toBeDeleted, ret);
861 } catch (ParseException e) {
862
863
864
865 throw new IOException(e);
866 }
867 prunePacked();
868 if (repo.getRefDatabase() instanceof RefDirectory) {
869
870 deleteEmptyRefsFolders();
871 }
872 deleteOrphans();
873 deleteTempPacksIdx();
874
875 lastPackedRefs = refsBefore;
876 lastRepackTime = time;
877 return ret;
878 }
879
880 private static boolean isHead(Ref ref) {
881 return ref.getName().startsWith(Constants.R_HEADS);
882 }
883
884 private static boolean isTag(Ref ref) {
885 return ref.getName().startsWith(Constants.R_TAGS);
886 }
887
888 private void deleteEmptyRefsFolders() throws IOException {
889 Path refs = repo.getDirectory().toPath().resolve(Constants.R_REFS);
890
891
892 Instant threshold = Instant.now().minus(30, ChronoUnit.SECONDS);
893 try (Stream<Path> entries = Files.list(refs)
894 .filter(Files::isDirectory)) {
895 Iterator<Path> iterator = entries.iterator();
896 while (iterator.hasNext()) {
897 try (Stream<Path> s = Files.list(iterator.next())) {
898 s.filter(path -> canBeSafelyDeleted(path, threshold)).forEach(this::deleteDir);
899 }
900 }
901 }
902 }
903
904 private boolean canBeSafelyDeleted(Path path, Instant threshold) {
905 try {
906 return Files.getLastModifiedTime(path).toInstant().isBefore(threshold);
907 }
908 catch (IOException e) {
909 LOG.warn(MessageFormat.format(
910 JGitText.get().cannotAccessLastModifiedForSafeDeletion,
911 path), e);
912 return false;
913 }
914 }
915
916 private void deleteDir(Path dir) {
917 try (Stream<Path> dirs = Files.walk(dir)) {
918 dirs.filter(this::isDirectory).sorted(Comparator.reverseOrder())
919 .forEach(this::delete);
920 } catch (IOException e) {
921 LOG.error(e.getMessage(), e);
922 }
923 }
924
925 private boolean isDirectory(Path p) {
926 return p.toFile().isDirectory();
927 }
928
929 private void delete(Path d) {
930 try {
931 Files.delete(d);
932 } catch (DirectoryNotEmptyException e) {
933
934 } catch (IOException e) {
935 LOG.error(MessageFormat.format(JGitText.get().cannotDeleteFile, d),
936 e);
937 }
938 }
939
940
941
942
943
944
945
946
947 private void deleteOrphans() {
948 Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
949 List<String> fileNames = null;
950 try (Stream<Path> files = Files.list(packDir)) {
951 fileNames = files.map(path -> path.getFileName().toString())
952 .filter(name -> (name.endsWith(PACK_EXT)
953 || name.endsWith(BITMAP_EXT)
954 || name.endsWith(INDEX_EXT)
955 || name.endsWith(KEEP_EXT)))
956
957
958 .sorted(Collections.reverseOrder())
959 .collect(Collectors.toList());
960 } catch (IOException e) {
961 LOG.error(e.getMessage(), e);
962 return;
963 }
964 if (fileNames == null) {
965 return;
966 }
967
968 String latestId = null;
969 for (String n : fileNames) {
970 PackFile pf = new PackFile(packDir.toFile(), n);
971 PackExt ext = pf.getPackExt();
972 if (ext.equals(PACK) || ext.equals(KEEP)) {
973 latestId = pf.getId();
974 }
975 if (latestId == null || !pf.getId().equals(latestId)) {
976
977 try {
978 FileUtils.delete(pf,
979 FileUtils.RETRY | FileUtils.SKIP_MISSING);
980 LOG.warn(JGitText.get().deletedOrphanInPackDir, pf);
981 } catch (IOException e) {
982 LOG.error(e.getMessage(), e);
983 }
984 }
985 }
986 }
987
988 private void deleteTempPacksIdx() {
989 Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
990 Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS);
991 if (!Files.exists(packDir)) {
992 return;
993 }
994 try (DirectoryStream<Path> stream =
995 Files.newDirectoryStream(packDir, "gc_*_tmp")) {
996 stream.forEach(t -> {
997 try {
998 Instant lastModified = Files.getLastModifiedTime(t)
999 .toInstant();
1000 if (lastModified.isBefore(threshold)) {
1001 Files.deleteIfExists(t);
1002 }
1003 } catch (IOException e) {
1004 LOG.error(e.getMessage(), e);
1005 }
1006 });
1007 } catch (IOException e) {
1008 LOG.error(e.getMessage(), e);
1009 }
1010 }
1011
1012
1013
1014
1015
1016
1017
1018
1019 private Set<ObjectId> listRefLogObjects(Ref ref, long minTime) throws IOException {
1020 ReflogReader reflogReader = repo.getReflogReader(ref.getName());
1021 if (reflogReader == null) {
1022 return Collections.emptySet();
1023 }
1024 List<ReflogEntry> rlEntries = reflogReader
1025 .getReverseEntries();
1026 if (rlEntries == null || rlEntries.isEmpty())
1027 return Collections.emptySet();
1028 Set<ObjectId> ret = new HashSet<>();
1029 for (ReflogEntry e : rlEntries) {
1030 if (e.getWho().getWhen().getTime() < minTime)
1031 break;
1032 ObjectId newId = e.getNewId();
1033 if (newId != null && !ObjectId.zeroId().equals(newId))
1034 ret.add(newId);
1035 ObjectId oldId = e.getOldId();
1036 if (oldId != null && !ObjectId.zeroId().equals(oldId))
1037 ret.add(oldId);
1038 }
1039 return ret;
1040 }
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053 private Collection<Ref> getAllRefs() throws IOException {
1054 RefDatabase refdb = repo.getRefDatabase();
1055 Collection<Ref> refs = refdb.getRefs();
1056 List<Ref> addl = refdb.getAdditionalRefs();
1057 if (!addl.isEmpty()) {
1058 List<Ref> all = new ArrayList<>(refs.size() + addl.size());
1059 all.addAll(refs);
1060
1061 for (Ref r : addl) {
1062 checkCancelled();
1063 if (r.getName().startsWith(Constants.R_REFS)) {
1064 all.add(r);
1065 }
1066 }
1067 return all;
1068 }
1069 return refs;
1070 }
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081 private Set<ObjectId> listNonHEADIndexObjects()
1082 throws CorruptObjectException, IOException {
1083 if (repo.isBare()) {
1084 return Collections.emptySet();
1085 }
1086 try (TreeWalk treeWalk = new TreeWalk(repo)) {
1087 treeWalk.addTree(new DirCacheIterator(repo.readDirCache()));
1088 ObjectId headID = repo.resolve(Constants.HEAD);
1089 if (headID != null) {
1090 try (RevWalk revWalk = new RevWalk(repo)) {
1091 treeWalk.addTree(revWalk.parseTree(headID));
1092 }
1093 }
1094
1095 treeWalk.setFilter(TreeFilter.ANY_DIFF);
1096 treeWalk.setRecursive(true);
1097 Set<ObjectId> ret = new HashSet<>();
1098
1099 while (treeWalk.next()) {
1100 checkCancelled();
1101 ObjectId objectId = treeWalk.getObjectId(0);
1102 switch (treeWalk.getRawMode(0) & FileMode.TYPE_MASK) {
1103 case FileMode.TYPE_MISSING:
1104 case FileMode.TYPE_GITLINK:
1105 continue;
1106 case FileMode.TYPE_TREE:
1107 case FileMode.TYPE_FILE:
1108 case FileMode.TYPE_SYMLINK:
1109 ret.add(objectId);
1110 continue;
1111 default:
1112 throw new IOException(MessageFormat.format(
1113 JGitText.get().corruptObjectInvalidMode3,
1114 String.format("%o",
1115 Integer.valueOf(treeWalk.getRawMode(0))),
1116 (objectId == null) ? "null" : objectId.name(),
1117 treeWalk.getPathString(),
1118 repo.getIndexFile()));
1119 }
1120 }
1121 return ret;
1122 }
1123 }
1124
1125 private Pack writePack(@NonNull Set<? extends ObjectId> want,
1126 @NonNull Set<? extends ObjectId> have, @NonNull Set<ObjectId> tags,
1127 Set<ObjectId> tagTargets, List<ObjectIdSet> excludeObjects)
1128 throws IOException {
1129 checkCancelled();
1130 File tmpPack = null;
1131 Map<PackExt, File> tmpExts = new TreeMap<>((o1, o2) -> {
1132
1133
1134
1135 if (o1 == o2) {
1136 return 0;
1137 }
1138 if (o1 == PackExt.INDEX) {
1139 return 1;
1140 }
1141 if (o2 == PackExt.INDEX) {
1142 return -1;
1143 }
1144 return Integer.signum(o1.hashCode() - o2.hashCode());
1145 });
1146 try (PackWriter pw = new PackWriter(
1147 pconfig,
1148 repo.newObjectReader())) {
1149
1150 pw.setDeltaBaseAsOffset(true);
1151 pw.setReuseDeltaCommits(false);
1152 if (tagTargets != null) {
1153 pw.setTagTargets(tagTargets);
1154 }
1155 if (excludeObjects != null)
1156 for (ObjectIdSet idx : excludeObjects)
1157 pw.excludeObjects(idx);
1158 pw.preparePack(pm, want, have, PackWriter.NONE, tags);
1159 if (pw.getObjectCount() == 0)
1160 return null;
1161 checkCancelled();
1162
1163
1164 ObjectId id = pw.computeName();
1165 File packdir = repo.getObjectDatabase().getPackDirectory();
1166 packdir.mkdirs();
1167 tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir);
1168 final String tmpBase = tmpPack.getName()
1169 .substring(0, tmpPack.getName().lastIndexOf('.'));
1170 File tmpIdx = new File(packdir, tmpBase + ".idx_tmp");
1171 tmpExts.put(INDEX, tmpIdx);
1172
1173 if (!tmpIdx.createNewFile())
1174 throw new IOException(MessageFormat.format(
1175 JGitText.get().cannotCreateIndexfile, tmpIdx.getPath()));
1176
1177
1178 try (FileOutputStream fos = new FileOutputStream(tmpPack);
1179 FileChannel channel = fos.getChannel();
1180 OutputStream channelStream = Channels
1181 .newOutputStream(channel)) {
1182 pw.writePack(pm, pm, channelStream);
1183 channel.force(true);
1184 }
1185
1186
1187 try (FileOutputStream fos = new FileOutputStream(tmpIdx);
1188 FileChannel idxChannel = fos.getChannel();
1189 OutputStream idxStream = Channels
1190 .newOutputStream(idxChannel)) {
1191 pw.writeIndex(idxStream);
1192 idxChannel.force(true);
1193 }
1194
1195 if (pw.prepareBitmapIndex(pm)) {
1196 File tmpBitmapIdx = new File(packdir, tmpBase + ".bitmap_tmp");
1197 tmpExts.put(BITMAP_INDEX, tmpBitmapIdx);
1198
1199 if (!tmpBitmapIdx.createNewFile())
1200 throw new IOException(MessageFormat.format(
1201 JGitText.get().cannotCreateIndexfile,
1202 tmpBitmapIdx.getPath()));
1203
1204 try (FileOutputStream fos = new FileOutputStream(tmpBitmapIdx);
1205 FileChannel idxChannel = fos.getChannel();
1206 OutputStream idxStream = Channels
1207 .newOutputStream(idxChannel)) {
1208 pw.writeBitmapIndex(idxStream);
1209 idxChannel.force(true);
1210 }
1211 }
1212
1213
1214 File packDir = repo.getObjectDatabase().getPackDirectory();
1215 PackFile realPack = new PackFile(packDir, id, PackExt.PACK);
1216
1217 repo.getObjectDatabase().closeAllPackHandles(realPack);
1218 tmpPack.setReadOnly();
1219
1220 FileUtils.rename(tmpPack, realPack, StandardCopyOption.ATOMIC_MOVE);
1221 for (Map.Entry<PackExt, File> tmpEntry : tmpExts.entrySet()) {
1222 File tmpExt = tmpEntry.getValue();
1223 tmpExt.setReadOnly();
1224
1225 PackFile realExt = new PackFile(packDir, id, tmpEntry.getKey());
1226 try {
1227 FileUtils.rename(tmpExt, realExt,
1228 StandardCopyOption.ATOMIC_MOVE);
1229 } catch (IOException e) {
1230 File newExt = new File(realExt.getParentFile(),
1231 realExt.getName() + ".new");
1232 try {
1233 FileUtils.rename(tmpExt, newExt,
1234 StandardCopyOption.ATOMIC_MOVE);
1235 } catch (IOException e2) {
1236 newExt = tmpExt;
1237 e = e2;
1238 }
1239 throw new IOException(MessageFormat.format(
1240 JGitText.get().panicCantRenameIndexFile, newExt,
1241 realExt), e);
1242 }
1243 }
1244 boolean interrupted = false;
1245 try {
1246 FileSnapshot snapshot = FileSnapshot.save(realPack);
1247 if (pconfig.doWaitPreventRacyPack(snapshot.size())) {
1248 snapshot.waitUntilNotRacy();
1249 }
1250 } catch (InterruptedException e) {
1251 interrupted = true;
1252 }
1253 try {
1254 return repo.getObjectDatabase().openPack(realPack);
1255 } finally {
1256 if (interrupted) {
1257
1258 Thread.currentThread().interrupt();
1259 }
1260 }
1261 } finally {
1262 if (tmpPack != null && tmpPack.exists())
1263 tmpPack.delete();
1264 for (File tmpExt : tmpExts.values()) {
1265 if (tmpExt.exists())
1266 tmpExt.delete();
1267 }
1268 }
1269 }
1270
1271 private void checkCancelled() throws CancelledException {
1272 if (pm.isCancelled() || Thread.currentThread().isInterrupted()) {
1273 throw new CancelledException(JGitText.get().operationCanceled);
1274 }
1275 }
1276
1277
1278
1279
1280
1281 public static class RepoStatistics {
1282
1283
1284
1285
1286
1287 public long numberOfPackedObjects;
1288
1289
1290
1291
1292 public long numberOfPackFiles;
1293
1294
1295
1296
1297 public long numberOfLooseObjects;
1298
1299
1300
1301
1302 public long sizeOfLooseObjects;
1303
1304
1305
1306
1307 public long sizeOfPackedObjects;
1308
1309
1310
1311
1312 public long numberOfLooseRefs;
1313
1314
1315
1316
1317 public long numberOfPackedRefs;
1318
1319
1320
1321
1322 public long numberOfBitmaps;
1323
1324 @Override
1325 public String toString() {
1326 final StringBuilder b = new StringBuilder();
1327 b.append("numberOfPackedObjects=").append(numberOfPackedObjects);
1328 b.append(", numberOfPackFiles=").append(numberOfPackFiles);
1329 b.append(", numberOfLooseObjects=").append(numberOfLooseObjects);
1330 b.append(", numberOfLooseRefs=").append(numberOfLooseRefs);
1331 b.append(", numberOfPackedRefs=").append(numberOfPackedRefs);
1332 b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects);
1333 b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects);
1334 b.append(", numberOfBitmaps=").append(numberOfBitmaps);
1335 return b.toString();
1336 }
1337 }
1338
1339
1340
1341
1342
1343
1344
1345 public RepoStatistics getStatistics() throws IOException {
1346 RepoStatistics ret = new RepoStatistics();
1347 Collection<Pack> packs = repo.getObjectDatabase().getPacks();
1348 for (Pack p : packs) {
1349 ret.numberOfPackedObjects += p.getIndex().getObjectCount();
1350 ret.numberOfPackFiles++;
1351 ret.sizeOfPackedObjects += p.getPackFile().length();
1352 if (p.getBitmapIndex() != null)
1353 ret.numberOfBitmaps += p.getBitmapIndex().getBitmapCount();
1354 }
1355 File objDir = repo.getObjectsDirectory();
1356 String[] fanout = objDir.list();
1357 if (fanout != null && fanout.length > 0) {
1358 for (String d : fanout) {
1359 if (d.length() != 2)
1360 continue;
1361 File[] entries = new File(objDir, d).listFiles();
1362 if (entries == null)
1363 continue;
1364 for (File f : entries) {
1365 if (f.getName().length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
1366 continue;
1367 ret.numberOfLooseObjects++;
1368 ret.sizeOfLooseObjects += f.length();
1369 }
1370 }
1371 }
1372
1373 RefDatabase refDb = repo.getRefDatabase();
1374 for (Ref r : refDb.getRefs()) {
1375 Storage storage = r.getStorage();
1376 if (storage == Storage.LOOSE || storage == Storage.LOOSE_PACKED)
1377 ret.numberOfLooseRefs++;
1378 if (storage == Storage.PACKED || storage == Storage.LOOSE_PACKED)
1379 ret.numberOfPackedRefs++;
1380 }
1381
1382 return ret;
1383 }
1384
1385
1386
1387
1388
1389
1390
1391 public GC setProgressMonitor(ProgressMonitor pm) {
1392 this.pm = (pm == null) ? NullProgressMonitor.INSTANCE : pm;
1393 return this;
1394 }
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405 public void setExpireAgeMillis(long expireAgeMillis) {
1406 this.expireAgeMillis = expireAgeMillis;
1407 expire = null;
1408 }
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419 public void setPackExpireAgeMillis(long packExpireAgeMillis) {
1420 this.packExpireAgeMillis = packExpireAgeMillis;
1421 expire = null;
1422 }
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433 public void setPackConfig(@NonNull PackConfig pconfig) {
1434 this.pconfig = pconfig;
1435 }
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449 public void setExpire(Date expire) {
1450 this.expire = expire;
1451 expireAgeMillis = -1;
1452 }
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463 public void setPackExpire(Date packExpire) {
1464 this.packExpire = packExpire;
1465 packExpireAgeMillis = -1;
1466 }
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503 public void setAuto(boolean auto) {
1504 this.automatic = auto;
1505 }
1506
1507
1508
1509
1510
1511 void setBackground(boolean background) {
1512 this.background = background;
1513 }
1514
1515 private boolean needGc() {
1516 if (tooManyPacks()) {
1517 addRepackAllOption();
1518 } else {
1519 return tooManyLooseObjects();
1520 }
1521
1522 return true;
1523 }
1524
1525 private void addRepackAllOption() {
1526
1527
1528 }
1529
1530
1531
1532
1533
1534 boolean tooManyPacks() {
1535 int autopacklimit = repo.getConfig().getInt(
1536 ConfigConstants.CONFIG_GC_SECTION,
1537 ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT,
1538 DEFAULT_AUTOPACKLIMIT);
1539 if (autopacklimit <= 0) {
1540 return false;
1541 }
1542
1543
1544 return repo.getObjectDatabase().getPacks().size() > (autopacklimit + 1);
1545 }
1546
1547
1548
1549
1550
1551
1552
1553
1554 boolean tooManyLooseObjects() {
1555 int auto = getLooseObjectLimit();
1556 if (auto <= 0) {
1557 return false;
1558 }
1559 int n = 0;
1560 int threshold = (auto + 255) / 256;
1561 Path dir = repo.getObjectsDirectory().toPath().resolve("17");
1562 if (!dir.toFile().exists()) {
1563 return false;
1564 }
1565 try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, file -> {
1566 Path fileName = file.getFileName();
1567 return file.toFile().isFile() && fileName != null
1568 && PATTERN_LOOSE_OBJECT.matcher(fileName.toString())
1569 .matches();
1570 })) {
1571 for (Iterator<Path> iter = stream.iterator(); iter.hasNext(); iter
1572 .next()) {
1573 if (++n > threshold) {
1574 return true;
1575 }
1576 }
1577 } catch (IOException e) {
1578 LOG.error(e.getMessage(), e);
1579 }
1580 return false;
1581 }
1582
1583 private int getLooseObjectLimit() {
1584 return repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION,
1585 ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT);
1586 }
1587 }