1
2
3
4
5
6
7
8
9
10
11 package org.eclipse.jgit.internal.storage.file;
12
13 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
14 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
15
16 import java.io.File;
17 import java.io.FileOutputStream;
18 import java.io.IOException;
19 import java.io.OutputStream;
20 import java.io.PrintWriter;
21 import java.io.StringWriter;
22 import java.nio.channels.Channels;
23 import java.nio.channels.FileChannel;
24 import java.nio.file.DirectoryNotEmptyException;
25 import java.nio.file.DirectoryStream;
26 import java.nio.file.Files;
27 import java.nio.file.Path;
28 import java.nio.file.StandardCopyOption;
29 import java.text.MessageFormat;
30 import java.text.ParseException;
31 import java.time.Instant;
32 import java.time.temporal.ChronoUnit;
33 import java.util.ArrayList;
34 import java.util.Collection;
35 import java.util.Collections;
36 import java.util.Comparator;
37 import java.util.Date;
38 import java.util.HashMap;
39 import java.util.HashSet;
40 import java.util.Iterator;
41 import java.util.LinkedList;
42 import java.util.List;
43 import java.util.Map;
44 import java.util.Objects;
45 import java.util.Set;
46 import java.util.TreeMap;
47 import java.util.concurrent.Callable;
48 import java.util.concurrent.ExecutorService;
49 import java.util.regex.Pattern;
50 import java.util.stream.Collectors;
51 import java.util.stream.Stream;
52
53 import org.eclipse.jgit.annotations.NonNull;
54 import org.eclipse.jgit.dircache.DirCacheIterator;
55 import org.eclipse.jgit.errors.CancelledException;
56 import org.eclipse.jgit.errors.CorruptObjectException;
57 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
58 import org.eclipse.jgit.errors.MissingObjectException;
59 import org.eclipse.jgit.errors.NoWorkTreeException;
60 import org.eclipse.jgit.internal.JGitText;
61 import org.eclipse.jgit.internal.storage.pack.PackExt;
62 import org.eclipse.jgit.internal.storage.pack.PackWriter;
63 import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
64 import org.eclipse.jgit.lib.ConfigConstants;
65 import org.eclipse.jgit.lib.Constants;
66 import org.eclipse.jgit.lib.FileMode;
67 import org.eclipse.jgit.lib.NullProgressMonitor;
68 import org.eclipse.jgit.lib.ObjectId;
69 import org.eclipse.jgit.lib.ObjectIdSet;
70 import org.eclipse.jgit.lib.ObjectLoader;
71 import org.eclipse.jgit.lib.ObjectReader;
72 import org.eclipse.jgit.lib.ProgressMonitor;
73 import org.eclipse.jgit.lib.Ref;
74 import org.eclipse.jgit.lib.Ref.Storage;
75 import org.eclipse.jgit.lib.RefDatabase;
76 import org.eclipse.jgit.lib.ReflogEntry;
77 import org.eclipse.jgit.lib.ReflogReader;
78 import org.eclipse.jgit.lib.internal.WorkQueue;
79 import org.eclipse.jgit.revwalk.ObjectWalk;
80 import org.eclipse.jgit.revwalk.RevObject;
81 import org.eclipse.jgit.revwalk.RevWalk;
82 import org.eclipse.jgit.storage.pack.PackConfig;
83 import org.eclipse.jgit.treewalk.TreeWalk;
84 import org.eclipse.jgit.treewalk.filter.TreeFilter;
85 import org.eclipse.jgit.util.FileUtils;
86 import org.eclipse.jgit.util.GitDateParser;
87 import org.eclipse.jgit.util.SystemReader;
88 import org.slf4j.Logger;
89 import org.slf4j.LoggerFactory;
90
91
92
93
94
95
96
97
98
99
100 public class GC {
101 private static final Logger LOG = LoggerFactory
102 .getLogger(GC.class);
103
104 private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago";
105
106 private static final String PRUNE_PACK_EXPIRE_DEFAULT = "1.hour.ago";
107
108 private static final Pattern PATTERN_LOOSE_OBJECT = Pattern
109 .compile("[0-9a-fA-F]{38}");
110
111 private static final String PACK_EXT = "." + PackExt.PACK.getExtension();
112
113 private static final String BITMAP_EXT = "."
114 + PackExt.BITMAP_INDEX.getExtension();
115
116 private static final String INDEX_EXT = "." + PackExt.INDEX.getExtension();
117
118 private static final String KEEP_EXT = "." + PackExt.KEEP.getExtension();
119
120 private static final int DEFAULT_AUTOPACKLIMIT = 50;
121
122 private static final int DEFAULT_AUTOLIMIT = 6700;
123
124 private static volatile ExecutorService executor;
125
126
127
128
129
130
131
132
133 public static void setExecutor(ExecutorService e) {
134 executor = e;
135 }
136
137 private final FileRepository repo;
138
139 private ProgressMonitor pm;
140
141 private long expireAgeMillis = -1;
142
143 private Date expire;
144
145 private long packExpireAgeMillis = -1;
146
147 private Date packExpire;
148
149 private PackConfig pconfig;
150
151
152
153
154
155
156
157 private Collection<Ref> lastPackedRefs;
158
159
160
161
162
163
164 private long lastRepackTime;
165
166
167
168
169 private boolean automatic;
170
171
172
173
174 private boolean background;
175
176
177
178
179
180
181
182
183 public GC(FileRepository repo) {
184 this.repo = repo;
185 this.pconfig = new PackConfig(repo);
186 this.pm = NullProgressMonitor.INSTANCE;
187 }
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217 @SuppressWarnings("FutureReturnValueIgnored")
218 public Collection<PackFile> gc() throws IOException, ParseException {
219 if (!background) {
220 return doGc();
221 }
222 final GcLogernal/storage/file/GcLog.html#GcLog">GcLog gcLog = new GcLog(repo);
223 if (!gcLog.lock()) {
224
225 return Collections.emptyList();
226 }
227
228 Callable<Collection<PackFile>> gcTask = () -> {
229 try {
230 Collection<PackFile> newPacks = doGc();
231 if (automatic && tooManyLooseObjects()) {
232 String message = JGitText.get().gcTooManyUnpruned;
233 gcLog.write(message);
234 gcLog.commit();
235 }
236 return newPacks;
237 } catch (IOException | ParseException e) {
238 try {
239 gcLog.write(e.getMessage());
240 StringWriter sw = new StringWriter();
241 e.printStackTrace(new PrintWriter(sw));
242 gcLog.write(sw.toString());
243 gcLog.commit();
244 } catch (IOException e2) {
245 e2.addSuppressed(e);
246 LOG.error(e2.getMessage(), e2);
247 }
248 } finally {
249 gcLog.unlock();
250 }
251 return Collections.emptyList();
252 };
253
254 executor().submit(gcTask);
255 return Collections.emptyList();
256 }
257
258 private ExecutorService executor() {
259 return (executor != null) ? executor : WorkQueue.getExecutor();
260 }
261
262 private Collection<PackFile> doGc() throws IOException, ParseException {
263 if (automatic && !needGc()) {
264 return Collections.emptyList();
265 }
266 pm.start(6 );
267 packRefs();
268
269 Collection<PackFile> newPacks = repack();
270 prune(Collections.emptySet());
271
272 return newPacks;
273 }
274
275
276
277
278
279
280
281
282
283
284
285 private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, PackFile pack, HashSet<ObjectId> existing)
286 throws IOException {
287 for (PackIndex.MutableEntry entry : pack) {
288 ObjectId oid = entry.toObjectId();
289 if (existing.contains(oid)) {
290 continue;
291 }
292 existing.add(oid);
293 ObjectLoader loader = reader.open(oid);
294 inserter.insert(loader.getType(),
295 loader.getSize(),
296 loader.openStream(),
297 true );
298 }
299 }
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317 private void deleteOldPacks(Collection<PackFile> oldPacks,
318 Collection<PackFile> newPacks) throws ParseException, IOException {
319 HashSet<ObjectId> ids = new HashSet<>();
320 for (PackFile pack : newPacks) {
321 for (PackIndex.MutableEntry entry : pack) {
322 ids.add(entry.toObjectId());
323 }
324 }
325 ObjectReader reader = repo.newObjectReader();
326 ObjectDirectory dir = repo.getObjectDatabase();
327 ObjectDirectoryInserter inserter = dir.newInserter();
328 boolean shouldLoosen = !"now".equals(getPruneExpireStr()) &&
329 getExpireDate() < Long.MAX_VALUE;
330
331 prunePreserved();
332 long packExpireDate = getPackExpireDate();
333 oldPackLoop: for (PackFile oldPack : oldPacks) {
334 checkCancelled();
335 String oldName = oldPack.getPackName();
336
337
338 for (PackFile newPack : newPacks)
339 if (oldName.equals(newPack.getPackName()))
340 continue oldPackLoop;
341
342 if (!oldPack.shouldBeKept()
343 && repo.getFS()
344 .lastModifiedInstant(oldPack.getPackFile())
345 .toEpochMilli() < packExpireDate) {
346 oldPack.close();
347 if (shouldLoosen) {
348 loosen(inserter, reader, oldPack, ids);
349 }
350 prunePack(oldName);
351 }
352 }
353
354
355
356 repo.getObjectDatabase().close();
357 }
358
359
360
361
362
363
364
365
366
367
368
369 private void removeOldPack(File packFile, String packName, PackExt ext,
370 int deleteOptions) throws IOException {
371 if (pconfig.isPreserveOldPacks()) {
372 File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
373 FileUtils.mkdir(oldPackDir, true);
374
375 String oldPackName = "pack-" + packName + ".old-" + ext.getExtension();
376 File oldPackFile = new File(oldPackDir, oldPackName);
377 FileUtils.rename(packFile, oldPackFile);
378 } else {
379 FileUtils.delete(packFile, deleteOptions);
380 }
381 }
382
383
384
385
386 private void prunePreserved() {
387 if (pconfig.isPrunePreserved()) {
388 try {
389 FileUtils.delete(repo.getObjectDatabase().getPreservedDirectory(),
390 FileUtils.RECURSIVE | FileUtils.RETRY | FileUtils.SKIP_MISSING);
391 } catch (IOException e) {
392
393 }
394 }
395 }
396
397
398
399
400
401
402
403
404
405
406
407 private void prunePack(String packName) {
408 PackExt[] extensions = PackExt.values();
409 try {
410
411
412 int deleteOptions = FileUtils.RETRY | FileUtils.SKIP_MISSING;
413 for (PackExt ext : extensions)
414 if (PackExt.PACK.equals(ext)) {
415 File f = nameFor(packName, "." + ext.getExtension());
416 removeOldPack(f, packName, ext, deleteOptions);
417 break;
418 }
419
420
421 deleteOptions |= FileUtils.IGNORE_ERRORS;
422 for (PackExt ext : extensions) {
423 if (!PackExt.PACK.equals(ext)) {
424 File f = nameFor(packName, "." + ext.getExtension());
425 removeOldPack(f, packName, ext, deleteOptions);
426 }
427 }
428 } catch (IOException e) {
429
430 }
431 }
432
433
434
435
436
437
438
439
440 public void prunePacked() throws IOException {
441 ObjectDirectory objdb = repo.getObjectDatabase();
442 Collection<PackFile> packs = objdb.getPacks();
443 File objects = repo.getObjectsDirectory();
444 String[] fanout = objects.list();
445
446 if (fanout != null && fanout.length > 0) {
447 pm.beginTask(JGitText.get().pruneLoosePackedObjects, fanout.length);
448 try {
449 for (String d : fanout) {
450 checkCancelled();
451 pm.update(1);
452 if (d.length() != 2)
453 continue;
454 String[] entries = new File(objects, d).list();
455 if (entries == null)
456 continue;
457 for (String e : entries) {
458 checkCancelled();
459 if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
460 continue;
461 ObjectId id;
462 try {
463 id = ObjectId.fromString(d + e);
464 } catch (IllegalArgumentException notAnObject) {
465
466
467 continue;
468 }
469 boolean found = false;
470 for (PackFile p : packs) {
471 checkCancelled();
472 if (p.hasObject(id)) {
473 found = true;
474 break;
475 }
476 }
477 if (found)
478 FileUtils.delete(objdb.fileFor(id), FileUtils.RETRY
479 | FileUtils.SKIP_MISSING
480 | FileUtils.IGNORE_ERRORS);
481 }
482 }
483 } finally {
484 pm.endTask();
485 }
486 }
487 }
488
489
490
491
492
493
494
495
496
497
498
499
500
501 public void prune(Set<ObjectId> objectsToKeep) throws IOException,
502 ParseException {
503 long expireDate = getExpireDate();
504
505
506
507 Map<ObjectId, File> deletionCandidates = new HashMap<>();
508 Set<ObjectId> indexObjects = null;
509 File objects = repo.getObjectsDirectory();
510 String[] fanout = objects.list();
511 if (fanout == null || fanout.length == 0) {
512 return;
513 }
514 pm.beginTask(JGitText.get().pruneLooseUnreferencedObjects,
515 fanout.length);
516 try {
517 for (String d : fanout) {
518 checkCancelled();
519 pm.update(1);
520 if (d.length() != 2)
521 continue;
522 File dir = new File(objects, d);
523 File[] entries = dir.listFiles();
524 if (entries == null || entries.length == 0) {
525 FileUtils.delete(dir, FileUtils.IGNORE_ERRORS);
526 continue;
527 }
528 for (File f : entries) {
529 checkCancelled();
530 String fName = f.getName();
531 if (fName.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
532 continue;
533 if (repo.getFS().lastModifiedInstant(f)
534 .toEpochMilli() >= expireDate) {
535 continue;
536 }
537 try {
538 ObjectId id = ObjectId.fromString(d + fName);
539 if (objectsToKeep.contains(id))
540 continue;
541 if (indexObjects == null)
542 indexObjects = listNonHEADIndexObjects();
543 if (indexObjects.contains(id))
544 continue;
545 deletionCandidates.put(id, f);
546 } catch (IllegalArgumentException notAnObject) {
547
548
549 }
550 }
551 }
552 } finally {
553 pm.endTask();
554 }
555
556 if (deletionCandidates.isEmpty()) {
557 return;
558 }
559
560 checkCancelled();
561
562
563
564
565
566 Collection<Ref> newRefs;
567 if (lastPackedRefs == null || lastPackedRefs.isEmpty())
568 newRefs = getAllRefs();
569 else {
570 Map<String, Ref> last = new HashMap<>();
571 for (Ref r : lastPackedRefs) {
572 last.put(r.getName(), r);
573 }
574 newRefs = new ArrayList<>();
575 for (Ref r : getAllRefs()) {
576 Ref old = last.get(r.getName());
577 if (!equals(r, old)) {
578 newRefs.add(r);
579 }
580 }
581 }
582
583 if (!newRefs.isEmpty()) {
584
585
586
587
588
589 ObjectWalk w = new ObjectWalk(repo);
590 try {
591 for (Ref cr : newRefs) {
592 checkCancelled();
593 w.markStart(w.parseAny(cr.getObjectId()));
594 }
595 if (lastPackedRefs != null)
596 for (Ref lpr : lastPackedRefs) {
597 w.markUninteresting(w.parseAny(lpr.getObjectId()));
598 }
599 removeReferenced(deletionCandidates, w);
600 } finally {
601 w.dispose();
602 }
603 }
604
605 if (deletionCandidates.isEmpty())
606 return;
607
608
609
610
611
612
613 ObjectWalk w = new ObjectWalk(repo);
614 try {
615 for (Ref ar : getAllRefs())
616 for (ObjectId id : listRefLogObjects(ar, lastRepackTime)) {
617 checkCancelled();
618 w.markStart(w.parseAny(id));
619 }
620 if (lastPackedRefs != null)
621 for (Ref lpr : lastPackedRefs) {
622 checkCancelled();
623 w.markUninteresting(w.parseAny(lpr.getObjectId()));
624 }
625 removeReferenced(deletionCandidates, w);
626 } finally {
627 w.dispose();
628 }
629
630 if (deletionCandidates.isEmpty())
631 return;
632
633 checkCancelled();
634
635
636
637
638
639 Set<File> touchedFanout = new HashSet<>();
640 for (File f : deletionCandidates.values()) {
641 if (f.lastModified() < expireDate) {
642 f.delete();
643 touchedFanout.add(f.getParentFile());
644 }
645 }
646
647 for (File f : touchedFanout) {
648 FileUtils.delete(f,
649 FileUtils.EMPTY_DIRECTORIES_ONLY | FileUtils.IGNORE_ERRORS);
650 }
651
652 repo.getObjectDatabase().close();
653 }
654
655 private long getExpireDate() throws ParseException {
656 long expireDate = Long.MAX_VALUE;
657
658 if (expire == null && expireAgeMillis == -1) {
659 String pruneExpireStr = getPruneExpireStr();
660 if (pruneExpireStr == null)
661 pruneExpireStr = PRUNE_EXPIRE_DEFAULT;
662 expire = GitDateParser.parse(pruneExpireStr, null, SystemReader
663 .getInstance().getLocale());
664 expireAgeMillis = -1;
665 }
666 if (expire != null)
667 expireDate = expire.getTime();
668 if (expireAgeMillis != -1)
669 expireDate = System.currentTimeMillis() - expireAgeMillis;
670 return expireDate;
671 }
672
673 private String getPruneExpireStr() {
674 return repo.getConfig().getString(
675 ConfigConstants.CONFIG_GC_SECTION, null,
676 ConfigConstants.CONFIG_KEY_PRUNEEXPIRE);
677 }
678
679 private long getPackExpireDate() throws ParseException {
680 long packExpireDate = Long.MAX_VALUE;
681
682 if (packExpire == null && packExpireAgeMillis == -1) {
683 String prunePackExpireStr = repo.getConfig().getString(
684 ConfigConstants.CONFIG_GC_SECTION, null,
685 ConfigConstants.CONFIG_KEY_PRUNEPACKEXPIRE);
686 if (prunePackExpireStr == null)
687 prunePackExpireStr = PRUNE_PACK_EXPIRE_DEFAULT;
688 packExpire = GitDateParser.parse(prunePackExpireStr, null,
689 SystemReader.getInstance().getLocale());
690 packExpireAgeMillis = -1;
691 }
692 if (packExpire != null)
693 packExpireDate = packExpire.getTime();
694 if (packExpireAgeMillis != -1)
695 packExpireDate = System.currentTimeMillis() - packExpireAgeMillis;
696 return packExpireDate;
697 }
698
699
700
701
702
703
704
705
706
707
708
709 private void removeReferenced(Map<ObjectId, File> id2File,
710 ObjectWalk w) throws MissingObjectException,
711 IncorrectObjectTypeException, IOException {
712 RevObject ro = w.next();
713 while (ro != null) {
714 checkCancelled();
715 if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
716 return;
717 }
718 ro = w.next();
719 }
720 ro = w.nextObject();
721 while (ro != null) {
722 checkCancelled();
723 if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
724 return;
725 }
726 ro = w.nextObject();
727 }
728 }
729
730 private static boolean equals(Reff" href="../../../../../../org/eclipse/jgit/lib/Ref.html#Ref">Ref r1, Ref r2) {
731 if (r1 == null || r2 == null) {
732 return false;
733 }
734 if (r1.isSymbolic()) {
735 return r2.isSymbolic() && r1.getTarget().getName()
736 .equals(r2.getTarget().getName());
737 }
738 return !r2.isSymbolic()
739 && Objects.equals(r1.getObjectId(), r2.getObjectId());
740 }
741
742
743
744
745
746
747
748
749 public void packRefs() throws IOException {
750 RefDatabase refDb = repo.getRefDatabase();
751 if (refDb instanceof FileReftableDatabase) {
752
753 pm.beginTask(JGitText.get().packRefs, 1);
754 try {
755 ((FileReftableDatabase) refDb).compactFully();
756 } finally {
757 pm.endTask();
758 }
759 return;
760 }
761
762 Collection<Ref> refs = refDb.getRefsByPrefix(Constants.R_REFS);
763 List<String> refsToBePacked = new ArrayList<>(refs.size());
764 pm.beginTask(JGitText.get().packRefs, refs.size());
765 try {
766 for (Ref ref : refs) {
767 checkCancelled();
768 if (!ref.isSymbolic() && ref.getStorage().isLoose())
769 refsToBePacked.add(ref.getName());
770 pm.update(1);
771 }
772 ((RefDirectory) repo.getRefDatabase()).pack(refsToBePacked);
773 } finally {
774 pm.endTask();
775 }
776 }
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792 public Collection<PackFile> repack() throws IOException {
793 Collection<PackFile> toBeDeleted = repo.getObjectDatabase().getPacks();
794
795 long time = System.currentTimeMillis();
796 Collection<Ref> refsBefore = getAllRefs();
797
798 Set<ObjectId> allHeadsAndTags = new HashSet<>();
799 Set<ObjectId> allHeads = new HashSet<>();
800 Set<ObjectId> allTags = new HashSet<>();
801 Set<ObjectId> nonHeads = new HashSet<>();
802 Set<ObjectId> txnHeads = new HashSet<>();
803 Set<ObjectId> tagTargets = new HashSet<>();
804 Set<ObjectId> indexObjects = listNonHEADIndexObjects();
805 RefDatabase refdb = repo.getRefDatabase();
806
807 for (Ref ref : refsBefore) {
808 checkCancelled();
809 nonHeads.addAll(listRefLogObjects(ref, 0));
810 if (ref.isSymbolic() || ref.getObjectId() == null) {
811 continue;
812 }
813 if (isHead(ref)) {
814 allHeads.add(ref.getObjectId());
815 } else if (isTag(ref)) {
816 allTags.add(ref.getObjectId());
817 } else if (RefTreeNames.isRefTree(refdb, ref.getName())) {
818 txnHeads.add(ref.getObjectId());
819 } else {
820 nonHeads.add(ref.getObjectId());
821 }
822 if (ref.getPeeledObjectId() != null) {
823 tagTargets.add(ref.getPeeledObjectId());
824 }
825 }
826
827 List<ObjectIdSet> excluded = new LinkedList<>();
828 for (PackFile f : repo.getObjectDatabase().getPacks()) {
829 checkCancelled();
830 if (f.shouldBeKept())
831 excluded.add(f.getIndex());
832 }
833
834
835 allTags.removeAll(allHeads);
836 allHeadsAndTags.addAll(allHeads);
837 allHeadsAndTags.addAll(allTags);
838
839
840 tagTargets.addAll(allHeadsAndTags);
841 nonHeads.addAll(indexObjects);
842
843
844 if (pconfig.getSinglePack()) {
845 allHeadsAndTags.addAll(nonHeads);
846 nonHeads.clear();
847 }
848
849 List<PackFile> ret = new ArrayList<>(2);
850 PackFile heads = null;
851 if (!allHeadsAndTags.isEmpty()) {
852 heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags,
853 tagTargets, excluded);
854 if (heads != null) {
855 ret.add(heads);
856 excluded.add(0, heads.getIndex());
857 }
858 }
859 if (!nonHeads.isEmpty()) {
860 PackFile rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE,
861 tagTargets, excluded);
862 if (rest != null)
863 ret.add(rest);
864 }
865 if (!txnHeads.isEmpty()) {
866 PackFile txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE,
867 null, excluded);
868 if (txn != null)
869 ret.add(txn);
870 }
871 try {
872 deleteOldPacks(toBeDeleted, ret);
873 } catch (ParseException e) {
874
875
876
877 throw new IOException(e);
878 }
879 prunePacked();
880 if (repo.getRefDatabase() instanceof RefDirectory) {
881
882 deleteEmptyRefsFolders();
883 }
884 deleteOrphans();
885 deleteTempPacksIdx();
886
887 lastPackedRefs = refsBefore;
888 lastRepackTime = time;
889 return ret;
890 }
891
892 private static boolean isHead(Ref ref) {
893 return ref.getName().startsWith(Constants.R_HEADS);
894 }
895
896 private static boolean isTag(Ref ref) {
897 return ref.getName().startsWith(Constants.R_TAGS);
898 }
899
900 private void deleteEmptyRefsFolders() throws IOException {
901 Path refs = repo.getDirectory().toPath().resolve(Constants.R_REFS);
902
903
904 Instant threshold = Instant.now().minus(30, ChronoUnit.SECONDS);
905 try (Stream<Path> entries = Files.list(refs)
906 .filter(Files::isDirectory)) {
907 Iterator<Path> iterator = entries.iterator();
908 while (iterator.hasNext()) {
909 try (Stream<Path> s = Files.list(iterator.next())) {
910 s.filter(path -> canBeSafelyDeleted(path, threshold)).forEach(this::deleteDir);
911 }
912 }
913 }
914 }
915
916 private boolean canBeSafelyDeleted(Path path, Instant threshold) {
917 try {
918 return Files.getLastModifiedTime(path).toInstant().isBefore(threshold);
919 }
920 catch (IOException e) {
921 LOG.warn(MessageFormat.format(
922 JGitText.get().cannotAccessLastModifiedForSafeDeletion,
923 path), e);
924 return false;
925 }
926 }
927
928 private void deleteDir(Path dir) {
929 try (Stream<Path> dirs = Files.walk(dir)) {
930 dirs.filter(this::isDirectory).sorted(Comparator.reverseOrder())
931 .forEach(this::delete);
932 } catch (IOException e) {
933 LOG.error(e.getMessage(), e);
934 }
935 }
936
937 private boolean isDirectory(Path p) {
938 return p.toFile().isDirectory();
939 }
940
941 private void delete(Path d) {
942 try {
943 Files.delete(d);
944 } catch (DirectoryNotEmptyException e) {
945
946 } catch (IOException e) {
947 LOG.error(MessageFormat.format(JGitText.get().cannotDeleteFile, d),
948 e);
949 }
950 }
951
952
953
954
955
956
957
958
959 private void deleteOrphans() {
960 Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
961 List<String> fileNames = null;
962 try (Stream<Path> files = Files.list(packDir)) {
963 fileNames = files.map(path -> path.getFileName().toString())
964 .filter(name -> (name.endsWith(PACK_EXT)
965 || name.endsWith(BITMAP_EXT)
966 || name.endsWith(INDEX_EXT)
967 || name.endsWith(KEEP_EXT)))
968
969
970 .sorted(Collections.reverseOrder())
971 .collect(Collectors.toList());
972 } catch (IOException e) {
973 LOG.error(e.getMessage(), e);
974 return;
975 }
976 if (fileNames == null) {
977 return;
978 }
979
980 String base = null;
981 for (String n : fileNames) {
982 if (n.endsWith(PACK_EXT) || n.endsWith(KEEP_EXT)) {
983 base = n.substring(0, n.lastIndexOf('.'));
984 } else {
985 if (base == null || !n.startsWith(base)) {
986 try {
987 Path delete = packDir.resolve(n);
988 FileUtils.delete(delete.toFile(),
989 FileUtils.RETRY | FileUtils.SKIP_MISSING);
990 LOG.warn(JGitText.get().deletedOrphanInPackDir, delete);
991 } catch (IOException e) {
992 LOG.error(e.getMessage(), e);
993 }
994 }
995 }
996 }
997 }
998
999 private void deleteTempPacksIdx() {
1000 Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
1001 Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS);
1002 if (!Files.exists(packDir)) {
1003 return;
1004 }
1005 try (DirectoryStream<Path> stream =
1006 Files.newDirectoryStream(packDir, "gc_*_tmp")) {
1007 stream.forEach(t -> {
1008 try {
1009 Instant lastModified = Files.getLastModifiedTime(t)
1010 .toInstant();
1011 if (lastModified.isBefore(threshold)) {
1012 Files.deleteIfExists(t);
1013 }
1014 } catch (IOException e) {
1015 LOG.error(e.getMessage(), e);
1016 }
1017 });
1018 } catch (IOException e) {
1019 LOG.error(e.getMessage(), e);
1020 }
1021 }
1022
1023
1024
1025
1026
1027
1028
1029
1030 private Set<ObjectId> listRefLogObjects(Ref ref, long minTime) throws IOException {
1031 ReflogReader reflogReader = repo.getReflogReader(ref.getName());
1032 if (reflogReader == null) {
1033 return Collections.emptySet();
1034 }
1035 List<ReflogEntry> rlEntries = reflogReader
1036 .getReverseEntries();
1037 if (rlEntries == null || rlEntries.isEmpty())
1038 return Collections.emptySet();
1039 Set<ObjectId> ret = new HashSet<>();
1040 for (ReflogEntry e : rlEntries) {
1041 if (e.getWho().getWhen().getTime() < minTime)
1042 break;
1043 ObjectId newId = e.getNewId();
1044 if (newId != null && !ObjectId.zeroId().equals(newId))
1045 ret.add(newId);
1046 ObjectId oldId = e.getOldId();
1047 if (oldId != null && !ObjectId.zeroId().equals(oldId))
1048 ret.add(oldId);
1049 }
1050 return ret;
1051 }
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064 private Collection<Ref> getAllRefs() throws IOException {
1065 RefDatabase refdb = repo.getRefDatabase();
1066 Collection<Ref> refs = refdb.getRefs();
1067 List<Ref> addl = refdb.getAdditionalRefs();
1068 if (!addl.isEmpty()) {
1069 List<Ref> all = new ArrayList<>(refs.size() + addl.size());
1070 all.addAll(refs);
1071
1072 for (Ref r : addl) {
1073 checkCancelled();
1074 if (r.getName().startsWith(Constants.R_REFS)) {
1075 all.add(r);
1076 }
1077 }
1078 return all;
1079 }
1080 return refs;
1081 }
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092 private Set<ObjectId> listNonHEADIndexObjects()
1093 throws CorruptObjectException, IOException {
1094 if (repo.isBare()) {
1095 return Collections.emptySet();
1096 }
1097 try (TreeWalkTreeWalk.html#TreeWalk">TreeWalk treeWalk = new TreeWalk(repo)) {
1098 treeWalk.addTree(new DirCacheIterator(repo.readDirCache()));
1099 ObjectId headID = repo.resolve(Constants.HEAD);
1100 if (headID != null) {
1101 try (RevWalk/RevWalk.html#RevWalk">RevWalk revWalk = new RevWalk(repo)) {
1102 treeWalk.addTree(revWalk.parseTree(headID));
1103 }
1104 }
1105
1106 treeWalk.setFilter(TreeFilter.ANY_DIFF);
1107 treeWalk.setRecursive(true);
1108 Set<ObjectId> ret = new HashSet<>();
1109
1110 while (treeWalk.next()) {
1111 checkCancelled();
1112 ObjectId objectId = treeWalk.getObjectId(0);
1113 switch (treeWalk.getRawMode(0) & FileMode.TYPE_MASK) {
1114 case FileMode.TYPE_MISSING:
1115 case FileMode.TYPE_GITLINK:
1116 continue;
1117 case FileMode.TYPE_TREE:
1118 case FileMode.TYPE_FILE:
1119 case FileMode.TYPE_SYMLINK:
1120 ret.add(objectId);
1121 continue;
1122 default:
1123 throw new IOException(MessageFormat.format(
1124 JGitText.get().corruptObjectInvalidMode3,
1125 String.format("%o",
1126 Integer.valueOf(treeWalk.getRawMode(0))),
1127 (objectId == null) ? "null" : objectId.name(),
1128 treeWalk.getPathString(),
1129 repo.getIndexFile()));
1130 }
1131 }
1132 return ret;
1133 }
1134 }
1135
1136 private PackFile writePack(@NonNull Set<? extends ObjectId> want,
1137 @NonNull Set<? extends ObjectId> have, @NonNull Set<ObjectId> tags,
1138 Set<ObjectId> tagTargets, List<ObjectIdSet> excludeObjects)
1139 throws IOException {
1140 checkCancelled();
1141 File tmpPack = null;
1142 Map<PackExt, File> tmpExts = new TreeMap<>((o1, o2) -> {
1143
1144
1145
1146 if (o1 == o2) {
1147 return 0;
1148 }
1149 if (o1 == PackExt.INDEX) {
1150 return 1;
1151 }
1152 if (o2 == PackExt.INDEX) {
1153 return -1;
1154 }
1155 return Integer.signum(o1.hashCode() - o2.hashCode());
1156 });
1157 try (PackWriternal/storage/pack/PackWriter.html#PackWriter">PackWriter pw = new PackWriter(
1158 pconfig,
1159 repo.newObjectReader())) {
1160
1161 pw.setDeltaBaseAsOffset(true);
1162 pw.setReuseDeltaCommits(false);
1163 if (tagTargets != null) {
1164 pw.setTagTargets(tagTargets);
1165 }
1166 if (excludeObjects != null)
1167 for (ObjectIdSet idx : excludeObjects)
1168 pw.excludeObjects(idx);
1169 pw.preparePack(pm, want, have, PackWriter.NONE, tags);
1170 if (pw.getObjectCount() == 0)
1171 return null;
1172 checkCancelled();
1173
1174
1175 String id = pw.computeName().getName();
1176 File packdir = repo.getObjectDatabase().getPackDirectory();
1177 packdir.mkdirs();
1178 tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir);
1179 final String tmpBase = tmpPack.getName()
1180 .substring(0, tmpPack.getName().lastIndexOf('.'));
1181 File tmpIdx = new File(packdir, tmpBase + ".idx_tmp");
1182 tmpExts.put(INDEX, tmpIdx);
1183
1184 if (!tmpIdx.createNewFile())
1185 throw new IOException(MessageFormat.format(
1186 JGitText.get().cannotCreateIndexfile, tmpIdx.getPath()));
1187
1188
1189 try (FileOutputStream fos = new FileOutputStream(tmpPack);
1190 FileChannel channel = fos.getChannel();
1191 OutputStream channelStream = Channels
1192 .newOutputStream(channel)) {
1193 pw.writePack(pm, pm, channelStream);
1194 channel.force(true);
1195 }
1196
1197
1198 try (FileOutputStream fos = new FileOutputStream(tmpIdx);
1199 FileChannel idxChannel = fos.getChannel();
1200 OutputStream idxStream = Channels
1201 .newOutputStream(idxChannel)) {
1202 pw.writeIndex(idxStream);
1203 idxChannel.force(true);
1204 }
1205
1206 if (pw.prepareBitmapIndex(pm)) {
1207 File tmpBitmapIdx = new File(packdir, tmpBase + ".bitmap_tmp");
1208 tmpExts.put(BITMAP_INDEX, tmpBitmapIdx);
1209
1210 if (!tmpBitmapIdx.createNewFile())
1211 throw new IOException(MessageFormat.format(
1212 JGitText.get().cannotCreateIndexfile,
1213 tmpBitmapIdx.getPath()));
1214
1215 try (FileOutputStream fos = new FileOutputStream(tmpBitmapIdx);
1216 FileChannel idxChannel = fos.getChannel();
1217 OutputStream idxStream = Channels
1218 .newOutputStream(idxChannel)) {
1219 pw.writeBitmapIndex(idxStream);
1220 idxChannel.force(true);
1221 }
1222 }
1223
1224
1225 File realPack = nameFor(id, ".pack");
1226
1227 repo.getObjectDatabase().closeAllPackHandles(realPack);
1228 tmpPack.setReadOnly();
1229
1230 FileUtils.rename(tmpPack, realPack, StandardCopyOption.ATOMIC_MOVE);
1231 for (Map.Entry<PackExt, File> tmpEntry : tmpExts.entrySet()) {
1232 File tmpExt = tmpEntry.getValue();
1233 tmpExt.setReadOnly();
1234
1235 File realExt = nameFor(id,
1236 "." + tmpEntry.getKey().getExtension());
1237 try {
1238 FileUtils.rename(tmpExt, realExt,
1239 StandardCopyOption.ATOMIC_MOVE);
1240 } catch (IOException e) {
1241 File newExt = new File(realExt.getParentFile(),
1242 realExt.getName() + ".new");
1243 try {
1244 FileUtils.rename(tmpExt, newExt,
1245 StandardCopyOption.ATOMIC_MOVE);
1246 } catch (IOException e2) {
1247 newExt = tmpExt;
1248 e = e2;
1249 }
1250 throw new IOException(MessageFormat.format(
1251 JGitText.get().panicCantRenameIndexFile, newExt,
1252 realExt), e);
1253 }
1254 }
1255 boolean interrupted = false;
1256 try {
1257 FileSnapshot snapshot = FileSnapshot.save(realPack);
1258 if (pconfig.doWaitPreventRacyPack(snapshot.size())) {
1259 snapshot.waitUntilNotRacy();
1260 }
1261 } catch (InterruptedException e) {
1262 interrupted = true;
1263 }
1264 try {
1265 return repo.getObjectDatabase().openPack(realPack);
1266 } finally {
1267 if (interrupted) {
1268
1269 Thread.currentThread().interrupt();
1270 }
1271 }
1272 } finally {
1273 if (tmpPack != null && tmpPack.exists())
1274 tmpPack.delete();
1275 for (File tmpExt : tmpExts.values()) {
1276 if (tmpExt.exists())
1277 tmpExt.delete();
1278 }
1279 }
1280 }
1281
1282 private File nameFor(String name, String ext) {
1283 File packdir = repo.getObjectDatabase().getPackDirectory();
1284 return new File(packdir, "pack-" + name + ext);
1285 }
1286
1287 private void checkCancelled() throws CancelledException {
1288 if (pm.isCancelled() || Thread.currentThread().isInterrupted()) {
1289 throw new CancelledException(JGitText.get().operationCanceled);
1290 }
1291 }
1292
1293
1294
1295
1296
1297 public static class RepoStatistics {
1298
1299
1300
1301
1302
1303 public long numberOfPackedObjects;
1304
1305
1306
1307
1308 public long numberOfPackFiles;
1309
1310
1311
1312
1313 public long numberOfLooseObjects;
1314
1315
1316
1317
1318 public long sizeOfLooseObjects;
1319
1320
1321
1322
1323 public long sizeOfPackedObjects;
1324
1325
1326
1327
1328 public long numberOfLooseRefs;
1329
1330
1331
1332
1333 public long numberOfPackedRefs;
1334
1335
1336
1337
1338 public long numberOfBitmaps;
1339
1340 @Override
1341 public String toString() {
1342 final StringBuilder b = new StringBuilder();
1343 b.append("numberOfPackedObjects=").append(numberOfPackedObjects);
1344 b.append(", numberOfPackFiles=").append(numberOfPackFiles);
1345 b.append(", numberOfLooseObjects=").append(numberOfLooseObjects);
1346 b.append(", numberOfLooseRefs=").append(numberOfLooseRefs);
1347 b.append(", numberOfPackedRefs=").append(numberOfPackedRefs);
1348 b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects);
1349 b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects);
1350 b.append(", numberOfBitmaps=").append(numberOfBitmaps);
1351 return b.toString();
1352 }
1353 }
1354
1355
1356
1357
1358
1359
1360
1361 public RepoStatistics getStatistics() throws IOException {
1362 RepoStatistics ret = new RepoStatistics();
1363 Collection<PackFile> packs = repo.getObjectDatabase().getPacks();
1364 for (PackFile f : packs) {
1365 ret.numberOfPackedObjects += f.getIndex().getObjectCount();
1366 ret.numberOfPackFiles++;
1367 ret.sizeOfPackedObjects += f.getPackFile().length();
1368 if (f.getBitmapIndex() != null)
1369 ret.numberOfBitmaps += f.getBitmapIndex().getBitmapCount();
1370 }
1371 File objDir = repo.getObjectsDirectory();
1372 String[] fanout = objDir.list();
1373 if (fanout != null && fanout.length > 0) {
1374 for (String d : fanout) {
1375 if (d.length() != 2)
1376 continue;
1377 File[] entries = new File(objDir, d).listFiles();
1378 if (entries == null)
1379 continue;
1380 for (File f : entries) {
1381 if (f.getName().length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
1382 continue;
1383 ret.numberOfLooseObjects++;
1384 ret.sizeOfLooseObjects += f.length();
1385 }
1386 }
1387 }
1388
1389 RefDatabase refDb = repo.getRefDatabase();
1390 for (Ref r : refDb.getRefs()) {
1391 Storage storage = r.getStorage();
1392 if (storage == Storage.LOOSE || storage == Storage.LOOSE_PACKED)
1393 ret.numberOfLooseRefs++;
1394 if (storage == Storage.PACKED || storage == Storage.LOOSE_PACKED)
1395 ret.numberOfPackedRefs++;
1396 }
1397
1398 return ret;
1399 }
1400
1401
1402
1403
1404
1405
1406
1407 public GC setProgressMonitor(ProgressMonitor pm) {
1408 this.pm = (pm == null) ? NullProgressMonitor.INSTANCE : pm;
1409 return this;
1410 }
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421 public void setExpireAgeMillis(long expireAgeMillis) {
1422 this.expireAgeMillis = expireAgeMillis;
1423 expire = null;
1424 }
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435 public void setPackExpireAgeMillis(long packExpireAgeMillis) {
1436 this.packExpireAgeMillis = packExpireAgeMillis;
1437 expire = null;
1438 }
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449 public void setPackConfig(@NonNull PackConfig pconfig) {
1450 this.pconfig = pconfig;
1451 }
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465 public void setExpire(Date expire) {
1466 this.expire = expire;
1467 expireAgeMillis = -1;
1468 }
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479 public void setPackExpire(Date packExpire) {
1480 this.packExpire = packExpire;
1481 packExpireAgeMillis = -1;
1482 }
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519 public void setAuto(boolean auto) {
1520 this.automatic = auto;
1521 }
1522
1523
1524
1525
1526
1527 void setBackground(boolean background) {
1528 this.background = background;
1529 }
1530
1531 private boolean needGc() {
1532 if (tooManyPacks()) {
1533 addRepackAllOption();
1534 } else {
1535 return tooManyLooseObjects();
1536 }
1537
1538 return true;
1539 }
1540
1541 private void addRepackAllOption() {
1542
1543
1544 }
1545
1546
1547
1548
1549 boolean tooManyPacks() {
1550 int autopacklimit = repo.getConfig().getInt(
1551 ConfigConstants.CONFIG_GC_SECTION,
1552 ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT,
1553 DEFAULT_AUTOPACKLIMIT);
1554 if (autopacklimit <= 0) {
1555 return false;
1556 }
1557
1558
1559 return repo.getObjectDatabase().getPacks().size() > (autopacklimit + 1);
1560 }
1561
1562
1563
1564
1565
1566
1567
1568 boolean tooManyLooseObjects() {
1569 int auto = getLooseObjectLimit();
1570 if (auto <= 0) {
1571 return false;
1572 }
1573 int n = 0;
1574 int threshold = (auto + 255) / 256;
1575 Path dir = repo.getObjectsDirectory().toPath().resolve("17");
1576 if (!dir.toFile().exists()) {
1577 return false;
1578 }
1579 try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, file -> {
1580 Path fileName = file.getFileName();
1581 return file.toFile().isFile() && fileName != null
1582 && PATTERN_LOOSE_OBJECT.matcher(fileName.toString())
1583 .matches();
1584 })) {
1585 for (Iterator<Path> iter = stream.iterator(); iter.hasNext(); iter
1586 .next()) {
1587 if (++n > threshold) {
1588 return true;
1589 }
1590 }
1591 } catch (IOException e) {
1592 LOG.error(e.getMessage(), e);
1593 }
1594 return false;
1595 }
1596
1597 private int getLooseObjectLimit() {
1598 return repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION,
1599 ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT);
1600 }
1601 }