1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44 package org.eclipse.jgit.internal.storage.file;
45
46 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
47 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
48
49 import java.io.File;
50 import java.io.FileOutputStream;
51 import java.io.IOException;
52 import java.io.OutputStream;
53 import java.io.PrintWriter;
54 import java.io.StringWriter;
55 import java.nio.channels.Channels;
56 import java.nio.channels.FileChannel;
57 import java.nio.file.DirectoryNotEmptyException;
58 import java.nio.file.DirectoryStream;
59 import java.nio.file.Files;
60 import java.nio.file.Path;
61 import java.nio.file.StandardCopyOption;
62 import java.text.MessageFormat;
63 import java.text.ParseException;
64 import java.time.Instant;
65 import java.time.temporal.ChronoUnit;
66 import java.util.ArrayList;
67 import java.util.Collection;
68 import java.util.Collections;
69 import java.util.Comparator;
70 import java.util.Date;
71 import java.util.HashMap;
72 import java.util.HashSet;
73 import java.util.Iterator;
74 import java.util.LinkedList;
75 import java.util.List;
76 import java.util.Map;
77 import java.util.Objects;
78 import java.util.Set;
79 import java.util.TreeMap;
80 import java.util.concurrent.Callable;
81 import java.util.concurrent.ExecutorService;
82 import java.util.regex.Pattern;
83 import java.util.stream.Collectors;
84 import java.util.stream.Stream;
85
86 import org.eclipse.jgit.annotations.NonNull;
87 import org.eclipse.jgit.dircache.DirCacheIterator;
88 import org.eclipse.jgit.errors.CancelledException;
89 import org.eclipse.jgit.errors.CorruptObjectException;
90 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
91 import org.eclipse.jgit.errors.MissingObjectException;
92 import org.eclipse.jgit.errors.NoWorkTreeException;
93 import org.eclipse.jgit.internal.JGitText;
94 import org.eclipse.jgit.internal.storage.pack.PackExt;
95 import org.eclipse.jgit.internal.storage.pack.PackWriter;
96 import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
97 import org.eclipse.jgit.lib.ConfigConstants;
98 import org.eclipse.jgit.lib.Constants;
99 import org.eclipse.jgit.lib.FileMode;
100 import org.eclipse.jgit.lib.NullProgressMonitor;
101 import org.eclipse.jgit.lib.ObjectId;
102 import org.eclipse.jgit.lib.ObjectIdSet;
103 import org.eclipse.jgit.lib.ObjectLoader;
104 import org.eclipse.jgit.lib.ObjectReader;
105 import org.eclipse.jgit.lib.ProgressMonitor;
106 import org.eclipse.jgit.lib.Ref;
107 import org.eclipse.jgit.lib.Ref.Storage;
108 import org.eclipse.jgit.lib.RefDatabase;
109 import org.eclipse.jgit.lib.ReflogEntry;
110 import org.eclipse.jgit.lib.ReflogReader;
111 import org.eclipse.jgit.lib.internal.WorkQueue;
112 import org.eclipse.jgit.revwalk.ObjectWalk;
113 import org.eclipse.jgit.revwalk.RevObject;
114 import org.eclipse.jgit.revwalk.RevWalk;
115 import org.eclipse.jgit.storage.pack.PackConfig;
116 import org.eclipse.jgit.treewalk.TreeWalk;
117 import org.eclipse.jgit.treewalk.filter.TreeFilter;
118 import org.eclipse.jgit.util.FileUtils;
119 import org.eclipse.jgit.util.GitDateParser;
120 import org.eclipse.jgit.util.SystemReader;
121 import org.slf4j.Logger;
122 import org.slf4j.LoggerFactory;
123
124
125
126
127
128
129
130
131
132
133 public class GC {
134 private final static Logger LOG = LoggerFactory
135 .getLogger(GC.class);
136
137 private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago";
138
139 private static final String PRUNE_PACK_EXPIRE_DEFAULT = "1.hour.ago";
140
141 private static final Pattern PATTERN_LOOSE_OBJECT = Pattern
142 .compile("[0-9a-fA-F]{38}");
143
144 private static final String PACK_EXT = "." + PackExt.PACK.getExtension();
145
146 private static final String BITMAP_EXT = "."
147 + PackExt.BITMAP_INDEX.getExtension();
148
149 private static final String INDEX_EXT = "." + PackExt.INDEX.getExtension();
150
151 private static final int DEFAULT_AUTOPACKLIMIT = 50;
152
153 private static final int DEFAULT_AUTOLIMIT = 6700;
154
155 private static volatile ExecutorService executor;
156
157
158
159
160
161
162
163
164 public static void setExecutor(ExecutorService e) {
165 executor = e;
166 }
167
168 private final FileRepository repo;
169
170 private ProgressMonitor pm;
171
172 private long expireAgeMillis = -1;
173
174 private Date expire;
175
176 private long packExpireAgeMillis = -1;
177
178 private Date packExpire;
179
180 private PackConfig pconfig;
181
182
183
184
185
186
187
188 private Collection<Ref> lastPackedRefs;
189
190
191
192
193
194
195 private long lastRepackTime;
196
197
198
199
200 private boolean automatic;
201
202
203
204
205 private boolean background;
206
207
208
209
210
211
212
213
214 public GC(FileRepository repo) {
215 this.repo = repo;
216 this.pconfig = new PackConfig(repo);
217 this.pm = NullProgressMonitor.INSTANCE;
218 }
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248 @SuppressWarnings("FutureReturnValueIgnored")
249 public Collection<PackFile> gc() throws IOException, ParseException {
250 if (!background) {
251 return doGc();
252 }
253 final GcLogernal/storage/file/GcLog.html#GcLog">GcLog gcLog = new GcLog(repo);
254 if (!gcLog.lock()) {
255
256 return Collections.emptyList();
257 }
258
259 Callable<Collection<PackFile>> gcTask = () -> {
260 try {
261 Collection<PackFile> newPacks = doGc();
262 if (automatic && tooManyLooseObjects()) {
263 String message = JGitText.get().gcTooManyUnpruned;
264 gcLog.write(message);
265 gcLog.commit();
266 }
267 return newPacks;
268 } catch (IOException | ParseException e) {
269 try {
270 gcLog.write(e.getMessage());
271 StringWriter sw = new StringWriter();
272 e.printStackTrace(new PrintWriter(sw));
273 gcLog.write(sw.toString());
274 gcLog.commit();
275 } catch (IOException e2) {
276 e2.addSuppressed(e);
277 LOG.error(e2.getMessage(), e2);
278 }
279 } finally {
280 gcLog.unlock();
281 }
282 return Collections.emptyList();
283 };
284
285 executor().submit(gcTask);
286 return Collections.emptyList();
287 }
288
289 private ExecutorService executor() {
290 return (executor != null) ? executor : WorkQueue.getExecutor();
291 }
292
293 private Collection<PackFile> doGc() throws IOException, ParseException {
294 if (automatic && !needGc()) {
295 return Collections.emptyList();
296 }
297 pm.start(6 );
298 packRefs();
299
300 Collection<PackFile> newPacks = repack();
301 prune(Collections.emptySet());
302
303 return newPacks;
304 }
305
306
307
308
309
310
311
312
313
314
315
316 private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, PackFile pack, HashSet<ObjectId> existing)
317 throws IOException {
318 for (PackIndex.MutableEntry entry : pack) {
319 ObjectId oid = entry.toObjectId();
320 if (existing.contains(oid)) {
321 continue;
322 }
323 existing.add(oid);
324 ObjectLoader loader = reader.open(oid);
325 inserter.insert(loader.getType(),
326 loader.getSize(),
327 loader.openStream(),
328 true );
329 }
330 }
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348 private void deleteOldPacks(Collection<PackFile> oldPacks,
349 Collection<PackFile> newPacks) throws ParseException, IOException {
350 HashSet<ObjectId> ids = new HashSet<>();
351 for (PackFile pack : newPacks) {
352 for (PackIndex.MutableEntry entry : pack) {
353 ids.add(entry.toObjectId());
354 }
355 }
356 ObjectReader reader = repo.newObjectReader();
357 ObjectDirectory dir = repo.getObjectDatabase();
358 ObjectDirectoryInserter inserter = dir.newInserter();
359 boolean shouldLoosen = !"now".equals(getPruneExpireStr()) &&
360 getExpireDate() < Long.MAX_VALUE;
361
362 prunePreserved();
363 long packExpireDate = getPackExpireDate();
364 oldPackLoop: for (PackFile oldPack : oldPacks) {
365 checkCancelled();
366 String oldName = oldPack.getPackName();
367
368
369 for (PackFile newPack : newPacks)
370 if (oldName.equals(newPack.getPackName()))
371 continue oldPackLoop;
372
373 if (!oldPack.shouldBeKept()
374 && repo.getFS()
375 .lastModifiedInstant(oldPack.getPackFile())
376 .toEpochMilli() < packExpireDate) {
377 oldPack.close();
378 if (shouldLoosen) {
379 loosen(inserter, reader, oldPack, ids);
380 }
381 prunePack(oldName);
382 }
383 }
384
385
386
387 repo.getObjectDatabase().close();
388 }
389
390
391
392
393
394
395
396
397
398
399
400 private void removeOldPack(File packFile, String packName, PackExt ext,
401 int deleteOptions) throws IOException {
402 if (pconfig.isPreserveOldPacks()) {
403 File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
404 FileUtils.mkdir(oldPackDir, true);
405
406 String oldPackName = "pack-" + packName + ".old-" + ext.getExtension();
407 File oldPackFile = new File(oldPackDir, oldPackName);
408 FileUtils.rename(packFile, oldPackFile);
409 } else {
410 FileUtils.delete(packFile, deleteOptions);
411 }
412 }
413
414
415
416
417 private void prunePreserved() {
418 if (pconfig.isPrunePreserved()) {
419 try {
420 FileUtils.delete(repo.getObjectDatabase().getPreservedDirectory(),
421 FileUtils.RECURSIVE | FileUtils.RETRY | FileUtils.SKIP_MISSING);
422 } catch (IOException e) {
423
424 }
425 }
426 }
427
428
429
430
431
432
433
434
435
436
437
438 private void prunePack(String packName) {
439 PackExt[] extensions = PackExt.values();
440 try {
441
442
443 int deleteOptions = FileUtils.RETRY | FileUtils.SKIP_MISSING;
444 for (PackExt ext : extensions)
445 if (PackExt.PACK.equals(ext)) {
446 File f = nameFor(packName, "." + ext.getExtension());
447 removeOldPack(f, packName, ext, deleteOptions);
448 break;
449 }
450
451
452 deleteOptions |= FileUtils.IGNORE_ERRORS;
453 for (PackExt ext : extensions) {
454 if (!PackExt.PACK.equals(ext)) {
455 File f = nameFor(packName, "." + ext.getExtension());
456 removeOldPack(f, packName, ext, deleteOptions);
457 }
458 }
459 } catch (IOException e) {
460
461 }
462 }
463
464
465
466
467
468
469
470
471 public void prunePacked() throws IOException {
472 ObjectDirectory objdb = repo.getObjectDatabase();
473 Collection<PackFile> packs = objdb.getPacks();
474 File objects = repo.getObjectsDirectory();
475 String[] fanout = objects.list();
476
477 if (fanout != null && fanout.length > 0) {
478 pm.beginTask(JGitText.get().pruneLoosePackedObjects, fanout.length);
479 try {
480 for (String d : fanout) {
481 checkCancelled();
482 pm.update(1);
483 if (d.length() != 2)
484 continue;
485 String[] entries = new File(objects, d).list();
486 if (entries == null)
487 continue;
488 for (String e : entries) {
489 checkCancelled();
490 if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
491 continue;
492 ObjectId id;
493 try {
494 id = ObjectId.fromString(d + e);
495 } catch (IllegalArgumentException notAnObject) {
496
497
498 continue;
499 }
500 boolean found = false;
501 for (PackFile p : packs) {
502 checkCancelled();
503 if (p.hasObject(id)) {
504 found = true;
505 break;
506 }
507 }
508 if (found)
509 FileUtils.delete(objdb.fileFor(id), FileUtils.RETRY
510 | FileUtils.SKIP_MISSING
511 | FileUtils.IGNORE_ERRORS);
512 }
513 }
514 } finally {
515 pm.endTask();
516 }
517 }
518 }
519
520
521
522
523
524
525
526
527
528
529
530
531
532 public void prune(Set<ObjectId> objectsToKeep) throws IOException,
533 ParseException {
534 long expireDate = getExpireDate();
535
536
537
538 Map<ObjectId, File> deletionCandidates = new HashMap<>();
539 Set<ObjectId> indexObjects = null;
540 File objects = repo.getObjectsDirectory();
541 String[] fanout = objects.list();
542 if (fanout == null || fanout.length == 0) {
543 return;
544 }
545 pm.beginTask(JGitText.get().pruneLooseUnreferencedObjects,
546 fanout.length);
547 try {
548 for (String d : fanout) {
549 checkCancelled();
550 pm.update(1);
551 if (d.length() != 2)
552 continue;
553 File dir = new File(objects, d);
554 File[] entries = dir.listFiles();
555 if (entries == null || entries.length == 0) {
556 FileUtils.delete(dir, FileUtils.IGNORE_ERRORS);
557 continue;
558 }
559 for (File f : entries) {
560 checkCancelled();
561 String fName = f.getName();
562 if (fName.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
563 continue;
564 if (repo.getFS().lastModifiedInstant(f)
565 .toEpochMilli() >= expireDate) {
566 continue;
567 }
568 try {
569 ObjectId id = ObjectId.fromString(d + fName);
570 if (objectsToKeep.contains(id))
571 continue;
572 if (indexObjects == null)
573 indexObjects = listNonHEADIndexObjects();
574 if (indexObjects.contains(id))
575 continue;
576 deletionCandidates.put(id, f);
577 } catch (IllegalArgumentException notAnObject) {
578
579
580 }
581 }
582 }
583 } finally {
584 pm.endTask();
585 }
586
587 if (deletionCandidates.isEmpty()) {
588 return;
589 }
590
591 checkCancelled();
592
593
594
595
596
597 Collection<Ref> newRefs;
598 if (lastPackedRefs == null || lastPackedRefs.isEmpty())
599 newRefs = getAllRefs();
600 else {
601 Map<String, Ref> last = new HashMap<>();
602 for (Ref r : lastPackedRefs) {
603 last.put(r.getName(), r);
604 }
605 newRefs = new ArrayList<>();
606 for (Ref r : getAllRefs()) {
607 Ref old = last.get(r.getName());
608 if (!equals(r, old)) {
609 newRefs.add(r);
610 }
611 }
612 }
613
614 if (!newRefs.isEmpty()) {
615
616
617
618
619
620 ObjectWalk w = new ObjectWalk(repo);
621 try {
622 for (Ref cr : newRefs) {
623 checkCancelled();
624 w.markStart(w.parseAny(cr.getObjectId()));
625 }
626 if (lastPackedRefs != null)
627 for (Ref lpr : lastPackedRefs) {
628 w.markUninteresting(w.parseAny(lpr.getObjectId()));
629 }
630 removeReferenced(deletionCandidates, w);
631 } finally {
632 w.dispose();
633 }
634 }
635
636 if (deletionCandidates.isEmpty())
637 return;
638
639
640
641
642
643
644 ObjectWalk w = new ObjectWalk(repo);
645 try {
646 for (Ref ar : getAllRefs())
647 for (ObjectId id : listRefLogObjects(ar, lastRepackTime)) {
648 checkCancelled();
649 w.markStart(w.parseAny(id));
650 }
651 if (lastPackedRefs != null)
652 for (Ref lpr : lastPackedRefs) {
653 checkCancelled();
654 w.markUninteresting(w.parseAny(lpr.getObjectId()));
655 }
656 removeReferenced(deletionCandidates, w);
657 } finally {
658 w.dispose();
659 }
660
661 if (deletionCandidates.isEmpty())
662 return;
663
664 checkCancelled();
665
666
667
668
669
670 Set<File> touchedFanout = new HashSet<>();
671 for (File f : deletionCandidates.values()) {
672 if (f.lastModified() < expireDate) {
673 f.delete();
674 touchedFanout.add(f.getParentFile());
675 }
676 }
677
678 for (File f : touchedFanout) {
679 FileUtils.delete(f,
680 FileUtils.EMPTY_DIRECTORIES_ONLY | FileUtils.IGNORE_ERRORS);
681 }
682
683 repo.getObjectDatabase().close();
684 }
685
686 private long getExpireDate() throws ParseException {
687 long expireDate = Long.MAX_VALUE;
688
689 if (expire == null && expireAgeMillis == -1) {
690 String pruneExpireStr = getPruneExpireStr();
691 if (pruneExpireStr == null)
692 pruneExpireStr = PRUNE_EXPIRE_DEFAULT;
693 expire = GitDateParser.parse(pruneExpireStr, null, SystemReader
694 .getInstance().getLocale());
695 expireAgeMillis = -1;
696 }
697 if (expire != null)
698 expireDate = expire.getTime();
699 if (expireAgeMillis != -1)
700 expireDate = System.currentTimeMillis() - expireAgeMillis;
701 return expireDate;
702 }
703
704 private String getPruneExpireStr() {
705 return repo.getConfig().getString(
706 ConfigConstants.CONFIG_GC_SECTION, null,
707 ConfigConstants.CONFIG_KEY_PRUNEEXPIRE);
708 }
709
710 private long getPackExpireDate() throws ParseException {
711 long packExpireDate = Long.MAX_VALUE;
712
713 if (packExpire == null && packExpireAgeMillis == -1) {
714 String prunePackExpireStr = repo.getConfig().getString(
715 ConfigConstants.CONFIG_GC_SECTION, null,
716 ConfigConstants.CONFIG_KEY_PRUNEPACKEXPIRE);
717 if (prunePackExpireStr == null)
718 prunePackExpireStr = PRUNE_PACK_EXPIRE_DEFAULT;
719 packExpire = GitDateParser.parse(prunePackExpireStr, null,
720 SystemReader.getInstance().getLocale());
721 packExpireAgeMillis = -1;
722 }
723 if (packExpire != null)
724 packExpireDate = packExpire.getTime();
725 if (packExpireAgeMillis != -1)
726 packExpireDate = System.currentTimeMillis() - packExpireAgeMillis;
727 return packExpireDate;
728 }
729
730
731
732
733
734
735
736
737
738
739
740 private void removeReferenced(Map<ObjectId, File> id2File,
741 ObjectWalk w) throws MissingObjectException,
742 IncorrectObjectTypeException, IOException {
743 RevObject ro = w.next();
744 while (ro != null) {
745 checkCancelled();
746 if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
747 return;
748 }
749 ro = w.next();
750 }
751 ro = w.nextObject();
752 while (ro != null) {
753 checkCancelled();
754 if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
755 return;
756 }
757 ro = w.nextObject();
758 }
759 }
760
761 private static boolean equals(Reff" href="../../../../../../org/eclipse/jgit/lib/Ref.html#Ref">Ref r1, Ref r2) {
762 if (r1 == null || r2 == null) {
763 return false;
764 }
765 if (r1.isSymbolic()) {
766 return r2.isSymbolic() && r1.getTarget().getName()
767 .equals(r2.getTarget().getName());
768 }
769 return !r2.isSymbolic()
770 && Objects.equals(r1.getObjectId(), r2.getObjectId());
771 }
772
773
774
775
776
777
778
779
780 public void packRefs() throws IOException {
781 RefDatabase refDb = repo.getRefDatabase();
782 if (refDb instanceof FileReftableDatabase) {
783
784 pm.beginTask(JGitText.get().packRefs, 1);
785 try {
786 ((FileReftableDatabase) refDb).compactFully();
787 } finally {
788 pm.endTask();
789 }
790 return;
791 }
792
793 Collection<Ref> refs = refDb.getRefsByPrefix(Constants.R_REFS);
794 List<String> refsToBePacked = new ArrayList<>(refs.size());
795 pm.beginTask(JGitText.get().packRefs, refs.size());
796 try {
797 for (Ref ref : refs) {
798 checkCancelled();
799 if (!ref.isSymbolic() && ref.getStorage().isLoose())
800 refsToBePacked.add(ref.getName());
801 pm.update(1);
802 }
803 ((RefDirectory) repo.getRefDatabase()).pack(refsToBePacked);
804 } finally {
805 pm.endTask();
806 }
807 }
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823 public Collection<PackFile> repack() throws IOException {
824 Collection<PackFile> toBeDeleted = repo.getObjectDatabase().getPacks();
825
826 long time = System.currentTimeMillis();
827 Collection<Ref> refsBefore = getAllRefs();
828
829 Set<ObjectId> allHeadsAndTags = new HashSet<>();
830 Set<ObjectId> allHeads = new HashSet<>();
831 Set<ObjectId> allTags = new HashSet<>();
832 Set<ObjectId> nonHeads = new HashSet<>();
833 Set<ObjectId> txnHeads = new HashSet<>();
834 Set<ObjectId> tagTargets = new HashSet<>();
835 Set<ObjectId> indexObjects = listNonHEADIndexObjects();
836 RefDatabase refdb = repo.getRefDatabase();
837
838 for (Ref ref : refsBefore) {
839 checkCancelled();
840 nonHeads.addAll(listRefLogObjects(ref, 0));
841 if (ref.isSymbolic() || ref.getObjectId() == null) {
842 continue;
843 }
844 if (isHead(ref)) {
845 allHeads.add(ref.getObjectId());
846 } else if (isTag(ref)) {
847 allTags.add(ref.getObjectId());
848 } else if (RefTreeNames.isRefTree(refdb, ref.getName())) {
849 txnHeads.add(ref.getObjectId());
850 } else {
851 nonHeads.add(ref.getObjectId());
852 }
853 if (ref.getPeeledObjectId() != null) {
854 tagTargets.add(ref.getPeeledObjectId());
855 }
856 }
857
858 List<ObjectIdSet> excluded = new LinkedList<>();
859 for (PackFile f : repo.getObjectDatabase().getPacks()) {
860 checkCancelled();
861 if (f.shouldBeKept())
862 excluded.add(f.getIndex());
863 }
864
865
866 allTags.removeAll(allHeads);
867 allHeadsAndTags.addAll(allHeads);
868 allHeadsAndTags.addAll(allTags);
869
870
871 tagTargets.addAll(allHeadsAndTags);
872 nonHeads.addAll(indexObjects);
873
874
875 if (pconfig.getSinglePack()) {
876 allHeadsAndTags.addAll(nonHeads);
877 nonHeads.clear();
878 }
879
880 List<PackFile> ret = new ArrayList<>(2);
881 PackFile heads = null;
882 if (!allHeadsAndTags.isEmpty()) {
883 heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags,
884 tagTargets, excluded);
885 if (heads != null) {
886 ret.add(heads);
887 excluded.add(0, heads.getIndex());
888 }
889 }
890 if (!nonHeads.isEmpty()) {
891 PackFile rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE,
892 tagTargets, excluded);
893 if (rest != null)
894 ret.add(rest);
895 }
896 if (!txnHeads.isEmpty()) {
897 PackFile txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE,
898 null, excluded);
899 if (txn != null)
900 ret.add(txn);
901 }
902 try {
903 deleteOldPacks(toBeDeleted, ret);
904 } catch (ParseException e) {
905
906
907
908 throw new IOException(e);
909 }
910 prunePacked();
911 if (repo.getRefDatabase() instanceof RefDirectory) {
912
913 deleteEmptyRefsFolders();
914 }
915 deleteOrphans();
916 deleteTempPacksIdx();
917
918 lastPackedRefs = refsBefore;
919 lastRepackTime = time;
920 return ret;
921 }
922
923 private static boolean isHead(Ref ref) {
924 return ref.getName().startsWith(Constants.R_HEADS);
925 }
926
927 private static boolean isTag(Ref ref) {
928 return ref.getName().startsWith(Constants.R_TAGS);
929 }
930
931 private void deleteEmptyRefsFolders() throws IOException {
932 Path refs = repo.getDirectory().toPath().resolve(Constants.R_REFS);
933
934
935 Instant threshold = Instant.now().minus(30, ChronoUnit.SECONDS);
936 try (Stream<Path> entries = Files.list(refs)
937 .filter(Files::isDirectory)) {
938 Iterator<Path> iterator = entries.iterator();
939 while (iterator.hasNext()) {
940 try (Stream<Path> s = Files.list(iterator.next())) {
941 s.filter(path -> canBeSafelyDeleted(path, threshold)).forEach(this::deleteDir);
942 }
943 }
944 }
945 }
946
947 private boolean canBeSafelyDeleted(Path path, Instant threshold) {
948 try {
949 return Files.getLastModifiedTime(path).toInstant().isBefore(threshold);
950 }
951 catch (IOException e) {
952 LOG.warn(MessageFormat.format(
953 JGitText.get().cannotAccessLastModifiedForSafeDeletion,
954 path), e);
955 return false;
956 }
957 }
958
959 private void deleteDir(Path dir) {
960 try (Stream<Path> dirs = Files.walk(dir)) {
961 dirs.filter(this::isDirectory).sorted(Comparator.reverseOrder())
962 .forEach(this::delete);
963 } catch (IOException e) {
964 LOG.error(e.getMessage(), e);
965 }
966 }
967
968 private boolean isDirectory(Path p) {
969 return p.toFile().isDirectory();
970 }
971
972 private void delete(Path d) {
973 try {
974 Files.delete(d);
975 } catch (DirectoryNotEmptyException e) {
976
977 } catch (IOException e) {
978 LOG.error(MessageFormat.format(JGitText.get().cannotDeleteFile, d),
979 e);
980 }
981 }
982
983
984
985
986
987
988
989
990 private void deleteOrphans() {
991 Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
992 List<String> fileNames = null;
993 try (Stream<Path> files = Files.list(packDir)) {
994 fileNames = files.map(path -> path.getFileName().toString())
995 .filter(name -> (name.endsWith(PACK_EXT)
996 || name.endsWith(BITMAP_EXT)
997 || name.endsWith(INDEX_EXT)))
998 .sorted(Collections.reverseOrder())
999 .collect(Collectors.toList());
1000 } catch (IOException e1) {
1001
1002 }
1003 if (fileNames == null) {
1004 return;
1005 }
1006
1007 String base = null;
1008 for (String n : fileNames) {
1009 if (n.endsWith(PACK_EXT)) {
1010 base = n.substring(0, n.lastIndexOf('.'));
1011 } else {
1012 if (base == null || !n.startsWith(base)) {
1013 try {
1014 Files.delete(packDir.resolve(n));
1015 } catch (IOException e) {
1016 LOG.error(e.getMessage(), e);
1017 }
1018 }
1019 }
1020 }
1021 }
1022
1023 private void deleteTempPacksIdx() {
1024 Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
1025 Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS);
1026 if (!Files.exists(packDir)) {
1027 return;
1028 }
1029 try (DirectoryStream<Path> stream =
1030 Files.newDirectoryStream(packDir, "gc_*_tmp")) {
1031 stream.forEach(t -> {
1032 try {
1033 Instant lastModified = Files.getLastModifiedTime(t)
1034 .toInstant();
1035 if (lastModified.isBefore(threshold)) {
1036 Files.deleteIfExists(t);
1037 }
1038 } catch (IOException e) {
1039 LOG.error(e.getMessage(), e);
1040 }
1041 });
1042 } catch (IOException e) {
1043 LOG.error(e.getMessage(), e);
1044 }
1045 }
1046
1047
1048
1049
1050
1051
1052
1053
1054 private Set<ObjectId> listRefLogObjects(Ref ref, long minTime) throws IOException {
1055 ReflogReader reflogReader = repo.getReflogReader(ref.getName());
1056 if (reflogReader == null) {
1057 return Collections.emptySet();
1058 }
1059 List<ReflogEntry> rlEntries = reflogReader
1060 .getReverseEntries();
1061 if (rlEntries == null || rlEntries.isEmpty())
1062 return Collections.emptySet();
1063 Set<ObjectId> ret = new HashSet<>();
1064 for (ReflogEntry e : rlEntries) {
1065 if (e.getWho().getWhen().getTime() < minTime)
1066 break;
1067 ObjectId newId = e.getNewId();
1068 if (newId != null && !ObjectId.zeroId().equals(newId))
1069 ret.add(newId);
1070 ObjectId oldId = e.getOldId();
1071 if (oldId != null && !ObjectId.zeroId().equals(oldId))
1072 ret.add(oldId);
1073 }
1074 return ret;
1075 }
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088 private Collection<Ref> getAllRefs() throws IOException {
1089 RefDatabase refdb = repo.getRefDatabase();
1090 Collection<Ref> refs = refdb.getRefs();
1091 List<Ref> addl = refdb.getAdditionalRefs();
1092 if (!addl.isEmpty()) {
1093 List<Ref> all = new ArrayList<>(refs.size() + addl.size());
1094 all.addAll(refs);
1095
1096 for (Ref r : addl) {
1097 checkCancelled();
1098 if (r.getName().startsWith(Constants.R_REFS)) {
1099 all.add(r);
1100 }
1101 }
1102 return all;
1103 }
1104 return refs;
1105 }
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116 private Set<ObjectId> listNonHEADIndexObjects()
1117 throws CorruptObjectException, IOException {
1118 if (repo.isBare()) {
1119 return Collections.emptySet();
1120 }
1121 try (TreeWalkTreeWalk.html#TreeWalk">TreeWalk treeWalk = new TreeWalk(repo)) {
1122 treeWalk.addTree(new DirCacheIterator(repo.readDirCache()));
1123 ObjectId headID = repo.resolve(Constants.HEAD);
1124 if (headID != null) {
1125 try (RevWalk/RevWalk.html#RevWalk">RevWalk revWalk = new RevWalk(repo)) {
1126 treeWalk.addTree(revWalk.parseTree(headID));
1127 }
1128 }
1129
1130 treeWalk.setFilter(TreeFilter.ANY_DIFF);
1131 treeWalk.setRecursive(true);
1132 Set<ObjectId> ret = new HashSet<>();
1133
1134 while (treeWalk.next()) {
1135 checkCancelled();
1136 ObjectId objectId = treeWalk.getObjectId(0);
1137 switch (treeWalk.getRawMode(0) & FileMode.TYPE_MASK) {
1138 case FileMode.TYPE_MISSING:
1139 case FileMode.TYPE_GITLINK:
1140 continue;
1141 case FileMode.TYPE_TREE:
1142 case FileMode.TYPE_FILE:
1143 case FileMode.TYPE_SYMLINK:
1144 ret.add(objectId);
1145 continue;
1146 default:
1147 throw new IOException(MessageFormat.format(
1148 JGitText.get().corruptObjectInvalidMode3,
1149 String.format("%o",
1150 Integer.valueOf(treeWalk.getRawMode(0))),
1151 (objectId == null) ? "null" : objectId.name(),
1152 treeWalk.getPathString(),
1153 repo.getIndexFile()));
1154 }
1155 }
1156 return ret;
1157 }
1158 }
1159
1160 private PackFile writePack(@NonNull Set<? extends ObjectId> want,
1161 @NonNull Set<? extends ObjectId> have, @NonNull Set<ObjectId> tags,
1162 Set<ObjectId> tagTargets, List<ObjectIdSet> excludeObjects)
1163 throws IOException {
1164 checkCancelled();
1165 File tmpPack = null;
1166 Map<PackExt, File> tmpExts = new TreeMap<>((o1, o2) -> {
1167
1168
1169
1170 if (o1 == o2) {
1171 return 0;
1172 }
1173 if (o1 == PackExt.INDEX) {
1174 return 1;
1175 }
1176 if (o2 == PackExt.INDEX) {
1177 return -1;
1178 }
1179 return Integer.signum(o1.hashCode() - o2.hashCode());
1180 });
1181 try (PackWriternal/storage/pack/PackWriter.html#PackWriter">PackWriter pw = new PackWriter(
1182 pconfig,
1183 repo.newObjectReader())) {
1184
1185 pw.setDeltaBaseAsOffset(true);
1186 pw.setReuseDeltaCommits(false);
1187 if (tagTargets != null) {
1188 pw.setTagTargets(tagTargets);
1189 }
1190 if (excludeObjects != null)
1191 for (ObjectIdSet idx : excludeObjects)
1192 pw.excludeObjects(idx);
1193 pw.preparePack(pm, want, have, PackWriter.NONE, tags);
1194 if (pw.getObjectCount() == 0)
1195 return null;
1196 checkCancelled();
1197
1198
1199 String id = pw.computeName().getName();
1200 File packdir = repo.getObjectDatabase().getPackDirectory();
1201 tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir);
1202 final String tmpBase = tmpPack.getName()
1203 .substring(0, tmpPack.getName().lastIndexOf('.'));
1204 File tmpIdx = new File(packdir, tmpBase + ".idx_tmp");
1205 tmpExts.put(INDEX, tmpIdx);
1206
1207 if (!tmpIdx.createNewFile())
1208 throw new IOException(MessageFormat.format(
1209 JGitText.get().cannotCreateIndexfile, tmpIdx.getPath()));
1210
1211
1212 try (FileOutputStream fos = new FileOutputStream(tmpPack);
1213 FileChannel channel = fos.getChannel();
1214 OutputStream channelStream = Channels
1215 .newOutputStream(channel)) {
1216 pw.writePack(pm, pm, channelStream);
1217 channel.force(true);
1218 }
1219
1220
1221 try (FileOutputStream fos = new FileOutputStream(tmpIdx);
1222 FileChannel idxChannel = fos.getChannel();
1223 OutputStream idxStream = Channels
1224 .newOutputStream(idxChannel)) {
1225 pw.writeIndex(idxStream);
1226 idxChannel.force(true);
1227 }
1228
1229 if (pw.prepareBitmapIndex(pm)) {
1230 File tmpBitmapIdx = new File(packdir, tmpBase + ".bitmap_tmp");
1231 tmpExts.put(BITMAP_INDEX, tmpBitmapIdx);
1232
1233 if (!tmpBitmapIdx.createNewFile())
1234 throw new IOException(MessageFormat.format(
1235 JGitText.get().cannotCreateIndexfile,
1236 tmpBitmapIdx.getPath()));
1237
1238 try (FileOutputStream fos = new FileOutputStream(tmpBitmapIdx);
1239 FileChannel idxChannel = fos.getChannel();
1240 OutputStream idxStream = Channels
1241 .newOutputStream(idxChannel)) {
1242 pw.writeBitmapIndex(idxStream);
1243 idxChannel.force(true);
1244 }
1245 }
1246
1247
1248 File realPack = nameFor(id, ".pack");
1249
1250 repo.getObjectDatabase().closeAllPackHandles(realPack);
1251 tmpPack.setReadOnly();
1252
1253 FileUtils.rename(tmpPack, realPack, StandardCopyOption.ATOMIC_MOVE);
1254 for (Map.Entry<PackExt, File> tmpEntry : tmpExts.entrySet()) {
1255 File tmpExt = tmpEntry.getValue();
1256 tmpExt.setReadOnly();
1257
1258 File realExt = nameFor(id,
1259 "." + tmpEntry.getKey().getExtension());
1260 try {
1261 FileUtils.rename(tmpExt, realExt,
1262 StandardCopyOption.ATOMIC_MOVE);
1263 } catch (IOException e) {
1264 File newExt = new File(realExt.getParentFile(),
1265 realExt.getName() + ".new");
1266 try {
1267 FileUtils.rename(tmpExt, newExt,
1268 StandardCopyOption.ATOMIC_MOVE);
1269 } catch (IOException e2) {
1270 newExt = tmpExt;
1271 e = e2;
1272 }
1273 throw new IOException(MessageFormat.format(
1274 JGitText.get().panicCantRenameIndexFile, newExt,
1275 realExt), e);
1276 }
1277 }
1278 boolean interrupted = false;
1279 try {
1280 FileSnapshot snapshot = FileSnapshot.save(realPack);
1281 if (pconfig.doWaitPreventRacyPack(snapshot.size())) {
1282 snapshot.waitUntilNotRacy();
1283 }
1284 } catch (InterruptedException e) {
1285 interrupted = true;
1286 }
1287 try {
1288 return repo.getObjectDatabase().openPack(realPack);
1289 } finally {
1290 if (interrupted) {
1291
1292 Thread.currentThread().interrupt();
1293 }
1294 }
1295 } finally {
1296 if (tmpPack != null && tmpPack.exists())
1297 tmpPack.delete();
1298 for (File tmpExt : tmpExts.values()) {
1299 if (tmpExt.exists())
1300 tmpExt.delete();
1301 }
1302 }
1303 }
1304
1305 private File nameFor(String name, String ext) {
1306 File packdir = repo.getObjectDatabase().getPackDirectory();
1307 return new File(packdir, "pack-" + name + ext);
1308 }
1309
1310 private void checkCancelled() throws CancelledException {
1311 if (pm.isCancelled() || Thread.currentThread().isInterrupted()) {
1312 throw new CancelledException(JGitText.get().operationCanceled);
1313 }
1314 }
1315
1316
1317
1318
1319
1320 public static class RepoStatistics {
1321
1322
1323
1324
1325
1326 public long numberOfPackedObjects;
1327
1328
1329
1330
1331 public long numberOfPackFiles;
1332
1333
1334
1335
1336 public long numberOfLooseObjects;
1337
1338
1339
1340
1341 public long sizeOfLooseObjects;
1342
1343
1344
1345
1346 public long sizeOfPackedObjects;
1347
1348
1349
1350
1351 public long numberOfLooseRefs;
1352
1353
1354
1355
1356 public long numberOfPackedRefs;
1357
1358
1359
1360
1361 public long numberOfBitmaps;
1362
1363 @Override
1364 public String toString() {
1365 final StringBuilder b = new StringBuilder();
1366 b.append("numberOfPackedObjects=").append(numberOfPackedObjects);
1367 b.append(", numberOfPackFiles=").append(numberOfPackFiles);
1368 b.append(", numberOfLooseObjects=").append(numberOfLooseObjects);
1369 b.append(", numberOfLooseRefs=").append(numberOfLooseRefs);
1370 b.append(", numberOfPackedRefs=").append(numberOfPackedRefs);
1371 b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects);
1372 b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects);
1373 b.append(", numberOfBitmaps=").append(numberOfBitmaps);
1374 return b.toString();
1375 }
1376 }
1377
1378
1379
1380
1381
1382
1383
1384 public RepoStatistics getStatistics() throws IOException {
1385 RepoStatistics ret = new RepoStatistics();
1386 Collection<PackFile> packs = repo.getObjectDatabase().getPacks();
1387 for (PackFile f : packs) {
1388 ret.numberOfPackedObjects += f.getIndex().getObjectCount();
1389 ret.numberOfPackFiles++;
1390 ret.sizeOfPackedObjects += f.getPackFile().length();
1391 if (f.getBitmapIndex() != null)
1392 ret.numberOfBitmaps += f.getBitmapIndex().getBitmapCount();
1393 }
1394 File objDir = repo.getObjectsDirectory();
1395 String[] fanout = objDir.list();
1396 if (fanout != null && fanout.length > 0) {
1397 for (String d : fanout) {
1398 if (d.length() != 2)
1399 continue;
1400 File[] entries = new File(objDir, d).listFiles();
1401 if (entries == null)
1402 continue;
1403 for (File f : entries) {
1404 if (f.getName().length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
1405 continue;
1406 ret.numberOfLooseObjects++;
1407 ret.sizeOfLooseObjects += f.length();
1408 }
1409 }
1410 }
1411
1412 RefDatabase refDb = repo.getRefDatabase();
1413 for (Ref r : refDb.getRefs()) {
1414 Storage storage = r.getStorage();
1415 if (storage == Storage.LOOSE || storage == Storage.LOOSE_PACKED)
1416 ret.numberOfLooseRefs++;
1417 if (storage == Storage.PACKED || storage == Storage.LOOSE_PACKED)
1418 ret.numberOfPackedRefs++;
1419 }
1420
1421 return ret;
1422 }
1423
1424
1425
1426
1427
1428
1429
1430 public GC setProgressMonitor(ProgressMonitor pm) {
1431 this.pm = (pm == null) ? NullProgressMonitor.INSTANCE : pm;
1432 return this;
1433 }
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444 public void setExpireAgeMillis(long expireAgeMillis) {
1445 this.expireAgeMillis = expireAgeMillis;
1446 expire = null;
1447 }
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458 public void setPackExpireAgeMillis(long packExpireAgeMillis) {
1459 this.packExpireAgeMillis = packExpireAgeMillis;
1460 expire = null;
1461 }
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472 public void setPackConfig(@NonNull PackConfig pconfig) {
1473 this.pconfig = pconfig;
1474 }
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488 public void setExpire(Date expire) {
1489 this.expire = expire;
1490 expireAgeMillis = -1;
1491 }
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502 public void setPackExpire(Date packExpire) {
1503 this.packExpire = packExpire;
1504 packExpireAgeMillis = -1;
1505 }
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542 public void setAuto(boolean auto) {
1543 this.automatic = auto;
1544 }
1545
1546
1547
1548
1549
1550 void setBackground(boolean background) {
1551 this.background = background;
1552 }
1553
1554 private boolean needGc() {
1555 if (tooManyPacks()) {
1556 addRepackAllOption();
1557 } else {
1558 return tooManyLooseObjects();
1559 }
1560
1561 return true;
1562 }
1563
1564 private void addRepackAllOption() {
1565
1566
1567 }
1568
1569
1570
1571
1572 boolean tooManyPacks() {
1573 int autopacklimit = repo.getConfig().getInt(
1574 ConfigConstants.CONFIG_GC_SECTION,
1575 ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT,
1576 DEFAULT_AUTOPACKLIMIT);
1577 if (autopacklimit <= 0) {
1578 return false;
1579 }
1580
1581
1582 return repo.getObjectDatabase().getPacks().size() > (autopacklimit + 1);
1583 }
1584
1585
1586
1587
1588
1589
1590
1591 boolean tooManyLooseObjects() {
1592 int auto = getLooseObjectLimit();
1593 if (auto <= 0) {
1594 return false;
1595 }
1596 int n = 0;
1597 int threshold = (auto + 255) / 256;
1598 Path dir = repo.getObjectsDirectory().toPath().resolve("17");
1599 if (!dir.toFile().exists()) {
1600 return false;
1601 }
1602 try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, file -> {
1603 Path fileName = file.getFileName();
1604 return file.toFile().isFile() && fileName != null
1605 && PATTERN_LOOSE_OBJECT.matcher(fileName.toString())
1606 .matches();
1607 })) {
1608 for (Iterator<Path> iter = stream.iterator(); iter.hasNext(); iter
1609 .next()) {
1610 if (++n > threshold) {
1611 return true;
1612 }
1613 }
1614 } catch (IOException e) {
1615 LOG.error(e.getMessage(), e);
1616 }
1617 return false;
1618 }
1619
1620 private int getLooseObjectLimit() {
1621 return repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION,
1622 ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT);
1623 }
1624 }