1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44 package org.eclipse.jgit.internal.storage.file;
45
46 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
47 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
48
49 import java.io.File;
50 import java.io.FileOutputStream;
51 import java.io.IOException;
52 import java.io.OutputStream;
53 import java.io.PrintWriter;
54 import java.io.StringWriter;
55 import java.nio.channels.Channels;
56 import java.nio.channels.FileChannel;
57 import java.nio.file.DirectoryNotEmptyException;
58 import java.nio.file.DirectoryStream;
59 import java.nio.file.Files;
60 import java.nio.file.Path;
61 import java.nio.file.StandardCopyOption;
62 import java.text.MessageFormat;
63 import java.text.ParseException;
64 import java.time.Instant;
65 import java.time.temporal.ChronoUnit;
66 import java.util.ArrayList;
67 import java.util.Collection;
68 import java.util.Collections;
69 import java.util.Comparator;
70 import java.util.Date;
71 import java.util.HashMap;
72 import java.util.HashSet;
73 import java.util.Iterator;
74 import java.util.LinkedList;
75 import java.util.List;
76 import java.util.Map;
77 import java.util.Objects;
78 import java.util.Set;
79 import java.util.TreeMap;
80 import java.util.concurrent.Callable;
81 import java.util.concurrent.ExecutorService;
82 import java.util.regex.Pattern;
83 import java.util.stream.Collectors;
84 import java.util.stream.Stream;
85
86 import org.eclipse.jgit.annotations.NonNull;
87 import org.eclipse.jgit.dircache.DirCacheIterator;
88 import org.eclipse.jgit.errors.CancelledException;
89 import org.eclipse.jgit.errors.CorruptObjectException;
90 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
91 import org.eclipse.jgit.errors.MissingObjectException;
92 import org.eclipse.jgit.errors.NoWorkTreeException;
93 import org.eclipse.jgit.internal.JGitText;
94 import org.eclipse.jgit.internal.storage.pack.PackExt;
95 import org.eclipse.jgit.internal.storage.pack.PackWriter;
96 import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
97 import org.eclipse.jgit.lib.ConfigConstants;
98 import org.eclipse.jgit.lib.Constants;
99 import org.eclipse.jgit.lib.FileMode;
100 import org.eclipse.jgit.lib.NullProgressMonitor;
101 import org.eclipse.jgit.lib.ObjectId;
102 import org.eclipse.jgit.lib.ObjectIdSet;
103 import org.eclipse.jgit.lib.ObjectLoader;
104 import org.eclipse.jgit.lib.ObjectReader;
105 import org.eclipse.jgit.lib.ProgressMonitor;
106 import org.eclipse.jgit.lib.Ref;
107 import org.eclipse.jgit.lib.Ref.Storage;
108 import org.eclipse.jgit.lib.RefDatabase;
109 import org.eclipse.jgit.lib.ReflogEntry;
110 import org.eclipse.jgit.lib.ReflogReader;
111 import org.eclipse.jgit.lib.internal.WorkQueue;
112 import org.eclipse.jgit.revwalk.ObjectWalk;
113 import org.eclipse.jgit.revwalk.RevObject;
114 import org.eclipse.jgit.revwalk.RevWalk;
115 import org.eclipse.jgit.storage.pack.PackConfig;
116 import org.eclipse.jgit.treewalk.TreeWalk;
117 import org.eclipse.jgit.treewalk.filter.TreeFilter;
118 import org.eclipse.jgit.util.FileUtils;
119 import org.eclipse.jgit.util.GitDateParser;
120 import org.eclipse.jgit.util.SystemReader;
121 import org.slf4j.Logger;
122 import org.slf4j.LoggerFactory;
123
124
125
126
127
128
129
130
131
132
133 public class GC {
134 private final static Logger LOG = LoggerFactory
135 .getLogger(GC.class);
136
137 private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago";
138
139 private static final String PRUNE_PACK_EXPIRE_DEFAULT = "1.hour.ago";
140
141 private static final Pattern PATTERN_LOOSE_OBJECT = Pattern
142 .compile("[0-9a-fA-F]{38}");
143
144 private static final String PACK_EXT = "." + PackExt.PACK.getExtension();
145
146 private static final String BITMAP_EXT = "."
147 + PackExt.BITMAP_INDEX.getExtension();
148
149 private static final String INDEX_EXT = "." + PackExt.INDEX.getExtension();
150
151 private static final int DEFAULT_AUTOPACKLIMIT = 50;
152
153 private static final int DEFAULT_AUTOLIMIT = 6700;
154
155 private static volatile ExecutorService executor;
156
157
158
159
160
161
162
163
164 public static void setExecutor(ExecutorService e) {
165 executor = e;
166 }
167
168 private final FileRepository repo;
169
170 private ProgressMonitor pm;
171
172 private long expireAgeMillis = -1;
173
174 private Date expire;
175
176 private long packExpireAgeMillis = -1;
177
178 private Date packExpire;
179
180 private PackConfig pconfig;
181
182
183
184
185
186
187
188 private Collection<Ref> lastPackedRefs;
189
190
191
192
193
194
195 private long lastRepackTime;
196
197
198
199
200 private boolean automatic;
201
202
203
204
205 private boolean background;
206
207
208
209
210
211
212
213
214 public GC(FileRepository repo) {
215 this.repo = repo;
216 this.pconfig = new PackConfig(repo);
217 this.pm = NullProgressMonitor.INSTANCE;
218 }
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248 @SuppressWarnings("FutureReturnValueIgnored")
249 public Collection<PackFile> gc() throws IOException, ParseException {
250 if (!background) {
251 return doGc();
252 }
253 final GcLogernal/storage/file/GcLog.html#GcLog">GcLog gcLog = new GcLog(repo);
254 if (!gcLog.lock()) {
255
256 return Collections.emptyList();
257 }
258
259 Callable<Collection<PackFile>> gcTask = () -> {
260 try {
261 Collection<PackFile> newPacks = doGc();
262 if (automatic && tooManyLooseObjects()) {
263 String message = JGitText.get().gcTooManyUnpruned;
264 gcLog.write(message);
265 gcLog.commit();
266 }
267 return newPacks;
268 } catch (IOException | ParseException e) {
269 try {
270 gcLog.write(e.getMessage());
271 StringWriter sw = new StringWriter();
272 e.printStackTrace(new PrintWriter(sw));
273 gcLog.write(sw.toString());
274 gcLog.commit();
275 } catch (IOException e2) {
276 e2.addSuppressed(e);
277 LOG.error(e2.getMessage(), e2);
278 }
279 } finally {
280 gcLog.unlock();
281 }
282 return Collections.emptyList();
283 };
284
285 executor().submit(gcTask);
286 return Collections.emptyList();
287 }
288
289 private ExecutorService executor() {
290 return (executor != null) ? executor : WorkQueue.getExecutor();
291 }
292
293 private Collection<PackFile> doGc() throws IOException, ParseException {
294 if (automatic && !needGc()) {
295 return Collections.emptyList();
296 }
297 pm.start(6 );
298 packRefs();
299
300 Collection<PackFile> newPacks = repack();
301 prune(Collections.emptySet());
302
303 return newPacks;
304 }
305
306
307
308
309
310
311
312
313
314
315
316 private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, PackFile pack, HashSet<ObjectId> existing)
317 throws IOException {
318 for (PackIndex.MutableEntry entry : pack) {
319 ObjectId oid = entry.toObjectId();
320 if (existing.contains(oid)) {
321 continue;
322 }
323 existing.add(oid);
324 ObjectLoader loader = reader.open(oid);
325 inserter.insert(loader.getType(),
326 loader.getSize(),
327 loader.openStream(),
328 true );
329 }
330 }
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348 private void deleteOldPacks(Collection<PackFile> oldPacks,
349 Collection<PackFile> newPacks) throws ParseException, IOException {
350 HashSet<ObjectId> ids = new HashSet<>();
351 for (PackFile pack : newPacks) {
352 for (PackIndex.MutableEntry entry : pack) {
353 ids.add(entry.toObjectId());
354 }
355 }
356 ObjectReader reader = repo.newObjectReader();
357 ObjectDirectory dir = repo.getObjectDatabase();
358 ObjectDirectoryInserter inserter = dir.newInserter();
359 boolean shouldLoosen = !"now".equals(getPruneExpireStr()) &&
360 getExpireDate() < Long.MAX_VALUE;
361
362 prunePreserved();
363 long packExpireDate = getPackExpireDate();
364 oldPackLoop: for (PackFile oldPack : oldPacks) {
365 checkCancelled();
366 String oldName = oldPack.getPackName();
367
368
369 for (PackFile newPack : newPacks)
370 if (oldName.equals(newPack.getPackName()))
371 continue oldPackLoop;
372
373 if (!oldPack.shouldBeKept()
374 && repo.getFS().lastModified(
375 oldPack.getPackFile()) < packExpireDate) {
376 oldPack.close();
377 if (shouldLoosen) {
378 loosen(inserter, reader, oldPack, ids);
379 }
380 prunePack(oldName);
381 }
382 }
383
384
385
386 repo.getObjectDatabase().close();
387 }
388
389
390
391
392
393
394
395
396
397
398
399 private void removeOldPack(File packFile, String packName, PackExt ext,
400 int deleteOptions) throws IOException {
401 if (pconfig.isPreserveOldPacks()) {
402 File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
403 FileUtils.mkdir(oldPackDir, true);
404
405 String oldPackName = "pack-" + packName + ".old-" + ext.getExtension();
406 File oldPackFile = new File(oldPackDir, oldPackName);
407 FileUtils.rename(packFile, oldPackFile);
408 } else {
409 FileUtils.delete(packFile, deleteOptions);
410 }
411 }
412
413
414
415
416 private void prunePreserved() {
417 if (pconfig.isPrunePreserved()) {
418 try {
419 FileUtils.delete(repo.getObjectDatabase().getPreservedDirectory(),
420 FileUtils.RECURSIVE | FileUtils.RETRY | FileUtils.SKIP_MISSING);
421 } catch (IOException e) {
422
423 }
424 }
425 }
426
427
428
429
430
431
432
433
434
435
436
437 private void prunePack(String packName) {
438 PackExt[] extensions = PackExt.values();
439 try {
440
441
442 int deleteOptions = FileUtils.RETRY | FileUtils.SKIP_MISSING;
443 for (PackExt ext : extensions)
444 if (PackExt.PACK.equals(ext)) {
445 File f = nameFor(packName, "." + ext.getExtension());
446 removeOldPack(f, packName, ext, deleteOptions);
447 break;
448 }
449
450
451 deleteOptions |= FileUtils.IGNORE_ERRORS;
452 for (PackExt ext : extensions) {
453 if (!PackExt.PACK.equals(ext)) {
454 File f = nameFor(packName, "." + ext.getExtension());
455 removeOldPack(f, packName, ext, deleteOptions);
456 }
457 }
458 } catch (IOException e) {
459
460 }
461 }
462
463
464
465
466
467
468
469
470 public void prunePacked() throws IOException {
471 ObjectDirectory objdb = repo.getObjectDatabase();
472 Collection<PackFile> packs = objdb.getPacks();
473 File objects = repo.getObjectsDirectory();
474 String[] fanout = objects.list();
475
476 if (fanout != null && fanout.length > 0) {
477 pm.beginTask(JGitText.get().pruneLoosePackedObjects, fanout.length);
478 try {
479 for (String d : fanout) {
480 checkCancelled();
481 pm.update(1);
482 if (d.length() != 2)
483 continue;
484 String[] entries = new File(objects, d).list();
485 if (entries == null)
486 continue;
487 for (String e : entries) {
488 checkCancelled();
489 if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
490 continue;
491 ObjectId id;
492 try {
493 id = ObjectId.fromString(d + e);
494 } catch (IllegalArgumentException notAnObject) {
495
496
497 continue;
498 }
499 boolean found = false;
500 for (PackFile p : packs) {
501 checkCancelled();
502 if (p.hasObject(id)) {
503 found = true;
504 break;
505 }
506 }
507 if (found)
508 FileUtils.delete(objdb.fileFor(id), FileUtils.RETRY
509 | FileUtils.SKIP_MISSING
510 | FileUtils.IGNORE_ERRORS);
511 }
512 }
513 } finally {
514 pm.endTask();
515 }
516 }
517 }
518
519
520
521
522
523
524
525
526
527
528
529
530
531 public void prune(Set<ObjectId> objectsToKeep) throws IOException,
532 ParseException {
533 long expireDate = getExpireDate();
534
535
536
537 Map<ObjectId, File> deletionCandidates = new HashMap<>();
538 Set<ObjectId> indexObjects = null;
539 File objects = repo.getObjectsDirectory();
540 String[] fanout = objects.list();
541 if (fanout == null || fanout.length == 0) {
542 return;
543 }
544 pm.beginTask(JGitText.get().pruneLooseUnreferencedObjects,
545 fanout.length);
546 try {
547 for (String d : fanout) {
548 checkCancelled();
549 pm.update(1);
550 if (d.length() != 2)
551 continue;
552 File dir = new File(objects, d);
553 File[] entries = dir.listFiles();
554 if (entries == null || entries.length == 0) {
555 FileUtils.delete(dir, FileUtils.IGNORE_ERRORS);
556 continue;
557 }
558 for (File f : entries) {
559 checkCancelled();
560 String fName = f.getName();
561 if (fName.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
562 continue;
563 if (repo.getFS().lastModified(f) >= expireDate)
564 continue;
565 try {
566 ObjectId id = ObjectId.fromString(d + fName);
567 if (objectsToKeep.contains(id))
568 continue;
569 if (indexObjects == null)
570 indexObjects = listNonHEADIndexObjects();
571 if (indexObjects.contains(id))
572 continue;
573 deletionCandidates.put(id, f);
574 } catch (IllegalArgumentException notAnObject) {
575
576
577 }
578 }
579 }
580 } finally {
581 pm.endTask();
582 }
583
584 if (deletionCandidates.isEmpty()) {
585 return;
586 }
587
588 checkCancelled();
589
590
591
592
593
594 Collection<Ref> newRefs;
595 if (lastPackedRefs == null || lastPackedRefs.isEmpty())
596 newRefs = getAllRefs();
597 else {
598 Map<String, Ref> last = new HashMap<>();
599 for (Ref r : lastPackedRefs) {
600 last.put(r.getName(), r);
601 }
602 newRefs = new ArrayList<>();
603 for (Ref r : getAllRefs()) {
604 Ref old = last.get(r.getName());
605 if (!equals(r, old)) {
606 newRefs.add(r);
607 }
608 }
609 }
610
611 if (!newRefs.isEmpty()) {
612
613
614
615
616
617 ObjectWalk w = new ObjectWalk(repo);
618 try {
619 for (Ref cr : newRefs) {
620 checkCancelled();
621 w.markStart(w.parseAny(cr.getObjectId()));
622 }
623 if (lastPackedRefs != null)
624 for (Ref lpr : lastPackedRefs) {
625 w.markUninteresting(w.parseAny(lpr.getObjectId()));
626 }
627 removeReferenced(deletionCandidates, w);
628 } finally {
629 w.dispose();
630 }
631 }
632
633 if (deletionCandidates.isEmpty())
634 return;
635
636
637
638
639
640
641 ObjectWalk w = new ObjectWalk(repo);
642 try {
643 for (Ref ar : getAllRefs())
644 for (ObjectId id : listRefLogObjects(ar, lastRepackTime)) {
645 checkCancelled();
646 w.markStart(w.parseAny(id));
647 }
648 if (lastPackedRefs != null)
649 for (Ref lpr : lastPackedRefs) {
650 checkCancelled();
651 w.markUninteresting(w.parseAny(lpr.getObjectId()));
652 }
653 removeReferenced(deletionCandidates, w);
654 } finally {
655 w.dispose();
656 }
657
658 if (deletionCandidates.isEmpty())
659 return;
660
661 checkCancelled();
662
663
664
665
666
667 Set<File> touchedFanout = new HashSet<>();
668 for (File f : deletionCandidates.values()) {
669 if (f.lastModified() < expireDate) {
670 f.delete();
671 touchedFanout.add(f.getParentFile());
672 }
673 }
674
675 for (File f : touchedFanout) {
676 FileUtils.delete(f,
677 FileUtils.EMPTY_DIRECTORIES_ONLY | FileUtils.IGNORE_ERRORS);
678 }
679
680 repo.getObjectDatabase().close();
681 }
682
683 private long getExpireDate() throws ParseException {
684 long expireDate = Long.MAX_VALUE;
685
686 if (expire == null && expireAgeMillis == -1) {
687 String pruneExpireStr = getPruneExpireStr();
688 if (pruneExpireStr == null)
689 pruneExpireStr = PRUNE_EXPIRE_DEFAULT;
690 expire = GitDateParser.parse(pruneExpireStr, null, SystemReader
691 .getInstance().getLocale());
692 expireAgeMillis = -1;
693 }
694 if (expire != null)
695 expireDate = expire.getTime();
696 if (expireAgeMillis != -1)
697 expireDate = System.currentTimeMillis() - expireAgeMillis;
698 return expireDate;
699 }
700
701 private String getPruneExpireStr() {
702 return repo.getConfig().getString(
703 ConfigConstants.CONFIG_GC_SECTION, null,
704 ConfigConstants.CONFIG_KEY_PRUNEEXPIRE);
705 }
706
707 private long getPackExpireDate() throws ParseException {
708 long packExpireDate = Long.MAX_VALUE;
709
710 if (packExpire == null && packExpireAgeMillis == -1) {
711 String prunePackExpireStr = repo.getConfig().getString(
712 ConfigConstants.CONFIG_GC_SECTION, null,
713 ConfigConstants.CONFIG_KEY_PRUNEPACKEXPIRE);
714 if (prunePackExpireStr == null)
715 prunePackExpireStr = PRUNE_PACK_EXPIRE_DEFAULT;
716 packExpire = GitDateParser.parse(prunePackExpireStr, null,
717 SystemReader.getInstance().getLocale());
718 packExpireAgeMillis = -1;
719 }
720 if (packExpire != null)
721 packExpireDate = packExpire.getTime();
722 if (packExpireAgeMillis != -1)
723 packExpireDate = System.currentTimeMillis() - packExpireAgeMillis;
724 return packExpireDate;
725 }
726
727
728
729
730
731
732
733
734
735
736
737 private void removeReferenced(Map<ObjectId, File> id2File,
738 ObjectWalk w) throws MissingObjectException,
739 IncorrectObjectTypeException, IOException {
740 RevObject ro = w.next();
741 while (ro != null) {
742 checkCancelled();
743 if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
744 return;
745 }
746 ro = w.next();
747 }
748 ro = w.nextObject();
749 while (ro != null) {
750 checkCancelled();
751 if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
752 return;
753 }
754 ro = w.nextObject();
755 }
756 }
757
758 private static boolean equals(Reff" href="../../../../../../org/eclipse/jgit/lib/Ref.html#Ref">Ref r1, Ref r2) {
759 if (r1 == null || r2 == null) {
760 return false;
761 }
762 if (r1.isSymbolic()) {
763 return r2.isSymbolic() && r1.getTarget().getName()
764 .equals(r2.getTarget().getName());
765 }
766 return !r2.isSymbolic()
767 && Objects.equals(r1.getObjectId(), r2.getObjectId());
768 }
769
770
771
772
773
774
775 public void packRefs() throws IOException {
776 Collection<Ref> refs = repo.getRefDatabase()
777 .getRefsByPrefix(Constants.R_REFS);
778 List<String> refsToBePacked = new ArrayList<>(refs.size());
779 pm.beginTask(JGitText.get().packRefs, refs.size());
780 try {
781 for (Ref ref : refs) {
782 checkCancelled();
783 if (!ref.isSymbolic() && ref.getStorage().isLoose())
784 refsToBePacked.add(ref.getName());
785 pm.update(1);
786 }
787 ((RefDirectory) repo.getRefDatabase()).pack(refsToBePacked);
788 } finally {
789 pm.endTask();
790 }
791 }
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807 public Collection<PackFile> repack() throws IOException {
808 Collection<PackFile> toBeDeleted = repo.getObjectDatabase().getPacks();
809
810 long time = System.currentTimeMillis();
811 Collection<Ref> refsBefore = getAllRefs();
812
813 Set<ObjectId> allHeadsAndTags = new HashSet<>();
814 Set<ObjectId> allHeads = new HashSet<>();
815 Set<ObjectId> allTags = new HashSet<>();
816 Set<ObjectId> nonHeads = new HashSet<>();
817 Set<ObjectId> txnHeads = new HashSet<>();
818 Set<ObjectId> tagTargets = new HashSet<>();
819 Set<ObjectId> indexObjects = listNonHEADIndexObjects();
820 RefDatabase refdb = repo.getRefDatabase();
821
822 for (Ref ref : refsBefore) {
823 checkCancelled();
824 nonHeads.addAll(listRefLogObjects(ref, 0));
825 if (ref.isSymbolic() || ref.getObjectId() == null) {
826 continue;
827 }
828 if (isHead(ref)) {
829 allHeads.add(ref.getObjectId());
830 } else if (isTag(ref)) {
831 allTags.add(ref.getObjectId());
832 } else if (RefTreeNames.isRefTree(refdb, ref.getName())) {
833 txnHeads.add(ref.getObjectId());
834 } else {
835 nonHeads.add(ref.getObjectId());
836 }
837 if (ref.getPeeledObjectId() != null) {
838 tagTargets.add(ref.getPeeledObjectId());
839 }
840 }
841
842 List<ObjectIdSet> excluded = new LinkedList<>();
843 for (PackFile f : repo.getObjectDatabase().getPacks()) {
844 checkCancelled();
845 if (f.shouldBeKept())
846 excluded.add(f.getIndex());
847 }
848
849
850 allTags.removeAll(allHeads);
851 allHeadsAndTags.addAll(allHeads);
852 allHeadsAndTags.addAll(allTags);
853
854
855 tagTargets.addAll(allHeadsAndTags);
856 nonHeads.addAll(indexObjects);
857
858
859 if (pconfig.getSinglePack()) {
860 allHeadsAndTags.addAll(nonHeads);
861 nonHeads.clear();
862 }
863
864 List<PackFile> ret = new ArrayList<>(2);
865 PackFile heads = null;
866 if (!allHeadsAndTags.isEmpty()) {
867 heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags,
868 tagTargets, excluded);
869 if (heads != null) {
870 ret.add(heads);
871 excluded.add(0, heads.getIndex());
872 }
873 }
874 if (!nonHeads.isEmpty()) {
875 PackFile rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE,
876 tagTargets, excluded);
877 if (rest != null)
878 ret.add(rest);
879 }
880 if (!txnHeads.isEmpty()) {
881 PackFile txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE,
882 null, excluded);
883 if (txn != null)
884 ret.add(txn);
885 }
886 try {
887 deleteOldPacks(toBeDeleted, ret);
888 } catch (ParseException e) {
889
890
891
892 throw new IOException(e);
893 }
894 prunePacked();
895 deleteEmptyRefsFolders();
896 deleteOrphans();
897 deleteTempPacksIdx();
898
899 lastPackedRefs = refsBefore;
900 lastRepackTime = time;
901 return ret;
902 }
903
904 private static boolean isHead(Ref ref) {
905 return ref.getName().startsWith(Constants.R_HEADS);
906 }
907
908 private static boolean isTag(Ref ref) {
909 return ref.getName().startsWith(Constants.R_TAGS);
910 }
911
912 private void deleteEmptyRefsFolders() throws IOException {
913 Path refs = repo.getDirectory().toPath().resolve(Constants.R_REFS);
914
915
916 Instant threshold = Instant.now().minus(30, ChronoUnit.SECONDS);
917 try (Stream<Path> entries = Files.list(refs)
918 .filter(Files::isDirectory)) {
919 Iterator<Path> iterator = entries.iterator();
920 while (iterator.hasNext()) {
921 try (Stream<Path> s = Files.list(iterator.next())) {
922 s.filter(path -> canBeSafelyDeleted(path, threshold)).forEach(this::deleteDir);
923 }
924 }
925 }
926 }
927
928 private boolean canBeSafelyDeleted(Path path, Instant threshold) {
929 try {
930 return Files.getLastModifiedTime(path).toInstant().isBefore(threshold);
931 }
932 catch (IOException e) {
933 LOG.warn(MessageFormat.format(
934 JGitText.get().cannotAccessLastModifiedForSafeDeletion,
935 path), e);
936 return false;
937 }
938 }
939
940 private void deleteDir(Path dir) {
941 try (Stream<Path> dirs = Files.walk(dir)) {
942 dirs.filter(this::isDirectory).sorted(Comparator.reverseOrder())
943 .forEach(this::delete);
944 } catch (IOException e) {
945 LOG.error(e.getMessage(), e);
946 }
947 }
948
949 private boolean isDirectory(Path p) {
950 return p.toFile().isDirectory();
951 }
952
953 private void delete(Path d) {
954 try {
955 Files.delete(d);
956 } catch (DirectoryNotEmptyException e) {
957
958 } catch (IOException e) {
959 LOG.error(MessageFormat.format(JGitText.get().cannotDeleteFile, d),
960 e);
961 }
962 }
963
964
965
966
967
968
969
970
971 private void deleteOrphans() {
972 Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
973 List<String> fileNames = null;
974 try (Stream<Path> files = Files.list(packDir)) {
975 fileNames = files.map(path -> path.getFileName().toString())
976 .filter(name -> (name.endsWith(PACK_EXT)
977 || name.endsWith(BITMAP_EXT)
978 || name.endsWith(INDEX_EXT)))
979 .sorted(Collections.reverseOrder())
980 .collect(Collectors.toList());
981 } catch (IOException e1) {
982
983 }
984 if (fileNames == null) {
985 return;
986 }
987
988 String base = null;
989 for (String n : fileNames) {
990 if (n.endsWith(PACK_EXT)) {
991 base = n.substring(0, n.lastIndexOf('.'));
992 } else {
993 if (base == null || !n.startsWith(base)) {
994 try {
995 Files.delete(packDir.resolve(n));
996 } catch (IOException e) {
997 LOG.error(e.getMessage(), e);
998 }
999 }
1000 }
1001 }
1002 }
1003
1004 private void deleteTempPacksIdx() {
1005 Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
1006 Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS);
1007 if (!Files.exists(packDir)) {
1008 return;
1009 }
1010 try (DirectoryStream<Path> stream =
1011 Files.newDirectoryStream(packDir, "gc_*_tmp")) {
1012 stream.forEach(t -> {
1013 try {
1014 Instant lastModified = Files.getLastModifiedTime(t)
1015 .toInstant();
1016 if (lastModified.isBefore(threshold)) {
1017 Files.deleteIfExists(t);
1018 }
1019 } catch (IOException e) {
1020 LOG.error(e.getMessage(), e);
1021 }
1022 });
1023 } catch (IOException e) {
1024 LOG.error(e.getMessage(), e);
1025 }
1026 }
1027
1028
1029
1030
1031
1032
1033
1034
1035 private Set<ObjectId> listRefLogObjects(Ref ref, long minTime) throws IOException {
1036 ReflogReader reflogReader = repo.getReflogReader(ref.getName());
1037 if (reflogReader == null) {
1038 return Collections.emptySet();
1039 }
1040 List<ReflogEntry> rlEntries = reflogReader
1041 .getReverseEntries();
1042 if (rlEntries == null || rlEntries.isEmpty())
1043 return Collections.emptySet();
1044 Set<ObjectId> ret = new HashSet<>();
1045 for (ReflogEntry e : rlEntries) {
1046 if (e.getWho().getWhen().getTime() < minTime)
1047 break;
1048 ObjectId newId = e.getNewId();
1049 if (newId != null && !ObjectId.zeroId().equals(newId))
1050 ret.add(newId);
1051 ObjectId oldId = e.getOldId();
1052 if (oldId != null && !ObjectId.zeroId().equals(oldId))
1053 ret.add(oldId);
1054 }
1055 return ret;
1056 }
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069 private Collection<Ref> getAllRefs() throws IOException {
1070 RefDatabase refdb = repo.getRefDatabase();
1071 Collection<Ref> refs = refdb.getRefs();
1072 List<Ref> addl = refdb.getAdditionalRefs();
1073 if (!addl.isEmpty()) {
1074 List<Ref> all = new ArrayList<>(refs.size() + addl.size());
1075 all.addAll(refs);
1076
1077 for (Ref r : addl) {
1078 checkCancelled();
1079 if (r.getName().startsWith(Constants.R_REFS)) {
1080 all.add(r);
1081 }
1082 }
1083 return all;
1084 }
1085 return refs;
1086 }
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097 private Set<ObjectId> listNonHEADIndexObjects()
1098 throws CorruptObjectException, IOException {
1099 if (repo.isBare()) {
1100 return Collections.emptySet();
1101 }
1102 try (TreeWalkTreeWalk.html#TreeWalk">TreeWalk treeWalk = new TreeWalk(repo)) {
1103 treeWalk.addTree(new DirCacheIterator(repo.readDirCache()));
1104 ObjectId headID = repo.resolve(Constants.HEAD);
1105 if (headID != null) {
1106 try (RevWalk/RevWalk.html#RevWalk">RevWalk revWalk = new RevWalk(repo)) {
1107 treeWalk.addTree(revWalk.parseTree(headID));
1108 }
1109 }
1110
1111 treeWalk.setFilter(TreeFilter.ANY_DIFF);
1112 treeWalk.setRecursive(true);
1113 Set<ObjectId> ret = new HashSet<>();
1114
1115 while (treeWalk.next()) {
1116 checkCancelled();
1117 ObjectId objectId = treeWalk.getObjectId(0);
1118 switch (treeWalk.getRawMode(0) & FileMode.TYPE_MASK) {
1119 case FileMode.TYPE_MISSING:
1120 case FileMode.TYPE_GITLINK:
1121 continue;
1122 case FileMode.TYPE_TREE:
1123 case FileMode.TYPE_FILE:
1124 case FileMode.TYPE_SYMLINK:
1125 ret.add(objectId);
1126 continue;
1127 default:
1128 throw new IOException(MessageFormat.format(
1129 JGitText.get().corruptObjectInvalidMode3,
1130 String.format("%o",
1131 Integer.valueOf(treeWalk.getRawMode(0))),
1132 (objectId == null) ? "null" : objectId.name(),
1133 treeWalk.getPathString(),
1134 repo.getIndexFile()));
1135 }
1136 }
1137 return ret;
1138 }
1139 }
1140
1141 private PackFile writePack(@NonNull Set<? extends ObjectId> want,
1142 @NonNull Set<? extends ObjectId> have, @NonNull Set<ObjectId> tags,
1143 Set<ObjectId> tagTargets, List<ObjectIdSet> excludeObjects)
1144 throws IOException {
1145 checkCancelled();
1146 File tmpPack = null;
1147 Map<PackExt, File> tmpExts = new TreeMap<>((o1, o2) -> {
1148
1149
1150
1151 if (o1 == o2) {
1152 return 0;
1153 }
1154 if (o1 == PackExt.INDEX) {
1155 return 1;
1156 }
1157 if (o2 == PackExt.INDEX) {
1158 return -1;
1159 }
1160 return Integer.signum(o1.hashCode() - o2.hashCode());
1161 });
1162 try (PackWriternal/storage/pack/PackWriter.html#PackWriter">PackWriter pw = new PackWriter(
1163 pconfig,
1164 repo.newObjectReader())) {
1165
1166 pw.setDeltaBaseAsOffset(true);
1167 pw.setReuseDeltaCommits(false);
1168 if (tagTargets != null) {
1169 pw.setTagTargets(tagTargets);
1170 }
1171 if (excludeObjects != null)
1172 for (ObjectIdSet idx : excludeObjects)
1173 pw.excludeObjects(idx);
1174 pw.preparePack(pm, want, have, PackWriter.NONE, tags);
1175 if (pw.getObjectCount() == 0)
1176 return null;
1177 checkCancelled();
1178
1179
1180 String id = pw.computeName().getName();
1181 File packdir = repo.getObjectDatabase().getPackDirectory();
1182 tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir);
1183 final String tmpBase = tmpPack.getName()
1184 .substring(0, tmpPack.getName().lastIndexOf('.'));
1185 File tmpIdx = new File(packdir, tmpBase + ".idx_tmp");
1186 tmpExts.put(INDEX, tmpIdx);
1187
1188 if (!tmpIdx.createNewFile())
1189 throw new IOException(MessageFormat.format(
1190 JGitText.get().cannotCreateIndexfile, tmpIdx.getPath()));
1191
1192
1193 try (FileOutputStream fos = new FileOutputStream(tmpPack);
1194 FileChannel channel = fos.getChannel();
1195 OutputStream channelStream = Channels
1196 .newOutputStream(channel)) {
1197 pw.writePack(pm, pm, channelStream);
1198 channel.force(true);
1199 }
1200
1201
1202 try (FileOutputStream fos = new FileOutputStream(tmpIdx);
1203 FileChannel idxChannel = fos.getChannel();
1204 OutputStream idxStream = Channels
1205 .newOutputStream(idxChannel)) {
1206 pw.writeIndex(idxStream);
1207 idxChannel.force(true);
1208 }
1209
1210 if (pw.prepareBitmapIndex(pm)) {
1211 File tmpBitmapIdx = new File(packdir, tmpBase + ".bitmap_tmp");
1212 tmpExts.put(BITMAP_INDEX, tmpBitmapIdx);
1213
1214 if (!tmpBitmapIdx.createNewFile())
1215 throw new IOException(MessageFormat.format(
1216 JGitText.get().cannotCreateIndexfile,
1217 tmpBitmapIdx.getPath()));
1218
1219 try (FileOutputStream fos = new FileOutputStream(tmpBitmapIdx);
1220 FileChannel idxChannel = fos.getChannel();
1221 OutputStream idxStream = Channels
1222 .newOutputStream(idxChannel)) {
1223 pw.writeBitmapIndex(idxStream);
1224 idxChannel.force(true);
1225 }
1226 }
1227
1228
1229 File realPack = nameFor(id, ".pack");
1230
1231 repo.getObjectDatabase().closeAllPackHandles(realPack);
1232 tmpPack.setReadOnly();
1233
1234 FileUtils.rename(tmpPack, realPack, StandardCopyOption.ATOMIC_MOVE);
1235 for (Map.Entry<PackExt, File> tmpEntry : tmpExts.entrySet()) {
1236 File tmpExt = tmpEntry.getValue();
1237 tmpExt.setReadOnly();
1238
1239 File realExt = nameFor(id,
1240 "." + tmpEntry.getKey().getExtension());
1241 try {
1242 FileUtils.rename(tmpExt, realExt,
1243 StandardCopyOption.ATOMIC_MOVE);
1244 } catch (IOException e) {
1245 File newExt = new File(realExt.getParentFile(),
1246 realExt.getName() + ".new");
1247 try {
1248 FileUtils.rename(tmpExt, newExt,
1249 StandardCopyOption.ATOMIC_MOVE);
1250 } catch (IOException e2) {
1251 newExt = tmpExt;
1252 e = e2;
1253 }
1254 throw new IOException(MessageFormat.format(
1255 JGitText.get().panicCantRenameIndexFile, newExt,
1256 realExt), e);
1257 }
1258 }
1259 boolean interrupted = false;
1260 try {
1261 FileSnapshot snapshot = FileSnapshot.save(realPack);
1262 if (pconfig.doWaitPreventRacyPack(snapshot.size())) {
1263 snapshot.waitUntilNotRacy();
1264 }
1265 } catch (InterruptedException e) {
1266 interrupted = true;
1267 }
1268 try {
1269 return repo.getObjectDatabase().openPack(realPack);
1270 } finally {
1271 if (interrupted) {
1272
1273 Thread.currentThread().interrupt();
1274 }
1275 }
1276 } finally {
1277 if (tmpPack != null && tmpPack.exists())
1278 tmpPack.delete();
1279 for (File tmpExt : tmpExts.values()) {
1280 if (tmpExt.exists())
1281 tmpExt.delete();
1282 }
1283 }
1284 }
1285
1286 private File nameFor(String name, String ext) {
1287 File packdir = repo.getObjectDatabase().getPackDirectory();
1288 return new File(packdir, "pack-" + name + ext);
1289 }
1290
1291 private void checkCancelled() throws CancelledException {
1292 if (pm.isCancelled() || Thread.currentThread().isInterrupted()) {
1293 throw new CancelledException(JGitText.get().operationCanceled);
1294 }
1295 }
1296
1297
1298
1299
1300
1301 public static class RepoStatistics {
1302
1303
1304
1305
1306
1307 public long numberOfPackedObjects;
1308
1309
1310
1311
1312 public long numberOfPackFiles;
1313
1314
1315
1316
1317 public long numberOfLooseObjects;
1318
1319
1320
1321
1322 public long sizeOfLooseObjects;
1323
1324
1325
1326
1327 public long sizeOfPackedObjects;
1328
1329
1330
1331
1332 public long numberOfLooseRefs;
1333
1334
1335
1336
1337 public long numberOfPackedRefs;
1338
1339
1340
1341
1342 public long numberOfBitmaps;
1343
1344 @Override
1345 public String toString() {
1346 final StringBuilder b = new StringBuilder();
1347 b.append("numberOfPackedObjects=").append(numberOfPackedObjects);
1348 b.append(", numberOfPackFiles=").append(numberOfPackFiles);
1349 b.append(", numberOfLooseObjects=").append(numberOfLooseObjects);
1350 b.append(", numberOfLooseRefs=").append(numberOfLooseRefs);
1351 b.append(", numberOfPackedRefs=").append(numberOfPackedRefs);
1352 b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects);
1353 b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects);
1354 b.append(", numberOfBitmaps=").append(numberOfBitmaps);
1355 return b.toString();
1356 }
1357 }
1358
1359
1360
1361
1362
1363
1364
1365 public RepoStatistics getStatistics() throws IOException {
1366 RepoStatistics ret = new RepoStatistics();
1367 Collection<PackFile> packs = repo.getObjectDatabase().getPacks();
1368 for (PackFile f : packs) {
1369 ret.numberOfPackedObjects += f.getIndex().getObjectCount();
1370 ret.numberOfPackFiles++;
1371 ret.sizeOfPackedObjects += f.getPackFile().length();
1372 if (f.getBitmapIndex() != null)
1373 ret.numberOfBitmaps += f.getBitmapIndex().getBitmapCount();
1374 }
1375 File objDir = repo.getObjectsDirectory();
1376 String[] fanout = objDir.list();
1377 if (fanout != null && fanout.length > 0) {
1378 for (String d : fanout) {
1379 if (d.length() != 2)
1380 continue;
1381 File[] entries = new File(objDir, d).listFiles();
1382 if (entries == null)
1383 continue;
1384 for (File f : entries) {
1385 if (f.getName().length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
1386 continue;
1387 ret.numberOfLooseObjects++;
1388 ret.sizeOfLooseObjects += f.length();
1389 }
1390 }
1391 }
1392
1393 RefDatabase refDb = repo.getRefDatabase();
1394 for (Ref r : refDb.getRefs()) {
1395 Storage storage = r.getStorage();
1396 if (storage == Storage.LOOSE || storage == Storage.LOOSE_PACKED)
1397 ret.numberOfLooseRefs++;
1398 if (storage == Storage.PACKED || storage == Storage.LOOSE_PACKED)
1399 ret.numberOfPackedRefs++;
1400 }
1401
1402 return ret;
1403 }
1404
1405
1406
1407
1408
1409
1410
1411 public GC setProgressMonitor(ProgressMonitor pm) {
1412 this.pm = (pm == null) ? NullProgressMonitor.INSTANCE : pm;
1413 return this;
1414 }
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425 public void setExpireAgeMillis(long expireAgeMillis) {
1426 this.expireAgeMillis = expireAgeMillis;
1427 expire = null;
1428 }
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439 public void setPackExpireAgeMillis(long packExpireAgeMillis) {
1440 this.packExpireAgeMillis = packExpireAgeMillis;
1441 expire = null;
1442 }
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453 public void setPackConfig(@NonNull PackConfig pconfig) {
1454 this.pconfig = pconfig;
1455 }
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469 public void setExpire(Date expire) {
1470 this.expire = expire;
1471 expireAgeMillis = -1;
1472 }
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483 public void setPackExpire(Date packExpire) {
1484 this.packExpire = packExpire;
1485 packExpireAgeMillis = -1;
1486 }
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523 public void setAuto(boolean auto) {
1524 this.automatic = auto;
1525 }
1526
1527
1528
1529
1530
1531 void setBackground(boolean background) {
1532 this.background = background;
1533 }
1534
1535 private boolean needGc() {
1536 if (tooManyPacks()) {
1537 addRepackAllOption();
1538 } else {
1539 return tooManyLooseObjects();
1540 }
1541
1542 return true;
1543 }
1544
1545 private void addRepackAllOption() {
1546
1547
1548 }
1549
1550
1551
1552
1553 boolean tooManyPacks() {
1554 int autopacklimit = repo.getConfig().getInt(
1555 ConfigConstants.CONFIG_GC_SECTION,
1556 ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT,
1557 DEFAULT_AUTOPACKLIMIT);
1558 if (autopacklimit <= 0) {
1559 return false;
1560 }
1561
1562
1563 return repo.getObjectDatabase().getPacks().size() > (autopacklimit + 1);
1564 }
1565
1566
1567
1568
1569
1570
1571
1572 boolean tooManyLooseObjects() {
1573 int auto = getLooseObjectLimit();
1574 if (auto <= 0) {
1575 return false;
1576 }
1577 int n = 0;
1578 int threshold = (auto + 255) / 256;
1579 Path dir = repo.getObjectsDirectory().toPath().resolve("17");
1580 if (!dir.toFile().exists()) {
1581 return false;
1582 }
1583 try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, file -> {
1584 Path fileName = file.getFileName();
1585 return file.toFile().isFile() && fileName != null
1586 && PATTERN_LOOSE_OBJECT.matcher(fileName.toString())
1587 .matches();
1588 })) {
1589 for (Iterator<Path> iter = stream.iterator(); iter.hasNext(); iter
1590 .next()) {
1591 if (++n > threshold) {
1592 return true;
1593 }
1594 }
1595 } catch (IOException e) {
1596 LOG.error(e.getMessage(), e);
1597 }
1598 return false;
1599 }
1600
1601 private int getLooseObjectLimit() {
1602 return repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION,
1603 ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT);
1604 }
1605 }