1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44 package org.eclipse.jgit.internal.storage.file;
45
46 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
47 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
48
49 import java.io.File;
50 import java.io.FileOutputStream;
51 import java.io.IOException;
52 import java.io.OutputStream;
53 import java.io.PrintWriter;
54 import java.io.StringWriter;
55 import java.nio.channels.Channels;
56 import java.nio.channels.FileChannel;
57 import java.nio.file.DirectoryNotEmptyException;
58 import java.nio.file.DirectoryStream;
59 import java.nio.file.Files;
60 import java.nio.file.Path;
61 import java.nio.file.StandardCopyOption;
62 import java.text.MessageFormat;
63 import java.text.ParseException;
64 import java.time.Instant;
65 import java.time.temporal.ChronoUnit;
66 import java.util.ArrayList;
67 import java.util.Collection;
68 import java.util.Collections;
69 import java.util.Comparator;
70 import java.util.Date;
71 import java.util.HashMap;
72 import java.util.HashSet;
73 import java.util.Iterator;
74 import java.util.LinkedList;
75 import java.util.List;
76 import java.util.Map;
77 import java.util.Objects;
78 import java.util.Set;
79 import java.util.TreeMap;
80 import java.util.concurrent.Callable;
81 import java.util.concurrent.ExecutorService;
82 import java.util.regex.Pattern;
83 import java.util.stream.Collectors;
84 import java.util.stream.Stream;
85
86 import org.eclipse.jgit.annotations.NonNull;
87 import org.eclipse.jgit.dircache.DirCacheIterator;
88 import org.eclipse.jgit.errors.CancelledException;
89 import org.eclipse.jgit.errors.CorruptObjectException;
90 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
91 import org.eclipse.jgit.errors.MissingObjectException;
92 import org.eclipse.jgit.errors.NoWorkTreeException;
93 import org.eclipse.jgit.internal.JGitText;
94 import org.eclipse.jgit.internal.storage.pack.PackExt;
95 import org.eclipse.jgit.internal.storage.pack.PackWriter;
96 import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
97 import org.eclipse.jgit.lib.ConfigConstants;
98 import org.eclipse.jgit.lib.Constants;
99 import org.eclipse.jgit.lib.FileMode;
100 import org.eclipse.jgit.lib.NullProgressMonitor;
101 import org.eclipse.jgit.lib.ObjectId;
102 import org.eclipse.jgit.lib.ObjectIdSet;
103 import org.eclipse.jgit.lib.ObjectLoader;
104 import org.eclipse.jgit.lib.ObjectReader;
105 import org.eclipse.jgit.lib.ProgressMonitor;
106 import org.eclipse.jgit.lib.Ref;
107 import org.eclipse.jgit.lib.Ref.Storage;
108 import org.eclipse.jgit.lib.RefDatabase;
109 import org.eclipse.jgit.lib.ReflogEntry;
110 import org.eclipse.jgit.lib.ReflogReader;
111 import org.eclipse.jgit.lib.internal.WorkQueue;
112 import org.eclipse.jgit.revwalk.ObjectWalk;
113 import org.eclipse.jgit.revwalk.RevObject;
114 import org.eclipse.jgit.revwalk.RevWalk;
115 import org.eclipse.jgit.storage.pack.PackConfig;
116 import org.eclipse.jgit.treewalk.TreeWalk;
117 import org.eclipse.jgit.treewalk.filter.TreeFilter;
118 import org.eclipse.jgit.util.FileUtils;
119 import org.eclipse.jgit.util.GitDateParser;
120 import org.eclipse.jgit.util.SystemReader;
121 import org.slf4j.Logger;
122 import org.slf4j.LoggerFactory;
123
124
125
126
127
128
129
130
131
132
133 public class GC {
134 private final static Logger LOG = LoggerFactory
135 .getLogger(GC.class);
136
137 private static final String PRUNE_EXPIRE_DEFAULT = "2.weeks.ago";
138
139 private static final String PRUNE_PACK_EXPIRE_DEFAULT = "1.hour.ago";
140
141 private static final Pattern PATTERN_LOOSE_OBJECT = Pattern
142 .compile("[0-9a-fA-F]{38}");
143
144 private static final String PACK_EXT = "." + PackExt.PACK.getExtension();
145
146 private static final String BITMAP_EXT = "."
147 + PackExt.BITMAP_INDEX.getExtension();
148
149 private static final String INDEX_EXT = "." + PackExt.INDEX.getExtension();
150
151 private static final int DEFAULT_AUTOPACKLIMIT = 50;
152
153 private static final int DEFAULT_AUTOLIMIT = 6700;
154
155 private static volatile ExecutorService executor;
156
157
158
159
160
161
162
163
164 public static void setExecutor(ExecutorService e) {
165 executor = e;
166 }
167
168 private final FileRepository repo;
169
170 private ProgressMonitor pm;
171
172 private long expireAgeMillis = -1;
173
174 private Date expire;
175
176 private long packExpireAgeMillis = -1;
177
178 private Date packExpire;
179
180 private PackConfig pconfig;
181
182
183
184
185
186
187
188 private Collection<Ref> lastPackedRefs;
189
190
191
192
193
194
195 private long lastRepackTime;
196
197
198
199
200 private boolean automatic;
201
202
203
204
205 private boolean background;
206
207
208
209
210
211
212
213
214 public GC(FileRepository repo) {
215 this.repo = repo;
216 this.pconfig = new PackConfig(repo);
217 this.pm = NullProgressMonitor.INSTANCE;
218 }
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248 @SuppressWarnings("FutureReturnValueIgnored")
249 public Collection<PackFile> gc() throws IOException, ParseException {
250 if (!background) {
251 return doGc();
252 }
253 final GcLogernal/storage/file/GcLog.html#GcLog">GcLog gcLog = new GcLog(repo);
254 if (!gcLog.lock()) {
255
256 return Collections.emptyList();
257 }
258
259 Callable<Collection<PackFile>> gcTask = () -> {
260 try {
261 Collection<PackFile> newPacks = doGc();
262 if (automatic && tooManyLooseObjects()) {
263 String message = JGitText.get().gcTooManyUnpruned;
264 gcLog.write(message);
265 gcLog.commit();
266 }
267 return newPacks;
268 } catch (IOException | ParseException e) {
269 try {
270 gcLog.write(e.getMessage());
271 StringWriter sw = new StringWriter();
272 e.printStackTrace(new PrintWriter(sw));
273 gcLog.write(sw.toString());
274 gcLog.commit();
275 } catch (IOException e2) {
276 e2.addSuppressed(e);
277 LOG.error(e2.getMessage(), e2);
278 }
279 } finally {
280 gcLog.unlock();
281 }
282 return Collections.emptyList();
283 };
284
285 executor().submit(gcTask);
286 return Collections.emptyList();
287 }
288
289 private ExecutorService executor() {
290 return (executor != null) ? executor : WorkQueue.getExecutor();
291 }
292
293 private Collection<PackFile> doGc() throws IOException, ParseException {
294 if (automatic && !needGc()) {
295 return Collections.emptyList();
296 }
297 pm.start(6 );
298 packRefs();
299
300 Collection<PackFile> newPacks = repack();
301 prune(Collections.emptySet());
302
303 return newPacks;
304 }
305
306
307
308
309
310
311
312
313
314
315
316 private void loosen(ObjectDirectoryInserter inserter, ObjectReader reader, PackFile pack, HashSet<ObjectId> existing)
317 throws IOException {
318 for (PackIndex.MutableEntry entry : pack) {
319 ObjectId oid = entry.toObjectId();
320 if (existing.contains(oid)) {
321 continue;
322 }
323 existing.add(oid);
324 ObjectLoader loader = reader.open(oid);
325 inserter.insert(loader.getType(),
326 loader.getSize(),
327 loader.openStream(),
328 true );
329 }
330 }
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348 private void deleteOldPacks(Collection<PackFile> oldPacks,
349 Collection<PackFile> newPacks) throws ParseException, IOException {
350 HashSet<ObjectId> ids = new HashSet<>();
351 for (PackFile pack : newPacks) {
352 for (PackIndex.MutableEntry entry : pack) {
353 ids.add(entry.toObjectId());
354 }
355 }
356 ObjectReader reader = repo.newObjectReader();
357 ObjectDirectory dir = repo.getObjectDatabase();
358 ObjectDirectoryInserter inserter = dir.newInserter();
359 boolean shouldLoosen = !"now".equals(getPruneExpireStr()) &&
360 getExpireDate() < Long.MAX_VALUE;
361
362 prunePreserved();
363 long packExpireDate = getPackExpireDate();
364 oldPackLoop: for (PackFile oldPack : oldPacks) {
365 checkCancelled();
366 String oldName = oldPack.getPackName();
367
368
369 for (PackFile newPack : newPacks)
370 if (oldName.equals(newPack.getPackName()))
371 continue oldPackLoop;
372
373 if (!oldPack.shouldBeKept()
374 && repo.getFS()
375 .lastModifiedInstant(oldPack.getPackFile())
376 .toEpochMilli() < packExpireDate) {
377 oldPack.close();
378 if (shouldLoosen) {
379 loosen(inserter, reader, oldPack, ids);
380 }
381 prunePack(oldName);
382 }
383 }
384
385
386
387 repo.getObjectDatabase().close();
388 }
389
390
391
392
393
394
395
396
397
398
399
400 private void removeOldPack(File packFile, String packName, PackExt ext,
401 int deleteOptions) throws IOException {
402 if (pconfig.isPreserveOldPacks()) {
403 File oldPackDir = repo.getObjectDatabase().getPreservedDirectory();
404 FileUtils.mkdir(oldPackDir, true);
405
406 String oldPackName = "pack-" + packName + ".old-" + ext.getExtension();
407 File oldPackFile = new File(oldPackDir, oldPackName);
408 FileUtils.rename(packFile, oldPackFile);
409 } else {
410 FileUtils.delete(packFile, deleteOptions);
411 }
412 }
413
414
415
416
417 private void prunePreserved() {
418 if (pconfig.isPrunePreserved()) {
419 try {
420 FileUtils.delete(repo.getObjectDatabase().getPreservedDirectory(),
421 FileUtils.RECURSIVE | FileUtils.RETRY | FileUtils.SKIP_MISSING);
422 } catch (IOException e) {
423
424 }
425 }
426 }
427
428
429
430
431
432
433
434
435
436
437
438 private void prunePack(String packName) {
439 PackExt[] extensions = PackExt.values();
440 try {
441
442
443 int deleteOptions = FileUtils.RETRY | FileUtils.SKIP_MISSING;
444 for (PackExt ext : extensions)
445 if (PackExt.PACK.equals(ext)) {
446 File f = nameFor(packName, "." + ext.getExtension());
447 removeOldPack(f, packName, ext, deleteOptions);
448 break;
449 }
450
451
452 deleteOptions |= FileUtils.IGNORE_ERRORS;
453 for (PackExt ext : extensions) {
454 if (!PackExt.PACK.equals(ext)) {
455 File f = nameFor(packName, "." + ext.getExtension());
456 removeOldPack(f, packName, ext, deleteOptions);
457 }
458 }
459 } catch (IOException e) {
460
461 }
462 }
463
464
465
466
467
468
469
470
471 public void prunePacked() throws IOException {
472 ObjectDirectory objdb = repo.getObjectDatabase();
473 Collection<PackFile> packs = objdb.getPacks();
474 File objects = repo.getObjectsDirectory();
475 String[] fanout = objects.list();
476
477 if (fanout != null && fanout.length > 0) {
478 pm.beginTask(JGitText.get().pruneLoosePackedObjects, fanout.length);
479 try {
480 for (String d : fanout) {
481 checkCancelled();
482 pm.update(1);
483 if (d.length() != 2)
484 continue;
485 String[] entries = new File(objects, d).list();
486 if (entries == null)
487 continue;
488 for (String e : entries) {
489 checkCancelled();
490 if (e.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
491 continue;
492 ObjectId id;
493 try {
494 id = ObjectId.fromString(d + e);
495 } catch (IllegalArgumentException notAnObject) {
496
497
498 continue;
499 }
500 boolean found = false;
501 for (PackFile p : packs) {
502 checkCancelled();
503 if (p.hasObject(id)) {
504 found = true;
505 break;
506 }
507 }
508 if (found)
509 FileUtils.delete(objdb.fileFor(id), FileUtils.RETRY
510 | FileUtils.SKIP_MISSING
511 | FileUtils.IGNORE_ERRORS);
512 }
513 }
514 } finally {
515 pm.endTask();
516 }
517 }
518 }
519
520
521
522
523
524
525
526
527
528
529
530
531
532 public void prune(Set<ObjectId> objectsToKeep) throws IOException,
533 ParseException {
534 long expireDate = getExpireDate();
535
536
537
538 Map<ObjectId, File> deletionCandidates = new HashMap<>();
539 Set<ObjectId> indexObjects = null;
540 File objects = repo.getObjectsDirectory();
541 String[] fanout = objects.list();
542 if (fanout == null || fanout.length == 0) {
543 return;
544 }
545 pm.beginTask(JGitText.get().pruneLooseUnreferencedObjects,
546 fanout.length);
547 try {
548 for (String d : fanout) {
549 checkCancelled();
550 pm.update(1);
551 if (d.length() != 2)
552 continue;
553 File dir = new File(objects, d);
554 File[] entries = dir.listFiles();
555 if (entries == null || entries.length == 0) {
556 FileUtils.delete(dir, FileUtils.IGNORE_ERRORS);
557 continue;
558 }
559 for (File f : entries) {
560 checkCancelled();
561 String fName = f.getName();
562 if (fName.length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
563 continue;
564 if (repo.getFS().lastModifiedInstant(f)
565 .toEpochMilli() >= expireDate) {
566 continue;
567 }
568 try {
569 ObjectId id = ObjectId.fromString(d + fName);
570 if (objectsToKeep.contains(id))
571 continue;
572 if (indexObjects == null)
573 indexObjects = listNonHEADIndexObjects();
574 if (indexObjects.contains(id))
575 continue;
576 deletionCandidates.put(id, f);
577 } catch (IllegalArgumentException notAnObject) {
578
579
580 }
581 }
582 }
583 } finally {
584 pm.endTask();
585 }
586
587 if (deletionCandidates.isEmpty()) {
588 return;
589 }
590
591 checkCancelled();
592
593
594
595
596
597 Collection<Ref> newRefs;
598 if (lastPackedRefs == null || lastPackedRefs.isEmpty())
599 newRefs = getAllRefs();
600 else {
601 Map<String, Ref> last = new HashMap<>();
602 for (Ref r : lastPackedRefs) {
603 last.put(r.getName(), r);
604 }
605 newRefs = new ArrayList<>();
606 for (Ref r : getAllRefs()) {
607 Ref old = last.get(r.getName());
608 if (!equals(r, old)) {
609 newRefs.add(r);
610 }
611 }
612 }
613
614 if (!newRefs.isEmpty()) {
615
616
617
618
619
620 ObjectWalk w = new ObjectWalk(repo);
621 try {
622 for (Ref cr : newRefs) {
623 checkCancelled();
624 w.markStart(w.parseAny(cr.getObjectId()));
625 }
626 if (lastPackedRefs != null)
627 for (Ref lpr : lastPackedRefs) {
628 w.markUninteresting(w.parseAny(lpr.getObjectId()));
629 }
630 removeReferenced(deletionCandidates, w);
631 } finally {
632 w.dispose();
633 }
634 }
635
636 if (deletionCandidates.isEmpty())
637 return;
638
639
640
641
642
643
644 ObjectWalk w = new ObjectWalk(repo);
645 try {
646 for (Ref ar : getAllRefs())
647 for (ObjectId id : listRefLogObjects(ar, lastRepackTime)) {
648 checkCancelled();
649 w.markStart(w.parseAny(id));
650 }
651 if (lastPackedRefs != null)
652 for (Ref lpr : lastPackedRefs) {
653 checkCancelled();
654 w.markUninteresting(w.parseAny(lpr.getObjectId()));
655 }
656 removeReferenced(deletionCandidates, w);
657 } finally {
658 w.dispose();
659 }
660
661 if (deletionCandidates.isEmpty())
662 return;
663
664 checkCancelled();
665
666
667
668
669
670 Set<File> touchedFanout = new HashSet<>();
671 for (File f : deletionCandidates.values()) {
672 if (f.lastModified() < expireDate) {
673 f.delete();
674 touchedFanout.add(f.getParentFile());
675 }
676 }
677
678 for (File f : touchedFanout) {
679 FileUtils.delete(f,
680 FileUtils.EMPTY_DIRECTORIES_ONLY | FileUtils.IGNORE_ERRORS);
681 }
682
683 repo.getObjectDatabase().close();
684 }
685
686 private long getExpireDate() throws ParseException {
687 long expireDate = Long.MAX_VALUE;
688
689 if (expire == null && expireAgeMillis == -1) {
690 String pruneExpireStr = getPruneExpireStr();
691 if (pruneExpireStr == null)
692 pruneExpireStr = PRUNE_EXPIRE_DEFAULT;
693 expire = GitDateParser.parse(pruneExpireStr, null, SystemReader
694 .getInstance().getLocale());
695 expireAgeMillis = -1;
696 }
697 if (expire != null)
698 expireDate = expire.getTime();
699 if (expireAgeMillis != -1)
700 expireDate = System.currentTimeMillis() - expireAgeMillis;
701 return expireDate;
702 }
703
704 private String getPruneExpireStr() {
705 return repo.getConfig().getString(
706 ConfigConstants.CONFIG_GC_SECTION, null,
707 ConfigConstants.CONFIG_KEY_PRUNEEXPIRE);
708 }
709
710 private long getPackExpireDate() throws ParseException {
711 long packExpireDate = Long.MAX_VALUE;
712
713 if (packExpire == null && packExpireAgeMillis == -1) {
714 String prunePackExpireStr = repo.getConfig().getString(
715 ConfigConstants.CONFIG_GC_SECTION, null,
716 ConfigConstants.CONFIG_KEY_PRUNEPACKEXPIRE);
717 if (prunePackExpireStr == null)
718 prunePackExpireStr = PRUNE_PACK_EXPIRE_DEFAULT;
719 packExpire = GitDateParser.parse(prunePackExpireStr, null,
720 SystemReader.getInstance().getLocale());
721 packExpireAgeMillis = -1;
722 }
723 if (packExpire != null)
724 packExpireDate = packExpire.getTime();
725 if (packExpireAgeMillis != -1)
726 packExpireDate = System.currentTimeMillis() - packExpireAgeMillis;
727 return packExpireDate;
728 }
729
730
731
732
733
734
735
736
737
738
739
740 private void removeReferenced(Map<ObjectId, File> id2File,
741 ObjectWalk w) throws MissingObjectException,
742 IncorrectObjectTypeException, IOException {
743 RevObject ro = w.next();
744 while (ro != null) {
745 checkCancelled();
746 if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
747 return;
748 }
749 ro = w.next();
750 }
751 ro = w.nextObject();
752 while (ro != null) {
753 checkCancelled();
754 if (id2File.remove(ro.getId()) != null && id2File.isEmpty()) {
755 return;
756 }
757 ro = w.nextObject();
758 }
759 }
760
761 private static boolean equals(Reff" href="../../../../../../org/eclipse/jgit/lib/Ref.html#Ref">Ref r1, Ref r2) {
762 if (r1 == null || r2 == null) {
763 return false;
764 }
765 if (r1.isSymbolic()) {
766 return r2.isSymbolic() && r1.getTarget().getName()
767 .equals(r2.getTarget().getName());
768 }
769 return !r2.isSymbolic()
770 && Objects.equals(r1.getObjectId(), r2.getObjectId());
771 }
772
773
774
775
776
777
778 public void packRefs() throws IOException {
779 Collection<Ref> refs = repo.getRefDatabase()
780 .getRefsByPrefix(Constants.R_REFS);
781 List<String> refsToBePacked = new ArrayList<>(refs.size());
782 pm.beginTask(JGitText.get().packRefs, refs.size());
783 try {
784 for (Ref ref : refs) {
785 checkCancelled();
786 if (!ref.isSymbolic() && ref.getStorage().isLoose())
787 refsToBePacked.add(ref.getName());
788 pm.update(1);
789 }
790 ((RefDirectory) repo.getRefDatabase()).pack(refsToBePacked);
791 } finally {
792 pm.endTask();
793 }
794 }
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810 public Collection<PackFile> repack() throws IOException {
811 Collection<PackFile> toBeDeleted = repo.getObjectDatabase().getPacks();
812
813 long time = System.currentTimeMillis();
814 Collection<Ref> refsBefore = getAllRefs();
815
816 Set<ObjectId> allHeadsAndTags = new HashSet<>();
817 Set<ObjectId> allHeads = new HashSet<>();
818 Set<ObjectId> allTags = new HashSet<>();
819 Set<ObjectId> nonHeads = new HashSet<>();
820 Set<ObjectId> txnHeads = new HashSet<>();
821 Set<ObjectId> tagTargets = new HashSet<>();
822 Set<ObjectId> indexObjects = listNonHEADIndexObjects();
823 RefDatabase refdb = repo.getRefDatabase();
824
825 for (Ref ref : refsBefore) {
826 checkCancelled();
827 nonHeads.addAll(listRefLogObjects(ref, 0));
828 if (ref.isSymbolic() || ref.getObjectId() == null) {
829 continue;
830 }
831 if (isHead(ref)) {
832 allHeads.add(ref.getObjectId());
833 } else if (isTag(ref)) {
834 allTags.add(ref.getObjectId());
835 } else if (RefTreeNames.isRefTree(refdb, ref.getName())) {
836 txnHeads.add(ref.getObjectId());
837 } else {
838 nonHeads.add(ref.getObjectId());
839 }
840 if (ref.getPeeledObjectId() != null) {
841 tagTargets.add(ref.getPeeledObjectId());
842 }
843 }
844
845 List<ObjectIdSet> excluded = new LinkedList<>();
846 for (PackFile f : repo.getObjectDatabase().getPacks()) {
847 checkCancelled();
848 if (f.shouldBeKept())
849 excluded.add(f.getIndex());
850 }
851
852
853 allTags.removeAll(allHeads);
854 allHeadsAndTags.addAll(allHeads);
855 allHeadsAndTags.addAll(allTags);
856
857
858 tagTargets.addAll(allHeadsAndTags);
859 nonHeads.addAll(indexObjects);
860
861
862 if (pconfig.getSinglePack()) {
863 allHeadsAndTags.addAll(nonHeads);
864 nonHeads.clear();
865 }
866
867 List<PackFile> ret = new ArrayList<>(2);
868 PackFile heads = null;
869 if (!allHeadsAndTags.isEmpty()) {
870 heads = writePack(allHeadsAndTags, PackWriter.NONE, allTags,
871 tagTargets, excluded);
872 if (heads != null) {
873 ret.add(heads);
874 excluded.add(0, heads.getIndex());
875 }
876 }
877 if (!nonHeads.isEmpty()) {
878 PackFile rest = writePack(nonHeads, allHeadsAndTags, PackWriter.NONE,
879 tagTargets, excluded);
880 if (rest != null)
881 ret.add(rest);
882 }
883 if (!txnHeads.isEmpty()) {
884 PackFile txn = writePack(txnHeads, PackWriter.NONE, PackWriter.NONE,
885 null, excluded);
886 if (txn != null)
887 ret.add(txn);
888 }
889 try {
890 deleteOldPacks(toBeDeleted, ret);
891 } catch (ParseException e) {
892
893
894
895 throw new IOException(e);
896 }
897 prunePacked();
898 deleteEmptyRefsFolders();
899 deleteOrphans();
900 deleteTempPacksIdx();
901
902 lastPackedRefs = refsBefore;
903 lastRepackTime = time;
904 return ret;
905 }
906
907 private static boolean isHead(Ref ref) {
908 return ref.getName().startsWith(Constants.R_HEADS);
909 }
910
911 private static boolean isTag(Ref ref) {
912 return ref.getName().startsWith(Constants.R_TAGS);
913 }
914
915 private void deleteEmptyRefsFolders() throws IOException {
916 Path refs = repo.getDirectory().toPath().resolve(Constants.R_REFS);
917
918
919 Instant threshold = Instant.now().minus(30, ChronoUnit.SECONDS);
920 try (Stream<Path> entries = Files.list(refs)
921 .filter(Files::isDirectory)) {
922 Iterator<Path> iterator = entries.iterator();
923 while (iterator.hasNext()) {
924 try (Stream<Path> s = Files.list(iterator.next())) {
925 s.filter(path -> canBeSafelyDeleted(path, threshold)).forEach(this::deleteDir);
926 }
927 }
928 }
929 }
930
931 private boolean canBeSafelyDeleted(Path path, Instant threshold) {
932 try {
933 return Files.getLastModifiedTime(path).toInstant().isBefore(threshold);
934 }
935 catch (IOException e) {
936 LOG.warn(MessageFormat.format(
937 JGitText.get().cannotAccessLastModifiedForSafeDeletion,
938 path), e);
939 return false;
940 }
941 }
942
943 private void deleteDir(Path dir) {
944 try (Stream<Path> dirs = Files.walk(dir)) {
945 dirs.filter(this::isDirectory).sorted(Comparator.reverseOrder())
946 .forEach(this::delete);
947 } catch (IOException e) {
948 LOG.error(e.getMessage(), e);
949 }
950 }
951
952 private boolean isDirectory(Path p) {
953 return p.toFile().isDirectory();
954 }
955
956 private void delete(Path d) {
957 try {
958 Files.delete(d);
959 } catch (DirectoryNotEmptyException e) {
960
961 } catch (IOException e) {
962 LOG.error(MessageFormat.format(JGitText.get().cannotDeleteFile, d),
963 e);
964 }
965 }
966
967
968
969
970
971
972
973
974 private void deleteOrphans() {
975 Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
976 List<String> fileNames = null;
977 try (Stream<Path> files = Files.list(packDir)) {
978 fileNames = files.map(path -> path.getFileName().toString())
979 .filter(name -> (name.endsWith(PACK_EXT)
980 || name.endsWith(BITMAP_EXT)
981 || name.endsWith(INDEX_EXT)))
982 .sorted(Collections.reverseOrder())
983 .collect(Collectors.toList());
984 } catch (IOException e1) {
985
986 }
987 if (fileNames == null) {
988 return;
989 }
990
991 String base = null;
992 for (String n : fileNames) {
993 if (n.endsWith(PACK_EXT)) {
994 base = n.substring(0, n.lastIndexOf('.'));
995 } else {
996 if (base == null || !n.startsWith(base)) {
997 try {
998 Files.delete(packDir.resolve(n));
999 } catch (IOException e) {
1000 LOG.error(e.getMessage(), e);
1001 }
1002 }
1003 }
1004 }
1005 }
1006
1007 private void deleteTempPacksIdx() {
1008 Path packDir = repo.getObjectDatabase().getPackDirectory().toPath();
1009 Instant threshold = Instant.now().minus(1, ChronoUnit.DAYS);
1010 if (!Files.exists(packDir)) {
1011 return;
1012 }
1013 try (DirectoryStream<Path> stream =
1014 Files.newDirectoryStream(packDir, "gc_*_tmp")) {
1015 stream.forEach(t -> {
1016 try {
1017 Instant lastModified = Files.getLastModifiedTime(t)
1018 .toInstant();
1019 if (lastModified.isBefore(threshold)) {
1020 Files.deleteIfExists(t);
1021 }
1022 } catch (IOException e) {
1023 LOG.error(e.getMessage(), e);
1024 }
1025 });
1026 } catch (IOException e) {
1027 LOG.error(e.getMessage(), e);
1028 }
1029 }
1030
1031
1032
1033
1034
1035
1036
1037
1038 private Set<ObjectId> listRefLogObjects(Ref ref, long minTime) throws IOException {
1039 ReflogReader reflogReader = repo.getReflogReader(ref.getName());
1040 if (reflogReader == null) {
1041 return Collections.emptySet();
1042 }
1043 List<ReflogEntry> rlEntries = reflogReader
1044 .getReverseEntries();
1045 if (rlEntries == null || rlEntries.isEmpty())
1046 return Collections.emptySet();
1047 Set<ObjectId> ret = new HashSet<>();
1048 for (ReflogEntry e : rlEntries) {
1049 if (e.getWho().getWhen().getTime() < minTime)
1050 break;
1051 ObjectId newId = e.getNewId();
1052 if (newId != null && !ObjectId.zeroId().equals(newId))
1053 ret.add(newId);
1054 ObjectId oldId = e.getOldId();
1055 if (oldId != null && !ObjectId.zeroId().equals(oldId))
1056 ret.add(oldId);
1057 }
1058 return ret;
1059 }
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072 private Collection<Ref> getAllRefs() throws IOException {
1073 RefDatabase refdb = repo.getRefDatabase();
1074 Collection<Ref> refs = refdb.getRefs();
1075 List<Ref> addl = refdb.getAdditionalRefs();
1076 if (!addl.isEmpty()) {
1077 List<Ref> all = new ArrayList<>(refs.size() + addl.size());
1078 all.addAll(refs);
1079
1080 for (Ref r : addl) {
1081 checkCancelled();
1082 if (r.getName().startsWith(Constants.R_REFS)) {
1083 all.add(r);
1084 }
1085 }
1086 return all;
1087 }
1088 return refs;
1089 }
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100 private Set<ObjectId> listNonHEADIndexObjects()
1101 throws CorruptObjectException, IOException {
1102 if (repo.isBare()) {
1103 return Collections.emptySet();
1104 }
1105 try (TreeWalkTreeWalk.html#TreeWalk">TreeWalk treeWalk = new TreeWalk(repo)) {
1106 treeWalk.addTree(new DirCacheIterator(repo.readDirCache()));
1107 ObjectId headID = repo.resolve(Constants.HEAD);
1108 if (headID != null) {
1109 try (RevWalk/RevWalk.html#RevWalk">RevWalk revWalk = new RevWalk(repo)) {
1110 treeWalk.addTree(revWalk.parseTree(headID));
1111 }
1112 }
1113
1114 treeWalk.setFilter(TreeFilter.ANY_DIFF);
1115 treeWalk.setRecursive(true);
1116 Set<ObjectId> ret = new HashSet<>();
1117
1118 while (treeWalk.next()) {
1119 checkCancelled();
1120 ObjectId objectId = treeWalk.getObjectId(0);
1121 switch (treeWalk.getRawMode(0) & FileMode.TYPE_MASK) {
1122 case FileMode.TYPE_MISSING:
1123 case FileMode.TYPE_GITLINK:
1124 continue;
1125 case FileMode.TYPE_TREE:
1126 case FileMode.TYPE_FILE:
1127 case FileMode.TYPE_SYMLINK:
1128 ret.add(objectId);
1129 continue;
1130 default:
1131 throw new IOException(MessageFormat.format(
1132 JGitText.get().corruptObjectInvalidMode3,
1133 String.format("%o",
1134 Integer.valueOf(treeWalk.getRawMode(0))),
1135 (objectId == null) ? "null" : objectId.name(),
1136 treeWalk.getPathString(),
1137 repo.getIndexFile()));
1138 }
1139 }
1140 return ret;
1141 }
1142 }
1143
1144 private PackFile writePack(@NonNull Set<? extends ObjectId> want,
1145 @NonNull Set<? extends ObjectId> have, @NonNull Set<ObjectId> tags,
1146 Set<ObjectId> tagTargets, List<ObjectIdSet> excludeObjects)
1147 throws IOException {
1148 checkCancelled();
1149 File tmpPack = null;
1150 Map<PackExt, File> tmpExts = new TreeMap<>((o1, o2) -> {
1151
1152
1153
1154 if (o1 == o2) {
1155 return 0;
1156 }
1157 if (o1 == PackExt.INDEX) {
1158 return 1;
1159 }
1160 if (o2 == PackExt.INDEX) {
1161 return -1;
1162 }
1163 return Integer.signum(o1.hashCode() - o2.hashCode());
1164 });
1165 try (PackWriternal/storage/pack/PackWriter.html#PackWriter">PackWriter pw = new PackWriter(
1166 pconfig,
1167 repo.newObjectReader())) {
1168
1169 pw.setDeltaBaseAsOffset(true);
1170 pw.setReuseDeltaCommits(false);
1171 if (tagTargets != null) {
1172 pw.setTagTargets(tagTargets);
1173 }
1174 if (excludeObjects != null)
1175 for (ObjectIdSet idx : excludeObjects)
1176 pw.excludeObjects(idx);
1177 pw.preparePack(pm, want, have, PackWriter.NONE, tags);
1178 if (pw.getObjectCount() == 0)
1179 return null;
1180 checkCancelled();
1181
1182
1183 String id = pw.computeName().getName();
1184 File packdir = repo.getObjectDatabase().getPackDirectory();
1185 tmpPack = File.createTempFile("gc_", ".pack_tmp", packdir);
1186 final String tmpBase = tmpPack.getName()
1187 .substring(0, tmpPack.getName().lastIndexOf('.'));
1188 File tmpIdx = new File(packdir, tmpBase + ".idx_tmp");
1189 tmpExts.put(INDEX, tmpIdx);
1190
1191 if (!tmpIdx.createNewFile())
1192 throw new IOException(MessageFormat.format(
1193 JGitText.get().cannotCreateIndexfile, tmpIdx.getPath()));
1194
1195
1196 try (FileOutputStream fos = new FileOutputStream(tmpPack);
1197 FileChannel channel = fos.getChannel();
1198 OutputStream channelStream = Channels
1199 .newOutputStream(channel)) {
1200 pw.writePack(pm, pm, channelStream);
1201 channel.force(true);
1202 }
1203
1204
1205 try (FileOutputStream fos = new FileOutputStream(tmpIdx);
1206 FileChannel idxChannel = fos.getChannel();
1207 OutputStream idxStream = Channels
1208 .newOutputStream(idxChannel)) {
1209 pw.writeIndex(idxStream);
1210 idxChannel.force(true);
1211 }
1212
1213 if (pw.prepareBitmapIndex(pm)) {
1214 File tmpBitmapIdx = new File(packdir, tmpBase + ".bitmap_tmp");
1215 tmpExts.put(BITMAP_INDEX, tmpBitmapIdx);
1216
1217 if (!tmpBitmapIdx.createNewFile())
1218 throw new IOException(MessageFormat.format(
1219 JGitText.get().cannotCreateIndexfile,
1220 tmpBitmapIdx.getPath()));
1221
1222 try (FileOutputStream fos = new FileOutputStream(tmpBitmapIdx);
1223 FileChannel idxChannel = fos.getChannel();
1224 OutputStream idxStream = Channels
1225 .newOutputStream(idxChannel)) {
1226 pw.writeBitmapIndex(idxStream);
1227 idxChannel.force(true);
1228 }
1229 }
1230
1231
1232 File realPack = nameFor(id, ".pack");
1233
1234 repo.getObjectDatabase().closeAllPackHandles(realPack);
1235 tmpPack.setReadOnly();
1236
1237 FileUtils.rename(tmpPack, realPack, StandardCopyOption.ATOMIC_MOVE);
1238 for (Map.Entry<PackExt, File> tmpEntry : tmpExts.entrySet()) {
1239 File tmpExt = tmpEntry.getValue();
1240 tmpExt.setReadOnly();
1241
1242 File realExt = nameFor(id,
1243 "." + tmpEntry.getKey().getExtension());
1244 try {
1245 FileUtils.rename(tmpExt, realExt,
1246 StandardCopyOption.ATOMIC_MOVE);
1247 } catch (IOException e) {
1248 File newExt = new File(realExt.getParentFile(),
1249 realExt.getName() + ".new");
1250 try {
1251 FileUtils.rename(tmpExt, newExt,
1252 StandardCopyOption.ATOMIC_MOVE);
1253 } catch (IOException e2) {
1254 newExt = tmpExt;
1255 e = e2;
1256 }
1257 throw new IOException(MessageFormat.format(
1258 JGitText.get().panicCantRenameIndexFile, newExt,
1259 realExt), e);
1260 }
1261 }
1262 boolean interrupted = false;
1263 try {
1264 FileSnapshot snapshot = FileSnapshot.save(realPack);
1265 if (pconfig.doWaitPreventRacyPack(snapshot.size())) {
1266 snapshot.waitUntilNotRacy();
1267 }
1268 } catch (InterruptedException e) {
1269 interrupted = true;
1270 }
1271 try {
1272 return repo.getObjectDatabase().openPack(realPack);
1273 } finally {
1274 if (interrupted) {
1275
1276 Thread.currentThread().interrupt();
1277 }
1278 }
1279 } finally {
1280 if (tmpPack != null && tmpPack.exists())
1281 tmpPack.delete();
1282 for (File tmpExt : tmpExts.values()) {
1283 if (tmpExt.exists())
1284 tmpExt.delete();
1285 }
1286 }
1287 }
1288
1289 private File nameFor(String name, String ext) {
1290 File packdir = repo.getObjectDatabase().getPackDirectory();
1291 return new File(packdir, "pack-" + name + ext);
1292 }
1293
1294 private void checkCancelled() throws CancelledException {
1295 if (pm.isCancelled() || Thread.currentThread().isInterrupted()) {
1296 throw new CancelledException(JGitText.get().operationCanceled);
1297 }
1298 }
1299
1300
1301
1302
1303
1304 public static class RepoStatistics {
1305
1306
1307
1308
1309
1310 public long numberOfPackedObjects;
1311
1312
1313
1314
1315 public long numberOfPackFiles;
1316
1317
1318
1319
1320 public long numberOfLooseObjects;
1321
1322
1323
1324
1325 public long sizeOfLooseObjects;
1326
1327
1328
1329
1330 public long sizeOfPackedObjects;
1331
1332
1333
1334
1335 public long numberOfLooseRefs;
1336
1337
1338
1339
1340 public long numberOfPackedRefs;
1341
1342
1343
1344
1345 public long numberOfBitmaps;
1346
1347 @Override
1348 public String toString() {
1349 final StringBuilder b = new StringBuilder();
1350 b.append("numberOfPackedObjects=").append(numberOfPackedObjects);
1351 b.append(", numberOfPackFiles=").append(numberOfPackFiles);
1352 b.append(", numberOfLooseObjects=").append(numberOfLooseObjects);
1353 b.append(", numberOfLooseRefs=").append(numberOfLooseRefs);
1354 b.append(", numberOfPackedRefs=").append(numberOfPackedRefs);
1355 b.append(", sizeOfLooseObjects=").append(sizeOfLooseObjects);
1356 b.append(", sizeOfPackedObjects=").append(sizeOfPackedObjects);
1357 b.append(", numberOfBitmaps=").append(numberOfBitmaps);
1358 return b.toString();
1359 }
1360 }
1361
1362
1363
1364
1365
1366
1367
1368 public RepoStatistics getStatistics() throws IOException {
1369 RepoStatistics ret = new RepoStatistics();
1370 Collection<PackFile> packs = repo.getObjectDatabase().getPacks();
1371 for (PackFile f : packs) {
1372 ret.numberOfPackedObjects += f.getIndex().getObjectCount();
1373 ret.numberOfPackFiles++;
1374 ret.sizeOfPackedObjects += f.getPackFile().length();
1375 if (f.getBitmapIndex() != null)
1376 ret.numberOfBitmaps += f.getBitmapIndex().getBitmapCount();
1377 }
1378 File objDir = repo.getObjectsDirectory();
1379 String[] fanout = objDir.list();
1380 if (fanout != null && fanout.length > 0) {
1381 for (String d : fanout) {
1382 if (d.length() != 2)
1383 continue;
1384 File[] entries = new File(objDir, d).listFiles();
1385 if (entries == null)
1386 continue;
1387 for (File f : entries) {
1388 if (f.getName().length() != Constants.OBJECT_ID_STRING_LENGTH - 2)
1389 continue;
1390 ret.numberOfLooseObjects++;
1391 ret.sizeOfLooseObjects += f.length();
1392 }
1393 }
1394 }
1395
1396 RefDatabase refDb = repo.getRefDatabase();
1397 for (Ref r : refDb.getRefs()) {
1398 Storage storage = r.getStorage();
1399 if (storage == Storage.LOOSE || storage == Storage.LOOSE_PACKED)
1400 ret.numberOfLooseRefs++;
1401 if (storage == Storage.PACKED || storage == Storage.LOOSE_PACKED)
1402 ret.numberOfPackedRefs++;
1403 }
1404
1405 return ret;
1406 }
1407
1408
1409
1410
1411
1412
1413
1414 public GC setProgressMonitor(ProgressMonitor pm) {
1415 this.pm = (pm == null) ? NullProgressMonitor.INSTANCE : pm;
1416 return this;
1417 }
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428 public void setExpireAgeMillis(long expireAgeMillis) {
1429 this.expireAgeMillis = expireAgeMillis;
1430 expire = null;
1431 }
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442 public void setPackExpireAgeMillis(long packExpireAgeMillis) {
1443 this.packExpireAgeMillis = packExpireAgeMillis;
1444 expire = null;
1445 }
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456 public void setPackConfig(@NonNull PackConfig pconfig) {
1457 this.pconfig = pconfig;
1458 }
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472 public void setExpire(Date expire) {
1473 this.expire = expire;
1474 expireAgeMillis = -1;
1475 }
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486 public void setPackExpire(Date packExpire) {
1487 this.packExpire = packExpire;
1488 packExpireAgeMillis = -1;
1489 }
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526 public void setAuto(boolean auto) {
1527 this.automatic = auto;
1528 }
1529
1530
1531
1532
1533
1534 void setBackground(boolean background) {
1535 this.background = background;
1536 }
1537
1538 private boolean needGc() {
1539 if (tooManyPacks()) {
1540 addRepackAllOption();
1541 } else {
1542 return tooManyLooseObjects();
1543 }
1544
1545 return true;
1546 }
1547
1548 private void addRepackAllOption() {
1549
1550
1551 }
1552
1553
1554
1555
1556 boolean tooManyPacks() {
1557 int autopacklimit = repo.getConfig().getInt(
1558 ConfigConstants.CONFIG_GC_SECTION,
1559 ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT,
1560 DEFAULT_AUTOPACKLIMIT);
1561 if (autopacklimit <= 0) {
1562 return false;
1563 }
1564
1565
1566 return repo.getObjectDatabase().getPacks().size() > (autopacklimit + 1);
1567 }
1568
1569
1570
1571
1572
1573
1574
1575 boolean tooManyLooseObjects() {
1576 int auto = getLooseObjectLimit();
1577 if (auto <= 0) {
1578 return false;
1579 }
1580 int n = 0;
1581 int threshold = (auto + 255) / 256;
1582 Path dir = repo.getObjectsDirectory().toPath().resolve("17");
1583 if (!dir.toFile().exists()) {
1584 return false;
1585 }
1586 try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, file -> {
1587 Path fileName = file.getFileName();
1588 return file.toFile().isFile() && fileName != null
1589 && PATTERN_LOOSE_OBJECT.matcher(fileName.toString())
1590 .matches();
1591 })) {
1592 for (Iterator<Path> iter = stream.iterator(); iter.hasNext(); iter
1593 .next()) {
1594 if (++n > threshold) {
1595 return true;
1596 }
1597 }
1598 } catch (IOException e) {
1599 LOG.error(e.getMessage(), e);
1600 }
1601 return false;
1602 }
1603
1604 private int getLooseObjectLimit() {
1605 return repo.getConfig().getInt(ConfigConstants.CONFIG_GC_SECTION,
1606 ConfigConstants.CONFIG_KEY_AUTO, DEFAULT_AUTOLIMIT);
1607 }
1608 }