1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45 package org.eclipse.jgit.internal.storage.pack;
46
47 import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_DELTA;
48 import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_WHOLE;
49 import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH;
50 import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
51 import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT;
52 import static org.eclipse.jgit.lib.Constants.OBJ_TAG;
53 import static org.eclipse.jgit.lib.Constants.OBJ_TREE;
54
55 import java.io.IOException;
56 import java.io.OutputStream;
57 import java.lang.ref.WeakReference;
58 import java.security.MessageDigest;
59 import java.text.MessageFormat;
60 import java.util.ArrayList;
61 import java.util.Arrays;
62 import java.util.Collection;
63 import java.util.Collections;
64 import java.util.Comparator;
65 import java.util.HashSet;
66 import java.util.Iterator;
67 import java.util.List;
68 import java.util.Map;
69 import java.util.NoSuchElementException;
70 import java.util.Set;
71 import java.util.concurrent.ConcurrentHashMap;
72 import java.util.concurrent.ExecutionException;
73 import java.util.concurrent.Executor;
74 import java.util.concurrent.ExecutorService;
75 import java.util.concurrent.Executors;
76 import java.util.concurrent.Future;
77 import java.util.concurrent.TimeUnit;
78 import java.util.zip.CRC32;
79 import java.util.zip.CheckedOutputStream;
80 import java.util.zip.Deflater;
81 import java.util.zip.DeflaterOutputStream;
82
83 import org.eclipse.jgit.annotations.NonNull;
84 import org.eclipse.jgit.annotations.Nullable;
85 import org.eclipse.jgit.errors.CorruptObjectException;
86 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
87 import org.eclipse.jgit.errors.LargeObjectException;
88 import org.eclipse.jgit.errors.MissingObjectException;
89 import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
90 import org.eclipse.jgit.internal.JGitText;
91 import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder;
92 import org.eclipse.jgit.internal.storage.file.PackBitmapIndexWriterV1;
93 import org.eclipse.jgit.internal.storage.file.PackIndexWriter;
94 import org.eclipse.jgit.lib.AnyObjectId;
95 import org.eclipse.jgit.lib.AsyncObjectSizeQueue;
96 import org.eclipse.jgit.lib.BatchingProgressMonitor;
97 import org.eclipse.jgit.lib.BitmapIndex;
98 import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder;
99 import org.eclipse.jgit.lib.BitmapObject;
100 import org.eclipse.jgit.lib.Constants;
101 import org.eclipse.jgit.lib.NullProgressMonitor;
102 import org.eclipse.jgit.lib.ObjectId;
103 import org.eclipse.jgit.lib.ObjectIdOwnerMap;
104 import org.eclipse.jgit.lib.ObjectIdSet;
105 import org.eclipse.jgit.lib.ObjectLoader;
106 import org.eclipse.jgit.lib.ObjectReader;
107 import org.eclipse.jgit.lib.ProgressMonitor;
108 import org.eclipse.jgit.lib.Repository;
109 import org.eclipse.jgit.lib.ThreadSafeProgressMonitor;
110 import org.eclipse.jgit.revwalk.AsyncRevObjectQueue;
111 import org.eclipse.jgit.revwalk.BitmapWalker;
112 import org.eclipse.jgit.revwalk.DepthWalk;
113 import org.eclipse.jgit.revwalk.ObjectWalk;
114 import org.eclipse.jgit.revwalk.RevCommit;
115 import org.eclipse.jgit.revwalk.RevFlag;
116 import org.eclipse.jgit.revwalk.RevObject;
117 import org.eclipse.jgit.revwalk.RevSort;
118 import org.eclipse.jgit.revwalk.RevTag;
119 import org.eclipse.jgit.revwalk.RevTree;
120 import org.eclipse.jgit.storage.pack.PackConfig;
121 import org.eclipse.jgit.storage.pack.PackStatistics;
122 import org.eclipse.jgit.transport.ObjectCountCallback;
123 import org.eclipse.jgit.transport.WriteAbortedException;
124 import org.eclipse.jgit.util.BlockList;
125 import org.eclipse.jgit.util.TemporaryBuffer;
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166 public class PackWriter implements AutoCloseable {
167 private static final int PACK_VERSION_GENERATED = 2;
168
169
170 public static final Set<ObjectId> NONE = Collections.emptySet();
171
172 private static final Map<WeakReference<PackWriter>, Boolean> instances =
173 new ConcurrentHashMap<>();
174
175 private static final Iterable<PackWriter> instancesIterable = new Iterable<PackWriter>() {
176 @Override
177 public Iterator<PackWriter> iterator() {
178 return new Iterator<PackWriter>() {
179 private final Iterator<WeakReference<PackWriter>> it =
180 instances.keySet().iterator();
181 private PackWriter next;
182
183 @Override
184 public boolean hasNext() {
185 if (next != null)
186 return true;
187 while (it.hasNext()) {
188 WeakReference<PackWriter> ref = it.next();
189 next = ref.get();
190 if (next != null)
191 return true;
192 it.remove();
193 }
194 return false;
195 }
196
197 @Override
198 public PackWriter next() {
199 if (hasNext()) {
200 PackWriter result = next;
201 next = null;
202 return result;
203 }
204 throw new NoSuchElementException();
205 }
206
207 @Override
208 public void remove() {
209 throw new UnsupportedOperationException();
210 }
211 };
212 }
213 };
214
215
216
217
218
219
220 public static Iterable<PackWriter> getInstances() {
221 return instancesIterable;
222 }
223
224 @SuppressWarnings("unchecked")
225 BlockList<ObjectToPack> objectsLists[] = new BlockList[OBJ_TAG + 1];
226 {
227 objectsLists[OBJ_COMMIT] = new BlockList<>();
228 objectsLists[OBJ_TREE] = new BlockList<>();
229 objectsLists[OBJ_BLOB] = new BlockList<>();
230 objectsLists[OBJ_TAG] = new BlockList<>();
231 }
232
233 private ObjectIdOwnerMap<ObjectToPack> objectsMap = new ObjectIdOwnerMap<>();
234
235
236 private List<ObjectToPack> edgeObjects = new BlockList<>();
237
238
239 private BitmapBuilder haveObjects;
240
241 private List<CachedPack> cachedPacks = new ArrayList<>(2);
242
243 private Set<ObjectId> tagTargets = NONE;
244
245 private Set<? extends ObjectId> excludeFromBitmapSelection = NONE;
246
247 private ObjectIdSet[] excludeInPacks;
248
249 private ObjectIdSet excludeInPackLast;
250
251 private Deflater myDeflater;
252
253 private final ObjectReader reader;
254
255
256 private final ObjectReuseAsIs reuseSupport;
257
258 final PackConfig config;
259
260 private final PackStatistics.Accumulator stats;
261
262 private final MutableState state;
263
264 private final WeakReference<PackWriter> selfRef;
265
266 private PackStatistics.ObjectType.Accumulator typeStats;
267
268 private List<ObjectToPack> sortedByName;
269
270 private byte packcsum[];
271
272 private boolean deltaBaseAsOffset;
273
274 private boolean reuseDeltas;
275
276 private boolean reuseDeltaCommits;
277
278 private boolean reuseValidate;
279
280 private boolean thin;
281
282 private boolean useCachedPacks;
283
284 private boolean useBitmaps;
285
286 private boolean ignoreMissingUninteresting = true;
287
288 private boolean pruneCurrentObjectList;
289
290 private boolean shallowPack;
291
292 private boolean canBuildBitmaps;
293
294 private boolean indexDisabled;
295
296 private int depth;
297
298 private Collection<? extends ObjectId> unshallowObjects;
299
300 private PackBitmapIndexBuilder writeBitmaps;
301
302 private CRC32 crc32;
303
304 private ObjectCountCallback callback;
305
306 private long filterBlobLimit = -1;
307
308
309
310
311
312
313
314
315
316
317 public PackWriter(Repository repo) {
318 this(repo, repo.newObjectReader());
319 }
320
321
322
323
324
325
326
327
328
329
330 public PackWriter(ObjectReader reader) {
331 this(new PackConfig(), reader);
332 }
333
334
335
336
337
338
339
340
341
342
343
344
345 public PackWriter(Repository repo, ObjectReader reader) {
346 this(new PackConfig(repo), reader);
347 }
348
349
350
351
352
353
354
355
356
357
358
359
360 public PackWriter(PackConfig config, ObjectReader reader) {
361 this(config, reader, null);
362 }
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377 public PackWriter(PackConfig config, final ObjectReader reader,
378 @Nullable PackStatistics.Accumulator statsAccumulator) {
379 this.config = config;
380 this.reader = reader;
381 if (reader instanceof ObjectReuseAsIs)
382 reuseSupport = ((ObjectReuseAsIs) reader);
383 else
384 reuseSupport = null;
385
386 deltaBaseAsOffset = config.isDeltaBaseAsOffset();
387 reuseDeltas = config.isReuseDeltas();
388 reuseValidate = true;
389 stats = statsAccumulator != null ? statsAccumulator
390 : new PackStatistics.Accumulator();
391 state = new MutableState();
392 selfRef = new WeakReference<>(this);
393 instances.put(selfRef, Boolean.TRUE);
394 }
395
396
397
398
399
400
401
402
403
404
405
406 public PackWriter setObjectCountCallback(ObjectCountCallback callback) {
407 this.callback = callback;
408 return this;
409 }
410
411
412
413
414
415
416
417 public void setClientShallowCommits(Set<ObjectId> clientShallowCommits) {
418 stats.clientShallowCommits = Collections
419 .unmodifiableSet(new HashSet<>(clientShallowCommits));
420 }
421
422
423
424
425
426
427
428
429
430
431
432 public boolean isDeltaBaseAsOffset() {
433 return deltaBaseAsOffset;
434 }
435
436
437
438
439
440
441
442
443
444
445
446
447 public void setDeltaBaseAsOffset(boolean deltaBaseAsOffset) {
448 this.deltaBaseAsOffset = deltaBaseAsOffset;
449 }
450
451
452
453
454
455
456
457 public boolean isReuseDeltaCommits() {
458 return reuseDeltaCommits;
459 }
460
461
462
463
464
465
466
467
468 public void setReuseDeltaCommits(boolean reuse) {
469 reuseDeltaCommits = reuse;
470 }
471
472
473
474
475
476
477
478 public boolean isReuseValidatingObjects() {
479 return reuseValidate;
480 }
481
482
483
484
485
486
487
488
489
490
491 public void setReuseValidatingObjects(boolean validate) {
492 reuseValidate = validate;
493 }
494
495
496
497
498
499
500 public boolean isThin() {
501 return thin;
502 }
503
504
505
506
507
508
509
510
511
512
513
514
515 public void setThin(boolean packthin) {
516 thin = packthin;
517 }
518
519
520
521
522
523
524
525 public boolean isUseCachedPacks() {
526 return useCachedPacks;
527 }
528
529
530
531
532
533
534
535
536
537
538
539
540 public void setUseCachedPacks(boolean useCached) {
541 useCachedPacks = useCached;
542 }
543
544
545
546
547
548
549 public boolean isUseBitmaps() {
550 return useBitmaps;
551 }
552
553
554
555
556
557
558
559 public void setUseBitmaps(boolean useBitmaps) {
560 this.useBitmaps = useBitmaps;
561 }
562
563
564
565
566
567
568
569 public boolean isIndexDisabled() {
570 return indexDisabled || !cachedPacks.isEmpty();
571 }
572
573
574
575
576
577
578
579 public void setIndexDisabled(boolean noIndex) {
580 this.indexDisabled = noIndex;
581 }
582
583
584
585
586
587
588
589
590
591
592
593 public boolean isIgnoreMissingUninteresting() {
594 return ignoreMissingUninteresting;
595 }
596
597
598
599
600
601
602
603
604
605
606 public void setIgnoreMissingUninteresting(boolean ignore) {
607 ignoreMissingUninteresting = ignore;
608 }
609
610
611
612
613
614
615
616
617
618
619
620
621
622 public void setTagTargets(Set<ObjectId> objects) {
623 tagTargets = objects;
624 }
625
626
627
628
629
630
631
632
633
634
635
636 public void setShallowPack(int depth,
637 Collection<? extends ObjectId> unshallow) {
638 this.shallowPack = true;
639 this.depth = depth;
640 this.unshallowObjects = unshallow;
641 }
642
643
644
645
646
647 public void setFilterBlobLimit(long bytes) {
648 filterBlobLimit = bytes;
649 }
650
651
652
653
654
655
656
657
658 public long getObjectCount() throws IOException {
659 if (stats.totalObjects == 0) {
660 long objCnt = 0;
661
662 objCnt += objectsLists[OBJ_COMMIT].size();
663 objCnt += objectsLists[OBJ_TREE].size();
664 objCnt += objectsLists[OBJ_BLOB].size();
665 objCnt += objectsLists[OBJ_TAG].size();
666
667 for (CachedPack pack : cachedPacks)
668 objCnt += pack.getObjectCount();
669 return objCnt;
670 }
671 return stats.totalObjects;
672 }
673
674
675
676
677
678
679
680
681
682
683
684
685 public ObjectIdOwnerMap<ObjectIdOwnerMap.Entry> getObjectSet()
686 throws IOException {
687 if (!cachedPacks.isEmpty())
688 throw new IOException(
689 JGitText.get().cachedPacksPreventsListingObjects);
690
691 if (writeBitmaps != null) {
692 return writeBitmaps.getObjectSet();
693 }
694
695 ObjectIdOwnerMap<ObjectIdOwnerMap.Entry> r = new ObjectIdOwnerMap<>();
696 for (BlockList<ObjectToPack> objList : objectsLists) {
697 if (objList != null) {
698 for (ObjectToPack otp : objList)
699 r.add(new ObjectIdOwnerMap.Entry(otp) {
700
701 });
702 }
703 }
704 return r;
705 }
706
707
708
709
710
711
712
713 public void excludeObjects(ObjectIdSet idx) {
714 if (excludeInPacks == null) {
715 excludeInPacks = new ObjectIdSet[] { idx };
716 excludeInPackLast = idx;
717 } else {
718 int cnt = excludeInPacks.length;
719 ObjectIdSet[] newList = new ObjectIdSet[cnt + 1];
720 System.arraycopy(excludeInPacks, 0, newList, 0, cnt);
721 newList[cnt] = idx;
722 excludeInPacks = newList;
723 }
724 }
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751 public void preparePack(@NonNull Iterator<RevObject> objectsSource)
752 throws IOException {
753 while (objectsSource.hasNext()) {
754 addObject(objectsSource.next());
755 }
756 }
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783 public void preparePack(ProgressMonitor countingMonitor,
784 @NonNull Set<? extends ObjectId> want,
785 @NonNull Set<? extends ObjectId> have) throws IOException {
786 preparePack(countingMonitor, want, have, NONE, NONE);
787 }
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818 public void preparePack(ProgressMonitor countingMonitor,
819 @NonNull Set<? extends ObjectId> want,
820 @NonNull Set<? extends ObjectId> have,
821 @NonNull Set<? extends ObjectId> shallow) throws IOException {
822 preparePack(countingMonitor, want, have, shallow, NONE);
823 }
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857 public void preparePack(ProgressMonitor countingMonitor,
858 @NonNull Set<? extends ObjectId> want,
859 @NonNull Set<? extends ObjectId> have,
860 @NonNull Set<? extends ObjectId> shallow,
861 @NonNull Set<? extends ObjectId> noBitmaps) throws IOException {
862 try (ObjectWalk ow = getObjectWalk()) {
863 ow.assumeShallow(shallow);
864 preparePack(countingMonitor, ow, want, have, noBitmaps);
865 }
866 }
867
868 private ObjectWalk getObjectWalk() {
869 return shallowPack ? new DepthWalk.ObjectWalk(reader, depth - 1)
870 : new ObjectWalk(reader);
871 }
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903 public void preparePack(ProgressMonitor countingMonitor,
904 @NonNull ObjectWalk walk,
905 @NonNull Set<? extends ObjectId> interestingObjects,
906 @NonNull Set<? extends ObjectId> uninterestingObjects,
907 @NonNull Set<? extends ObjectId> noBitmaps)
908 throws IOException {
909 if (countingMonitor == null)
910 countingMonitor = NullProgressMonitor.INSTANCE;
911 if (shallowPack && !(walk instanceof DepthWalk.ObjectWalk))
912 throw new IllegalArgumentException(
913 JGitText.get().shallowPacksRequireDepthWalk);
914 findObjectsToPack(countingMonitor, walk, interestingObjects,
915 uninterestingObjects, noBitmaps);
916 }
917
918
919
920
921
922
923
924
925
926
927 public boolean willInclude(AnyObjectId id) throws IOException {
928 ObjectToPack obj = objectsMap.get(id);
929 return obj != null && !obj.isEdge();
930 }
931
932
933
934
935
936
937
938
939 public ObjectToPack get(AnyObjectId id) {
940 ObjectToPack obj = objectsMap.get(id);
941 return obj != null && !obj.isEdge() ? obj : null;
942 }
943
944
945
946
947
948
949
950 public ObjectId computeName() {
951 final byte[] buf = new byte[OBJECT_ID_LENGTH];
952 final MessageDigest md = Constants.newMessageDigest();
953 for (ObjectToPack otp : sortByName()) {
954 otp.copyRawTo(buf, 0);
955 md.update(buf, 0, OBJECT_ID_LENGTH);
956 }
957 return ObjectId.fromRaw(md.digest());
958 }
959
960
961
962
963
964
965
966
967
968
969 public int getIndexVersion() {
970 int indexVersion = config.getIndexVersion();
971 if (indexVersion <= 0) {
972 for (BlockList<ObjectToPack> objs : objectsLists)
973 indexVersion = Math.max(indexVersion,
974 PackIndexWriter.oldestPossibleFormat(objs));
975 }
976 return indexVersion;
977 }
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994 public void writeIndex(OutputStream indexStream) throws IOException {
995 if (isIndexDisabled())
996 throw new IOException(JGitText.get().cachedPacksPreventsIndexCreation);
997
998 long writeStart = System.currentTimeMillis();
999 final PackIndexWriter iw = PackIndexWriter.createVersion(
1000 indexStream, getIndexVersion());
1001 iw.write(sortByName(), packcsum);
1002 stats.timeWriting += System.currentTimeMillis() - writeStart;
1003 }
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016 public void writeBitmapIndex(OutputStream bitmapIndexStream)
1017 throws IOException {
1018 if (writeBitmaps == null)
1019 throw new IOException(JGitText.get().bitmapsMustBePrepared);
1020
1021 long writeStart = System.currentTimeMillis();
1022 final PackBitmapIndexWriterV1 iw = new PackBitmapIndexWriterV1(bitmapIndexStream);
1023 iw.write(writeBitmaps, packcsum);
1024 stats.timeWriting += System.currentTimeMillis() - writeStart;
1025 }
1026
1027 private List<ObjectToPack> sortByName() {
1028 if (sortedByName == null) {
1029 int cnt = 0;
1030 cnt += objectsLists[OBJ_COMMIT].size();
1031 cnt += objectsLists[OBJ_TREE].size();
1032 cnt += objectsLists[OBJ_BLOB].size();
1033 cnt += objectsLists[OBJ_TAG].size();
1034
1035 sortedByName = new BlockList<>(cnt);
1036 sortedByName.addAll(objectsLists[OBJ_COMMIT]);
1037 sortedByName.addAll(objectsLists[OBJ_TREE]);
1038 sortedByName.addAll(objectsLists[OBJ_BLOB]);
1039 sortedByName.addAll(objectsLists[OBJ_TAG]);
1040 Collections.sort(sortedByName);
1041 }
1042 return sortedByName;
1043 }
1044
1045 private void beginPhase(PackingPhase phase, ProgressMonitor monitor,
1046 long cnt) {
1047 state.phase = phase;
1048 String task;
1049 switch (phase) {
1050 case COUNTING:
1051 task = JGitText.get().countingObjects;
1052 break;
1053 case GETTING_SIZES:
1054 task = JGitText.get().searchForSizes;
1055 break;
1056 case FINDING_SOURCES:
1057 task = JGitText.get().searchForReuse;
1058 break;
1059 case COMPRESSING:
1060 task = JGitText.get().compressingObjects;
1061 break;
1062 case WRITING:
1063 task = JGitText.get().writingObjects;
1064 break;
1065 case BUILDING_BITMAPS:
1066 task = JGitText.get().buildingBitmaps;
1067 break;
1068 default:
1069 throw new IllegalArgumentException(
1070 MessageFormat.format(JGitText.get().illegalPackingPhase, phase));
1071 }
1072 monitor.beginTask(task, (int) cnt);
1073 }
1074
1075 private void endPhase(ProgressMonitor monitor) {
1076 monitor.endTask();
1077 }
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106 public void writePack(ProgressMonitor compressMonitor,
1107 ProgressMonitor writeMonitor, OutputStream packStream)
1108 throws IOException {
1109 if (compressMonitor == null)
1110 compressMonitor = NullProgressMonitor.INSTANCE;
1111 if (writeMonitor == null)
1112 writeMonitor = NullProgressMonitor.INSTANCE;
1113
1114 excludeInPacks = null;
1115 excludeInPackLast = null;
1116
1117 boolean needSearchForReuse = reuseSupport != null && (
1118 reuseDeltas
1119 || config.isReuseObjects()
1120 || !cachedPacks.isEmpty());
1121
1122 if (compressMonitor instanceof BatchingProgressMonitor) {
1123 long delay = 1000;
1124 if (needSearchForReuse && config.isDeltaCompress())
1125 delay = 500;
1126 ((BatchingProgressMonitor) compressMonitor).setDelayStart(
1127 delay,
1128 TimeUnit.MILLISECONDS);
1129 }
1130
1131 if (needSearchForReuse)
1132 searchForReuse(compressMonitor);
1133 if (config.isDeltaCompress())
1134 searchForDeltas(compressMonitor);
1135
1136 crc32 = new CRC32();
1137 final PackOutputStream out = new PackOutputStream(
1138 writeMonitor,
1139 isIndexDisabled()
1140 ? packStream
1141 : new CheckedOutputStream(packStream, crc32),
1142 this);
1143
1144 long objCnt = getObjectCount();
1145 stats.totalObjects = objCnt;
1146 if (callback != null)
1147 callback.setObjectCount(objCnt);
1148 beginPhase(PackingPhase.WRITING, writeMonitor, objCnt);
1149 long writeStart = System.currentTimeMillis();
1150 try {
1151 out.writeFileHeader(PACK_VERSION_GENERATED, objCnt);
1152 out.flush();
1153
1154 writeObjects(out);
1155 if (!edgeObjects.isEmpty() || !cachedPacks.isEmpty()) {
1156 for (PackStatistics.ObjectType.Accumulator typeStat : stats.objectTypes) {
1157 if (typeStat == null)
1158 continue;
1159 stats.thinPackBytes += typeStat.bytes;
1160 }
1161 }
1162
1163 stats.reusedPacks = Collections.unmodifiableList(cachedPacks);
1164 for (CachedPack pack : cachedPacks) {
1165 long deltaCnt = pack.getDeltaCount();
1166 stats.reusedObjects += pack.getObjectCount();
1167 stats.reusedDeltas += deltaCnt;
1168 stats.totalDeltas += deltaCnt;
1169 reuseSupport.copyPackAsIs(out, pack);
1170 }
1171 writeChecksum(out);
1172 out.flush();
1173 } finally {
1174 stats.timeWriting = System.currentTimeMillis() - writeStart;
1175 stats.depth = depth;
1176
1177 for (PackStatistics.ObjectType.Accumulator typeStat : stats.objectTypes) {
1178 if (typeStat == null)
1179 continue;
1180 typeStat.cntDeltas += typeStat.reusedDeltas;
1181 stats.reusedObjects += typeStat.reusedObjects;
1182 stats.reusedDeltas += typeStat.reusedDeltas;
1183 stats.totalDeltas += typeStat.cntDeltas;
1184 }
1185 }
1186
1187 stats.totalBytes = out.length();
1188 reader.close();
1189 endPhase(writeMonitor);
1190 }
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200 public PackStatistics getStatistics() {
1201 return new PackStatistics(stats);
1202 }
1203
1204
1205
1206
1207
1208
1209 public State getState() {
1210 return state.snapshot();
1211 }
1212
1213
1214
1215
1216
1217
1218 @Override
1219 public void close() {
1220 reader.close();
1221 if (myDeflater != null) {
1222 myDeflater.end();
1223 myDeflater = null;
1224 }
1225 instances.remove(selfRef);
1226 }
1227
1228 private void searchForReuse(ProgressMonitor monitor) throws IOException {
1229 long cnt = 0;
1230 cnt += objectsLists[OBJ_COMMIT].size();
1231 cnt += objectsLists[OBJ_TREE].size();
1232 cnt += objectsLists[OBJ_BLOB].size();
1233 cnt += objectsLists[OBJ_TAG].size();
1234
1235 long start = System.currentTimeMillis();
1236 beginPhase(PackingPhase.FINDING_SOURCES, monitor, cnt);
1237 if (cnt <= 4096) {
1238
1239 BlockList<ObjectToPack> tmp = new BlockList<>((int) cnt);
1240 tmp.addAll(objectsLists[OBJ_TAG]);
1241 tmp.addAll(objectsLists[OBJ_COMMIT]);
1242 tmp.addAll(objectsLists[OBJ_TREE]);
1243 tmp.addAll(objectsLists[OBJ_BLOB]);
1244 searchForReuse(monitor, tmp);
1245 if (pruneCurrentObjectList) {
1246
1247 pruneEdgesFromObjectList(objectsLists[OBJ_COMMIT]);
1248 pruneEdgesFromObjectList(objectsLists[OBJ_TREE]);
1249 pruneEdgesFromObjectList(objectsLists[OBJ_BLOB]);
1250 pruneEdgesFromObjectList(objectsLists[OBJ_TAG]);
1251 }
1252 } else {
1253 searchForReuse(monitor, objectsLists[OBJ_TAG]);
1254 searchForReuse(monitor, objectsLists[OBJ_COMMIT]);
1255 searchForReuse(monitor, objectsLists[OBJ_TREE]);
1256 searchForReuse(monitor, objectsLists[OBJ_BLOB]);
1257 }
1258 endPhase(monitor);
1259 stats.timeSearchingForReuse = System.currentTimeMillis() - start;
1260
1261 if (config.isReuseDeltas() && config.getCutDeltaChains()) {
1262 cutDeltaChains(objectsLists[OBJ_TREE]);
1263 cutDeltaChains(objectsLists[OBJ_BLOB]);
1264 }
1265 }
1266
1267 private void searchForReuse(ProgressMonitor monitor, List<ObjectToPack> list)
1268 throws IOException, MissingObjectException {
1269 pruneCurrentObjectList = false;
1270 reuseSupport.selectObjectRepresentation(this, monitor, list);
1271 if (pruneCurrentObjectList)
1272 pruneEdgesFromObjectList(list);
1273 }
1274
1275 private void cutDeltaChains(BlockList<ObjectToPack> list)
1276 throws IOException {
1277 int max = config.getMaxDeltaDepth();
1278 for (int idx = list.size() - 1; idx >= 0; idx--) {
1279 int d = 0;
1280 ObjectToPack b = list.get(idx).getDeltaBase();
1281 while (b != null) {
1282 if (d < b.getChainLength())
1283 break;
1284 b.setChainLength(++d);
1285 if (d >= max && b.isDeltaRepresentation()) {
1286 reselectNonDelta(b);
1287 break;
1288 }
1289 b = b.getDeltaBase();
1290 }
1291 }
1292 if (config.isDeltaCompress()) {
1293 for (ObjectToPack otp : list)
1294 otp.clearChainLength();
1295 }
1296 }
1297
1298 private void searchForDeltas(ProgressMonitor monitor)
1299 throws MissingObjectException, IncorrectObjectTypeException,
1300 IOException {
1301
1302
1303
1304
1305 ObjectToPack[] list = new ObjectToPack[
1306 objectsLists[OBJ_TREE].size()
1307 + objectsLists[OBJ_BLOB].size()
1308 + edgeObjects.size()];
1309 int cnt = 0;
1310 cnt = findObjectsNeedingDelta(list, cnt, OBJ_TREE);
1311 cnt = findObjectsNeedingDelta(list, cnt, OBJ_BLOB);
1312 if (cnt == 0)
1313 return;
1314 int nonEdgeCnt = cnt;
1315
1316
1317
1318
1319
1320 for (ObjectToPack eo : edgeObjects) {
1321 eo.setWeight(0);
1322 list[cnt++] = eo;
1323 }
1324
1325
1326
1327
1328
1329
1330
1331
1332 final long sizingStart = System.currentTimeMillis();
1333 beginPhase(PackingPhase.GETTING_SIZES, monitor, cnt);
1334 AsyncObjectSizeQueue<ObjectToPack> sizeQueue = reader.getObjectSize(
1335 Arrays.<ObjectToPack> asList(list).subList(0, cnt), false);
1336 try {
1337 final long limit = Math.min(
1338 config.getBigFileThreshold(),
1339 Integer.MAX_VALUE);
1340 for (;;) {
1341 try {
1342 if (!sizeQueue.next())
1343 break;
1344 } catch (MissingObjectException notFound) {
1345 monitor.update(1);
1346 if (ignoreMissingUninteresting) {
1347 ObjectToPack otp = sizeQueue.getCurrent();
1348 if (otp != null && otp.isEdge()) {
1349 otp.setDoNotDelta();
1350 continue;
1351 }
1352
1353 otp = objectsMap.get(notFound.getObjectId());
1354 if (otp != null && otp.isEdge()) {
1355 otp.setDoNotDelta();
1356 continue;
1357 }
1358 }
1359 throw notFound;
1360 }
1361
1362 ObjectToPack otp = sizeQueue.getCurrent();
1363 if (otp == null)
1364 otp = objectsMap.get(sizeQueue.getObjectId());
1365
1366 long sz = sizeQueue.getSize();
1367 if (DeltaIndex.BLKSZ < sz && sz < limit)
1368 otp.setWeight((int) sz);
1369 else
1370 otp.setDoNotDelta();
1371 monitor.update(1);
1372 }
1373 } finally {
1374 sizeQueue.release();
1375 }
1376 endPhase(monitor);
1377 stats.timeSearchingForSizes = System.currentTimeMillis() - sizingStart;
1378
1379
1380
1381
1382
1383
1384 Arrays.sort(list, 0, cnt, new Comparator<ObjectToPack>() {
1385 @Override
1386 public int compare(ObjectToPack a, ObjectToPack b) {
1387 int cmp = (a.isDoNotDelta() ? 1 : 0)
1388 - (b.isDoNotDelta() ? 1 : 0);
1389 if (cmp != 0)
1390 return cmp;
1391
1392 cmp = a.getType() - b.getType();
1393 if (cmp != 0)
1394 return cmp;
1395
1396 cmp = (a.getPathHash() >>> 1) - (b.getPathHash() >>> 1);
1397 if (cmp != 0)
1398 return cmp;
1399
1400 cmp = (a.getPathHash() & 1) - (b.getPathHash() & 1);
1401 if (cmp != 0)
1402 return cmp;
1403
1404 cmp = (a.isEdge() ? 0 : 1) - (b.isEdge() ? 0 : 1);
1405 if (cmp != 0)
1406 return cmp;
1407
1408 return b.getWeight() - a.getWeight();
1409 }
1410 });
1411
1412
1413
1414 while (0 < cnt && list[cnt - 1].isDoNotDelta()) {
1415 if (!list[cnt - 1].isEdge())
1416 nonEdgeCnt--;
1417 cnt--;
1418 }
1419 if (cnt == 0)
1420 return;
1421
1422 final long searchStart = System.currentTimeMillis();
1423 searchForDeltas(monitor, list, cnt);
1424 stats.deltaSearchNonEdgeObjects = nonEdgeCnt;
1425 stats.timeCompressing = System.currentTimeMillis() - searchStart;
1426
1427 for (int i = 0; i < cnt; i++)
1428 if (!list[i].isEdge() && list[i].isDeltaRepresentation())
1429 stats.deltasFound++;
1430 }
1431
1432 private int findObjectsNeedingDelta(ObjectToPack[] list, int cnt, int type) {
1433 for (ObjectToPack otp : objectsLists[type]) {
1434 if (otp.isDoNotDelta())
1435 continue;
1436 if (otp.isDeltaRepresentation())
1437 continue;
1438 otp.setWeight(0);
1439 list[cnt++] = otp;
1440 }
1441 return cnt;
1442 }
1443
1444 private void reselectNonDelta(ObjectToPack otp) throws IOException {
1445 otp.clearDeltaBase();
1446 otp.clearReuseAsIs();
1447 boolean old = reuseDeltas;
1448 reuseDeltas = false;
1449 reuseSupport.selectObjectRepresentation(this,
1450 NullProgressMonitor.INSTANCE,
1451 Collections.singleton(otp));
1452 reuseDeltas = old;
1453 }
1454
1455 private void searchForDeltas(final ProgressMonitor monitor,
1456 final ObjectToPack[] list, final int cnt)
1457 throws MissingObjectException, IncorrectObjectTypeException,
1458 LargeObjectException, IOException {
1459 int threads = config.getThreads();
1460 if (threads == 0)
1461 threads = Runtime.getRuntime().availableProcessors();
1462 if (threads <= 1 || cnt <= config.getDeltaSearchWindowSize())
1463 singleThreadDeltaSearch(monitor, list, cnt);
1464 else
1465 parallelDeltaSearch(monitor, list, cnt, threads);
1466 }
1467
1468 private void singleThreadDeltaSearch(ProgressMonitor monitor,
1469 ObjectToPack[] list, int cnt) throws IOException {
1470 long totalWeight = 0;
1471 for (int i = 0; i < cnt; i++) {
1472 ObjectToPack o = list[i];
1473 totalWeight += DeltaTask.getAdjustedWeight(o);
1474 }
1475
1476 long bytesPerUnit = 1;
1477 while (DeltaTask.MAX_METER <= (totalWeight / bytesPerUnit))
1478 bytesPerUnit <<= 10;
1479 int cost = (int) (totalWeight / bytesPerUnit);
1480 if (totalWeight % bytesPerUnit != 0)
1481 cost++;
1482
1483 beginPhase(PackingPhase.COMPRESSING, monitor, cost);
1484 new DeltaWindow(config, new DeltaCache(config), reader,
1485 monitor, bytesPerUnit,
1486 list, 0, cnt).search();
1487 endPhase(monitor);
1488 }
1489
1490 private void parallelDeltaSearch(ProgressMonitor monitor,
1491 ObjectToPack[] list, int cnt, int threads) throws IOException {
1492 DeltaCache dc = new ThreadSafeDeltaCache(config);
1493 ThreadSafeProgressMonitor pm = new ThreadSafeProgressMonitor(monitor);
1494 DeltaTask.Block taskBlock = new DeltaTask.Block(threads, config,
1495 reader, dc, pm,
1496 list, 0, cnt);
1497 taskBlock.partitionTasks();
1498 beginPhase(PackingPhase.COMPRESSING, monitor, taskBlock.cost());
1499 pm.startWorkers(taskBlock.tasks.size());
1500
1501 Executor executor = config.getExecutor();
1502 final List<Throwable> errors =
1503 Collections.synchronizedList(new ArrayList<Throwable>(threads));
1504 if (executor instanceof ExecutorService) {
1505
1506 runTasks((ExecutorService) executor, pm, taskBlock, errors);
1507 } else if (executor == null) {
1508
1509
1510 ExecutorService pool = Executors.newFixedThreadPool(threads);
1511 try {
1512 runTasks(pool, pm, taskBlock, errors);
1513 } finally {
1514 pool.shutdown();
1515 for (;;) {
1516 try {
1517 if (pool.awaitTermination(60, TimeUnit.SECONDS))
1518 break;
1519 } catch (InterruptedException e) {
1520 throw new IOException(
1521 JGitText.get().packingCancelledDuringObjectsWriting);
1522 }
1523 }
1524 }
1525 } else {
1526
1527
1528
1529 for (DeltaTask task : taskBlock.tasks) {
1530 executor.execute(new Runnable() {
1531 @Override
1532 public void run() {
1533 try {
1534 task.call();
1535 } catch (Throwable failure) {
1536 errors.add(failure);
1537 }
1538 }
1539 });
1540 }
1541 try {
1542 pm.waitForCompletion();
1543 } catch (InterruptedException ie) {
1544
1545
1546
1547 throw new IOException(
1548 JGitText.get().packingCancelledDuringObjectsWriting);
1549 }
1550 }
1551
1552
1553
1554
1555 if (!errors.isEmpty()) {
1556 Throwable err = errors.get(0);
1557 if (err instanceof Error)
1558 throw (Error) err;
1559 if (err instanceof RuntimeException)
1560 throw (RuntimeException) err;
1561 if (err instanceof IOException)
1562 throw (IOException) err;
1563
1564 throw new IOException(err.getMessage(), err);
1565 }
1566 endPhase(monitor);
1567 }
1568
1569 private static void runTasks(ExecutorService pool,
1570 ThreadSafeProgressMonitor pm,
1571 DeltaTask.Block tb, List<Throwable> errors) throws IOException {
1572 List<Future<?>> futures = new ArrayList<>(tb.tasks.size());
1573 for (DeltaTask task : tb.tasks)
1574 futures.add(pool.submit(task));
1575
1576 try {
1577 pm.waitForCompletion();
1578 for (Future<?> f : futures) {
1579 try {
1580 f.get();
1581 } catch (ExecutionException failed) {
1582 errors.add(failed.getCause());
1583 }
1584 }
1585 } catch (InterruptedException ie) {
1586 for (Future<?> f : futures)
1587 f.cancel(true);
1588 throw new IOException(
1589 JGitText.get().packingCancelledDuringObjectsWriting);
1590 }
1591 }
1592
1593 private void writeObjects(PackOutputStream out) throws IOException {
1594 writeObjects(out, objectsLists[OBJ_COMMIT]);
1595 writeObjects(out, objectsLists[OBJ_TAG]);
1596 writeObjects(out, objectsLists[OBJ_TREE]);
1597 writeObjects(out, objectsLists[OBJ_BLOB]);
1598 }
1599
1600 private void writeObjects(PackOutputStream out, List<ObjectToPack> list)
1601 throws IOException {
1602 if (list.isEmpty())
1603 return;
1604
1605 typeStats = stats.objectTypes[list.get(0).getType()];
1606 long beginOffset = out.length();
1607
1608 if (reuseSupport != null) {
1609 reuseSupport.writeObjects(out, list);
1610 } else {
1611 for (ObjectToPack otp : list)
1612 out.writeObject(otp);
1613 }
1614
1615 typeStats.bytes += out.length() - beginOffset;
1616 typeStats.cntObjects = list.size();
1617 }
1618
1619 void writeObject(PackOutputStream out, ObjectToPack otp) throws IOException {
1620 if (!otp.isWritten())
1621 writeObjectImpl(out, otp);
1622 }
1623
1624 private void writeObjectImpl(PackOutputStream out, ObjectToPack otp)
1625 throws IOException {
1626 if (otp.wantWrite()) {
1627
1628
1629
1630
1631
1632 reselectNonDelta(otp);
1633 }
1634 otp.markWantWrite();
1635
1636 while (otp.isReuseAsIs()) {
1637 writeBase(out, otp.getDeltaBase());
1638 if (otp.isWritten())
1639 return;
1640
1641 crc32.reset();
1642 otp.setOffset(out.length());
1643 try {
1644 reuseSupport.copyObjectAsIs(out, otp, reuseValidate);
1645 out.endObject();
1646 otp.setCRC((int) crc32.getValue());
1647 typeStats.reusedObjects++;
1648 if (otp.isDeltaRepresentation()) {
1649 typeStats.reusedDeltas++;
1650 typeStats.deltaBytes += out.length() - otp.getOffset();
1651 }
1652 return;
1653 } catch (StoredObjectRepresentationNotAvailableException gone) {
1654 if (otp.getOffset() == out.length()) {
1655 otp.setOffset(0);
1656 otp.clearDeltaBase();
1657 otp.clearReuseAsIs();
1658 reuseSupport.selectObjectRepresentation(this,
1659 NullProgressMonitor.INSTANCE,
1660 Collections.singleton(otp));
1661 continue;
1662 } else {
1663
1664
1665 CorruptObjectException coe;
1666 coe = new CorruptObjectException(otp, "");
1667 coe.initCause(gone);
1668 throw coe;
1669 }
1670 }
1671 }
1672
1673
1674
1675 if (otp.isDeltaRepresentation())
1676 writeDeltaObjectDeflate(out, otp);
1677 else
1678 writeWholeObjectDeflate(out, otp);
1679 out.endObject();
1680 otp.setCRC((int) crc32.getValue());
1681 }
1682
1683 private void writeBase(PackOutputStream out, ObjectToPack base)
1684 throws IOException {
1685 if (base != null && !base.isWritten() && !base.isEdge())
1686 writeObjectImpl(out, base);
1687 }
1688
1689 private void writeWholeObjectDeflate(PackOutputStream out,
1690 final ObjectToPack otp) throws IOException {
1691 final Deflater deflater = deflater();
1692 final ObjectLoader ldr = reader.open(otp, otp.getType());
1693
1694 crc32.reset();
1695 otp.setOffset(out.length());
1696 out.writeHeader(otp, ldr.getSize());
1697
1698 deflater.reset();
1699 DeflaterOutputStream dst = new DeflaterOutputStream(out, deflater);
1700 ldr.copyTo(dst);
1701 dst.finish();
1702 }
1703
1704 private void writeDeltaObjectDeflate(PackOutputStream out,
1705 final ObjectToPack otp) throws IOException {
1706 writeBase(out, otp.getDeltaBase());
1707
1708 crc32.reset();
1709 otp.setOffset(out.length());
1710
1711 DeltaCache.Ref ref = otp.popCachedDelta();
1712 if (ref != null) {
1713 byte[] zbuf = ref.get();
1714 if (zbuf != null) {
1715 out.writeHeader(otp, otp.getCachedSize());
1716 out.write(zbuf);
1717 typeStats.cntDeltas++;
1718 typeStats.deltaBytes += out.length() - otp.getOffset();
1719 return;
1720 }
1721 }
1722
1723 try (TemporaryBuffer.Heap delta = delta(otp)) {
1724 out.writeHeader(otp, delta.length());
1725
1726 Deflater deflater = deflater();
1727 deflater.reset();
1728 DeflaterOutputStream dst = new DeflaterOutputStream(out, deflater);
1729 delta.writeTo(dst, null);
1730 dst.finish();
1731 }
1732 typeStats.cntDeltas++;
1733 typeStats.deltaBytes += out.length() - otp.getOffset();
1734 }
1735
1736 private TemporaryBuffer.Heap delta(ObjectToPack otp)
1737 throws IOException {
1738 DeltaIndex index = new DeltaIndex(buffer(otp.getDeltaBaseId()));
1739 byte[] res = buffer(otp);
1740
1741
1742
1743
1744
1745 TemporaryBuffer.Heap delta = new TemporaryBuffer.Heap(res.length);
1746 index.encode(delta, res);
1747 return delta;
1748 }
1749
1750 private byte[] buffer(AnyObjectId objId) throws IOException {
1751 return buffer(config, reader, objId);
1752 }
1753
1754 static byte[] buffer(PackConfig config, ObjectReader or, AnyObjectId objId)
1755 throws IOException {
1756
1757
1758
1759
1760
1761 return or.open(objId).getCachedBytes(config.getBigFileThreshold());
1762 }
1763
1764 private Deflater deflater() {
1765 if (myDeflater == null)
1766 myDeflater = new Deflater(config.getCompressionLevel());
1767 return myDeflater;
1768 }
1769
1770 private void writeChecksum(PackOutputStream out) throws IOException {
1771 packcsum = out.getDigest();
1772 out.write(packcsum);
1773 }
1774
1775 private void findObjectsToPack(@NonNull ProgressMonitor countingMonitor,
1776 @NonNull ObjectWalk walker, @NonNull Set<? extends ObjectId> want,
1777 @NonNull Set<? extends ObjectId> have,
1778 @NonNull Set<? extends ObjectId> noBitmaps) throws IOException {
1779 final long countingStart = System.currentTimeMillis();
1780 beginPhase(PackingPhase.COUNTING, countingMonitor, ProgressMonitor.UNKNOWN);
1781
1782 stats.interestingObjects = Collections.unmodifiableSet(new HashSet<ObjectId>(want));
1783 stats.uninterestingObjects = Collections.unmodifiableSet(new HashSet<ObjectId>(have));
1784 excludeFromBitmapSelection = noBitmaps;
1785
1786 canBuildBitmaps = config.isBuildBitmaps()
1787 && !shallowPack
1788 && have.isEmpty()
1789 && (excludeInPacks == null || excludeInPacks.length == 0);
1790 if (!shallowPack && useBitmaps) {
1791 BitmapIndex bitmapIndex = reader.getBitmapIndex();
1792 if (bitmapIndex != null) {
1793 BitmapWalker bitmapWalker = new BitmapWalker(
1794 walker, bitmapIndex, countingMonitor);
1795 findObjectsToPackUsingBitmaps(bitmapWalker, want, have);
1796 endPhase(countingMonitor);
1797 stats.timeCounting = System.currentTimeMillis() - countingStart;
1798 stats.bitmapIndexMisses = bitmapWalker.getCountOfBitmapIndexMisses();
1799 return;
1800 }
1801 }
1802
1803 List<ObjectId> all = new ArrayList<>(want.size() + have.size());
1804 all.addAll(want);
1805 all.addAll(have);
1806
1807 final RevFlag include = walker.newFlag("include");
1808 final RevFlag added = walker.newFlag("added");
1809
1810 walker.carry(include);
1811
1812 int haveEst = have.size();
1813 if (have.isEmpty()) {
1814 walker.sort(RevSort.COMMIT_TIME_DESC);
1815 } else {
1816 walker.sort(RevSort.TOPO);
1817 if (thin)
1818 walker.sort(RevSort.BOUNDARY, true);
1819 }
1820
1821 List<RevObject> wantObjs = new ArrayList<>(want.size());
1822 List<RevObject> haveObjs = new ArrayList<>(haveEst);
1823 List<RevTag> wantTags = new ArrayList<>(want.size());
1824
1825
1826
1827 AsyncRevObjectQueue q = walker.parseAny(all, true);
1828 try {
1829 for (;;) {
1830 try {
1831 RevObject o = q.next();
1832 if (o == null)
1833 break;
1834 if (have.contains(o))
1835 haveObjs.add(o);
1836 if (want.contains(o)) {
1837 o.add(include);
1838 wantObjs.add(o);
1839 if (o instanceof RevTag)
1840 wantTags.add((RevTag) o);
1841 }
1842 } catch (MissingObjectException e) {
1843 if (ignoreMissingUninteresting
1844 && have.contains(e.getObjectId()))
1845 continue;
1846 throw e;
1847 }
1848 }
1849 } finally {
1850 q.release();
1851 }
1852
1853 if (!wantTags.isEmpty()) {
1854 all = new ArrayList<>(wantTags.size());
1855 for (RevTag tag : wantTags)
1856 all.add(tag.getObject());
1857 q = walker.parseAny(all, true);
1858 try {
1859 while (q.next() != null) {
1860
1861 }
1862 } finally {
1863 q.release();
1864 }
1865 }
1866
1867 if (walker instanceof DepthWalk.ObjectWalk) {
1868 DepthWalk.ObjectWalk depthWalk = (DepthWalk.ObjectWalk) walker;
1869 for (RevObject obj : wantObjs) {
1870 depthWalk.markRoot(obj);
1871 }
1872
1873
1874
1875
1876
1877 for (RevObject obj : haveObjs) {
1878 if (obj instanceof RevCommit) {
1879 RevTree t = ((RevCommit) obj).getTree();
1880 depthWalk.markUninteresting(t);
1881 }
1882 }
1883
1884 if (unshallowObjects != null) {
1885 for (ObjectId id : unshallowObjects) {
1886 depthWalk.markUnshallow(walker.parseAny(id));
1887 }
1888 }
1889 } else {
1890 for (RevObject obj : wantObjs)
1891 walker.markStart(obj);
1892 }
1893 for (RevObject obj : haveObjs)
1894 walker.markUninteresting(obj);
1895
1896 final int maxBases = config.getDeltaSearchWindowSize();
1897 Set<RevTree> baseTrees = new HashSet<>();
1898 BlockList<RevCommit> commits = new BlockList<>();
1899 Set<ObjectId> roots = new HashSet<>();
1900 RevCommit c;
1901 while ((c = walker.next()) != null) {
1902 if (exclude(c))
1903 continue;
1904 if (c.has(RevFlag.UNINTERESTING)) {
1905 if (baseTrees.size() <= maxBases)
1906 baseTrees.add(c.getTree());
1907 continue;
1908 }
1909
1910 commits.add(c);
1911 if (c.getParentCount() == 0) {
1912 roots.add(c.copy());
1913 }
1914 countingMonitor.update(1);
1915 }
1916 stats.rootCommits = Collections.unmodifiableSet(roots);
1917
1918 if (shallowPack) {
1919 for (RevCommit cmit : commits) {
1920 addObject(cmit, 0);
1921 }
1922 } else {
1923 int commitCnt = 0;
1924 boolean putTagTargets = false;
1925 for (RevCommit cmit : commits) {
1926 if (!cmit.has(added)) {
1927 cmit.add(added);
1928 addObject(cmit, 0);
1929 commitCnt++;
1930 }
1931
1932 for (int i = 0; i < cmit.getParentCount(); i++) {
1933 RevCommit p = cmit.getParent(i);
1934 if (!p.has(added) && !p.has(RevFlag.UNINTERESTING)
1935 && !exclude(p)) {
1936 p.add(added);
1937 addObject(p, 0);
1938 commitCnt++;
1939 }
1940 }
1941
1942 if (!putTagTargets && 4096 < commitCnt) {
1943 for (ObjectId id : tagTargets) {
1944 RevObject obj = walker.lookupOrNull(id);
1945 if (obj instanceof RevCommit
1946 && obj.has(include)
1947 && !obj.has(RevFlag.UNINTERESTING)
1948 && !obj.has(added)) {
1949 obj.add(added);
1950 addObject(obj, 0);
1951 }
1952 }
1953 putTagTargets = true;
1954 }
1955 }
1956 }
1957 commits = null;
1958
1959 if (thin && !baseTrees.isEmpty()) {
1960 BaseSearch bases = new BaseSearch(countingMonitor, baseTrees,
1961 objectsMap, edgeObjects, reader);
1962 RevObject o;
1963 while ((o = walker.nextObject()) != null) {
1964 if (o.has(RevFlag.UNINTERESTING))
1965 continue;
1966 if (exclude(o))
1967 continue;
1968
1969 int pathHash = walker.getPathHashCode();
1970 byte[] pathBuf = walker.getPathBuffer();
1971 int pathLen = walker.getPathLength();
1972 bases.addBase(o.getType(), pathBuf, pathLen, pathHash);
1973 filterAndAddObject(o, o.getType(), pathHash);
1974 countingMonitor.update(1);
1975 }
1976 } else {
1977 RevObject o;
1978 while ((o = walker.nextObject()) != null) {
1979 if (o.has(RevFlag.UNINTERESTING))
1980 continue;
1981 if (exclude(o))
1982 continue;
1983 filterAndAddObject(o, o.getType(), walker.getPathHashCode());
1984 countingMonitor.update(1);
1985 }
1986 }
1987
1988 for (CachedPack pack : cachedPacks)
1989 countingMonitor.update((int) pack.getObjectCount());
1990 endPhase(countingMonitor);
1991 stats.timeCounting = System.currentTimeMillis() - countingStart;
1992 stats.bitmapIndexMisses = -1;
1993 }
1994
1995 private void findObjectsToPackUsingBitmaps(
1996 BitmapWalker bitmapWalker, Set<? extends ObjectId> want,
1997 Set<? extends ObjectId> have)
1998 throws MissingObjectException, IncorrectObjectTypeException,
1999 IOException {
2000 BitmapBuilder haveBitmap = bitmapWalker.findObjects(have, null, true);
2001 BitmapBuilder wantBitmap = bitmapWalker.findObjects(want, haveBitmap,
2002 false);
2003 BitmapBuilder needBitmap = wantBitmap.andNot(haveBitmap);
2004
2005 if (useCachedPacks && reuseSupport != null && !reuseValidate
2006 && (excludeInPacks == null || excludeInPacks.length == 0))
2007 cachedPacks.addAll(
2008 reuseSupport.getCachedPacksAndUpdate(needBitmap));
2009
2010 for (BitmapObject obj : needBitmap) {
2011 ObjectId objectId = obj.getObjectId();
2012 if (exclude(objectId)) {
2013 needBitmap.remove(objectId);
2014 continue;
2015 }
2016 filterAndAddObject(objectId, obj.getType(), 0);
2017 }
2018
2019 if (thin)
2020 haveObjects = haveBitmap;
2021 }
2022
2023 private static void pruneEdgesFromObjectList(List<ObjectToPack> list) {
2024 final int size = list.size();
2025 int src = 0;
2026 int dst = 0;
2027
2028 for (; src < size; src++) {
2029 ObjectToPack obj = list.get(src);
2030 if (obj.isEdge())
2031 continue;
2032 if (dst != src)
2033 list.set(dst, obj);
2034 dst++;
2035 }
2036
2037 while (dst < list.size())
2038 list.remove(list.size() - 1);
2039 }
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053 public void addObject(RevObject object)
2054 throws IncorrectObjectTypeException {
2055 if (!exclude(object))
2056 addObject(object, 0);
2057 }
2058
2059 private void addObject(RevObject object, int pathHashCode) {
2060 addObject(object, object.getType(), pathHashCode);
2061 }
2062
2063 private void addObject(
2064 final AnyObjectId src, final int type, final int pathHashCode) {
2065 final ObjectToPack otp;
2066 if (reuseSupport != null)
2067 otp = reuseSupport.newObjectToPack(src, type);
2068 else
2069 otp = new ObjectToPack(src, type);
2070 otp.setPathHash(pathHashCode);
2071 objectsLists[type].add(otp);
2072 objectsMap.add(otp);
2073 }
2074
2075
2076
2077 private void filterAndAddObject(@NonNull AnyObjectId src, int type,
2078 int pathHashCode) throws IOException {
2079
2080
2081
2082 boolean reject = filterBlobLimit >= 0 &&
2083 type == OBJ_BLOB &&
2084 reader.getObjectSize(src, OBJ_BLOB) > filterBlobLimit;
2085 if (!reject) {
2086 addObject(src, type, pathHashCode);
2087 }
2088 }
2089
2090 private boolean exclude(AnyObjectId objectId) {
2091 if (excludeInPacks == null)
2092 return false;
2093 if (excludeInPackLast.contains(objectId))
2094 return true;
2095 for (ObjectIdSet idx : excludeInPacks) {
2096 if (idx.contains(objectId)) {
2097 excludeInPackLast = idx;
2098 return true;
2099 }
2100 }
2101 return false;
2102 }
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116 public void select(ObjectToPack otp, StoredObjectRepresentation next) {
2117 int nFmt = next.getFormat();
2118
2119 if (!cachedPacks.isEmpty()) {
2120 if (otp.isEdge())
2121 return;
2122 if ((nFmt == PACK_WHOLE) | (nFmt == PACK_DELTA)) {
2123 for (CachedPack pack : cachedPacks) {
2124 if (pack.hasObject(otp, next)) {
2125 otp.setEdge();
2126 otp.clearDeltaBase();
2127 otp.clearReuseAsIs();
2128 pruneCurrentObjectList = true;
2129 return;
2130 }
2131 }
2132 }
2133 }
2134
2135 if (nFmt == PACK_DELTA && reuseDeltas && reuseDeltaFor(otp)) {
2136 ObjectId baseId = next.getDeltaBase();
2137 ObjectToPack ptr = objectsMap.get(baseId);
2138 if (ptr != null && !ptr.isEdge()) {
2139 otp.setDeltaBase(ptr);
2140 otp.setReuseAsIs();
2141 } else if (thin && have(ptr, baseId)) {
2142 otp.setDeltaBase(baseId);
2143 otp.setReuseAsIs();
2144 } else {
2145 otp.clearDeltaBase();
2146 otp.clearReuseAsIs();
2147 }
2148 } else if (nFmt == PACK_WHOLE && config.isReuseObjects()) {
2149 int nWeight = next.getWeight();
2150 if (otp.isReuseAsIs() && !otp.isDeltaRepresentation()) {
2151
2152
2153
2154 if (otp.getWeight() <= nWeight)
2155 return;
2156 }
2157 otp.clearDeltaBase();
2158 otp.setReuseAsIs();
2159 otp.setWeight(nWeight);
2160 } else {
2161 otp.clearDeltaBase();
2162 otp.clearReuseAsIs();
2163 }
2164
2165 otp.setDeltaAttempted(reuseDeltas & next.wasDeltaAttempted());
2166 otp.select(next);
2167 }
2168
2169 private final boolean have(ObjectToPack ptr, AnyObjectId objectId) {
2170 return (ptr != null && ptr.isEdge())
2171 || (haveObjects != null && haveObjects.contains(objectId));
2172 }
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193 public boolean prepareBitmapIndex(ProgressMonitor pm) throws IOException {
2194 if (!canBuildBitmaps || getObjectCount() > Integer.MAX_VALUE
2195 || !cachedPacks.isEmpty())
2196 return false;
2197
2198 if (pm == null)
2199 pm = NullProgressMonitor.INSTANCE;
2200
2201 int numCommits = objectsLists[OBJ_COMMIT].size();
2202 List<ObjectToPack> byName = sortByName();
2203 sortedByName = null;
2204 objectsLists = null;
2205 objectsMap = null;
2206 writeBitmaps = new PackBitmapIndexBuilder(byName);
2207 byName = null;
2208
2209 PackWriterBitmapPreparer bitmapPreparer = new PackWriterBitmapPreparer(
2210 reader, writeBitmaps, pm, stats.interestingObjects, config);
2211
2212 Collection<PackWriterBitmapPreparer.BitmapCommit> selectedCommits = bitmapPreparer
2213 .selectCommits(numCommits, excludeFromBitmapSelection);
2214
2215 beginPhase(PackingPhase.BUILDING_BITMAPS, pm, selectedCommits.size());
2216
2217 BitmapWalker walker = bitmapPreparer.newBitmapWalker();
2218 AnyObjectId last = null;
2219 for (PackWriterBitmapPreparer.BitmapCommit cmit : selectedCommits) {
2220 if (!cmit.isReuseWalker()) {
2221 walker = bitmapPreparer.newBitmapWalker();
2222 }
2223 BitmapBuilder bitmap = walker.findObjects(
2224 Collections.singleton(cmit), null, false);
2225
2226 if (last != null && cmit.isReuseWalker() && !bitmap.contains(last))
2227 throw new IllegalStateException(MessageFormat.format(
2228 JGitText.get().bitmapMissingObject, cmit.name(),
2229 last.name()));
2230 last = cmit;
2231 writeBitmaps.addBitmap(cmit, bitmap.build(), cmit.getFlags());
2232
2233 pm.update(1);
2234 }
2235
2236 endPhase(pm);
2237 return true;
2238 }
2239
2240 private boolean reuseDeltaFor(ObjectToPack otp) {
2241 int type = otp.getType();
2242 if ((type & 2) != 0)
2243 return true;
2244 if (type == OBJ_COMMIT)
2245 return reuseDeltaCommits;
2246 if (type == OBJ_TAG)
2247 return false;
2248 return true;
2249 }
2250
2251 private class MutableState {
2252
2253
2254 private static final long OBJECT_TO_PACK_SIZE =
2255 (2 * 8)
2256 + (2 * 8) + (2 * 8)
2257 + (8 + 8)
2258 + 8
2259 + 40
2260 + 8;
2261
2262 private final long totalDeltaSearchBytes;
2263
2264 private volatile PackingPhase phase;
2265
2266 MutableState() {
2267 phase = PackingPhase.COUNTING;
2268 if (config.isDeltaCompress()) {
2269 int threads = config.getThreads();
2270 if (threads <= 0)
2271 threads = Runtime.getRuntime().availableProcessors();
2272 totalDeltaSearchBytes = (threads * config.getDeltaSearchMemoryLimit())
2273 + config.getBigFileThreshold();
2274 } else
2275 totalDeltaSearchBytes = 0;
2276 }
2277
2278 State snapshot() {
2279 long objCnt = 0;
2280 BlockList<ObjectToPack>[] lists = objectsLists;
2281 if (lists != null) {
2282 objCnt += lists[OBJ_COMMIT].size();
2283 objCnt += lists[OBJ_TREE].size();
2284 objCnt += lists[OBJ_BLOB].size();
2285 objCnt += lists[OBJ_TAG].size();
2286
2287 }
2288
2289 long bytesUsed = OBJECT_TO_PACK_SIZE * objCnt;
2290 PackingPhase curr = phase;
2291 if (curr == PackingPhase.COMPRESSING)
2292 bytesUsed += totalDeltaSearchBytes;
2293 return new State(curr, bytesUsed);
2294 }
2295 }
2296
2297
2298 public static enum PackingPhase {
2299
2300 COUNTING,
2301
2302
2303 GETTING_SIZES,
2304
2305
2306 FINDING_SOURCES,
2307
2308
2309 COMPRESSING,
2310
2311
2312 WRITING,
2313
2314
2315 BUILDING_BITMAPS;
2316 }
2317
2318
2319 public class State {
2320 private final PackingPhase phase;
2321
2322 private final long bytesUsed;
2323
2324 State(PackingPhase phase, long bytesUsed) {
2325 this.phase = phase;
2326 this.bytesUsed = bytesUsed;
2327 }
2328
2329
2330 public PackConfig getConfig() {
2331 return config;
2332 }
2333
2334
2335 public PackingPhase getPhase() {
2336 return phase;
2337 }
2338
2339
2340 public long estimateBytesUsed() {
2341 return bytesUsed;
2342 }
2343
2344 @SuppressWarnings("nls")
2345 @Override
2346 public String toString() {
2347 return "PackWriter.State[" + phase + ", memory=" + bytesUsed + "]";
2348 }
2349 }
2350 }