1
2
3
4
5
6
7
8
9
10
11
12 package org.eclipse.jgit.internal.storage.pack;
13
14 import static java.util.Objects.requireNonNull;
15 import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_DELTA;
16 import static org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation.PACK_WHOLE;
17 import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH;
18 import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
19 import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT;
20 import static org.eclipse.jgit.lib.Constants.OBJ_TAG;
21 import static org.eclipse.jgit.lib.Constants.OBJ_TREE;
22
23 import java.io.IOException;
24 import java.io.OutputStream;
25 import java.lang.ref.WeakReference;
26 import java.security.MessageDigest;
27 import java.text.MessageFormat;
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.Collection;
31 import java.util.Collections;
32 import java.util.HashMap;
33 import java.util.HashSet;
34 import java.util.Iterator;
35 import java.util.List;
36 import java.util.Map;
37 import java.util.NoSuchElementException;
38 import java.util.Set;
39 import java.util.concurrent.ConcurrentHashMap;
40 import java.util.concurrent.ExecutionException;
41 import java.util.concurrent.Executor;
42 import java.util.concurrent.ExecutorService;
43 import java.util.concurrent.Executors;
44 import java.util.concurrent.Future;
45 import java.util.concurrent.TimeUnit;
46 import java.util.zip.CRC32;
47 import java.util.zip.CheckedOutputStream;
48 import java.util.zip.Deflater;
49 import java.util.zip.DeflaterOutputStream;
50
51 import org.eclipse.jgit.annotations.NonNull;
52 import org.eclipse.jgit.annotations.Nullable;
53 import org.eclipse.jgit.errors.CorruptObjectException;
54 import org.eclipse.jgit.errors.IncorrectObjectTypeException;
55 import org.eclipse.jgit.errors.LargeObjectException;
56 import org.eclipse.jgit.errors.MissingObjectException;
57 import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
58 import org.eclipse.jgit.internal.JGitText;
59 import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder;
60 import org.eclipse.jgit.internal.storage.file.PackBitmapIndexWriterV1;
61 import org.eclipse.jgit.internal.storage.file.PackIndexWriter;
62 import org.eclipse.jgit.lib.AnyObjectId;
63 import org.eclipse.jgit.lib.AsyncObjectSizeQueue;
64 import org.eclipse.jgit.lib.BatchingProgressMonitor;
65 import org.eclipse.jgit.lib.BitmapIndex;
66 import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder;
67 import org.eclipse.jgit.lib.BitmapObject;
68 import org.eclipse.jgit.lib.Constants;
69 import org.eclipse.jgit.lib.NullProgressMonitor;
70 import org.eclipse.jgit.lib.ObjectId;
71 import org.eclipse.jgit.lib.ObjectIdOwnerMap;
72 import org.eclipse.jgit.lib.ObjectIdSet;
73 import org.eclipse.jgit.lib.ObjectLoader;
74 import org.eclipse.jgit.lib.ObjectReader;
75 import org.eclipse.jgit.lib.ProgressMonitor;
76 import org.eclipse.jgit.lib.Repository;
77 import org.eclipse.jgit.lib.ThreadSafeProgressMonitor;
78 import org.eclipse.jgit.revwalk.AsyncRevObjectQueue;
79 import org.eclipse.jgit.revwalk.BitmapWalker;
80 import org.eclipse.jgit.revwalk.DepthWalk;
81 import org.eclipse.jgit.revwalk.ObjectWalk;
82 import org.eclipse.jgit.revwalk.RevCommit;
83 import org.eclipse.jgit.revwalk.RevFlag;
84 import org.eclipse.jgit.revwalk.RevObject;
85 import org.eclipse.jgit.revwalk.RevSort;
86 import org.eclipse.jgit.revwalk.RevTag;
87 import org.eclipse.jgit.revwalk.RevTree;
88 import org.eclipse.jgit.storage.pack.PackConfig;
89 import org.eclipse.jgit.storage.pack.PackStatistics;
90 import org.eclipse.jgit.transport.FilterSpec;
91 import org.eclipse.jgit.transport.ObjectCountCallback;
92 import org.eclipse.jgit.transport.PacketLineOut;
93 import org.eclipse.jgit.transport.WriteAbortedException;
94 import org.eclipse.jgit.util.BlockList;
95 import org.eclipse.jgit.util.TemporaryBuffer;
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136 public class PackWriter implements AutoCloseable {
137 private static final int PACK_VERSION_GENERATED = 2;
138
139
140 public static final Set<ObjectId> NONE = Collections.emptySet();
141
142 private static final Map<WeakReference<PackWriter>, Boolean> instances =
143 new ConcurrentHashMap<>();
144
145 private static final Iterable<PackWriter> instancesIterable = () -> new Iterator<PackWriter>() {
146
147 private final Iterator<WeakReference<PackWriter>> it = instances
148 .keySet().iterator();
149
150 private PackWriter next;
151
152 @Override
153 public boolean hasNext() {
154 if (next != null) {
155 return true;
156 }
157 while (it.hasNext()) {
158 WeakReference<PackWriter> ref = it.next();
159 next = ref.get();
160 if (next != null) {
161 return true;
162 }
163 it.remove();
164 }
165 return false;
166 }
167
168 @Override
169 public PackWriter next() {
170 if (hasNext()) {
171 PackWriter result = next;
172 next = null;
173 return result;
174 }
175 throw new NoSuchElementException();
176 }
177
178 @Override
179 public void remove() {
180 throw new UnsupportedOperationException();
181 }
182 };
183
184
185
186
187
188
189 public static Iterable<PackWriter> getInstances() {
190 return instancesIterable;
191 }
192
193 @SuppressWarnings("unchecked")
194 BlockList<ObjectToPack>[] objectsLists = new BlockList[OBJ_TAG + 1];
195 {
196 objectsLists[OBJ_COMMIT] = new BlockList<>();
197 objectsLists[OBJ_TREE] = new BlockList<>();
198 objectsLists[OBJ_BLOB] = new BlockList<>();
199 objectsLists[OBJ_TAG] = new BlockList<>();
200 }
201
202 private ObjectIdOwnerMap<ObjectToPack> objectsMap = new ObjectIdOwnerMap<>();
203
204
205 private List<ObjectToPack> edgeObjects = new BlockList<>();
206
207
208 private BitmapBuilder haveObjects;
209
210 private List<CachedPack> cachedPacks = new ArrayList<>(2);
211
212 private Set<ObjectId> tagTargets = NONE;
213
214 private Set<? extends ObjectId> excludeFromBitmapSelection = NONE;
215
216 private ObjectIdSet[] excludeInPacks;
217
218 private ObjectIdSet excludeInPackLast;
219
220 private Deflater myDeflater;
221
222 private final ObjectReader reader;
223
224
225 private final ObjectReuseAsIs reuseSupport;
226
227 final PackConfig config;
228
229 private final PackStatistics.Accumulator stats;
230
231 private final MutableState state;
232
233 private final WeakReference<PackWriter> selfRef;
234
235 private PackStatistics.ObjectType.Accumulator typeStats;
236
237 private List<ObjectToPack> sortedByName;
238
239 private byte[] packcsum;
240
241 private boolean deltaBaseAsOffset;
242
243 private boolean reuseDeltas;
244
245 private boolean reuseDeltaCommits;
246
247 private boolean reuseValidate;
248
249 private boolean thin;
250
251 private boolean useCachedPacks;
252
253 private boolean useBitmaps;
254
255 private boolean ignoreMissingUninteresting = true;
256
257 private boolean pruneCurrentObjectList;
258
259 private boolean shallowPack;
260
261 private boolean canBuildBitmaps;
262
263 private boolean indexDisabled;
264
265 private int depth;
266
267 private Collection<? extends ObjectId> unshallowObjects;
268
269 private PackBitmapIndexBuilder writeBitmaps;
270
271 private CRC32 crc32;
272
273 private ObjectCountCallback callback;
274
275 private FilterSpec filterSpec = FilterSpec.NO_FILTER;
276
277 private PackfileUriConfig packfileUriConfig;
278
279
280
281
282
283
284
285
286
287
288 public PackWriter(Repository repo) {
289 this(repo, repo.newObjectReader());
290 }
291
292
293
294
295
296
297
298
299
300
301 public PackWriter(ObjectReader reader) {
302 this(new PackConfig(), reader);
303 }
304
305
306
307
308
309
310
311
312
313
314
315
316 public PackWriter(Repository repo, ObjectReader reader) {
317 this(new PackConfig(repo), reader);
318 }
319
320
321
322
323
324
325
326
327
328
329
330
331 public PackWriter(PackConfig config, ObjectReader reader) {
332 this(config, reader, null);
333 }
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348 public PackWriter(PackConfig config, final ObjectReader reader,
349 @Nullable PackStatistics.Accumulator statsAccumulator) {
350 this.config = config;
351 this.reader = reader;
352 if (reader instanceof ObjectReuseAsIs)
353 reuseSupport = ((ObjectReuseAsIs) reader);
354 else
355 reuseSupport = null;
356
357 deltaBaseAsOffset = config.isDeltaBaseAsOffset();
358 reuseDeltas = config.isReuseDeltas();
359 reuseValidate = true;
360 stats = statsAccumulator != null ? statsAccumulator
361 : new PackStatistics.Accumulator();
362 state = new MutableState();
363 selfRef = new WeakReference<>(this);
364 instances.put(selfRef, Boolean.TRUE);
365 }
366
367
368
369
370
371
372
373
374
375
376
377 public PackWriter setObjectCountCallback(ObjectCountCallback callback) {
378 this.callback = callback;
379 return this;
380 }
381
382
383
384
385
386
387
388 public void setClientShallowCommits(Set<ObjectId> clientShallowCommits) {
389 stats.clientShallowCommits = Collections
390 .unmodifiableSet(new HashSet<>(clientShallowCommits));
391 }
392
393
394
395
396
397
398
399
400
401
402
403 public boolean isDeltaBaseAsOffset() {
404 return deltaBaseAsOffset;
405 }
406
407
408
409
410
411
412
413
414
415
416
417
418 public void setDeltaBaseAsOffset(boolean deltaBaseAsOffset) {
419 this.deltaBaseAsOffset = deltaBaseAsOffset;
420 }
421
422
423
424
425
426
427
428 public boolean isReuseDeltaCommits() {
429 return reuseDeltaCommits;
430 }
431
432
433
434
435
436
437
438
439 public void setReuseDeltaCommits(boolean reuse) {
440 reuseDeltaCommits = reuse;
441 }
442
443
444
445
446
447
448
449 public boolean isReuseValidatingObjects() {
450 return reuseValidate;
451 }
452
453
454
455
456
457
458
459
460
461
462 public void setReuseValidatingObjects(boolean validate) {
463 reuseValidate = validate;
464 }
465
466
467
468
469
470
471 public boolean isThin() {
472 return thin;
473 }
474
475
476
477
478
479
480
481
482
483
484
485
486 public void setThin(boolean packthin) {
487 thin = packthin;
488 }
489
490
491
492
493
494
495
496 public boolean isUseCachedPacks() {
497 return useCachedPacks;
498 }
499
500
501
502
503
504
505
506
507
508
509
510
511 public void setUseCachedPacks(boolean useCached) {
512 useCachedPacks = useCached;
513 }
514
515
516
517
518
519
520 public boolean isUseBitmaps() {
521 return useBitmaps;
522 }
523
524
525
526
527
528
529
530 public void setUseBitmaps(boolean useBitmaps) {
531 this.useBitmaps = useBitmaps;
532 }
533
534
535
536
537
538
539
540 public boolean isIndexDisabled() {
541 return indexDisabled || !cachedPacks.isEmpty();
542 }
543
544
545
546
547
548
549
550 public void setIndexDisabled(boolean noIndex) {
551 this.indexDisabled = noIndex;
552 }
553
554
555
556
557
558
559
560
561
562
563
564 public boolean isIgnoreMissingUninteresting() {
565 return ignoreMissingUninteresting;
566 }
567
568
569
570
571
572
573
574
575
576
577 public void setIgnoreMissingUninteresting(boolean ignore) {
578 ignoreMissingUninteresting = ignore;
579 }
580
581
582
583
584
585
586
587
588
589
590
591
592
593 public void setTagTargets(Set<ObjectId> objects) {
594 tagTargets = objects;
595 }
596
597
598
599
600
601
602
603
604
605
606
607 public void setShallowPack(int depth,
608 Collection<? extends ObjectId> unshallow) {
609 this.shallowPack = true;
610 this.depth = depth;
611 this.unshallowObjects = unshallow;
612 }
613
614
615
616
617
618 public void setFilterSpec(@NonNull FilterSpec filter) {
619 filterSpec = requireNonNull(filter);
620 }
621
622
623
624
625
626 public void setPackfileUriConfig(PackfileUriConfig config) {
627 packfileUriConfig = config;
628 }
629
630
631
632
633
634
635
636
637 public long getObjectCount() throws IOException {
638 if (stats.totalObjects == 0) {
639 long objCnt = 0;
640
641 objCnt += objectsLists[OBJ_COMMIT].size();
642 objCnt += objectsLists[OBJ_TREE].size();
643 objCnt += objectsLists[OBJ_BLOB].size();
644 objCnt += objectsLists[OBJ_TAG].size();
645
646 for (CachedPack pack : cachedPacks)
647 objCnt += pack.getObjectCount();
648 return objCnt;
649 }
650 return stats.totalObjects;
651 }
652
653 private long getUnoffloadedObjectCount() throws IOException {
654 long objCnt = 0;
655
656 objCnt += objectsLists[OBJ_COMMIT].size();
657 objCnt += objectsLists[OBJ_TREE].size();
658 objCnt += objectsLists[OBJ_BLOB].size();
659 objCnt += objectsLists[OBJ_TAG].size();
660
661 for (CachedPack pack : cachedPacks) {
662 CachedPackUriProvider.PackInfo packInfo =
663 packfileUriConfig.cachedPackUriProvider.getInfo(
664 pack, packfileUriConfig.protocolsSupported);
665 if (packInfo == null) {
666 objCnt += pack.getObjectCount();
667 }
668 }
669
670 return objCnt;
671 }
672
673
674
675
676
677
678
679
680
681
682
683
684 public ObjectIdOwnerMap<ObjectIdOwnerMap.Entry> getObjectSet()
685 throws IOException {
686 if (!cachedPacks.isEmpty())
687 throw new IOException(
688 JGitText.get().cachedPacksPreventsListingObjects);
689
690 if (writeBitmaps != null) {
691 return writeBitmaps.getObjectSet();
692 }
693
694 ObjectIdOwnerMap<ObjectIdOwnerMap.Entry> r = new ObjectIdOwnerMap<>();
695 for (BlockList<ObjectToPack> objList : objectsLists) {
696 if (objList != null) {
697 for (ObjectToPack otp : objList)
698 r.add(new ObjectIdOwnerMap.Entry(otp) {
699
700 });
701 }
702 }
703 return r;
704 }
705
706
707
708
709
710
711
712 public void excludeObjects(ObjectIdSet idx) {
713 if (excludeInPacks == null) {
714 excludeInPacks = new ObjectIdSet[] { idx };
715 excludeInPackLast = idx;
716 } else {
717 int cnt = excludeInPacks.length;
718 ObjectIdSet[] newList = new ObjectIdSet[cnt + 1];
719 System.arraycopy(excludeInPacks, 0, newList, 0, cnt);
720 newList[cnt] = idx;
721 excludeInPacks = newList;
722 }
723 }
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750 public void preparePack(@NonNull Iterator<RevObject> objectsSource)
751 throws IOException {
752 while (objectsSource.hasNext()) {
753 addObject(objectsSource.next());
754 }
755 }
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782 public void preparePack(ProgressMonitor countingMonitor,
783 @NonNull Set<? extends ObjectId> want,
784 @NonNull Set<? extends ObjectId> have) throws IOException {
785 preparePack(countingMonitor, want, have, NONE, NONE);
786 }
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817 public void preparePack(ProgressMonitor countingMonitor,
818 @NonNull Set<? extends ObjectId> want,
819 @NonNull Set<? extends ObjectId> have,
820 @NonNull Set<? extends ObjectId> shallow) throws IOException {
821 preparePack(countingMonitor, want, have, shallow, NONE);
822 }
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856 public void preparePack(ProgressMonitor countingMonitor,
857 @NonNull Set<? extends ObjectId> want,
858 @NonNull Set<? extends ObjectId> have,
859 @NonNull Set<? extends ObjectId> shallow,
860 @NonNull Set<? extends ObjectId> noBitmaps) throws IOException {
861 try (ObjectWalk ow = getObjectWalk()) {
862 ow.assumeShallow(shallow);
863 preparePack(countingMonitor, ow, want, have, noBitmaps);
864 }
865 }
866
867 private ObjectWalk getObjectWalk() {
868 return shallowPack ? new DepthWalk.ObjectWalk(reader, depth - 1)
869 : new ObjectWalk(reader);
870 }
871
872
873
874
875
876
877
878 private static class DepthAwareVisitationPolicy
879 implements ObjectWalk.VisitationPolicy {
880 private final Map<ObjectId, Integer> lowestDepthVisited = new HashMap<>();
881
882 private final ObjectWalk walk;
883
884 DepthAwareVisitationPolicy(ObjectWalk walk) {
885 this.walk = requireNonNull(walk);
886 }
887
888 @Override
889 public boolean shouldVisit(RevObject o) {
890 Integer lastDepth = lowestDepthVisited.get(o);
891 if (lastDepth == null) {
892 return true;
893 }
894 return walk.getTreeDepth() < lastDepth.intValue();
895 }
896
897 @Override
898 public void visited(RevObject o) {
899 lowestDepthVisited.put(o, Integer.valueOf(walk.getTreeDepth()));
900 }
901 }
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933 public void preparePack(ProgressMonitor countingMonitor,
934 @NonNull ObjectWalk walk,
935 @NonNull Set<? extends ObjectId> interestingObjects,
936 @NonNull Set<? extends ObjectId> uninterestingObjects,
937 @NonNull Set<? extends ObjectId> noBitmaps)
938 throws IOException {
939 if (countingMonitor == null)
940 countingMonitor = NullProgressMonitor.INSTANCE;
941 if (shallowPack && !(walk instanceof DepthWalk.ObjectWalk))
942 throw new IllegalArgumentException(
943 JGitText.get().shallowPacksRequireDepthWalk);
944 if (filterSpec.getTreeDepthLimit() >= 0) {
945 walk.setVisitationPolicy(new DepthAwareVisitationPolicy(walk));
946 }
947 findObjectsToPack(countingMonitor, walk, interestingObjects,
948 uninterestingObjects, noBitmaps);
949 }
950
951
952
953
954
955
956
957
958
959
960 public boolean willInclude(AnyObjectId id) throws IOException {
961 ObjectToPack obj = objectsMap.get(id);
962 return obj != null && !obj.isEdge();
963 }
964
965
966
967
968
969
970
971
972 public ObjectToPack get(AnyObjectId id) {
973 ObjectToPack obj = objectsMap.get(id);
974 return obj != null && !obj.isEdge() ? obj : null;
975 }
976
977
978
979
980
981
982
983 public ObjectId computeName() {
984 final byte[] buf = new byte[OBJECT_ID_LENGTH];
985 final MessageDigest md = Constants.newMessageDigest();
986 for (ObjectToPack otp : sortByName()) {
987 otp.copyRawTo(buf, 0);
988 md.update(buf, 0, OBJECT_ID_LENGTH);
989 }
990 return ObjectId.fromRaw(md.digest());
991 }
992
993
994
995
996
997
998
999
1000
1001
1002 public int getIndexVersion() {
1003 int indexVersion = config.getIndexVersion();
1004 if (indexVersion <= 0) {
1005 for (BlockList<ObjectToPack> objs : objectsLists)
1006 indexVersion = Math.max(indexVersion,
1007 PackIndexWriter.oldestPossibleFormat(objs));
1008 }
1009 return indexVersion;
1010 }
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027 public void writeIndex(OutputStream indexStream) throws IOException {
1028 if (isIndexDisabled())
1029 throw new IOException(JGitText.get().cachedPacksPreventsIndexCreation);
1030
1031 long writeStart = System.currentTimeMillis();
1032 final PackIndexWriter iw = PackIndexWriter.createVersion(
1033 indexStream, getIndexVersion());
1034 iw.write(sortByName(), packcsum);
1035 stats.timeWriting += System.currentTimeMillis() - writeStart;
1036 }
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049 public void writeBitmapIndex(OutputStream bitmapIndexStream)
1050 throws IOException {
1051 if (writeBitmaps == null)
1052 throw new IOException(JGitText.get().bitmapsMustBePrepared);
1053
1054 long writeStart = System.currentTimeMillis();
1055 final PackBitmapIndexWriterV1ile/PackBitmapIndexWriterV1.html#PackBitmapIndexWriterV1">PackBitmapIndexWriterV1 iw = new PackBitmapIndexWriterV1(bitmapIndexStream);
1056 iw.write(writeBitmaps, packcsum);
1057 stats.timeWriting += System.currentTimeMillis() - writeStart;
1058 }
1059
1060 private List<ObjectToPack> sortByName() {
1061 if (sortedByName == null) {
1062 int cnt = 0;
1063 cnt += objectsLists[OBJ_COMMIT].size();
1064 cnt += objectsLists[OBJ_TREE].size();
1065 cnt += objectsLists[OBJ_BLOB].size();
1066 cnt += objectsLists[OBJ_TAG].size();
1067
1068 sortedByName = new BlockList<>(cnt);
1069 sortedByName.addAll(objectsLists[OBJ_COMMIT]);
1070 sortedByName.addAll(objectsLists[OBJ_TREE]);
1071 sortedByName.addAll(objectsLists[OBJ_BLOB]);
1072 sortedByName.addAll(objectsLists[OBJ_TAG]);
1073 Collections.sort(sortedByName);
1074 }
1075 return sortedByName;
1076 }
1077
1078 private void beginPhase(PackingPhase phase, ProgressMonitor monitor,
1079 long cnt) {
1080 state.phase = phase;
1081 String task;
1082 switch (phase) {
1083 case COUNTING:
1084 task = JGitText.get().countingObjects;
1085 break;
1086 case GETTING_SIZES:
1087 task = JGitText.get().searchForSizes;
1088 break;
1089 case FINDING_SOURCES:
1090 task = JGitText.get().searchForReuse;
1091 break;
1092 case COMPRESSING:
1093 task = JGitText.get().compressingObjects;
1094 break;
1095 case WRITING:
1096 task = JGitText.get().writingObjects;
1097 break;
1098 case BUILDING_BITMAPS:
1099 task = JGitText.get().buildingBitmaps;
1100 break;
1101 default:
1102 throw new IllegalArgumentException(
1103 MessageFormat.format(JGitText.get().illegalPackingPhase, phase));
1104 }
1105 monitor.beginTask(task, (int) cnt);
1106 }
1107
1108 private void endPhase(ProgressMonitor monitor) {
1109 monitor.endTask();
1110 }
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139 public void writePack(ProgressMonitor compressMonitor,
1140 ProgressMonitor writeMonitor, OutputStream packStream)
1141 throws IOException {
1142 if (compressMonitor == null)
1143 compressMonitor = NullProgressMonitor.INSTANCE;
1144 if (writeMonitor == null)
1145 writeMonitor = NullProgressMonitor.INSTANCE;
1146
1147 excludeInPacks = null;
1148 excludeInPackLast = null;
1149
1150 boolean needSearchForReuse = reuseSupport != null && (
1151 reuseDeltas
1152 || config.isReuseObjects()
1153 || !cachedPacks.isEmpty());
1154
1155 if (compressMonitor instanceof BatchingProgressMonitor) {
1156 long delay = 1000;
1157 if (needSearchForReuse && config.isDeltaCompress())
1158 delay = 500;
1159 ((BatchingProgressMonitor) compressMonitor).setDelayStart(
1160 delay,
1161 TimeUnit.MILLISECONDS);
1162 }
1163
1164 if (needSearchForReuse)
1165 searchForReuse(compressMonitor);
1166 if (config.isDeltaCompress())
1167 searchForDeltas(compressMonitor);
1168
1169 crc32 = new CRC32();
1170 final PackOutputStreamrage/pack/PackOutputStream.html#PackOutputStream">PackOutputStream out = new PackOutputStream(
1171 writeMonitor,
1172 isIndexDisabled()
1173 ? packStream
1174 : new CheckedOutputStream(packStream, crc32),
1175 this);
1176
1177 long objCnt = packfileUriConfig == null ? getObjectCount() :
1178 getUnoffloadedObjectCount();
1179 stats.totalObjects = objCnt;
1180 if (callback != null)
1181 callback.setObjectCount(objCnt);
1182 beginPhase(PackingPhase.WRITING, writeMonitor, objCnt);
1183 long writeStart = System.currentTimeMillis();
1184 try {
1185 List<CachedPack> unwrittenCachedPacks;
1186
1187 if (packfileUriConfig != null) {
1188 unwrittenCachedPacks = new ArrayList<>();
1189 CachedPackUriProvider p = packfileUriConfig.cachedPackUriProvider;
1190 PacketLineOut o = packfileUriConfig.pckOut;
1191
1192 o.writeString("packfile-uris\n");
1193 for (CachedPack pack : cachedPacks) {
1194 CachedPackUriProvider.PackInfo packInfo = p.getInfo(
1195 pack, packfileUriConfig.protocolsSupported);
1196 if (packInfo != null) {
1197 o.writeString(packInfo.getHash() + ' ' +
1198 packInfo.getUri() + '\n');
1199 stats.offloadedPackfiles += 1;
1200 stats.offloadedPackfileSize += packInfo.getSize();
1201 } else {
1202 unwrittenCachedPacks.add(pack);
1203 }
1204 }
1205 packfileUriConfig.pckOut.writeDelim();
1206 packfileUriConfig.pckOut.writeString("packfile\n");
1207 } else {
1208 unwrittenCachedPacks = cachedPacks;
1209 }
1210
1211 out.writeFileHeader(PACK_VERSION_GENERATED, objCnt);
1212 out.flush();
1213
1214 writeObjects(out);
1215 if (!edgeObjects.isEmpty() || !cachedPacks.isEmpty()) {
1216 for (PackStatistics.ObjectType.Accumulator typeStat : stats.objectTypes) {
1217 if (typeStat == null)
1218 continue;
1219 stats.thinPackBytes += typeStat.bytes;
1220 }
1221 }
1222
1223 stats.reusedPacks = Collections.unmodifiableList(cachedPacks);
1224 for (CachedPack pack : unwrittenCachedPacks) {
1225 long deltaCnt = pack.getDeltaCount();
1226 stats.reusedObjects += pack.getObjectCount();
1227 stats.reusedDeltas += deltaCnt;
1228 stats.totalDeltas += deltaCnt;
1229 reuseSupport.copyPackAsIs(out, pack);
1230 }
1231 writeChecksum(out);
1232 out.flush();
1233 } finally {
1234 stats.timeWriting = System.currentTimeMillis() - writeStart;
1235 stats.depth = depth;
1236
1237 for (PackStatistics.ObjectType.Accumulator typeStat : stats.objectTypes) {
1238 if (typeStat == null)
1239 continue;
1240 typeStat.cntDeltas += typeStat.reusedDeltas;
1241 stats.reusedObjects += typeStat.reusedObjects;
1242 stats.reusedDeltas += typeStat.reusedDeltas;
1243 stats.totalDeltas += typeStat.cntDeltas;
1244 }
1245 }
1246
1247 stats.totalBytes = out.length();
1248 reader.close();
1249 endPhase(writeMonitor);
1250 }
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260 public PackStatistics getStatistics() {
1261 return new PackStatistics(stats);
1262 }
1263
1264
1265
1266
1267
1268
1269 public State getState() {
1270 return state.snapshot();
1271 }
1272
1273
1274
1275
1276
1277
1278 @Override
1279 public void close() {
1280 reader.close();
1281 if (myDeflater != null) {
1282 myDeflater.end();
1283 myDeflater = null;
1284 }
1285 instances.remove(selfRef);
1286 }
1287
1288 private void searchForReuse(ProgressMonitor monitor) throws IOException {
1289 long cnt = 0;
1290 cnt += objectsLists[OBJ_COMMIT].size();
1291 cnt += objectsLists[OBJ_TREE].size();
1292 cnt += objectsLists[OBJ_BLOB].size();
1293 cnt += objectsLists[OBJ_TAG].size();
1294
1295 long start = System.currentTimeMillis();
1296 beginPhase(PackingPhase.FINDING_SOURCES, monitor, cnt);
1297 if (cnt <= 4096) {
1298
1299 BlockList<ObjectToPack> tmp = new BlockList<>((int) cnt);
1300 tmp.addAll(objectsLists[OBJ_TAG]);
1301 tmp.addAll(objectsLists[OBJ_COMMIT]);
1302 tmp.addAll(objectsLists[OBJ_TREE]);
1303 tmp.addAll(objectsLists[OBJ_BLOB]);
1304 searchForReuse(monitor, tmp);
1305 if (pruneCurrentObjectList) {
1306
1307 pruneEdgesFromObjectList(objectsLists[OBJ_COMMIT]);
1308 pruneEdgesFromObjectList(objectsLists[OBJ_TREE]);
1309 pruneEdgesFromObjectList(objectsLists[OBJ_BLOB]);
1310 pruneEdgesFromObjectList(objectsLists[OBJ_TAG]);
1311 }
1312 } else {
1313 searchForReuse(monitor, objectsLists[OBJ_TAG]);
1314 searchForReuse(monitor, objectsLists[OBJ_COMMIT]);
1315 searchForReuse(monitor, objectsLists[OBJ_TREE]);
1316 searchForReuse(monitor, objectsLists[OBJ_BLOB]);
1317 }
1318 endPhase(monitor);
1319 stats.timeSearchingForReuse = System.currentTimeMillis() - start;
1320
1321 if (config.isReuseDeltas() && config.getCutDeltaChains()) {
1322 cutDeltaChains(objectsLists[OBJ_TREE]);
1323 cutDeltaChains(objectsLists[OBJ_BLOB]);
1324 }
1325 }
1326
1327 private void searchForReuse(ProgressMonitor monitor, List<ObjectToPack> list)
1328 throws IOException, MissingObjectException {
1329 pruneCurrentObjectList = false;
1330 reuseSupport.selectObjectRepresentation(this, monitor, list);
1331 if (pruneCurrentObjectList)
1332 pruneEdgesFromObjectList(list);
1333 }
1334
1335 private void cutDeltaChains(BlockList<ObjectToPack> list)
1336 throws IOException {
1337 int max = config.getMaxDeltaDepth();
1338 for (int idx = list.size() - 1; idx >= 0; idx--) {
1339 int d = 0;
1340 ObjectToPack b = list.get(idx).getDeltaBase();
1341 while (b != null) {
1342 if (d < b.getChainLength())
1343 break;
1344 b.setChainLength(++d);
1345 if (d >= max && b.isDeltaRepresentation()) {
1346 reselectNonDelta(b);
1347 break;
1348 }
1349 b = b.getDeltaBase();
1350 }
1351 }
1352 if (config.isDeltaCompress()) {
1353 for (ObjectToPack otp : list)
1354 otp.clearChainLength();
1355 }
1356 }
1357
1358 private void searchForDeltas(ProgressMonitor monitor)
1359 throws MissingObjectException, IncorrectObjectTypeException,
1360 IOException {
1361
1362
1363
1364
1365 ObjectToPack[] list = new ObjectToPack[
1366 objectsLists[OBJ_TREE].size()
1367 + objectsLists[OBJ_BLOB].size()
1368 + edgeObjects.size()];
1369 int cnt = 0;
1370 cnt = findObjectsNeedingDelta(list, cnt, OBJ_TREE);
1371 cnt = findObjectsNeedingDelta(list, cnt, OBJ_BLOB);
1372 if (cnt == 0)
1373 return;
1374 int nonEdgeCnt = cnt;
1375
1376
1377
1378
1379
1380 for (ObjectToPack eo : edgeObjects) {
1381 eo.setWeight(0);
1382 list[cnt++] = eo;
1383 }
1384
1385
1386
1387
1388
1389
1390
1391
1392 final long sizingStart = System.currentTimeMillis();
1393 beginPhase(PackingPhase.GETTING_SIZES, monitor, cnt);
1394 AsyncObjectSizeQueue<ObjectToPack> sizeQueue = reader.getObjectSize(
1395 Arrays.<ObjectToPack> asList(list).subList(0, cnt), false);
1396 try {
1397 final long limit = Math.min(
1398 config.getBigFileThreshold(),
1399 Integer.MAX_VALUE);
1400 for (;;) {
1401 try {
1402 if (!sizeQueue.next())
1403 break;
1404 } catch (MissingObjectException notFound) {
1405 monitor.update(1);
1406 if (ignoreMissingUninteresting) {
1407 ObjectToPack otp = sizeQueue.getCurrent();
1408 if (otp != null && otp.isEdge()) {
1409 otp.setDoNotDelta();
1410 continue;
1411 }
1412
1413 otp = objectsMap.get(notFound.getObjectId());
1414 if (otp != null && otp.isEdge()) {
1415 otp.setDoNotDelta();
1416 continue;
1417 }
1418 }
1419 throw notFound;
1420 }
1421
1422 ObjectToPack otp = sizeQueue.getCurrent();
1423 if (otp == null)
1424 otp = objectsMap.get(sizeQueue.getObjectId());
1425
1426 long sz = sizeQueue.getSize();
1427 if (DeltaIndex.BLKSZ < sz && sz < limit)
1428 otp.setWeight((int) sz);
1429 else
1430 otp.setDoNotDelta();
1431 monitor.update(1);
1432 }
1433 } finally {
1434 sizeQueue.release();
1435 }
1436 endPhase(monitor);
1437 stats.timeSearchingForSizes = System.currentTimeMillis() - sizingStart;
1438
1439
1440
1441
1442
1443
1444 Arrays.sort(list, 0, cnt, (ObjectToPack"../../../../../../org/eclipse/jgit/internal/storage/pack/ObjectToPack.html#ObjectToPack">ObjectToPack a, ObjectToPack b) -> {
1445 int cmp = (a.isDoNotDelta() ? 1 : 0) - (b.isDoNotDelta() ? 1 : 0);
1446 if (cmp != 0) {
1447 return cmp;
1448 }
1449
1450 cmp = a.getType() - b.getType();
1451 if (cmp != 0) {
1452 return cmp;
1453 }
1454
1455 cmp = (a.getPathHash() >>> 1) - (b.getPathHash() >>> 1);
1456 if (cmp != 0) {
1457 return cmp;
1458 }
1459
1460 cmp = (a.getPathHash() & 1) - (b.getPathHash() & 1);
1461 if (cmp != 0) {
1462 return cmp;
1463 }
1464
1465 cmp = (a.isEdge() ? 0 : 1) - (b.isEdge() ? 0 : 1);
1466 if (cmp != 0) {
1467 return cmp;
1468 }
1469
1470 return b.getWeight() - a.getWeight();
1471 });
1472
1473
1474
1475 while (0 < cnt && list[cnt - 1].isDoNotDelta()) {
1476 if (!list[cnt - 1].isEdge())
1477 nonEdgeCnt--;
1478 cnt--;
1479 }
1480 if (cnt == 0)
1481 return;
1482
1483 final long searchStart = System.currentTimeMillis();
1484 searchForDeltas(monitor, list, cnt);
1485 stats.deltaSearchNonEdgeObjects = nonEdgeCnt;
1486 stats.timeCompressing = System.currentTimeMillis() - searchStart;
1487
1488 for (int i = 0; i < cnt; i++)
1489 if (!list[i].isEdge() && list[i].isDeltaRepresentation())
1490 stats.deltasFound++;
1491 }
1492
1493 private int findObjectsNeedingDelta(ObjectToPack[] list, int cnt, int type) {
1494 for (ObjectToPack otp : objectsLists[type]) {
1495 if (otp.isDoNotDelta())
1496 continue;
1497 if (otp.isDeltaRepresentation())
1498 continue;
1499 otp.setWeight(0);
1500 list[cnt++] = otp;
1501 }
1502 return cnt;
1503 }
1504
1505 private void reselectNonDelta(ObjectToPack otp) throws IOException {
1506 otp.clearDeltaBase();
1507 otp.clearReuseAsIs();
1508 boolean old = reuseDeltas;
1509 reuseDeltas = false;
1510 reuseSupport.selectObjectRepresentation(this,
1511 NullProgressMonitor.INSTANCE,
1512 Collections.singleton(otp));
1513 reuseDeltas = old;
1514 }
1515
1516 private void searchForDeltas(final ProgressMonitor monitor,
1517 final ObjectToPack[] list, final int cnt)
1518 throws MissingObjectException, IncorrectObjectTypeException,
1519 LargeObjectException, IOException {
1520 int threads = config.getThreads();
1521 if (threads == 0)
1522 threads = Runtime.getRuntime().availableProcessors();
1523 if (threads <= 1 || cnt <= config.getDeltaSearchWindowSize())
1524 singleThreadDeltaSearch(monitor, list, cnt);
1525 else
1526 parallelDeltaSearch(monitor, list, cnt, threads);
1527 }
1528
1529 private void singleThreadDeltaSearch(ProgressMonitor monitor,
1530 ObjectToPack[] list, int cnt) throws IOException {
1531 long totalWeight = 0;
1532 for (int i = 0; i < cnt; i++) {
1533 ObjectToPack o = list[i];
1534 totalWeight += DeltaTask.getAdjustedWeight(o);
1535 }
1536
1537 long bytesPerUnit = 1;
1538 while (DeltaTask.MAX_METER <= (totalWeight / bytesPerUnit))
1539 bytesPerUnit <<= 10;
1540 int cost = (int) (totalWeight / bytesPerUnit);
1541 if (totalWeight % bytesPerUnit != 0)
1542 cost++;
1543
1544 beginPhase(PackingPhase.COMPRESSING, monitor, cost);
1545 new DeltaWindow(config, new DeltaCache(config), reader,
1546 monitor, bytesPerUnit,
1547 list, 0, cnt).search();
1548 endPhase(monitor);
1549 }
1550
1551 private void parallelDeltaSearch(ProgressMonitor monitor,
1552 ObjectToPack[] list, int cnt, int threads) throws IOException {
1553 DeltaCache dc = new ThreadSafeDeltaCache(config);
1554 ThreadSafeProgressMonitor pm = new ThreadSafeProgressMonitor(monitor);
1555 DeltaTask.Block taskBlock = new DeltaTask.Block(threads, config,
1556 reader, dc, pm,
1557 list, 0, cnt);
1558 taskBlock.partitionTasks();
1559 beginPhase(PackingPhase.COMPRESSING, monitor, taskBlock.cost());
1560 pm.startWorkers(taskBlock.tasks.size());
1561
1562 Executor executor = config.getExecutor();
1563 final List<Throwable> errors =
1564 Collections.synchronizedList(new ArrayList<>(threads));
1565 if (executor instanceof ExecutorService) {
1566
1567 runTasks((ExecutorService) executor, pm, taskBlock, errors);
1568 } else if (executor == null) {
1569
1570
1571 ExecutorService pool = Executors.newFixedThreadPool(threads);
1572 try {
1573 runTasks(pool, pm, taskBlock, errors);
1574 } finally {
1575 pool.shutdown();
1576 for (;;) {
1577 try {
1578 if (pool.awaitTermination(60, TimeUnit.SECONDS))
1579 break;
1580 } catch (InterruptedException e) {
1581 throw new IOException(JGitText
1582 .get().packingCancelledDuringObjectsWriting, e);
1583 }
1584 }
1585 }
1586 } else {
1587
1588
1589
1590 for (DeltaTask task : taskBlock.tasks) {
1591 executor.execute(() -> {
1592 try {
1593 task.call();
1594 } catch (Throwable failure) {
1595 errors.add(failure);
1596 }
1597 });
1598 }
1599 try {
1600 pm.waitForCompletion();
1601 } catch (InterruptedException ie) {
1602
1603
1604
1605 throw new IOException(
1606 JGitText.get().packingCancelledDuringObjectsWriting,
1607 ie);
1608 }
1609 }
1610
1611
1612
1613
1614 if (!errors.isEmpty()) {
1615 Throwable err = errors.get(0);
1616 if (err instanceof Error)
1617 throw (Error) err;
1618 if (err instanceof RuntimeException)
1619 throw (RuntimeException) err;
1620 if (err instanceof IOException)
1621 throw (IOException) err;
1622
1623 throw new IOException(err.getMessage(), err);
1624 }
1625 endPhase(monitor);
1626 }
1627
1628 private static void runTasks(ExecutorService pool,
1629 ThreadSafeProgressMonitor pm,
1630 DeltaTask.Block tb, List<Throwable> errors) throws IOException {
1631 List<Future<?>> futures = new ArrayList<>(tb.tasks.size());
1632 for (DeltaTask task : tb.tasks)
1633 futures.add(pool.submit(task));
1634
1635 try {
1636 pm.waitForCompletion();
1637 for (Future<?> f : futures) {
1638 try {
1639 f.get();
1640 } catch (ExecutionException failed) {
1641 errors.add(failed.getCause());
1642 }
1643 }
1644 } catch (InterruptedException ie) {
1645 for (Future<?> f : futures)
1646 f.cancel(true);
1647 throw new IOException(
1648 JGitText.get().packingCancelledDuringObjectsWriting, ie);
1649 }
1650 }
1651
1652 private void writeObjects(PackOutputStream out) throws IOException {
1653 writeObjects(out, objectsLists[OBJ_COMMIT]);
1654 writeObjects(out, objectsLists[OBJ_TAG]);
1655 writeObjects(out, objectsLists[OBJ_TREE]);
1656 writeObjects(out, objectsLists[OBJ_BLOB]);
1657 }
1658
1659 private void writeObjects(PackOutputStream out, List<ObjectToPack> list)
1660 throws IOException {
1661 if (list.isEmpty())
1662 return;
1663
1664 typeStats = stats.objectTypes[list.get(0).getType()];
1665 long beginOffset = out.length();
1666
1667 if (reuseSupport != null) {
1668 reuseSupport.writeObjects(out, list);
1669 } else {
1670 for (ObjectToPack otp : list)
1671 out.writeObject(otp);
1672 }
1673
1674 typeStats.bytes += out.length() - beginOffset;
1675 typeStats.cntObjects = list.size();
1676 }
1677
1678 void writeObject(PackOutputStream out, ObjectToPack otp) throws IOException {
1679 if (!otp.isWritten())
1680 writeObjectImpl(out, otp);
1681 }
1682
1683 private void writeObjectImpl(PackOutputStream out, ObjectToPack otp)
1684 throws IOException {
1685 if (otp.wantWrite()) {
1686
1687
1688
1689
1690
1691 reselectNonDelta(otp);
1692 }
1693 otp.markWantWrite();
1694
1695 while (otp.isReuseAsIs()) {
1696 writeBase(out, otp.getDeltaBase());
1697 if (otp.isWritten())
1698 return;
1699
1700 crc32.reset();
1701 otp.setOffset(out.length());
1702 try {
1703 reuseSupport.copyObjectAsIs(out, otp, reuseValidate);
1704 out.endObject();
1705 otp.setCRC((int) crc32.getValue());
1706 typeStats.reusedObjects++;
1707 if (otp.isDeltaRepresentation()) {
1708 typeStats.reusedDeltas++;
1709 typeStats.deltaBytes += out.length() - otp.getOffset();
1710 }
1711 return;
1712 } catch (StoredObjectRepresentationNotAvailableException gone) {
1713 if (otp.getOffset() == out.length()) {
1714 otp.setOffset(0);
1715 otp.clearDeltaBase();
1716 otp.clearReuseAsIs();
1717 reuseSupport.selectObjectRepresentation(this,
1718 NullProgressMonitor.INSTANCE,
1719 Collections.singleton(otp));
1720 continue;
1721 }
1722
1723
1724 CorruptObjectException coe;
1725 coe = new CorruptObjectException(otp, "");
1726 coe.initCause(gone);
1727 throw coe;
1728 }
1729 }
1730
1731
1732
1733 if (otp.isDeltaRepresentation()) {
1734 writeDeltaObjectDeflate(out, otp);
1735 } else {
1736 writeWholeObjectDeflate(out, otp);
1737 }
1738 out.endObject();
1739 otp.setCRC((int) crc32.getValue());
1740 }
1741
1742 private void writeBase(PackOutputStream out, ObjectToPack base)
1743 throws IOException {
1744 if (base != null && !base.isWritten() && !base.isEdge())
1745 writeObjectImpl(out, base);
1746 }
1747
1748 private void writeWholeObjectDeflate(PackOutputStream out,
1749 final ObjectToPack otp) throws IOException {
1750 final Deflater deflater = deflater();
1751 final ObjectLoader ldr = reader.open(otp, otp.getType());
1752
1753 crc32.reset();
1754 otp.setOffset(out.length());
1755 out.writeHeader(otp, ldr.getSize());
1756
1757 deflater.reset();
1758 DeflaterOutputStream dst = new DeflaterOutputStream(out, deflater);
1759 ldr.copyTo(dst);
1760 dst.finish();
1761 }
1762
1763 private void writeDeltaObjectDeflate(PackOutputStream out,
1764 final ObjectToPack otp) throws IOException {
1765 writeBase(out, otp.getDeltaBase());
1766
1767 crc32.reset();
1768 otp.setOffset(out.length());
1769
1770 DeltaCache.Ref ref = otp.popCachedDelta();
1771 if (ref != null) {
1772 byte[] zbuf = ref.get();
1773 if (zbuf != null) {
1774 out.writeHeader(otp, otp.getCachedSize());
1775 out.write(zbuf);
1776 typeStats.cntDeltas++;
1777 typeStats.deltaBytes += out.length() - otp.getOffset();
1778 return;
1779 }
1780 }
1781
1782 try (TemporaryBuffer.Heap delta = delta(otp)) {
1783 out.writeHeader(otp, delta.length());
1784
1785 Deflater deflater = deflater();
1786 deflater.reset();
1787 DeflaterOutputStream dst = new DeflaterOutputStream(out, deflater);
1788 delta.writeTo(dst, null);
1789 dst.finish();
1790 }
1791 typeStats.cntDeltas++;
1792 typeStats.deltaBytes += out.length() - otp.getOffset();
1793 }
1794
1795 private TemporaryBuffer.Heap delta(ObjectToPack otp)
1796 throws IOException {
1797 DeltaIndex index = new DeltaIndex(buffer(otp.getDeltaBaseId()));
1798 byte[] res = buffer(otp);
1799
1800
1801
1802
1803
1804 TemporaryBuffer.Heap delta = new TemporaryBuffer.Heap(res.length);
1805 index.encode(delta, res);
1806 return delta;
1807 }
1808
1809 private byte[] buffer(AnyObjectId objId) throws IOException {
1810 return buffer(config, reader, objId);
1811 }
1812
1813 static byte[] buffer(PackConfig config, ObjectReader or, AnyObjectId objId)
1814 throws IOException {
1815
1816
1817
1818
1819
1820 return or.open(objId).getCachedBytes(config.getBigFileThreshold());
1821 }
1822
1823 private Deflater deflater() {
1824 if (myDeflater == null)
1825 myDeflater = new Deflater(config.getCompressionLevel());
1826 return myDeflater;
1827 }
1828
1829 private void writeChecksum(PackOutputStream out) throws IOException {
1830 packcsum = out.getDigest();
1831 out.write(packcsum);
1832 }
1833
1834 private void findObjectsToPack(@NonNull ProgressMonitor countingMonitor,
1835 @NonNull ObjectWalk walker, @NonNull Set<? extends ObjectId> want,
1836 @NonNull Set<? extends ObjectId> have,
1837 @NonNull Set<? extends ObjectId> noBitmaps) throws IOException {
1838 final long countingStart = System.currentTimeMillis();
1839 beginPhase(PackingPhase.COUNTING, countingMonitor, ProgressMonitor.UNKNOWN);
1840
1841 stats.interestingObjects = Collections.unmodifiableSet(new HashSet<ObjectId>(want));
1842 stats.uninterestingObjects = Collections.unmodifiableSet(new HashSet<ObjectId>(have));
1843 excludeFromBitmapSelection = noBitmaps;
1844
1845 canBuildBitmaps = config.isBuildBitmaps()
1846 && !shallowPack
1847 && have.isEmpty()
1848 && (excludeInPacks == null || excludeInPacks.length == 0);
1849 if (!shallowPack && useBitmaps) {
1850 BitmapIndex bitmapIndex = reader.getBitmapIndex();
1851 if (bitmapIndex != null) {
1852 BitmapWalker bitmapWalker = new BitmapWalker(
1853 walker, bitmapIndex, countingMonitor);
1854 findObjectsToPackUsingBitmaps(bitmapWalker, want, have);
1855 endPhase(countingMonitor);
1856 stats.timeCounting = System.currentTimeMillis() - countingStart;
1857 stats.bitmapIndexMisses = bitmapWalker.getCountOfBitmapIndexMisses();
1858 return;
1859 }
1860 }
1861
1862 List<ObjectId> all = new ArrayList<>(want.size() + have.size());
1863 all.addAll(want);
1864 all.addAll(have);
1865
1866 final RevFlag include = walker.newFlag("include");
1867 final RevFlag added = walker.newFlag("added");
1868
1869 walker.carry(include);
1870
1871 int haveEst = have.size();
1872 if (have.isEmpty()) {
1873 walker.sort(RevSort.COMMIT_TIME_DESC);
1874 } else {
1875 walker.sort(RevSort.TOPO);
1876 if (thin)
1877 walker.sort(RevSort.BOUNDARY, true);
1878 }
1879
1880 List<RevObject> wantObjs = new ArrayList<>(want.size());
1881 List<RevObject> haveObjs = new ArrayList<>(haveEst);
1882 List<RevTag> wantTags = new ArrayList<>(want.size());
1883
1884
1885
1886 AsyncRevObjectQueue q = walker.parseAny(all, true);
1887 try {
1888 for (;;) {
1889 try {
1890 RevObject o = q.next();
1891 if (o == null)
1892 break;
1893 if (have.contains(o))
1894 haveObjs.add(o);
1895 if (want.contains(o)) {
1896 o.add(include);
1897 wantObjs.add(o);
1898 if (o instanceof RevTag)
1899 wantTags.add((RevTag) o);
1900 }
1901 } catch (MissingObjectException e) {
1902 if (ignoreMissingUninteresting
1903 && have.contains(e.getObjectId()))
1904 continue;
1905 throw e;
1906 }
1907 }
1908 } finally {
1909 q.release();
1910 }
1911
1912 if (!wantTags.isEmpty()) {
1913 all = new ArrayList<>(wantTags.size());
1914 for (RevTag tag : wantTags)
1915 all.add(tag.getObject());
1916 q = walker.parseAny(all, true);
1917 try {
1918 while (q.next() != null) {
1919
1920 }
1921 } finally {
1922 q.release();
1923 }
1924 }
1925
1926 if (walker instanceof DepthWalk.ObjectWalk) {
1927 DepthWalk.ObjectWalk depthWalk = (DepthWalk.ObjectWalk) walker;
1928 for (RevObject obj : wantObjs) {
1929 depthWalk.markRoot(obj);
1930 }
1931
1932
1933
1934
1935
1936 for (RevObject obj : haveObjs) {
1937 if (obj instanceof RevCommit) {
1938 RevTree t = ((RevCommit) obj).getTree();
1939 depthWalk.markUninteresting(t);
1940 }
1941 }
1942
1943 if (unshallowObjects != null) {
1944 for (ObjectId id : unshallowObjects) {
1945 depthWalk.markUnshallow(walker.parseAny(id));
1946 }
1947 }
1948 } else {
1949 for (RevObject obj : wantObjs)
1950 walker.markStart(obj);
1951 }
1952 for (RevObject obj : haveObjs)
1953 walker.markUninteresting(obj);
1954
1955 final int maxBases = config.getDeltaSearchWindowSize();
1956 Set<RevTree> baseTrees = new HashSet<>();
1957 BlockList<RevCommit> commits = new BlockList<>();
1958 Set<ObjectId> roots = new HashSet<>();
1959 RevCommit c;
1960 while ((c = walker.next()) != null) {
1961 if (exclude(c))
1962 continue;
1963 if (c.has(RevFlag.UNINTERESTING)) {
1964 if (baseTrees.size() <= maxBases)
1965 baseTrees.add(c.getTree());
1966 continue;
1967 }
1968
1969 commits.add(c);
1970 if (c.getParentCount() == 0) {
1971 roots.add(c.copy());
1972 }
1973 countingMonitor.update(1);
1974 }
1975 stats.rootCommits = Collections.unmodifiableSet(roots);
1976
1977 if (shallowPack) {
1978 for (RevCommit cmit : commits) {
1979 addObject(cmit, 0);
1980 }
1981 } else {
1982 int commitCnt = 0;
1983 boolean putTagTargets = false;
1984 for (RevCommit cmit : commits) {
1985 if (!cmit.has(added)) {
1986 cmit.add(added);
1987 addObject(cmit, 0);
1988 commitCnt++;
1989 }
1990
1991 for (int i = 0; i < cmit.getParentCount(); i++) {
1992 RevCommit p = cmit.getParent(i);
1993 if (!p.has(added) && !p.has(RevFlag.UNINTERESTING)
1994 && !exclude(p)) {
1995 p.add(added);
1996 addObject(p, 0);
1997 commitCnt++;
1998 }
1999 }
2000
2001 if (!putTagTargets && 4096 < commitCnt) {
2002 for (ObjectId id : tagTargets) {
2003 RevObject obj = walker.lookupOrNull(id);
2004 if (obj instanceof RevCommit
2005 && obj.has(include)
2006 && !obj.has(RevFlag.UNINTERESTING)
2007 && !obj.has(added)) {
2008 obj.add(added);
2009 addObject(obj, 0);
2010 }
2011 }
2012 putTagTargets = true;
2013 }
2014 }
2015 }
2016 commits = null;
2017
2018 if (thin && !baseTrees.isEmpty()) {
2019 BaseSearch bases = new BaseSearch(countingMonitor, baseTrees,
2020 objectsMap, edgeObjects, reader);
2021 RevObject o;
2022 while ((o = walker.nextObject()) != null) {
2023 if (o.has(RevFlag.UNINTERESTING))
2024 continue;
2025 if (exclude(o))
2026 continue;
2027
2028 int pathHash = walker.getPathHashCode();
2029 byte[] pathBuf = walker.getPathBuffer();
2030 int pathLen = walker.getPathLength();
2031 bases.addBase(o.getType(), pathBuf, pathLen, pathHash);
2032 if (!depthSkip(o, walker)) {
2033 filterAndAddObject(o, o.getType(), pathHash, want);
2034 }
2035 countingMonitor.update(1);
2036 }
2037 } else {
2038 RevObject o;
2039 while ((o = walker.nextObject()) != null) {
2040 if (o.has(RevFlag.UNINTERESTING))
2041 continue;
2042 if (exclude(o))
2043 continue;
2044 if (!depthSkip(o, walker)) {
2045 filterAndAddObject(o, o.getType(), walker.getPathHashCode(),
2046 want);
2047 }
2048 countingMonitor.update(1);
2049 }
2050 }
2051
2052 for (CachedPack pack : cachedPacks)
2053 countingMonitor.update((int) pack.getObjectCount());
2054 endPhase(countingMonitor);
2055 stats.timeCounting = System.currentTimeMillis() - countingStart;
2056 stats.bitmapIndexMisses = -1;
2057 }
2058
2059 private void findObjectsToPackUsingBitmaps(
2060 BitmapWalker bitmapWalker, Set<? extends ObjectId> want,
2061 Set<? extends ObjectId> have)
2062 throws MissingObjectException, IncorrectObjectTypeException,
2063 IOException {
2064 BitmapBuilder haveBitmap = bitmapWalker.findObjects(have, null, true);
2065 BitmapBuilder wantBitmap = bitmapWalker.findObjects(want, haveBitmap,
2066 false);
2067 BitmapBuilder needBitmap = wantBitmap.andNot(haveBitmap);
2068
2069 if (useCachedPacks && reuseSupport != null && !reuseValidate
2070 && (excludeInPacks == null || excludeInPacks.length == 0))
2071 cachedPacks.addAll(
2072 reuseSupport.getCachedPacksAndUpdate(needBitmap));
2073
2074 for (BitmapObject obj : needBitmap) {
2075 ObjectId objectId = obj.getObjectId();
2076 if (exclude(objectId)) {
2077 needBitmap.remove(objectId);
2078 continue;
2079 }
2080 filterAndAddObject(objectId, obj.getType(), 0, want);
2081 }
2082
2083 if (thin)
2084 haveObjects = haveBitmap;
2085 }
2086
2087 private static void pruneEdgesFromObjectList(List<ObjectToPack> list) {
2088 final int size = list.size();
2089 int src = 0;
2090 int dst = 0;
2091
2092 for (; src < size; src++) {
2093 ObjectToPack obj = list.get(src);
2094 if (obj.isEdge())
2095 continue;
2096 if (dst != src)
2097 list.set(dst, obj);
2098 dst++;
2099 }
2100
2101 while (dst < list.size())
2102 list.remove(list.size() - 1);
2103 }
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117 public void addObject(RevObject object)
2118 throws IncorrectObjectTypeException {
2119 if (!exclude(object))
2120 addObject(object, 0);
2121 }
2122
2123 private void addObject(RevObject object, int pathHashCode) {
2124 addObject(object, object.getType(), pathHashCode);
2125 }
2126
2127 private void addObject(
2128 final AnyObjectId src, final int type, final int pathHashCode) {
2129 final ObjectToPack otp;
2130 if (reuseSupport != null)
2131 otp = reuseSupport.newObjectToPack(src, type);
2132 else
2133 otp = new ObjectToPack(src, type);
2134 otp.setPathHash(pathHashCode);
2135 objectsLists[type].add(otp);
2136 objectsMap.add(otp);
2137 }
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155 private boolean depthSkip(@NonNull RevObject obj, ObjectWalk walker) {
2156 long treeDepth = walker.getTreeDepth();
2157
2158
2159
2160
2161
2162 if (obj.getType() == OBJ_BLOB) {
2163 treeDepth++;
2164 } else {
2165 stats.treesTraversed++;
2166 }
2167
2168 if (filterSpec.getTreeDepthLimit() < 0 ||
2169 treeDepth <= filterSpec.getTreeDepthLimit()) {
2170 return false;
2171 }
2172
2173 walker.skipTree();
2174 return true;
2175 }
2176
2177
2178
2179 private void filterAndAddObject(@NonNull AnyObjectId src, int type,
2180 int pathHashCode, @NonNull Set<? extends AnyObjectId> want)
2181 throws IOException {
2182
2183
2184
2185 boolean reject = filterSpec.getBlobLimit() >= 0 &&
2186 type == OBJ_BLOB &&
2187 !want.contains(src) &&
2188 reader.getObjectSize(src, OBJ_BLOB) > filterSpec.getBlobLimit();
2189 if (!reject) {
2190 addObject(src, type, pathHashCode);
2191 }
2192 }
2193
2194 private boolean exclude(AnyObjectId objectId) {
2195 if (excludeInPacks == null)
2196 return false;
2197 if (excludeInPackLast.contains(objectId))
2198 return true;
2199 for (ObjectIdSet idx : excludeInPacks) {
2200 if (idx.contains(objectId)) {
2201 excludeInPackLast = idx;
2202 return true;
2203 }
2204 }
2205 return false;
2206 }
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220 public void select(ObjectToPack otp, StoredObjectRepresentation next) {
2221 int nFmt = next.getFormat();
2222
2223 if (!cachedPacks.isEmpty()) {
2224 if (otp.isEdge())
2225 return;
2226 if (nFmt == PACK_WHOLE || nFmt == PACK_DELTA) {
2227 for (CachedPack pack : cachedPacks) {
2228 if (pack.hasObject(otp, next)) {
2229 otp.setEdge();
2230 otp.clearDeltaBase();
2231 otp.clearReuseAsIs();
2232 pruneCurrentObjectList = true;
2233 return;
2234 }
2235 }
2236 }
2237 }
2238
2239 if (nFmt == PACK_DELTA && reuseDeltas && reuseDeltaFor(otp)) {
2240 ObjectId baseId = next.getDeltaBase();
2241 ObjectToPack ptr = objectsMap.get(baseId);
2242 if (ptr != null && !ptr.isEdge()) {
2243 otp.setDeltaBase(ptr);
2244 otp.setReuseAsIs();
2245 } else if (thin && have(ptr, baseId)) {
2246 otp.setDeltaBase(baseId);
2247 otp.setReuseAsIs();
2248 } else {
2249 otp.clearDeltaBase();
2250 otp.clearReuseAsIs();
2251 }
2252 } else if (nFmt == PACK_WHOLE && config.isReuseObjects()) {
2253 int nWeight = next.getWeight();
2254 if (otp.isReuseAsIs() && !otp.isDeltaRepresentation()) {
2255
2256
2257
2258 if (otp.getWeight() <= nWeight)
2259 return;
2260 }
2261 otp.clearDeltaBase();
2262 otp.setReuseAsIs();
2263 otp.setWeight(nWeight);
2264 } else {
2265 otp.clearDeltaBase();
2266 otp.clearReuseAsIs();
2267 }
2268
2269 otp.setDeltaAttempted(reuseDeltas && next.wasDeltaAttempted());
2270 otp.select(next);
2271 }
2272
2273 private final boolean have(ObjectToPack ptr, AnyObjectId objectId) {
2274 return (ptr != null && ptr.isEdge())
2275 || (haveObjects != null && haveObjects.contains(objectId));
2276 }
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297 public boolean prepareBitmapIndex(ProgressMonitor pm) throws IOException {
2298 if (!canBuildBitmaps || getObjectCount() > Integer.MAX_VALUE
2299 || !cachedPacks.isEmpty())
2300 return false;
2301
2302 if (pm == null)
2303 pm = NullProgressMonitor.INSTANCE;
2304
2305 int numCommits = objectsLists[OBJ_COMMIT].size();
2306 List<ObjectToPack> byName = sortByName();
2307 sortedByName = null;
2308 objectsLists = null;
2309 objectsMap = null;
2310 writeBitmaps = new PackBitmapIndexBuilder(byName);
2311 byName = null;
2312
2313 PackWriterBitmapPreparer bitmapPreparer = new PackWriterBitmapPreparer(
2314 reader, writeBitmaps, pm, stats.interestingObjects, config);
2315
2316 Collection<PackWriterBitmapPreparer.BitmapCommit> selectedCommits = bitmapPreparer
2317 .selectCommits(numCommits, excludeFromBitmapSelection);
2318
2319 beginPhase(PackingPhase.BUILDING_BITMAPS, pm, selectedCommits.size());
2320
2321 BitmapWalker walker = bitmapPreparer.newBitmapWalker();
2322 AnyObjectId last = null;
2323 for (PackWriterBitmapPreparer.BitmapCommit cmit : selectedCommits) {
2324 if (!cmit.isReuseWalker()) {
2325 walker = bitmapPreparer.newBitmapWalker();
2326 }
2327 BitmapBuilder bitmap = walker.findObjects(
2328 Collections.singleton(cmit), null, false);
2329
2330 if (last != null && cmit.isReuseWalker() && !bitmap.contains(last))
2331 throw new IllegalStateException(MessageFormat.format(
2332 JGitText.get().bitmapMissingObject, cmit.name(),
2333 last.name()));
2334 last = cmit;
2335 writeBitmaps.addBitmap(cmit, bitmap.build(), cmit.getFlags());
2336
2337 pm.update(1);
2338 }
2339
2340 endPhase(pm);
2341 return true;
2342 }
2343
2344 private boolean reuseDeltaFor(ObjectToPack otp) {
2345 int type = otp.getType();
2346 if ((type & 2) != 0)
2347 return true;
2348 if (type == OBJ_COMMIT)
2349 return reuseDeltaCommits;
2350 if (type == OBJ_TAG)
2351 return false;
2352 return true;
2353 }
2354
2355 private class MutableState {
2356
2357
2358 private static final long OBJECT_TO_PACK_SIZE =
2359 (2 * 8)
2360 + (2 * 8) + (2 * 8)
2361 + (8 + 8)
2362 + 8
2363 + 40
2364 + 8;
2365
2366 private final long totalDeltaSearchBytes;
2367
2368 private volatile PackingPhase phase;
2369
2370 MutableState() {
2371 phase = PackingPhase.COUNTING;
2372 if (config.isDeltaCompress()) {
2373 int threads = config.getThreads();
2374 if (threads <= 0)
2375 threads = Runtime.getRuntime().availableProcessors();
2376 totalDeltaSearchBytes = (threads * config.getDeltaSearchMemoryLimit())
2377 + config.getBigFileThreshold();
2378 } else
2379 totalDeltaSearchBytes = 0;
2380 }
2381
2382 State snapshot() {
2383 long objCnt = 0;
2384 BlockList<ObjectToPack>[] lists = objectsLists;
2385 if (lists != null) {
2386 objCnt += lists[OBJ_COMMIT].size();
2387 objCnt += lists[OBJ_TREE].size();
2388 objCnt += lists[OBJ_BLOB].size();
2389 objCnt += lists[OBJ_TAG].size();
2390
2391 }
2392
2393 long bytesUsed = OBJECT_TO_PACK_SIZE * objCnt;
2394 PackingPhase curr = phase;
2395 if (curr == PackingPhase.COMPRESSING)
2396 bytesUsed += totalDeltaSearchBytes;
2397 return new State(curr, bytesUsed);
2398 }
2399 }
2400
2401
2402 public enum PackingPhase {
2403
2404 COUNTING,
2405
2406
2407 GETTING_SIZES,
2408
2409
2410 FINDING_SOURCES,
2411
2412
2413 COMPRESSING,
2414
2415
2416 WRITING,
2417
2418
2419 BUILDING_BITMAPS;
2420 }
2421
2422
2423 public class State {
2424 private final PackingPhase phase;
2425
2426 private final long bytesUsed;
2427
2428 State(PackingPhase phase, long bytesUsed) {
2429 this.phase = phase;
2430 this.bytesUsed = bytesUsed;
2431 }
2432
2433
2434 public PackConfig getConfig() {
2435 return config;
2436 }
2437
2438
2439 public PackingPhase getPhase() {
2440 return phase;
2441 }
2442
2443
2444 public long estimateBytesUsed() {
2445 return bytesUsed;
2446 }
2447
2448 @SuppressWarnings("nls")
2449 @Override
2450 public String toString() {
2451 return "PackWriter.State[" + phase + ", memory=" + bytesUsed + "]";
2452 }
2453 }
2454
2455
2456
2457
2458
2459
2460 public static class PackfileUriConfig {
2461 @NonNull
2462 private final PacketLineOut pckOut;
2463
2464 @NonNull
2465 private final Collection<String> protocolsSupported;
2466
2467 @NonNull
2468 private final CachedPackUriProvider cachedPackUriProvider;
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479 public PackfileUriConfig(@NonNull PacketLineOut pckOut,
2480 @NonNull Collection<String> protocolsSupported,
2481 @NonNull CachedPackUriProvider cachedPackUriProvider) {
2482 this.pckOut = pckOut;
2483 this.protocolsSupported = protocolsSupported;
2484 this.cachedPackUriProvider = cachedPackUriProvider;
2485 }
2486 }
2487 }