1
2
3
4
5
6
7
8
9
10
11
12
13 package org.eclipse.jgit.transport;
14
15 import java.io.EOFException;
16 import java.io.IOException;
17 import java.io.InputStream;
18 import java.security.MessageDigest;
19 import java.text.MessageFormat;
20 import java.util.ArrayList;
21 import java.util.Arrays;
22 import java.util.Comparator;
23 import java.util.List;
24 import java.util.concurrent.TimeUnit;
25 import java.util.zip.DataFormatException;
26 import java.util.zip.Inflater;
27
28 import org.eclipse.jgit.errors.CorruptObjectException;
29 import org.eclipse.jgit.errors.MissingObjectException;
30 import org.eclipse.jgit.errors.TooLargeObjectInPackException;
31 import org.eclipse.jgit.internal.JGitText;
32 import org.eclipse.jgit.internal.storage.file.PackLock;
33 import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
34 import org.eclipse.jgit.lib.AnyObjectId;
35 import org.eclipse.jgit.lib.BatchingProgressMonitor;
36 import org.eclipse.jgit.lib.BlobObjectChecker;
37 import org.eclipse.jgit.lib.Constants;
38 import org.eclipse.jgit.lib.InflaterCache;
39 import org.eclipse.jgit.lib.MutableObjectId;
40 import org.eclipse.jgit.lib.NullProgressMonitor;
41 import org.eclipse.jgit.lib.ObjectChecker;
42 import org.eclipse.jgit.lib.ObjectDatabase;
43 import org.eclipse.jgit.lib.ObjectId;
44 import org.eclipse.jgit.lib.ObjectIdOwnerMap;
45 import org.eclipse.jgit.lib.ObjectIdSubclassMap;
46 import org.eclipse.jgit.lib.ObjectLoader;
47 import org.eclipse.jgit.lib.ObjectReader;
48 import org.eclipse.jgit.lib.ObjectStream;
49 import org.eclipse.jgit.lib.ProgressMonitor;
50 import org.eclipse.jgit.util.BlockList;
51 import org.eclipse.jgit.util.IO;
52 import org.eclipse.jgit.util.LongMap;
53 import org.eclipse.jgit.util.NB;
54 import org.eclipse.jgit.util.sha1.SHA1;
55
56
57
58
59
60
61
62
63
64
65
66
67
68 public abstract class PackParser {
69
70 private static final int BUFFER_SIZE = 8192;
71
72
73 public enum Source {
74
75 INPUT,
76
77
78 DATABASE;
79 }
80
81
82 private final ObjectDatabase objectDatabase;
83
84 private InflaterStream inflater;
85
86 private byte[] tempBuffer;
87
88 private byte[] hdrBuf;
89
90 private final SHA1 objectHasher = SHA1.newInstance();
91 private final MutableObjectId tempObjectId;
92
93 private InputStream in;
94
95 byte[] buf;
96
97
98 private long bBase;
99
100 private int bOffset;
101
102 int bAvail;
103
104 private ObjectChecker objCheck;
105
106 private boolean allowThin;
107
108 private boolean checkObjectCollisions;
109
110 private boolean needBaseObjectIds;
111
112 private boolean checkEofAfterPackFooter;
113
114 private boolean expectDataAfterPackFooter;
115
116 private long expectedObjectCount;
117
118 private PackedObjectInfo[] entries;
119
120
121
122
123
124
125
126
127 private ObjectIdSubclassMap<ObjectId> newObjectIds;
128
129 private int deltaCount;
130
131 private int entryCount;
132
133 private ObjectIdOwnerMap<DeltaChain> baseById;
134
135
136
137
138
139
140
141
142 private ObjectIdSubclassMap<ObjectId> baseObjectIds;
143
144 private LongMap<UnresolvedDelta> baseByPos;
145
146
147 private BlockList<PackedObjectInfo> collisionCheckObjs;
148
149 private MessageDigest packDigest;
150
151 private ObjectReader readCurs;
152
153
154 private String lockMessage;
155
156
157 private long maxObjectSizeLimit;
158
159 private final ReceivedPackStatistics.Builder stats =
160 new ReceivedPackStatistics.Builder();
161
162
163
164
165
166
167
168
169
170 protected PackParser(ObjectDatabase odb, InputStream src) {
171 objectDatabase = odb.newCachedDatabase();
172 in = src;
173
174 inflater = new InflaterStream();
175 readCurs = objectDatabase.newReader();
176 buf = new byte[BUFFER_SIZE];
177 tempBuffer = new byte[BUFFER_SIZE];
178 hdrBuf = new byte[64];
179 tempObjectId = new MutableObjectId();
180 packDigest = Constants.newMessageDigest();
181 checkObjectCollisions = true;
182 }
183
184
185
186
187
188
189 public boolean isAllowThin() {
190 return allowThin;
191 }
192
193
194
195
196
197
198
199
200
201
202 public void setAllowThin(boolean allow) {
203 allowThin = allow;
204 }
205
206
207
208
209
210
211
212 protected boolean isCheckObjectCollisions() {
213 return checkObjectCollisions;
214 }
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237 protected void setCheckObjectCollisions(boolean check) {
238 checkObjectCollisions = check;
239 }
240
241
242
243
244
245
246
247
248
249
250
251 public void setNeedNewObjectIds(boolean b) {
252 if (b)
253 newObjectIds = new ObjectIdSubclassMap<>();
254 else
255 newObjectIds = null;
256 }
257
258 private boolean needNewObjectIds() {
259 return newObjectIds != null;
260 }
261
262
263
264
265
266
267
268
269
270
271
272
273 public void setNeedBaseObjectIds(boolean b) {
274 this.needBaseObjectIds = b;
275 }
276
277
278
279
280
281
282 public boolean isCheckEofAfterPackFooter() {
283 return checkEofAfterPackFooter;
284 }
285
286
287
288
289
290
291
292 public void setCheckEofAfterPackFooter(boolean b) {
293 checkEofAfterPackFooter = b;
294 }
295
296
297
298
299
300
301 public boolean isExpectDataAfterPackFooter() {
302 return expectDataAfterPackFooter;
303 }
304
305
306
307
308
309
310
311
312
313 public void setExpectDataAfterPackFooter(boolean e) {
314 expectDataAfterPackFooter = e;
315 }
316
317
318
319
320
321
322 public ObjectIdSubclassMap<ObjectId> getNewObjectIds() {
323 if (newObjectIds != null)
324 return newObjectIds;
325 return new ObjectIdSubclassMap<>();
326 }
327
328
329
330
331
332
333 public ObjectIdSubclassMap<ObjectId> getBaseObjectIds() {
334 if (baseObjectIds != null)
335 return baseObjectIds;
336 return new ObjectIdSubclassMap<>();
337 }
338
339
340
341
342
343
344
345
346
347
348
349 public void setObjectChecker(ObjectChecker oc) {
350 objCheck = oc;
351 }
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369 public void setObjectChecking(boolean on) {
370 setObjectChecker(on ? new ObjectChecker() : null);
371 }
372
373
374
375
376
377
378 public String getLockMessage() {
379 return lockMessage;
380 }
381
382
383
384
385
386
387
388
389 public void setLockMessage(String msg) {
390 lockMessage = msg;
391 }
392
393
394
395
396
397
398
399
400
401
402 public void setMaxObjectSizeLimit(long limit) {
403 maxObjectSizeLimit = limit;
404 }
405
406
407
408
409
410
411
412
413
414
415 public int getObjectCount() {
416 return entryCount;
417 }
418
419
420
421
422
423
424
425
426
427
428
429
430 public PackedObjectInfo getObject(int nth) {
431 return entries[nth];
432 }
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448 public List<PackedObjectInfo> getSortedObjectList(
449 Comparator<PackedObjectInfo> cmp) {
450 Arrays.sort(entries, 0, entryCount, cmp);
451 List<PackedObjectInfo> list = Arrays.asList(entries);
452 if (entryCount < entries.length)
453 list = list.subList(0, entryCount);
454 return list;
455 }
456
457
458
459
460
461
462
463
464
465
466
467 public long getPackSize() {
468 return -1;
469 }
470
471
472
473
474
475
476
477
478
479 public ReceivedPackStatistics getReceivedPackStatistics() {
480 return stats.build();
481 }
482
483
484
485
486
487
488
489
490
491
492
493
494
495 public final PackLock parse(ProgressMonitor progress) throws IOException {
496 return parse(progress, progress);
497 }
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514 public PackLock parse(ProgressMonitor./org/eclipse/jgit/lib/ProgressMonitor.html#ProgressMonitor">ProgressMonitor receiving, ProgressMonitor resolving)
515 throws IOException {
516 if (receiving == null)
517 receiving = NullProgressMonitor.INSTANCE;
518 if (resolving == null)
519 resolving = NullProgressMonitor.INSTANCE;
520
521 if (receiving == resolving)
522 receiving.start(2 );
523 try {
524 readPackHeader();
525
526 entries = new PackedObjectInfo[(int) expectedObjectCount];
527 baseById = new ObjectIdOwnerMap<>();
528 baseByPos = new LongMap<>();
529 collisionCheckObjs = new BlockList<>();
530
531 receiving.beginTask(JGitText.get().receivingObjects,
532 (int) expectedObjectCount);
533 try {
534 for (int done = 0; done < expectedObjectCount; done++) {
535 indexOneObject();
536 receiving.update(1);
537 if (receiving.isCancelled())
538 throw new IOException(JGitText.get().downloadCancelled);
539 }
540 readPackFooter();
541 endInput();
542 } finally {
543 receiving.endTask();
544 }
545
546 if (!collisionCheckObjs.isEmpty()) {
547 checkObjectCollision();
548 }
549
550 if (deltaCount > 0) {
551 processDeltas(resolving);
552 }
553
554 packDigest = null;
555 baseById = null;
556 baseByPos = null;
557 } finally {
558 try {
559 if (readCurs != null)
560 readCurs.close();
561 } finally {
562 readCurs = null;
563 }
564
565 try {
566 inflater.release();
567 } finally {
568 inflater = null;
569 }
570 }
571 return null;
572 }
573
574 private void processDeltas(ProgressMonitor resolving) throws IOException {
575 if (resolving instanceof BatchingProgressMonitor) {
576 ((BatchingProgressMonitor) resolving).setDelayStart(1000,
577 TimeUnit.MILLISECONDS);
578 }
579 resolving.beginTask(JGitText.get().resolvingDeltas, deltaCount);
580 resolveDeltas(resolving);
581 if (entryCount < expectedObjectCount) {
582 if (!isAllowThin()) {
583 throw new IOException(MessageFormat.format(
584 JGitText.get().packHasUnresolvedDeltas,
585 Long.valueOf(expectedObjectCount - entryCount)));
586 }
587
588 resolveDeltasWithExternalBases(resolving);
589
590 if (entryCount < expectedObjectCount) {
591 throw new IOException(MessageFormat.format(
592 JGitText.get().packHasUnresolvedDeltas,
593 Long.valueOf(expectedObjectCount - entryCount)));
594 }
595 }
596 resolving.endTask();
597 }
598
599 private void resolveDeltas(ProgressMonitor progress)
600 throws IOException {
601 final int last = entryCount;
602 for (int i = 0; i < last; i++) {
603 resolveDeltas(entries[i], progress);
604 if (progress.isCancelled())
605 throw new IOException(
606 JGitText.get().downloadCancelledDuringIndexing);
607 }
608 }
609
610 private void resolveDeltas(final PackedObjectInfo oe,
611 ProgressMonitor progress) throws IOException {
612 UnresolvedDelta children = firstChildOf(oe);
613 if (children == null)
614 return;
615
616 DeltaVisit visit = new DeltaVisit();
617 visit.nextChild = children;
618
619 ObjectTypeAndSize info = openDatabase(oe, new ObjectTypeAndSize());
620 switch (info.type) {
621 case Constants.OBJ_COMMIT:
622 case Constants.OBJ_TREE:
623 case Constants.OBJ_BLOB:
624 case Constants.OBJ_TAG:
625 visit.data = inflateAndReturn(Source.DATABASE, info.size);
626 visit.id = oe;
627 break;
628 default:
629 throw new IOException(MessageFormat.format(
630 JGitText.get().unknownObjectType,
631 Integer.valueOf(info.type)));
632 }
633
634 if (!checkCRC(oe.getCRC())) {
635 throw new IOException(MessageFormat.format(
636 JGitText.get().corruptionDetectedReReadingAt,
637 Long.valueOf(oe.getOffset())));
638 }
639
640 resolveDeltas(visit.next(), info.type, info, progress);
641 }
642
643 private void resolveDeltas(DeltaVisit visit, final int type,
644 ObjectTypeAndSize info, ProgressMonitor progress)
645 throws IOException {
646 stats.addDeltaObject(type);
647 do {
648 progress.update(1);
649 info = openDatabase(visit.delta, info);
650 switch (info.type) {
651 case Constants.OBJ_OFS_DELTA:
652 case Constants.OBJ_REF_DELTA:
653 break;
654
655 default:
656 throw new IOException(MessageFormat.format(
657 JGitText.get().unknownObjectType,
658 Integer.valueOf(info.type)));
659 }
660
661 byte[] delta = inflateAndReturn(Source.DATABASE, info.size);
662 checkIfTooLarge(type, BinaryDelta.getResultSize(delta));
663
664 visit.data = BinaryDelta.apply(visit.parent.data, delta);
665 delta = null;
666
667 if (!checkCRC(visit.delta.crc))
668 throw new IOException(MessageFormat.format(
669 JGitText.get().corruptionDetectedReReadingAt,
670 Long.valueOf(visit.delta.position)));
671
672 SHA1 objectDigest = objectHasher.reset();
673 objectDigest.update(Constants.encodedTypeString(type));
674 objectDigest.update((byte) ' ');
675 objectDigest.update(Constants.encodeASCII(visit.data.length));
676 objectDigest.update((byte) 0);
677 objectDigest.update(visit.data);
678 objectDigest.digest(tempObjectId);
679
680 verifySafeObject(tempObjectId, type, visit.data);
681 if (isCheckObjectCollisions() && readCurs.has(tempObjectId)) {
682 checkObjectCollision(tempObjectId, type, visit.data,
683 visit.delta.sizeBeforeInflating);
684 }
685
686 PackedObjectInfo oe;
687 oe = newInfo(tempObjectId, visit.delta, visit.parent.id);
688 oe.setOffset(visit.delta.position);
689 oe.setType(type);
690 onInflatedObjectData(oe, type, visit.data);
691 addObjectAndTrack(oe);
692 visit.id = oe;
693
694 visit.nextChild = firstChildOf(oe);
695 visit = visit.next();
696 } while (visit != null);
697 }
698
699 private final void checkIfTooLarge(int typeCode, long size)
700 throws IOException {
701 if (0 < maxObjectSizeLimit && maxObjectSizeLimit < size) {
702 switch (typeCode) {
703 case Constants.OBJ_COMMIT:
704 case Constants.OBJ_TREE:
705 case Constants.OBJ_BLOB:
706 case Constants.OBJ_TAG:
707 throw new TooLargeObjectInPackException(size, maxObjectSizeLimit);
708
709 case Constants.OBJ_OFS_DELTA:
710 case Constants.OBJ_REF_DELTA:
711 throw new TooLargeObjectInPackException(size, maxObjectSizeLimit);
712
713 default:
714 throw new IOException(MessageFormat.format(
715 JGitText.get().unknownObjectType,
716 Integer.valueOf(typeCode)));
717 }
718 }
719 if (size > Integer.MAX_VALUE - 8) {
720 throw new TooLargeObjectInPackException(size, Integer.MAX_VALUE - 8);
721 }
722 }
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740 protected ObjectTypeAndSize readObjectHeader(ObjectTypeAndSize info)
741 throws IOException {
742 int hdrPtr = 0;
743 int c = readFrom(Source.DATABASE);
744 hdrBuf[hdrPtr++] = (byte) c;
745
746 info.type = (c >> 4) & 7;
747 long sz = c & 15;
748 int shift = 4;
749 while ((c & 0x80) != 0) {
750 c = readFrom(Source.DATABASE);
751 hdrBuf[hdrPtr++] = (byte) c;
752 sz += ((long) (c & 0x7f)) << shift;
753 shift += 7;
754 }
755 info.size = sz;
756
757 switch (info.type) {
758 case Constants.OBJ_COMMIT:
759 case Constants.OBJ_TREE:
760 case Constants.OBJ_BLOB:
761 case Constants.OBJ_TAG:
762 onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
763 break;
764
765 case Constants.OBJ_OFS_DELTA:
766 c = readFrom(Source.DATABASE);
767 hdrBuf[hdrPtr++] = (byte) c;
768 while ((c & 128) != 0) {
769 c = readFrom(Source.DATABASE);
770 hdrBuf[hdrPtr++] = (byte) c;
771 }
772 onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
773 break;
774
775 case Constants.OBJ_REF_DELTA:
776 System.arraycopy(buf, fill(Source.DATABASE, 20), hdrBuf, hdrPtr, 20);
777 hdrPtr += 20;
778 use(20);
779 onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
780 break;
781
782 default:
783 throw new IOException(MessageFormat.format(
784 JGitText.get().unknownObjectType,
785 Integer.valueOf(info.type)));
786 }
787 return info;
788 }
789
790 private UnresolvedDelta removeBaseById(AnyObjectId id) {
791 final DeltaChain d = baseById.get(id);
792 return d != null ? d.remove() : null;
793 }
794
795 private static UnresolvedDelta reverse(UnresolvedDelta c) {
796 UnresolvedDelta tail = null;
797 while (c != null) {
798 final UnresolvedDelta n = c.next;
799 c.next = tail;
800 tail = c;
801 c = n;
802 }
803 return tail;
804 }
805
806 private UnresolvedDelta firstChildOf(PackedObjectInfo oe) {
807 UnresolvedDelta a = reverse(removeBaseById(oe));
808 UnresolvedDelta b = reverse(baseByPos.remove(oe.getOffset()));
809
810 if (a == null)
811 return b;
812 if (b == null)
813 return a;
814
815 UnresolvedDelta first = null;
816 UnresolvedDelta last = null;
817 while (a != null || b != null) {
818 UnresolvedDelta curr;
819 if (b == null || (a != null && a.position < b.position)) {
820 curr = a;
821 a = a.next;
822 } else {
823 curr = b;
824 b = b.next;
825 }
826 if (last != null)
827 last.next = curr;
828 else
829 first = curr;
830 last = curr;
831 curr.next = null;
832 }
833 return first;
834 }
835
836 private void resolveDeltasWithExternalBases(ProgressMonitor progress)
837 throws IOException {
838 growEntries(baseById.size());
839
840 if (needBaseObjectIds)
841 baseObjectIds = new ObjectIdSubclassMap<>();
842
843 final List<DeltaChain> missing = new ArrayList<>(64);
844 for (DeltaChain baseId : baseById) {
845 if (baseId.head == null)
846 continue;
847
848 if (needBaseObjectIds)
849 baseObjectIds.add(baseId);
850
851 final ObjectLoader ldr;
852 try {
853 ldr = readCurs.open(baseId);
854 } catch (MissingObjectException notFound) {
855 missing.add(baseId);
856 continue;
857 }
858
859 final DeltaVisit visit = new DeltaVisit();
860 visit.data = ldr.getCachedBytes(Integer.MAX_VALUE);
861 visit.id = baseId;
862 final int typeCode = ldr.getType();
863 final PackedObjectInfo oe = newInfo(baseId, null, null);
864 oe.setType(typeCode);
865 if (onAppendBase(typeCode, visit.data, oe))
866 entries[entryCount++] = oe;
867 visit.nextChild = firstChildOf(oe);
868 resolveDeltas(visit.next(), typeCode,
869 new ObjectTypeAndSize(), progress);
870
871 if (progress.isCancelled())
872 throw new IOException(
873 JGitText.get().downloadCancelledDuringIndexing);
874 }
875
876 for (DeltaChain base : missing) {
877 if (base.head != null)
878 throw new MissingObjectException(base, "delta base");
879 }
880
881 onEndThinPack();
882 }
883
884 private void growEntries(int extraObjects) {
885 final PackedObjectInfo[] ne;
886
887 ne = new PackedObjectInfo[(int) expectedObjectCount + extraObjects];
888 System.arraycopy(entries, 0, ne, 0, entryCount);
889 entries = ne;
890 }
891
892 private void readPackHeader() throws IOException {
893 if (expectDataAfterPackFooter) {
894 if (!in.markSupported())
895 throw new IOException(
896 JGitText.get().inputStreamMustSupportMark);
897 in.mark(buf.length);
898 }
899
900 final int hdrln = Constants.PACK_SIGNATURE.length + 4 + 4;
901 final int p = fill(Source.INPUT, hdrln);
902 for (int k = 0; k < Constants.PACK_SIGNATURE.length; k++)
903 if (buf[p + k] != Constants.PACK_SIGNATURE[k])
904 throw new IOException(JGitText.get().notAPACKFile);
905
906 final long vers = NB.decodeUInt32(buf, p + 4);
907 if (vers != 2 && vers != 3)
908 throw new IOException(MessageFormat.format(
909 JGitText.get().unsupportedPackVersion, Long.valueOf(vers)));
910 final long objectCount = NB.decodeUInt32(buf, p + 8);
911 use(hdrln);
912 setExpectedObjectCount(objectCount);
913 onPackHeader(objectCount);
914 }
915
916 private void readPackFooter() throws IOException {
917 sync();
918 final byte[] actHash = packDigest.digest();
919
920 final int c = fill(Source.INPUT, 20);
921 final byte[] srcHash = new byte[20];
922 System.arraycopy(buf, c, srcHash, 0, 20);
923 use(20);
924
925 if (bAvail != 0 && !expectDataAfterPackFooter)
926 throw new CorruptObjectException(MessageFormat.format(
927 JGitText.get().expectedEOFReceived,
928 "\\x" + Integer.toHexString(buf[bOffset] & 0xff)));
929 if (isCheckEofAfterPackFooter()) {
930 int eof = in.read();
931 if (0 <= eof)
932 throw new CorruptObjectException(MessageFormat.format(
933 JGitText.get().expectedEOFReceived,
934 "\\x" + Integer.toHexString(eof)));
935 } else if (bAvail > 0 && expectDataAfterPackFooter) {
936 in.reset();
937 IO.skipFully(in, bOffset);
938 }
939
940 if (!Arrays.equals(actHash, srcHash))
941 throw new CorruptObjectException(
942 JGitText.get().corruptObjectPackfileChecksumIncorrect);
943
944 onPackFooter(srcHash);
945 }
946
947
948 private void endInput() {
949 stats.setNumBytesRead(streamPosition());
950 in = null;
951 }
952
953
954 private void indexOneObject() throws IOException {
955 final long streamPosition = streamPosition();
956
957 int hdrPtr = 0;
958 int c = readFrom(Source.INPUT);
959 hdrBuf[hdrPtr++] = (byte) c;
960
961 final int typeCode = (c >> 4) & 7;
962 long sz = c & 15;
963 int shift = 4;
964 while ((c & 0x80) != 0) {
965 c = readFrom(Source.INPUT);
966 hdrBuf[hdrPtr++] = (byte) c;
967 sz += ((long) (c & 0x7f)) << shift;
968 shift += 7;
969 }
970
971 checkIfTooLarge(typeCode, sz);
972
973 switch (typeCode) {
974 case Constants.OBJ_COMMIT:
975 case Constants.OBJ_TREE:
976 case Constants.OBJ_BLOB:
977 case Constants.OBJ_TAG:
978 stats.addWholeObject(typeCode);
979 onBeginWholeObject(streamPosition, typeCode, sz);
980 onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
981 whole(streamPosition, typeCode, sz);
982 break;
983
984 case Constants.OBJ_OFS_DELTA: {
985 stats.addOffsetDelta();
986 c = readFrom(Source.INPUT);
987 hdrBuf[hdrPtr++] = (byte) c;
988 long ofs = c & 127;
989 while ((c & 128) != 0) {
990 ofs += 1;
991 c = readFrom(Source.INPUT);
992 hdrBuf[hdrPtr++] = (byte) c;
993 ofs <<= 7;
994 ofs += (c & 127);
995 }
996 final long base = streamPosition - ofs;
997 onBeginOfsDelta(streamPosition, base, sz);
998 onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
999 inflateAndSkip(Source.INPUT, sz);
1000 UnresolvedDelta n = onEndDelta();
1001 n.position = streamPosition;
1002 n.next = baseByPos.put(base, n);
1003 n.sizeBeforeInflating = streamPosition() - streamPosition;
1004 deltaCount++;
1005 break;
1006 }
1007
1008 case Constants.OBJ_REF_DELTA: {
1009 stats.addRefDelta();
1010 c = fill(Source.INPUT, 20);
1011 final ObjectId base = ObjectId.fromRaw(buf, c);
1012 System.arraycopy(buf, c, hdrBuf, hdrPtr, 20);
1013 hdrPtr += 20;
1014 use(20);
1015 DeltaChain r = baseById.get(base);
1016 if (r == null) {
1017 r = new DeltaChain(base);
1018 baseById.add(r);
1019 }
1020 onBeginRefDelta(streamPosition, base, sz);
1021 onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
1022 inflateAndSkip(Source.INPUT, sz);
1023 UnresolvedDelta n = onEndDelta();
1024 n.position = streamPosition;
1025 n.sizeBeforeInflating = streamPosition() - streamPosition;
1026 r.add(n);
1027 deltaCount++;
1028 break;
1029 }
1030
1031 default:
1032 throw new IOException(
1033 MessageFormat.format(JGitText.get().unknownObjectType,
1034 Integer.valueOf(typeCode)));
1035 }
1036 }
1037
1038 private void whole(long pos, int type, long sz)
1039 throws IOException {
1040 SHA1 objectDigest = objectHasher.reset();
1041 objectDigest.update(Constants.encodedTypeString(type));
1042 objectDigest.update((byte) ' ');
1043 objectDigest.update(Constants.encodeASCII(sz));
1044 objectDigest.update((byte) 0);
1045
1046 final byte[] data;
1047 if (type == Constants.OBJ_BLOB) {
1048 byte[] readBuffer = buffer();
1049 BlobObjectChecker checker = null;
1050 if (objCheck != null) {
1051 checker = objCheck.newBlobObjectChecker();
1052 }
1053 if (checker == null) {
1054 checker = BlobObjectChecker.NULL_CHECKER;
1055 }
1056 long cnt = 0;
1057 try (InputStream inf = inflate(Source.INPUT, sz)) {
1058 while (cnt < sz) {
1059 int r = inf.read(readBuffer);
1060 if (r <= 0)
1061 break;
1062 objectDigest.update(readBuffer, 0, r);
1063 checker.update(readBuffer, 0, r);
1064 cnt += r;
1065 }
1066 }
1067 objectDigest.digest(tempObjectId);
1068 checker.endBlob(tempObjectId);
1069 data = null;
1070 } else {
1071 data = inflateAndReturn(Source.INPUT, sz);
1072 objectDigest.update(data);
1073 objectDigest.digest(tempObjectId);
1074 verifySafeObject(tempObjectId, type, data);
1075 }
1076
1077 long sizeBeforeInflating = streamPosition() - pos;
1078 PackedObjectInfo obj = newInfo(tempObjectId, null, null);
1079 obj.setOffset(pos);
1080 obj.setType(type);
1081 obj.setSize(sizeBeforeInflating);
1082 onEndWholeObject(obj);
1083 if (data != null)
1084 onInflatedObjectData(obj, type, data);
1085 addObjectAndTrack(obj);
1086
1087 if (isCheckObjectCollisions()) {
1088 collisionCheckObjs.add(obj);
1089 }
1090 }
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104 protected void verifySafeObject(final AnyObjectId id, final int type,
1105 final byte[] data) throws CorruptObjectException {
1106 if (objCheck != null) {
1107 try {
1108 objCheck.check(id, type, data);
1109 } catch (CorruptObjectException e) {
1110 if (e.getErrorType() != null) {
1111 throw e;
1112 }
1113 throw new CorruptObjectException(
1114 MessageFormat.format(JGitText.get().invalidObject,
1115 Constants.typeString(type), id.name(),
1116 e.getMessage()),
1117 e);
1118 }
1119 }
1120 }
1121
1122 private void checkObjectCollision() throws IOException {
1123 for (PackedObjectInfo obj : collisionCheckObjs) {
1124 if (!readCurs.has(obj)) {
1125 continue;
1126 }
1127 checkObjectCollision(obj);
1128 }
1129 }
1130
1131 private void checkObjectCollision(PackedObjectInfo obj)
1132 throws IOException {
1133 ObjectTypeAndSize info = openDatabase(obj, new ObjectTypeAndSize());
1134 final byte[] readBuffer = buffer();
1135 final byte[] curBuffer = new byte[readBuffer.length];
1136 long sz = info.size;
1137 try (ObjectStream cur = readCurs.open(obj, info.type).openStream()) {
1138 if (cur.getSize() != sz) {
1139 throw new IOException(MessageFormat.format(
1140 JGitText.get().collisionOn, obj.name()));
1141 }
1142 try (InputStream pck = inflate(Source.DATABASE, sz)) {
1143 while (0 < sz) {
1144 int n = (int) Math.min(readBuffer.length, sz);
1145 IO.readFully(cur, curBuffer, 0, n);
1146 IO.readFully(pck, readBuffer, 0, n);
1147 for (int i = 0; i < n; i++) {
1148 if (curBuffer[i] != readBuffer[i]) {
1149 throw new IOException(MessageFormat.format(
1150 JGitText.get().collisionOn, obj.name()));
1151 }
1152 }
1153 sz -= n;
1154 }
1155 }
1156 stats.incrementObjectsDuplicated();
1157 stats.incrementNumBytesDuplicated(obj.getSize());
1158 } catch (MissingObjectException notLocal) {
1159
1160
1161
1162 }
1163 }
1164
1165 private void checkObjectCollision(AnyObjectId obj, int type, byte[] data,
1166 long sizeBeforeInflating) throws IOException {
1167 try {
1168 final ObjectLoader ldr = readCurs.open(obj, type);
1169 final byte[] existingData = ldr.getCachedBytes(data.length);
1170 if (!Arrays.equals(data, existingData)) {
1171 throw new IOException(MessageFormat
1172 .format(JGitText.get().collisionOn, obj.name()));
1173 }
1174 stats.incrementObjectsDuplicated();
1175 stats.incrementNumBytesDuplicated(sizeBeforeInflating);
1176 } catch (MissingObjectException notLocal) {
1177
1178
1179
1180 }
1181 }
1182
1183
1184 private long streamPosition() {
1185 return bBase + bOffset;
1186 }
1187
1188 private ObjectTypeAndSize openDatabase(PackedObjectInfo obj,
1189 ObjectTypeAndSize info) throws IOException {
1190 bOffset = 0;
1191 bAvail = 0;
1192 return seekDatabase(obj, info);
1193 }
1194
1195 private ObjectTypeAndSize openDatabase(UnresolvedDelta delta,
1196 ObjectTypeAndSize info) throws IOException {
1197 bOffset = 0;
1198 bAvail = 0;
1199 return seekDatabase(delta, info);
1200 }
1201
1202
1203 private int readFrom(Source src) throws IOException {
1204 if (bAvail == 0)
1205 fill(src, 1);
1206 bAvail--;
1207 return buf[bOffset++] & 0xff;
1208 }
1209
1210
1211 void use(int cnt) {
1212 bOffset += cnt;
1213 bAvail -= cnt;
1214 }
1215
1216
1217 int fill(Source src, int need) throws IOException {
1218 while (bAvail < need) {
1219 int next = bOffset + bAvail;
1220 int free = buf.length - next;
1221 if (free + bAvail < need) {
1222 switch (src) {
1223 case INPUT:
1224 sync();
1225 break;
1226 case DATABASE:
1227 if (bAvail > 0)
1228 System.arraycopy(buf, bOffset, buf, 0, bAvail);
1229 bOffset = 0;
1230 break;
1231 }
1232 next = bAvail;
1233 free = buf.length - next;
1234 }
1235 switch (src) {
1236 case INPUT:
1237 next = in.read(buf, next, free);
1238 break;
1239 case DATABASE:
1240 next = readDatabase(buf, next, free);
1241 break;
1242 }
1243 if (next <= 0)
1244 throw new EOFException(
1245 JGitText.get().packfileIsTruncatedNoParam);
1246 bAvail += next;
1247 }
1248 return bOffset;
1249 }
1250
1251
1252 private void sync() throws IOException {
1253 packDigest.update(buf, 0, bOffset);
1254 onStoreStream(buf, 0, bOffset);
1255 if (expectDataAfterPackFooter) {
1256 if (bAvail > 0) {
1257 in.reset();
1258 IO.skipFully(in, bOffset);
1259 bAvail = 0;
1260 }
1261 in.mark(buf.length);
1262 } else if (bAvail > 0)
1263 System.arraycopy(buf, bOffset, buf, 0, bAvail);
1264 bBase += bOffset;
1265 bOffset = 0;
1266 }
1267
1268
1269
1270
1271
1272
1273 protected byte[] buffer() {
1274 return tempBuffer;
1275 }
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291 protected PackedObjectInfo newInfo(AnyObjectId id, UnresolvedDelta delta,
1292 ObjectId deltaBase) {
1293 PackedObjectInfo oe = new PackedObjectInfo(id);
1294 if (delta != null)
1295 oe.setCRC(delta.crc);
1296 return oe;
1297 }
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312 protected void setExpectedObjectCount(long expectedObjectCount) {
1313 this.expectedObjectCount = expectedObjectCount;
1314 }
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337 protected abstract void onStoreStream(byte[] raw, int pos, int len)
1338 throws IOException;
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357 protected abstract void onObjectHeader(Source src, byte[] raw, int pos,
1358 int len) throws IOException;
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380 protected abstract void onObjectData(Source src, byte[] raw, int pos,
1381 int len) throws IOException;
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395 protected abstract void onInflatedObjectData(PackedObjectInfo obj,
1396 int typeCode, byte[] data) throws IOException;
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406 protected abstract void onPackHeader(long objCnt) throws IOException;
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417 protected abstract void onPackFooter(byte[] hash) throws IOException;
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440 protected abstract boolean onAppendBase(int typeCode, byte[] data,
1441 PackedObjectInfo info) throws IOException;
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453 protected abstract void onEndThinPack() throws IOException;
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470 protected abstract ObjectTypeAndSize seekDatabase(PackedObjectInfo obj,
1471 ObjectTypeAndSize info) throws IOException;
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488 protected abstract ObjectTypeAndSize seekDatabase(UnresolvedDelta delta,
1489 ObjectTypeAndSize info) throws IOException;
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505 protected abstract int readDatabase(byte[] dst, int pos, int cnt)
1506 throws IOException;
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524 protected abstract boolean checkCRC(int oldCRC);
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543 protected abstract void onBeginWholeObject(long streamPosition, int type,
1544 long inflatedSize) throws IOException;
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554 protected abstract void onEndWholeObject(PackedObjectInfo info)
1555 throws IOException;
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573 protected abstract void onBeginOfsDelta(long deltaStreamPosition,
1574 long baseStreamPosition, long inflatedSize) throws IOException;
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591 protected abstract void onBeginRefDelta(long deltaStreamPosition,
1592 AnyObjectId baseId, long inflatedSize) throws IOException;
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602 protected UnresolvedDelta onEndDelta() throws IOException {
1603 return new UnresolvedDelta();
1604 }
1605
1606
1607 public static class ObjectTypeAndSize {
1608
1609 public int type;
1610
1611
1612 public long size;
1613 }
1614
1615 private void inflateAndSkip(Source src, long inflatedSize)
1616 throws IOException {
1617 try (InputStream inf = inflate(src, inflatedSize)) {
1618 IO.skipFully(inf, inflatedSize);
1619 }
1620 }
1621
1622 private byte[] inflateAndReturn(Source src, long inflatedSize)
1623 throws IOException {
1624 final byte[] dst = new byte[(int) inflatedSize];
1625 try (InputStream inf = inflate(src, inflatedSize)) {
1626 IO.readFully(inf, dst, 0, dst.length);
1627 }
1628 return dst;
1629 }
1630
1631 private InputStream inflate(Source src, long inflatedSize)
1632 throws IOException {
1633 inflater.open(src, inflatedSize);
1634 return inflater;
1635 }
1636
1637 private static class DeltaChain extends ObjectIdOwnerMap.Entry {
1638 UnresolvedDelta head;
1639
1640 DeltaChain(AnyObjectId id) {
1641 super(id);
1642 }
1643
1644 UnresolvedDelta remove() {
1645 final UnresolvedDelta r = head;
1646 if (r != null)
1647 head = null;
1648 return r;
1649 }
1650
1651 void add(UnresolvedDelta d) {
1652 d.next = head;
1653 head = d;
1654 }
1655 }
1656
1657
1658 public static class UnresolvedDelta {
1659 long position;
1660
1661 int crc;
1662
1663 UnresolvedDelta next;
1664
1665 long sizeBeforeInflating;
1666
1667
1668 public long getOffset() {
1669 return position;
1670 }
1671
1672
1673 public int getCRC() {
1674 return crc;
1675 }
1676
1677
1678
1679
1680
1681 public void setCRC(int crc32) {
1682 crc = crc32;
1683 }
1684 }
1685
1686 private static class DeltaVisit {
1687 final UnresolvedDelta delta;
1688
1689 ObjectId id;
1690
1691 byte[] data;
1692
1693 DeltaVisit parent;
1694
1695 UnresolvedDelta nextChild;
1696
1697 DeltaVisit() {
1698 this.delta = null;
1699 }
1700
1701 DeltaVisit(DeltaVisit parent) {
1702 this.parent = parent;
1703 this.delta = parent.nextChild;
1704 parent.nextChild = delta.next;
1705 }
1706
1707 DeltaVisit next() {
1708
1709 if (parent != null && parent.nextChild == null) {
1710 parent.data = null;
1711 parent = parent.parent;
1712 }
1713
1714 if (nextChild != null)
1715 return new DeltaVisit(this);
1716
1717
1718
1719 if (parent != null)
1720 return new DeltaVisit(parent);
1721 return null;
1722 }
1723 }
1724
1725 private void addObjectAndTrack(PackedObjectInfo oe) {
1726 entries[entryCount++] = oe;
1727 if (needNewObjectIds())
1728 newObjectIds.add(oe);
1729 }
1730
1731 private class InflaterStream extends InputStream {
1732 private final Inflater inf;
1733
1734 private final byte[] skipBuffer;
1735
1736 private Source src;
1737
1738 private long expectedSize;
1739
1740 private long actualSize;
1741
1742 private int p;
1743
1744 InflaterStream() {
1745 inf = InflaterCache.get();
1746 skipBuffer = new byte[512];
1747 }
1748
1749 void release() {
1750 inf.reset();
1751 InflaterCache.release(inf);
1752 }
1753
1754 void open(Source source, long inflatedSize) throws IOException {
1755 src = source;
1756 expectedSize = inflatedSize;
1757 actualSize = 0;
1758
1759 p = fill(src, 1);
1760 inf.setInput(buf, p, bAvail);
1761 }
1762
1763 @Override
1764 public long skip(long toSkip) throws IOException {
1765 long n = 0;
1766 while (n < toSkip) {
1767 final int cnt = (int) Math.min(skipBuffer.length, toSkip - n);
1768 final int r = read(skipBuffer, 0, cnt);
1769 if (r <= 0)
1770 break;
1771 n += r;
1772 }
1773 return n;
1774 }
1775
1776 @Override
1777 public int read() throws IOException {
1778 int n = read(skipBuffer, 0, 1);
1779 return n == 1 ? skipBuffer[0] & 0xff : -1;
1780 }
1781
1782 @Override
1783 public int read(byte[] dst, int pos, int cnt) throws IOException {
1784 try {
1785 int n = 0;
1786 while (n < cnt) {
1787 int r = inf.inflate(dst, pos + n, cnt - n);
1788 n += r;
1789 if (inf.finished())
1790 break;
1791 if (inf.needsInput()) {
1792 onObjectData(src, buf, p, bAvail);
1793 use(bAvail);
1794
1795 p = fill(src, 1);
1796 inf.setInput(buf, p, bAvail);
1797 } else if (r == 0) {
1798 throw new CorruptObjectException(MessageFormat.format(
1799 JGitText.get().packfileCorruptionDetected,
1800 JGitText.get().unknownZlibError));
1801 }
1802 }
1803 actualSize += n;
1804 return 0 < n ? n : -1;
1805 } catch (DataFormatException dfe) {
1806 throw new CorruptObjectException(MessageFormat.format(JGitText
1807 .get().packfileCorruptionDetected, dfe.getMessage()));
1808 }
1809 }
1810
1811 @Override
1812 public void close() throws IOException {
1813
1814
1815
1816
1817 if (read(skipBuffer) != -1 || actualSize != expectedSize) {
1818 throw new CorruptObjectException(MessageFormat.format(JGitText
1819 .get().packfileCorruptionDetected,
1820 JGitText.get().wrongDecompressedLength));
1821 }
1822
1823 int used = bAvail - inf.getRemaining();
1824 if (0 < used) {
1825 onObjectData(src, buf, p, used);
1826 use(used);
1827 }
1828
1829 inf.reset();
1830 }
1831 }
1832 }