1
2
3
4
5
6
7
8
9
10
11
12
13 package org.eclipse.jgit.transport;
14
15 import java.io.EOFException;
16 import java.io.IOException;
17 import java.io.InputStream;
18 import java.security.MessageDigest;
19 import java.text.MessageFormat;
20 import java.util.ArrayList;
21 import java.util.Arrays;
22 import java.util.Comparator;
23 import java.util.List;
24 import java.util.concurrent.TimeUnit;
25 import java.util.zip.DataFormatException;
26 import java.util.zip.Inflater;
27
28 import org.eclipse.jgit.errors.CorruptObjectException;
29 import org.eclipse.jgit.errors.MissingObjectException;
30 import org.eclipse.jgit.errors.TooLargeObjectInPackException;
31 import org.eclipse.jgit.internal.JGitText;
32 import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
33 import org.eclipse.jgit.lib.AnyObjectId;
34 import org.eclipse.jgit.lib.BatchingProgressMonitor;
35 import org.eclipse.jgit.lib.BlobObjectChecker;
36 import org.eclipse.jgit.lib.Constants;
37 import org.eclipse.jgit.lib.InflaterCache;
38 import org.eclipse.jgit.lib.MutableObjectId;
39 import org.eclipse.jgit.lib.NullProgressMonitor;
40 import org.eclipse.jgit.lib.ObjectChecker;
41 import org.eclipse.jgit.lib.ObjectDatabase;
42 import org.eclipse.jgit.lib.ObjectId;
43 import org.eclipse.jgit.lib.ObjectIdOwnerMap;
44 import org.eclipse.jgit.lib.ObjectIdSubclassMap;
45 import org.eclipse.jgit.lib.ObjectLoader;
46 import org.eclipse.jgit.lib.ObjectReader;
47 import org.eclipse.jgit.lib.ObjectStream;
48 import org.eclipse.jgit.lib.ProgressMonitor;
49 import org.eclipse.jgit.util.BlockList;
50 import org.eclipse.jgit.util.IO;
51 import org.eclipse.jgit.util.LongMap;
52 import org.eclipse.jgit.util.NB;
53 import org.eclipse.jgit.util.sha1.SHA1;
54
55
56
57
58
59
60
61
62
63
64
65
66
67 public abstract class PackParser {
68
69 private static final int BUFFER_SIZE = 8192;
70
71
72 public enum Source {
73
74 INPUT,
75
76
77 DATABASE;
78 }
79
80
81 private final ObjectDatabase objectDatabase;
82
83 private InflaterStream inflater;
84
85 private byte[] tempBuffer;
86
87 private byte[] hdrBuf;
88
89 private final SHA1 objectHasher = SHA1.newInstance();
90 private final MutableObjectId tempObjectId;
91
92 private InputStream in;
93
94 byte[] buf;
95
96
97 private long bBase;
98
99 private int bOffset;
100
101 int bAvail;
102
103 private ObjectChecker objCheck;
104
105 private boolean allowThin;
106
107 private boolean checkObjectCollisions;
108
109 private boolean needBaseObjectIds;
110
111 private boolean checkEofAfterPackFooter;
112
113 private boolean expectDataAfterPackFooter;
114
115 private long expectedObjectCount;
116
117 private PackedObjectInfo[] entries;
118
119
120
121
122
123
124
125
126 private ObjectIdSubclassMap<ObjectId> newObjectIds;
127
128 private int deltaCount;
129
130 private int entryCount;
131
132 private ObjectIdOwnerMap<DeltaChain> baseById;
133
134
135
136
137
138
139
140
141 private ObjectIdSubclassMap<ObjectId> baseObjectIds;
142
143 private LongMap<UnresolvedDelta> baseByPos;
144
145
146 private BlockList<PackedObjectInfo> collisionCheckObjs;
147
148 private MessageDigest packDigest;
149
150 private ObjectReader readCurs;
151
152
153 private String lockMessage;
154
155
156 private long maxObjectSizeLimit;
157
158 private final ReceivedPackStatistics.Builder stats =
159 new ReceivedPackStatistics.Builder();
160
161
162
163
164
165
166
167
168
169 protected PackParser(ObjectDatabase odb, InputStream src) {
170 objectDatabase = odb.newCachedDatabase();
171 in = src;
172
173 inflater = new InflaterStream();
174 readCurs = objectDatabase.newReader();
175 buf = new byte[BUFFER_SIZE];
176 tempBuffer = new byte[BUFFER_SIZE];
177 hdrBuf = new byte[64];
178 tempObjectId = new MutableObjectId();
179 packDigest = Constants.newMessageDigest();
180 checkObjectCollisions = true;
181 }
182
183
184
185
186
187
188 public boolean isAllowThin() {
189 return allowThin;
190 }
191
192
193
194
195
196
197
198
199
200
201 public void setAllowThin(boolean allow) {
202 allowThin = allow;
203 }
204
205
206
207
208
209
210
211 protected boolean isCheckObjectCollisions() {
212 return checkObjectCollisions;
213 }
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236 protected void setCheckObjectCollisions(boolean check) {
237 checkObjectCollisions = check;
238 }
239
240
241
242
243
244
245
246
247
248
249
250 public void setNeedNewObjectIds(boolean b) {
251 if (b)
252 newObjectIds = new ObjectIdSubclassMap<>();
253 else
254 newObjectIds = null;
255 }
256
257 private boolean needNewObjectIds() {
258 return newObjectIds != null;
259 }
260
261
262
263
264
265
266
267
268
269
270
271
272 public void setNeedBaseObjectIds(boolean b) {
273 this.needBaseObjectIds = b;
274 }
275
276
277
278
279
280
281 public boolean isCheckEofAfterPackFooter() {
282 return checkEofAfterPackFooter;
283 }
284
285
286
287
288
289
290
291 public void setCheckEofAfterPackFooter(boolean b) {
292 checkEofAfterPackFooter = b;
293 }
294
295
296
297
298
299
300 public boolean isExpectDataAfterPackFooter() {
301 return expectDataAfterPackFooter;
302 }
303
304
305
306
307
308
309
310
311
312 public void setExpectDataAfterPackFooter(boolean e) {
313 expectDataAfterPackFooter = e;
314 }
315
316
317
318
319
320
321 public ObjectIdSubclassMap<ObjectId> getNewObjectIds() {
322 if (newObjectIds != null)
323 return newObjectIds;
324 return new ObjectIdSubclassMap<>();
325 }
326
327
328
329
330
331
332 public ObjectIdSubclassMap<ObjectId> getBaseObjectIds() {
333 if (baseObjectIds != null)
334 return baseObjectIds;
335 return new ObjectIdSubclassMap<>();
336 }
337
338
339
340
341
342
343
344
345
346
347
348 public void setObjectChecker(ObjectChecker oc) {
349 objCheck = oc;
350 }
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368 public void setObjectChecking(boolean on) {
369 setObjectChecker(on ? new ObjectChecker() : null);
370 }
371
372
373
374
375
376
377 public String getLockMessage() {
378 return lockMessage;
379 }
380
381
382
383
384
385
386
387
388 public void setLockMessage(String msg) {
389 lockMessage = msg;
390 }
391
392
393
394
395
396
397
398
399
400
401 public void setMaxObjectSizeLimit(long limit) {
402 maxObjectSizeLimit = limit;
403 }
404
405
406
407
408
409
410
411
412
413
414 public int getObjectCount() {
415 return entryCount;
416 }
417
418
419
420
421
422
423
424
425
426
427
428
429 public PackedObjectInfo getObject(int nth) {
430 return entries[nth];
431 }
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447 public List<PackedObjectInfo> getSortedObjectList(
448 Comparator<PackedObjectInfo> cmp) {
449 Arrays.sort(entries, 0, entryCount, cmp);
450 List<PackedObjectInfo> list = Arrays.asList(entries);
451 if (entryCount < entries.length)
452 list = list.subList(0, entryCount);
453 return list;
454 }
455
456
457
458
459
460
461
462
463
464
465
466 public long getPackSize() {
467 return -1;
468 }
469
470
471
472
473
474
475
476
477
478 public ReceivedPackStatistics getReceivedPackStatistics() {
479 return stats.build();
480 }
481
482
483
484
485
486
487
488
489
490
491
492
493
494 public final PackLock parse(ProgressMonitor progress) throws IOException {
495 return parse(progress, progress);
496 }
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513 public PackLock parse(ProgressMonitor receiving, ProgressMonitor resolving)
514 throws IOException {
515 if (receiving == null)
516 receiving = NullProgressMonitor.INSTANCE;
517 if (resolving == null)
518 resolving = NullProgressMonitor.INSTANCE;
519
520 if (receiving == resolving)
521 receiving.start(2 );
522 try {
523 readPackHeader();
524
525 entries = new PackedObjectInfo[(int) expectedObjectCount];
526 baseById = new ObjectIdOwnerMap<>();
527 baseByPos = new LongMap<>();
528 collisionCheckObjs = new BlockList<>();
529
530 receiving.beginTask(JGitText.get().receivingObjects,
531 (int) expectedObjectCount);
532 try {
533 for (int done = 0; done < expectedObjectCount; done++) {
534 indexOneObject();
535 receiving.update(1);
536 if (receiving.isCancelled())
537 throw new IOException(JGitText.get().downloadCancelled);
538 }
539 readPackFooter();
540 endInput();
541 } finally {
542 receiving.endTask();
543 }
544
545 if (!collisionCheckObjs.isEmpty()) {
546 checkObjectCollision();
547 }
548
549 if (deltaCount > 0) {
550 processDeltas(resolving);
551 }
552
553 packDigest = null;
554 baseById = null;
555 baseByPos = null;
556 } finally {
557 try {
558 if (readCurs != null)
559 readCurs.close();
560 } finally {
561 readCurs = null;
562 }
563
564 try {
565 inflater.release();
566 } finally {
567 inflater = null;
568 }
569 }
570 return null;
571 }
572
573 private void processDeltas(ProgressMonitor resolving) throws IOException {
574 if (resolving instanceof BatchingProgressMonitor) {
575 ((BatchingProgressMonitor) resolving).setDelayStart(1000,
576 TimeUnit.MILLISECONDS);
577 }
578 resolving.beginTask(JGitText.get().resolvingDeltas, deltaCount);
579 resolveDeltas(resolving);
580 if (entryCount < expectedObjectCount) {
581 if (!isAllowThin()) {
582 throw new IOException(MessageFormat.format(
583 JGitText.get().packHasUnresolvedDeltas,
584 Long.valueOf(expectedObjectCount - entryCount)));
585 }
586
587 resolveDeltasWithExternalBases(resolving);
588
589 if (entryCount < expectedObjectCount) {
590 throw new IOException(MessageFormat.format(
591 JGitText.get().packHasUnresolvedDeltas,
592 Long.valueOf(expectedObjectCount - entryCount)));
593 }
594 }
595 resolving.endTask();
596 }
597
598 private void resolveDeltas(ProgressMonitor progress)
599 throws IOException {
600 final int last = entryCount;
601 for (int i = 0; i < last; i++) {
602 resolveDeltas(entries[i], progress);
603 if (progress.isCancelled())
604 throw new IOException(
605 JGitText.get().downloadCancelledDuringIndexing);
606 }
607 }
608
609 private void resolveDeltas(final PackedObjectInfo oe,
610 ProgressMonitor progress) throws IOException {
611 UnresolvedDelta children = firstChildOf(oe);
612 if (children == null)
613 return;
614
615 DeltaVisit visit = new DeltaVisit();
616 visit.nextChild = children;
617
618 ObjectTypeAndSize info = openDatabase(oe, new ObjectTypeAndSize());
619 switch (info.type) {
620 case Constants.OBJ_COMMIT:
621 case Constants.OBJ_TREE:
622 case Constants.OBJ_BLOB:
623 case Constants.OBJ_TAG:
624 visit.data = inflateAndReturn(Source.DATABASE, info.size);
625 visit.id = oe;
626 break;
627 default:
628 throw new IOException(MessageFormat.format(
629 JGitText.get().unknownObjectType,
630 Integer.valueOf(info.type)));
631 }
632
633 if (!checkCRC(oe.getCRC())) {
634 throw new IOException(MessageFormat.format(
635 JGitText.get().corruptionDetectedReReadingAt,
636 Long.valueOf(oe.getOffset())));
637 }
638
639 resolveDeltas(visit.next(), info.type, info, progress);
640 }
641
642 private void resolveDeltas(DeltaVisit visit, final int type,
643 ObjectTypeAndSize info, ProgressMonitor progress)
644 throws IOException {
645 stats.addDeltaObject(type);
646 do {
647 progress.update(1);
648 info = openDatabase(visit.delta, info);
649 switch (info.type) {
650 case Constants.OBJ_OFS_DELTA:
651 case Constants.OBJ_REF_DELTA:
652 break;
653
654 default:
655 throw new IOException(MessageFormat.format(
656 JGitText.get().unknownObjectType,
657 Integer.valueOf(info.type)));
658 }
659
660 byte[] delta = inflateAndReturn(Source.DATABASE, info.size);
661 checkIfTooLarge(type, BinaryDelta.getResultSize(delta));
662
663 visit.data = BinaryDelta.apply(visit.parent.data, delta);
664 delta = null;
665
666 if (!checkCRC(visit.delta.crc))
667 throw new IOException(MessageFormat.format(
668 JGitText.get().corruptionDetectedReReadingAt,
669 Long.valueOf(visit.delta.position)));
670
671 SHA1 objectDigest = objectHasher.reset();
672 objectDigest.update(Constants.encodedTypeString(type));
673 objectDigest.update((byte) ' ');
674 objectDigest.update(Constants.encodeASCII(visit.data.length));
675 objectDigest.update((byte) 0);
676 objectDigest.update(visit.data);
677 objectDigest.digest(tempObjectId);
678
679 verifySafeObject(tempObjectId, type, visit.data);
680 if (isCheckObjectCollisions() && readCurs.has(tempObjectId)) {
681 checkObjectCollision(tempObjectId, type, visit.data,
682 visit.delta.sizeBeforeInflating);
683 }
684
685 PackedObjectInfo oe;
686 oe = newInfo(tempObjectId, visit.delta, visit.parent.id);
687 oe.setOffset(visit.delta.position);
688 oe.setType(type);
689 onInflatedObjectData(oe, type, visit.data);
690 addObjectAndTrack(oe);
691 visit.id = oe;
692
693 visit.nextChild = firstChildOf(oe);
694 visit = visit.next();
695 } while (visit != null);
696 }
697
698 private final void checkIfTooLarge(int typeCode, long size)
699 throws IOException {
700 if (0 < maxObjectSizeLimit && maxObjectSizeLimit < size) {
701 switch (typeCode) {
702 case Constants.OBJ_COMMIT:
703 case Constants.OBJ_TREE:
704 case Constants.OBJ_BLOB:
705 case Constants.OBJ_TAG:
706 throw new TooLargeObjectInPackException(size, maxObjectSizeLimit);
707
708 case Constants.OBJ_OFS_DELTA:
709 case Constants.OBJ_REF_DELTA:
710 throw new TooLargeObjectInPackException(size, maxObjectSizeLimit);
711
712 default:
713 throw new IOException(MessageFormat.format(
714 JGitText.get().unknownObjectType,
715 Integer.valueOf(typeCode)));
716 }
717 }
718 if (size > Integer.MAX_VALUE - 8) {
719 throw new TooLargeObjectInPackException(size, Integer.MAX_VALUE - 8);
720 }
721 }
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739 protected ObjectTypeAndSize readObjectHeader(ObjectTypeAndSize info)
740 throws IOException {
741 int hdrPtr = 0;
742 int c = readFrom(Source.DATABASE);
743 hdrBuf[hdrPtr++] = (byte) c;
744
745 info.type = (c >> 4) & 7;
746 long sz = c & 15;
747 int shift = 4;
748 while ((c & 0x80) != 0) {
749 c = readFrom(Source.DATABASE);
750 hdrBuf[hdrPtr++] = (byte) c;
751 sz += ((long) (c & 0x7f)) << shift;
752 shift += 7;
753 }
754 info.size = sz;
755
756 switch (info.type) {
757 case Constants.OBJ_COMMIT:
758 case Constants.OBJ_TREE:
759 case Constants.OBJ_BLOB:
760 case Constants.OBJ_TAG:
761 onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
762 break;
763
764 case Constants.OBJ_OFS_DELTA:
765 c = readFrom(Source.DATABASE);
766 hdrBuf[hdrPtr++] = (byte) c;
767 while ((c & 128) != 0) {
768 c = readFrom(Source.DATABASE);
769 hdrBuf[hdrPtr++] = (byte) c;
770 }
771 onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
772 break;
773
774 case Constants.OBJ_REF_DELTA:
775 System.arraycopy(buf, fill(Source.DATABASE, 20), hdrBuf, hdrPtr, 20);
776 hdrPtr += 20;
777 use(20);
778 onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
779 break;
780
781 default:
782 throw new IOException(MessageFormat.format(
783 JGitText.get().unknownObjectType,
784 Integer.valueOf(info.type)));
785 }
786 return info;
787 }
788
789 private UnresolvedDelta removeBaseById(AnyObjectId id) {
790 final DeltaChain d = baseById.get(id);
791 return d != null ? d.remove() : null;
792 }
793
794 private static UnresolvedDelta reverse(UnresolvedDelta c) {
795 UnresolvedDelta tail = null;
796 while (c != null) {
797 final UnresolvedDelta n = c.next;
798 c.next = tail;
799 tail = c;
800 c = n;
801 }
802 return tail;
803 }
804
805 private UnresolvedDelta firstChildOf(PackedObjectInfo oe) {
806 UnresolvedDelta a = reverse(removeBaseById(oe));
807 UnresolvedDelta b = reverse(baseByPos.remove(oe.getOffset()));
808
809 if (a == null)
810 return b;
811 if (b == null)
812 return a;
813
814 UnresolvedDelta first = null;
815 UnresolvedDelta last = null;
816 while (a != null || b != null) {
817 UnresolvedDelta curr;
818 if (b == null || (a != null && a.position < b.position)) {
819 curr = a;
820 a = a.next;
821 } else {
822 curr = b;
823 b = b.next;
824 }
825 if (last != null)
826 last.next = curr;
827 else
828 first = curr;
829 last = curr;
830 curr.next = null;
831 }
832 return first;
833 }
834
835 private void resolveDeltasWithExternalBases(ProgressMonitor progress)
836 throws IOException {
837 growEntries(baseById.size());
838
839 if (needBaseObjectIds)
840 baseObjectIds = new ObjectIdSubclassMap<>();
841
842 final List<DeltaChain> missing = new ArrayList<>(64);
843 for (DeltaChain baseId : baseById) {
844 if (baseId.head == null)
845 continue;
846
847 if (needBaseObjectIds)
848 baseObjectIds.add(baseId);
849
850 final ObjectLoader ldr;
851 try {
852 ldr = readCurs.open(baseId);
853 } catch (MissingObjectException notFound) {
854 missing.add(baseId);
855 continue;
856 }
857
858 final DeltaVisit visit = new DeltaVisit();
859 visit.data = ldr.getCachedBytes(Integer.MAX_VALUE);
860 visit.id = baseId;
861 final int typeCode = ldr.getType();
862 final PackedObjectInfo oe = newInfo(baseId, null, null);
863 oe.setType(typeCode);
864 if (onAppendBase(typeCode, visit.data, oe))
865 entries[entryCount++] = oe;
866 visit.nextChild = firstChildOf(oe);
867 resolveDeltas(visit.next(), typeCode,
868 new ObjectTypeAndSize(), progress);
869
870 if (progress.isCancelled())
871 throw new IOException(
872 JGitText.get().downloadCancelledDuringIndexing);
873 }
874
875 for (DeltaChain base : missing) {
876 if (base.head != null)
877 throw new MissingObjectException(base, "delta base");
878 }
879
880 onEndThinPack();
881 }
882
883 private void growEntries(int extraObjects) {
884 final PackedObjectInfo[] ne;
885
886 ne = new PackedObjectInfo[(int) expectedObjectCount + extraObjects];
887 System.arraycopy(entries, 0, ne, 0, entryCount);
888 entries = ne;
889 }
890
891 private void readPackHeader() throws IOException {
892 if (expectDataAfterPackFooter) {
893 if (!in.markSupported())
894 throw new IOException(
895 JGitText.get().inputStreamMustSupportMark);
896 in.mark(buf.length);
897 }
898
899 final int hdrln = Constants.PACK_SIGNATURE.length + 4 + 4;
900 final int p = fill(Source.INPUT, hdrln);
901 for (int k = 0; k < Constants.PACK_SIGNATURE.length; k++)
902 if (buf[p + k] != Constants.PACK_SIGNATURE[k])
903 throw new IOException(JGitText.get().notAPACKFile);
904
905 final long vers = NB.decodeUInt32(buf, p + 4);
906 if (vers != 2 && vers != 3)
907 throw new IOException(MessageFormat.format(
908 JGitText.get().unsupportedPackVersion, Long.valueOf(vers)));
909 final long objectCount = NB.decodeUInt32(buf, p + 8);
910 use(hdrln);
911 setExpectedObjectCount(objectCount);
912 onPackHeader(objectCount);
913 }
914
915 private void readPackFooter() throws IOException {
916 sync();
917 final byte[] actHash = packDigest.digest();
918
919 final int c = fill(Source.INPUT, 20);
920 final byte[] srcHash = new byte[20];
921 System.arraycopy(buf, c, srcHash, 0, 20);
922 use(20);
923
924 if (bAvail != 0 && !expectDataAfterPackFooter)
925 throw new CorruptObjectException(MessageFormat.format(
926 JGitText.get().expectedEOFReceived,
927 "\\x" + Integer.toHexString(buf[bOffset] & 0xff)));
928 if (isCheckEofAfterPackFooter()) {
929 int eof = in.read();
930 if (0 <= eof)
931 throw new CorruptObjectException(MessageFormat.format(
932 JGitText.get().expectedEOFReceived,
933 "\\x" + Integer.toHexString(eof)));
934 } else if (bAvail > 0 && expectDataAfterPackFooter) {
935 in.reset();
936 IO.skipFully(in, bOffset);
937 }
938
939 if (!Arrays.equals(actHash, srcHash))
940 throw new CorruptObjectException(
941 JGitText.get().corruptObjectPackfileChecksumIncorrect);
942
943 onPackFooter(srcHash);
944 }
945
946
947 private void endInput() {
948 stats.setNumBytesRead(streamPosition());
949 in = null;
950 }
951
952
953 private void indexOneObject() throws IOException {
954 final long streamPosition = streamPosition();
955
956 int hdrPtr = 0;
957 int c = readFrom(Source.INPUT);
958 hdrBuf[hdrPtr++] = (byte) c;
959
960 final int typeCode = (c >> 4) & 7;
961 long sz = c & 15;
962 int shift = 4;
963 while ((c & 0x80) != 0) {
964 c = readFrom(Source.INPUT);
965 hdrBuf[hdrPtr++] = (byte) c;
966 sz += ((long) (c & 0x7f)) << shift;
967 shift += 7;
968 }
969
970 checkIfTooLarge(typeCode, sz);
971
972 switch (typeCode) {
973 case Constants.OBJ_COMMIT:
974 case Constants.OBJ_TREE:
975 case Constants.OBJ_BLOB:
976 case Constants.OBJ_TAG:
977 stats.addWholeObject(typeCode);
978 onBeginWholeObject(streamPosition, typeCode, sz);
979 onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
980 whole(streamPosition, typeCode, sz);
981 break;
982
983 case Constants.OBJ_OFS_DELTA: {
984 stats.addOffsetDelta();
985 c = readFrom(Source.INPUT);
986 hdrBuf[hdrPtr++] = (byte) c;
987 long ofs = c & 127;
988 while ((c & 128) != 0) {
989 ofs += 1;
990 c = readFrom(Source.INPUT);
991 hdrBuf[hdrPtr++] = (byte) c;
992 ofs <<= 7;
993 ofs += (c & 127);
994 }
995 final long base = streamPosition - ofs;
996 onBeginOfsDelta(streamPosition, base, sz);
997 onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
998 inflateAndSkip(Source.INPUT, sz);
999 UnresolvedDelta n = onEndDelta();
1000 n.position = streamPosition;
1001 n.next = baseByPos.put(base, n);
1002 n.sizeBeforeInflating = streamPosition() - streamPosition;
1003 deltaCount++;
1004 break;
1005 }
1006
1007 case Constants.OBJ_REF_DELTA: {
1008 stats.addRefDelta();
1009 c = fill(Source.INPUT, 20);
1010 final ObjectId base = ObjectId.fromRaw(buf, c);
1011 System.arraycopy(buf, c, hdrBuf, hdrPtr, 20);
1012 hdrPtr += 20;
1013 use(20);
1014 DeltaChain r = baseById.get(base);
1015 if (r == null) {
1016 r = new DeltaChain(base);
1017 baseById.add(r);
1018 }
1019 onBeginRefDelta(streamPosition, base, sz);
1020 onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
1021 inflateAndSkip(Source.INPUT, sz);
1022 UnresolvedDelta n = onEndDelta();
1023 n.position = streamPosition;
1024 n.sizeBeforeInflating = streamPosition() - streamPosition;
1025 r.add(n);
1026 deltaCount++;
1027 break;
1028 }
1029
1030 default:
1031 throw new IOException(
1032 MessageFormat.format(JGitText.get().unknownObjectType,
1033 Integer.valueOf(typeCode)));
1034 }
1035 }
1036
1037 private void whole(long pos, int type, long sz)
1038 throws IOException {
1039 SHA1 objectDigest = objectHasher.reset();
1040 objectDigest.update(Constants.encodedTypeString(type));
1041 objectDigest.update((byte) ' ');
1042 objectDigest.update(Constants.encodeASCII(sz));
1043 objectDigest.update((byte) 0);
1044
1045 final byte[] data;
1046 if (type == Constants.OBJ_BLOB) {
1047 byte[] readBuffer = buffer();
1048 BlobObjectChecker checker = null;
1049 if (objCheck != null) {
1050 checker = objCheck.newBlobObjectChecker();
1051 }
1052 if (checker == null) {
1053 checker = BlobObjectChecker.NULL_CHECKER;
1054 }
1055 long cnt = 0;
1056 try (InputStream inf = inflate(Source.INPUT, sz)) {
1057 while (cnt < sz) {
1058 int r = inf.read(readBuffer);
1059 if (r <= 0)
1060 break;
1061 objectDigest.update(readBuffer, 0, r);
1062 checker.update(readBuffer, 0, r);
1063 cnt += r;
1064 }
1065 }
1066 objectDigest.digest(tempObjectId);
1067 checker.endBlob(tempObjectId);
1068 data = null;
1069 } else {
1070 data = inflateAndReturn(Source.INPUT, sz);
1071 objectDigest.update(data);
1072 objectDigest.digest(tempObjectId);
1073 verifySafeObject(tempObjectId, type, data);
1074 }
1075
1076 long sizeBeforeInflating = streamPosition() - pos;
1077 PackedObjectInfo obj = newInfo(tempObjectId, null, null);
1078 obj.setOffset(pos);
1079 obj.setType(type);
1080 obj.setSize(sizeBeforeInflating);
1081 onEndWholeObject(obj);
1082 if (data != null)
1083 onInflatedObjectData(obj, type, data);
1084 addObjectAndTrack(obj);
1085
1086 if (isCheckObjectCollisions()) {
1087 collisionCheckObjs.add(obj);
1088 }
1089 }
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103 protected void verifySafeObject(final AnyObjectId id, final int type,
1104 final byte[] data) throws CorruptObjectException {
1105 if (objCheck != null) {
1106 try {
1107 objCheck.check(id, type, data);
1108 } catch (CorruptObjectException e) {
1109 if (e.getErrorType() != null) {
1110 throw e;
1111 }
1112 throw new CorruptObjectException(
1113 MessageFormat.format(JGitText.get().invalidObject,
1114 Constants.typeString(type), id.name(),
1115 e.getMessage()),
1116 e);
1117 }
1118 }
1119 }
1120
1121 private void checkObjectCollision() throws IOException {
1122 for (PackedObjectInfo obj : collisionCheckObjs) {
1123 if (!readCurs.has(obj)) {
1124 continue;
1125 }
1126 checkObjectCollision(obj);
1127 }
1128 }
1129
1130 private void checkObjectCollision(PackedObjectInfo obj)
1131 throws IOException {
1132 ObjectTypeAndSize info = openDatabase(obj, new ObjectTypeAndSize());
1133 final byte[] readBuffer = buffer();
1134 final byte[] curBuffer = new byte[readBuffer.length];
1135 long sz = info.size;
1136 try (ObjectStream cur = readCurs.open(obj, info.type).openStream()) {
1137 if (cur.getSize() != sz) {
1138 throw new IOException(MessageFormat.format(
1139 JGitText.get().collisionOn, obj.name()));
1140 }
1141 try (InputStream pck = inflate(Source.DATABASE, sz)) {
1142 while (0 < sz) {
1143 int n = (int) Math.min(readBuffer.length, sz);
1144 IO.readFully(cur, curBuffer, 0, n);
1145 IO.readFully(pck, readBuffer, 0, n);
1146 for (int i = 0; i < n; i++) {
1147 if (curBuffer[i] != readBuffer[i]) {
1148 throw new IOException(MessageFormat.format(
1149 JGitText.get().collisionOn, obj.name()));
1150 }
1151 }
1152 sz -= n;
1153 }
1154 }
1155 stats.incrementObjectsDuplicated();
1156 stats.incrementNumBytesDuplicated(obj.getSize());
1157 } catch (MissingObjectException notLocal) {
1158
1159
1160
1161 }
1162 }
1163
1164 private void checkObjectCollision(AnyObjectId obj, int type, byte[] data,
1165 long sizeBeforeInflating) throws IOException {
1166 try {
1167 final ObjectLoader ldr = readCurs.open(obj, type);
1168 final byte[] existingData = ldr.getCachedBytes(data.length);
1169 if (!Arrays.equals(data, existingData)) {
1170 throw new IOException(MessageFormat
1171 .format(JGitText.get().collisionOn, obj.name()));
1172 }
1173 stats.incrementObjectsDuplicated();
1174 stats.incrementNumBytesDuplicated(sizeBeforeInflating);
1175 } catch (MissingObjectException notLocal) {
1176
1177
1178
1179 }
1180 }
1181
1182
1183 private long streamPosition() {
1184 return bBase + bOffset;
1185 }
1186
1187 private ObjectTypeAndSize openDatabase(PackedObjectInfo obj,
1188 ObjectTypeAndSize info) throws IOException {
1189 bOffset = 0;
1190 bAvail = 0;
1191 return seekDatabase(obj, info);
1192 }
1193
1194 private ObjectTypeAndSize openDatabase(UnresolvedDelta delta,
1195 ObjectTypeAndSize info) throws IOException {
1196 bOffset = 0;
1197 bAvail = 0;
1198 return seekDatabase(delta, info);
1199 }
1200
1201
1202 private int readFrom(Source src) throws IOException {
1203 if (bAvail == 0)
1204 fill(src, 1);
1205 bAvail--;
1206 return buf[bOffset++] & 0xff;
1207 }
1208
1209
1210 void use(int cnt) {
1211 bOffset += cnt;
1212 bAvail -= cnt;
1213 }
1214
1215
1216 int fill(Source src, int need) throws IOException {
1217 while (bAvail < need) {
1218 int next = bOffset + bAvail;
1219 int free = buf.length - next;
1220 if (free + bAvail < need) {
1221 switch (src) {
1222 case INPUT:
1223 sync();
1224 break;
1225 case DATABASE:
1226 if (bAvail > 0)
1227 System.arraycopy(buf, bOffset, buf, 0, bAvail);
1228 bOffset = 0;
1229 break;
1230 }
1231 next = bAvail;
1232 free = buf.length - next;
1233 }
1234 switch (src) {
1235 case INPUT:
1236 next = in.read(buf, next, free);
1237 break;
1238 case DATABASE:
1239 next = readDatabase(buf, next, free);
1240 break;
1241 }
1242 if (next <= 0)
1243 throw new EOFException(
1244 JGitText.get().packfileIsTruncatedNoParam);
1245 bAvail += next;
1246 }
1247 return bOffset;
1248 }
1249
1250
1251 private void sync() throws IOException {
1252 packDigest.update(buf, 0, bOffset);
1253 onStoreStream(buf, 0, bOffset);
1254 if (expectDataAfterPackFooter) {
1255 if (bAvail > 0) {
1256 in.reset();
1257 IO.skipFully(in, bOffset);
1258 bAvail = 0;
1259 }
1260 in.mark(buf.length);
1261 } else if (bAvail > 0)
1262 System.arraycopy(buf, bOffset, buf, 0, bAvail);
1263 bBase += bOffset;
1264 bOffset = 0;
1265 }
1266
1267
1268
1269
1270
1271
1272 protected byte[] buffer() {
1273 return tempBuffer;
1274 }
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290 protected PackedObjectInfo newInfo(AnyObjectId id, UnresolvedDelta delta,
1291 ObjectId deltaBase) {
1292 PackedObjectInfo oe = new PackedObjectInfo(id);
1293 if (delta != null)
1294 oe.setCRC(delta.crc);
1295 return oe;
1296 }
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311 protected void setExpectedObjectCount(long expectedObjectCount) {
1312 this.expectedObjectCount = expectedObjectCount;
1313 }
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336 protected abstract void onStoreStream(byte[] raw, int pos, int len)
1337 throws IOException;
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356 protected abstract void onObjectHeader(Source src, byte[] raw, int pos,
1357 int len) throws IOException;
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379 protected abstract void onObjectData(Source src, byte[] raw, int pos,
1380 int len) throws IOException;
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394 protected abstract void onInflatedObjectData(PackedObjectInfo obj,
1395 int typeCode, byte[] data) throws IOException;
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405 protected abstract void onPackHeader(long objCnt) throws IOException;
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416 protected abstract void onPackFooter(byte[] hash) throws IOException;
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439 protected abstract boolean onAppendBase(int typeCode, byte[] data,
1440 PackedObjectInfo info) throws IOException;
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452 protected abstract void onEndThinPack() throws IOException;
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469 protected abstract ObjectTypeAndSize seekDatabase(PackedObjectInfo obj,
1470 ObjectTypeAndSize info) throws IOException;
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487 protected abstract ObjectTypeAndSize seekDatabase(UnresolvedDelta delta,
1488 ObjectTypeAndSize info) throws IOException;
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504 protected abstract int readDatabase(byte[] dst, int pos, int cnt)
1505 throws IOException;
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523 protected abstract boolean checkCRC(int oldCRC);
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542 protected abstract void onBeginWholeObject(long streamPosition, int type,
1543 long inflatedSize) throws IOException;
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553 protected abstract void onEndWholeObject(PackedObjectInfo info)
1554 throws IOException;
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572 protected abstract void onBeginOfsDelta(long deltaStreamPosition,
1573 long baseStreamPosition, long inflatedSize) throws IOException;
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590 protected abstract void onBeginRefDelta(long deltaStreamPosition,
1591 AnyObjectId baseId, long inflatedSize) throws IOException;
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601 protected UnresolvedDelta onEndDelta() throws IOException {
1602 return new UnresolvedDelta();
1603 }
1604
1605
1606 public static class ObjectTypeAndSize {
1607
1608 public int type;
1609
1610
1611 public long size;
1612 }
1613
1614 private void inflateAndSkip(Source src, long inflatedSize)
1615 throws IOException {
1616 try (InputStream inf = inflate(src, inflatedSize)) {
1617 IO.skipFully(inf, inflatedSize);
1618 }
1619 }
1620
1621 private byte[] inflateAndReturn(Source src, long inflatedSize)
1622 throws IOException {
1623 final byte[] dst = new byte[(int) inflatedSize];
1624 try (InputStream inf = inflate(src, inflatedSize)) {
1625 IO.readFully(inf, dst, 0, dst.length);
1626 }
1627 return dst;
1628 }
1629
1630 private InputStream inflate(Source src, long inflatedSize)
1631 throws IOException {
1632 inflater.open(src, inflatedSize);
1633 return inflater;
1634 }
1635
1636 private static class DeltaChain extends ObjectIdOwnerMap.Entry {
1637 UnresolvedDelta head;
1638
1639 DeltaChain(AnyObjectId id) {
1640 super(id);
1641 }
1642
1643 UnresolvedDelta remove() {
1644 final UnresolvedDelta r = head;
1645 if (r != null)
1646 head = null;
1647 return r;
1648 }
1649
1650 void add(UnresolvedDelta d) {
1651 d.next = head;
1652 head = d;
1653 }
1654 }
1655
1656
1657 public static class UnresolvedDelta {
1658 long position;
1659
1660 int crc;
1661
1662 UnresolvedDelta next;
1663
1664 long sizeBeforeInflating;
1665
1666
1667 public long getOffset() {
1668 return position;
1669 }
1670
1671
1672 public int getCRC() {
1673 return crc;
1674 }
1675
1676
1677
1678
1679
1680 public void setCRC(int crc32) {
1681 crc = crc32;
1682 }
1683 }
1684
1685 private static class DeltaVisit {
1686 final UnresolvedDelta delta;
1687
1688 ObjectId id;
1689
1690 byte[] data;
1691
1692 DeltaVisit parent;
1693
1694 UnresolvedDelta nextChild;
1695
1696 DeltaVisit() {
1697 this.delta = null;
1698 }
1699
1700 DeltaVisit(DeltaVisit parent) {
1701 this.parent = parent;
1702 this.delta = parent.nextChild;
1703 parent.nextChild = delta.next;
1704 }
1705
1706 DeltaVisit next() {
1707
1708 if (parent != null && parent.nextChild == null) {
1709 parent.data = null;
1710 parent = parent.parent;
1711 }
1712
1713 if (nextChild != null)
1714 return new DeltaVisit(this);
1715
1716
1717
1718 if (parent != null)
1719 return new DeltaVisit(parent);
1720 return null;
1721 }
1722 }
1723
1724 private void addObjectAndTrack(PackedObjectInfo oe) {
1725 entries[entryCount++] = oe;
1726 if (needNewObjectIds())
1727 newObjectIds.add(oe);
1728 }
1729
1730 private class InflaterStream extends InputStream {
1731 private final Inflater inf;
1732
1733 private final byte[] skipBuffer;
1734
1735 private Source src;
1736
1737 private long expectedSize;
1738
1739 private long actualSize;
1740
1741 private int p;
1742
1743 InflaterStream() {
1744 inf = InflaterCache.get();
1745 skipBuffer = new byte[512];
1746 }
1747
1748 void release() {
1749 inf.reset();
1750 InflaterCache.release(inf);
1751 }
1752
1753 void open(Source source, long inflatedSize) throws IOException {
1754 src = source;
1755 expectedSize = inflatedSize;
1756 actualSize = 0;
1757
1758 p = fill(src, 1);
1759 inf.setInput(buf, p, bAvail);
1760 }
1761
1762 @Override
1763 public long skip(long toSkip) throws IOException {
1764 long n = 0;
1765 while (n < toSkip) {
1766 final int cnt = (int) Math.min(skipBuffer.length, toSkip - n);
1767 final int r = read(skipBuffer, 0, cnt);
1768 if (r <= 0)
1769 break;
1770 n += r;
1771 }
1772 return n;
1773 }
1774
1775 @Override
1776 public int read() throws IOException {
1777 int n = read(skipBuffer, 0, 1);
1778 return n == 1 ? skipBuffer[0] & 0xff : -1;
1779 }
1780
1781 @Override
1782 public int read(byte[] dst, int pos, int cnt) throws IOException {
1783 try {
1784 int n = 0;
1785 while (n < cnt) {
1786 int r = inf.inflate(dst, pos + n, cnt - n);
1787 n += r;
1788 if (inf.finished())
1789 break;
1790 if (inf.needsInput()) {
1791 onObjectData(src, buf, p, bAvail);
1792 use(bAvail);
1793
1794 p = fill(src, 1);
1795 inf.setInput(buf, p, bAvail);
1796 } else if (r == 0) {
1797 throw new CorruptObjectException(MessageFormat.format(
1798 JGitText.get().packfileCorruptionDetected,
1799 JGitText.get().unknownZlibError));
1800 }
1801 }
1802 actualSize += n;
1803 return 0 < n ? n : -1;
1804 } catch (DataFormatException dfe) {
1805 throw new CorruptObjectException(MessageFormat.format(JGitText
1806 .get().packfileCorruptionDetected, dfe.getMessage()));
1807 }
1808 }
1809
1810 @Override
1811 public void close() throws IOException {
1812
1813
1814
1815
1816 if (read(skipBuffer) != -1 || actualSize != expectedSize) {
1817 throw new CorruptObjectException(MessageFormat.format(JGitText
1818 .get().packfileCorruptionDetected,
1819 JGitText.get().wrongDecompressedLength));
1820 }
1821
1822 int used = bAvail - inf.getRemaining();
1823 if (0 < used) {
1824 onObjectData(src, buf, p, used);
1825 use(used);
1826 }
1827
1828 inf.reset();
1829 }
1830 }
1831 }