1
2
3
4
5
6
7
8
9
10
11
12
13 package org.eclipse.jgit.transport;
14
15 import java.io.EOFException;
16 import java.io.IOException;
17 import java.io.InputStream;
18 import java.security.MessageDigest;
19 import java.text.MessageFormat;
20 import java.util.ArrayList;
21 import java.util.Arrays;
22 import java.util.Comparator;
23 import java.util.List;
24 import java.util.concurrent.TimeUnit;
25 import java.util.zip.DataFormatException;
26 import java.util.zip.Inflater;
27
28 import org.eclipse.jgit.errors.CorruptObjectException;
29 import org.eclipse.jgit.errors.MissingObjectException;
30 import org.eclipse.jgit.errors.TooLargeObjectInPackException;
31 import org.eclipse.jgit.internal.JGitText;
32 import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
33 import org.eclipse.jgit.lib.AnyObjectId;
34 import org.eclipse.jgit.lib.BatchingProgressMonitor;
35 import org.eclipse.jgit.lib.BlobObjectChecker;
36 import org.eclipse.jgit.lib.Constants;
37 import org.eclipse.jgit.lib.InflaterCache;
38 import org.eclipse.jgit.lib.MutableObjectId;
39 import org.eclipse.jgit.lib.NullProgressMonitor;
40 import org.eclipse.jgit.lib.ObjectChecker;
41 import org.eclipse.jgit.lib.ObjectDatabase;
42 import org.eclipse.jgit.lib.ObjectId;
43 import org.eclipse.jgit.lib.ObjectIdOwnerMap;
44 import org.eclipse.jgit.lib.ObjectIdSubclassMap;
45 import org.eclipse.jgit.lib.ObjectLoader;
46 import org.eclipse.jgit.lib.ObjectReader;
47 import org.eclipse.jgit.lib.ObjectStream;
48 import org.eclipse.jgit.lib.ProgressMonitor;
49 import org.eclipse.jgit.util.BlockList;
50 import org.eclipse.jgit.util.IO;
51 import org.eclipse.jgit.util.LongMap;
52 import org.eclipse.jgit.util.NB;
53 import org.eclipse.jgit.util.sha1.SHA1;
54
55
56
57
58
59
60
61
62
63
64
65
66
67 public abstract class PackParser {
68
69 private static final int BUFFER_SIZE = 8192;
70
71
72 public enum Source {
73
74 INPUT,
75
76
77 DATABASE;
78 }
79
80
81 private final ObjectDatabase objectDatabase;
82
83 private InflaterStream inflater;
84
85 private byte[] tempBuffer;
86
87 private byte[] hdrBuf;
88
89 private final SHA1 objectHasher = SHA1.newInstance();
90 private final MutableObjectId tempObjectId;
91
92 private InputStream in;
93
94 byte[] buf;
95
96
97 private long bBase;
98
99 private int bOffset;
100
101 int bAvail;
102
103 private ObjectChecker objCheck;
104
105 private boolean allowThin;
106
107 private boolean checkObjectCollisions;
108
109 private boolean needBaseObjectIds;
110
111 private boolean checkEofAfterPackFooter;
112
113 private boolean expectDataAfterPackFooter;
114
115 private long expectedObjectCount;
116
117 private PackedObjectInfo[] entries;
118
119
120
121
122
123
124
125
126 private ObjectIdSubclassMap<ObjectId> newObjectIds;
127
128 private int deltaCount;
129
130 private int entryCount;
131
132 private ObjectIdOwnerMap<DeltaChain> baseById;
133
134
135
136
137
138
139
140
141 private ObjectIdSubclassMap<ObjectId> baseObjectIds;
142
143 private LongMap<UnresolvedDelta> baseByPos;
144
145
146 private BlockList<PackedObjectInfo> collisionCheckObjs;
147
148 private MessageDigest packDigest;
149
150 private ObjectReader readCurs;
151
152
153 private String lockMessage;
154
155
156 private long maxObjectSizeLimit;
157
158 private final ReceivedPackStatistics.Builder stats =
159 new ReceivedPackStatistics.Builder();
160
161
162
163
164
165
166
167
168
169 protected PackParser(ObjectDatabase odb, InputStream src) {
170 objectDatabase = odb.newCachedDatabase();
171 in = src;
172
173 inflater = new InflaterStream();
174 readCurs = objectDatabase.newReader();
175 buf = new byte[BUFFER_SIZE];
176 tempBuffer = new byte[BUFFER_SIZE];
177 hdrBuf = new byte[64];
178 tempObjectId = new MutableObjectId();
179 packDigest = Constants.newMessageDigest();
180 checkObjectCollisions = true;
181 }
182
183
184
185
186
187
188 public boolean isAllowThin() {
189 return allowThin;
190 }
191
192
193
194
195
196
197
198
199
200
201 public void setAllowThin(boolean allow) {
202 allowThin = allow;
203 }
204
205
206
207
208
209
210
211 protected boolean isCheckObjectCollisions() {
212 return checkObjectCollisions;
213 }
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236 protected void setCheckObjectCollisions(boolean check) {
237 checkObjectCollisions = check;
238 }
239
240
241
242
243
244
245
246
247
248
249
250 public void setNeedNewObjectIds(boolean b) {
251 if (b)
252 newObjectIds = new ObjectIdSubclassMap<>();
253 else
254 newObjectIds = null;
255 }
256
257 private boolean needNewObjectIds() {
258 return newObjectIds != null;
259 }
260
261
262
263
264
265
266
267
268
269
270
271
272 public void setNeedBaseObjectIds(boolean b) {
273 this.needBaseObjectIds = b;
274 }
275
276
277
278
279
280
281 public boolean isCheckEofAfterPackFooter() {
282 return checkEofAfterPackFooter;
283 }
284
285
286
287
288
289
290
291 public void setCheckEofAfterPackFooter(boolean b) {
292 checkEofAfterPackFooter = b;
293 }
294
295
296
297
298
299
300 public boolean isExpectDataAfterPackFooter() {
301 return expectDataAfterPackFooter;
302 }
303
304
305
306
307
308
309
310
311
312 public void setExpectDataAfterPackFooter(boolean e) {
313 expectDataAfterPackFooter = e;
314 }
315
316
317
318
319
320
321 public ObjectIdSubclassMap<ObjectId> getNewObjectIds() {
322 if (newObjectIds != null)
323 return newObjectIds;
324 return new ObjectIdSubclassMap<>();
325 }
326
327
328
329
330
331
332 public ObjectIdSubclassMap<ObjectId> getBaseObjectIds() {
333 if (baseObjectIds != null)
334 return baseObjectIds;
335 return new ObjectIdSubclassMap<>();
336 }
337
338
339
340
341
342
343
344
345
346
347
348 public void setObjectChecker(ObjectChecker oc) {
349 objCheck = oc;
350 }
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368 public void setObjectChecking(boolean on) {
369 setObjectChecker(on ? new ObjectChecker() : null);
370 }
371
372
373
374
375
376
377 public String getLockMessage() {
378 return lockMessage;
379 }
380
381
382
383
384
385
386
387
388 public void setLockMessage(String msg) {
389 lockMessage = msg;
390 }
391
392
393
394
395
396
397
398
399
400
401 public void setMaxObjectSizeLimit(long limit) {
402 maxObjectSizeLimit = limit;
403 }
404
405
406
407
408
409
410
411
412
413
414 public int getObjectCount() {
415 return entryCount;
416 }
417
418
419
420
421
422
423
424
425
426
427
428
429 public PackedObjectInfo getObject(int nth) {
430 return entries[nth];
431 }
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447 public List<PackedObjectInfo> getSortedObjectList(
448 Comparator<PackedObjectInfo> cmp) {
449 Arrays.sort(entries, 0, entryCount, cmp);
450 List<PackedObjectInfo> list = Arrays.asList(entries);
451 if (entryCount < entries.length)
452 list = list.subList(0, entryCount);
453 return list;
454 }
455
456
457
458
459
460
461
462
463
464
465
466 public long getPackSize() {
467 return -1;
468 }
469
470
471
472
473
474
475
476
477
478 public ReceivedPackStatistics getReceivedPackStatistics() {
479 return stats.build();
480 }
481
482
483
484
485
486
487
488
489
490
491
492
493
494 public final PackLock parse(ProgressMonitor progress) throws IOException {
495 return parse(progress, progress);
496 }
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513 public PackLock parse(ProgressMonitor receiving, ProgressMonitor resolving)
514 throws IOException {
515 if (receiving == null)
516 receiving = NullProgressMonitor.INSTANCE;
517 if (resolving == null)
518 resolving = NullProgressMonitor.INSTANCE;
519
520 if (receiving == resolving)
521 receiving.start(2 );
522 try {
523 readPackHeader();
524
525 entries = new PackedObjectInfo[(int) expectedObjectCount];
526 baseById = new ObjectIdOwnerMap<>();
527 baseByPos = new LongMap<>();
528 collisionCheckObjs = new BlockList<>();
529
530 receiving.beginTask(JGitText.get().receivingObjects,
531 (int) expectedObjectCount);
532 try {
533 for (int done = 0; done < expectedObjectCount; done++) {
534 indexOneObject();
535 receiving.update(1);
536 if (receiving.isCancelled())
537 throw new IOException(JGitText.get().downloadCancelled);
538 }
539 readPackFooter();
540 endInput();
541 } finally {
542 receiving.endTask();
543 }
544
545 if (!collisionCheckObjs.isEmpty()) {
546 checkObjectCollision();
547 }
548
549 if (deltaCount > 0) {
550 processDeltas(resolving);
551 }
552
553 packDigest = null;
554 baseById = null;
555 baseByPos = null;
556 } finally {
557 try {
558 if (readCurs != null)
559 readCurs.close();
560 } finally {
561 readCurs = null;
562 }
563
564 try {
565 inflater.release();
566 } finally {
567 inflater = null;
568 }
569 }
570 return null;
571 }
572
573 private void processDeltas(ProgressMonitor resolving) throws IOException {
574 if (resolving instanceof BatchingProgressMonitor) {
575 ((BatchingProgressMonitor) resolving).setDelayStart(1000,
576 TimeUnit.MILLISECONDS);
577 }
578 resolving.beginTask(JGitText.get().resolvingDeltas, deltaCount);
579 resolveDeltas(resolving);
580 if (entryCount < expectedObjectCount) {
581 if (!isAllowThin()) {
582 throw new IOException(MessageFormat.format(
583 JGitText.get().packHasUnresolvedDeltas,
584 Long.valueOf(expectedObjectCount - entryCount)));
585 }
586
587 resolveDeltasWithExternalBases(resolving);
588
589 if (entryCount < expectedObjectCount) {
590 throw new IOException(MessageFormat.format(
591 JGitText.get().packHasUnresolvedDeltas,
592 Long.valueOf(expectedObjectCount - entryCount)));
593 }
594 }
595 resolving.endTask();
596 }
597
598 private void resolveDeltas(ProgressMonitor progress)
599 throws IOException {
600 final int last = entryCount;
601 for (int i = 0; i < last; i++) {
602 resolveDeltas(entries[i], progress);
603 if (progress.isCancelled())
604 throw new IOException(
605 JGitText.get().downloadCancelledDuringIndexing);
606 }
607 }
608
609 private void resolveDeltas(final PackedObjectInfo oe,
610 ProgressMonitor progress) throws IOException {
611 UnresolvedDelta children = firstChildOf(oe);
612 if (children == null)
613 return;
614
615 DeltaVisit visit = new DeltaVisit();
616 visit.nextChild = children;
617
618 ObjectTypeAndSize info = openDatabase(oe, new ObjectTypeAndSize());
619 switch (info.type) {
620 case Constants.OBJ_COMMIT:
621 case Constants.OBJ_TREE:
622 case Constants.OBJ_BLOB:
623 case Constants.OBJ_TAG:
624 visit.data = inflateAndReturn(Source.DATABASE, info.size);
625 visit.id = oe;
626 break;
627 default:
628 throw new IOException(MessageFormat.format(
629 JGitText.get().unknownObjectType,
630 Integer.valueOf(info.type)));
631 }
632
633 if (!checkCRC(oe.getCRC())) {
634 throw new IOException(MessageFormat.format(
635 JGitText.get().corruptionDetectedReReadingAt,
636 Long.valueOf(oe.getOffset())));
637 }
638
639 resolveDeltas(visit.next(), info.type, info, progress);
640 }
641
642 private void resolveDeltas(DeltaVisit visit, final int type,
643 ObjectTypeAndSize info, ProgressMonitor progress)
644 throws IOException {
645 stats.addDeltaObject(type);
646 do {
647 progress.update(1);
648 info = openDatabase(visit.delta, info);
649 switch (info.type) {
650 case Constants.OBJ_OFS_DELTA:
651 case Constants.OBJ_REF_DELTA:
652 break;
653
654 default:
655 throw new IOException(MessageFormat.format(
656 JGitText.get().unknownObjectType,
657 Integer.valueOf(info.type)));
658 }
659
660 byte[] delta = inflateAndReturn(Source.DATABASE, info.size);
661 long finalSz = BinaryDelta.getResultSize(delta);
662 checkIfTooLarge(type, finalSz);
663
664 visit.data = BinaryDelta.apply(visit.parent.data, delta);
665 delta = null;
666
667 if (!checkCRC(visit.delta.crc))
668 throw new IOException(MessageFormat.format(
669 JGitText.get().corruptionDetectedReReadingAt,
670 Long.valueOf(visit.delta.position)));
671
672 SHA1 objectDigest = objectHasher.reset();
673 objectDigest.update(Constants.encodedTypeString(type));
674 objectDigest.update((byte) ' ');
675 objectDigest.update(Constants.encodeASCII(visit.data.length));
676 objectDigest.update((byte) 0);
677 objectDigest.update(visit.data);
678 objectDigest.digest(tempObjectId);
679
680 verifySafeObject(tempObjectId, type, visit.data);
681 if (isCheckObjectCollisions() && readCurs.has(tempObjectId)) {
682 checkObjectCollision(tempObjectId, type, visit.data,
683 visit.delta.sizeBeforeInflating);
684 }
685
686 PackedObjectInfo oe;
687 oe = newInfo(tempObjectId, visit.delta, visit.parent.id);
688 oe.setFullSize(finalSz);
689 oe.setOffset(visit.delta.position);
690 oe.setType(type);
691 onInflatedObjectData(oe, type, visit.data);
692 addObjectAndTrack(oe);
693 visit.id = oe;
694
695 visit.nextChild = firstChildOf(oe);
696 visit = visit.next();
697 } while (visit != null);
698 }
699
700 private final void checkIfTooLarge(int typeCode, long size)
701 throws IOException {
702 if (0 < maxObjectSizeLimit && maxObjectSizeLimit < size) {
703 switch (typeCode) {
704 case Constants.OBJ_COMMIT:
705 case Constants.OBJ_TREE:
706 case Constants.OBJ_BLOB:
707 case Constants.OBJ_TAG:
708 throw new TooLargeObjectInPackException(size, maxObjectSizeLimit);
709
710 case Constants.OBJ_OFS_DELTA:
711 case Constants.OBJ_REF_DELTA:
712 throw new TooLargeObjectInPackException(size, maxObjectSizeLimit);
713
714 default:
715 throw new IOException(MessageFormat.format(
716 JGitText.get().unknownObjectType,
717 Integer.valueOf(typeCode)));
718 }
719 }
720 if (size > Integer.MAX_VALUE - 8) {
721 throw new TooLargeObjectInPackException(size, Integer.MAX_VALUE - 8);
722 }
723 }
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741 protected ObjectTypeAndSize readObjectHeader(ObjectTypeAndSize info)
742 throws IOException {
743 int hdrPtr = 0;
744 int c = readFrom(Source.DATABASE);
745 hdrBuf[hdrPtr++] = (byte) c;
746
747 info.type = (c >> 4) & 7;
748 long sz = c & 15;
749 int shift = 4;
750 while ((c & 0x80) != 0) {
751 c = readFrom(Source.DATABASE);
752 hdrBuf[hdrPtr++] = (byte) c;
753 sz += ((long) (c & 0x7f)) << shift;
754 shift += 7;
755 }
756 info.size = sz;
757
758 switch (info.type) {
759 case Constants.OBJ_COMMIT:
760 case Constants.OBJ_TREE:
761 case Constants.OBJ_BLOB:
762 case Constants.OBJ_TAG:
763 onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
764 break;
765
766 case Constants.OBJ_OFS_DELTA:
767 c = readFrom(Source.DATABASE);
768 hdrBuf[hdrPtr++] = (byte) c;
769 while ((c & 128) != 0) {
770 c = readFrom(Source.DATABASE);
771 hdrBuf[hdrPtr++] = (byte) c;
772 }
773 onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
774 break;
775
776 case Constants.OBJ_REF_DELTA:
777 System.arraycopy(buf, fill(Source.DATABASE, 20), hdrBuf, hdrPtr, 20);
778 hdrPtr += 20;
779 use(20);
780 onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
781 break;
782
783 default:
784 throw new IOException(MessageFormat.format(
785 JGitText.get().unknownObjectType,
786 Integer.valueOf(info.type)));
787 }
788 return info;
789 }
790
791 private UnresolvedDelta removeBaseById(AnyObjectId id) {
792 final DeltaChain d = baseById.get(id);
793 return d != null ? d.remove() : null;
794 }
795
796 private static UnresolvedDelta reverse(UnresolvedDelta c) {
797 UnresolvedDelta tail = null;
798 while (c != null) {
799 final UnresolvedDelta n = c.next;
800 c.next = tail;
801 tail = c;
802 c = n;
803 }
804 return tail;
805 }
806
807 private UnresolvedDelta firstChildOf(PackedObjectInfo oe) {
808 UnresolvedDelta a = reverse(removeBaseById(oe));
809 UnresolvedDelta b = reverse(baseByPos.remove(oe.getOffset()));
810
811 if (a == null)
812 return b;
813 if (b == null)
814 return a;
815
816 UnresolvedDelta first = null;
817 UnresolvedDelta last = null;
818 while (a != null || b != null) {
819 UnresolvedDelta curr;
820 if (b == null || (a != null && a.position < b.position)) {
821 curr = a;
822 a = a.next;
823 } else {
824 curr = b;
825 b = b.next;
826 }
827 if (last != null)
828 last.next = curr;
829 else
830 first = curr;
831 last = curr;
832 curr.next = null;
833 }
834 return first;
835 }
836
837 private void resolveDeltasWithExternalBases(ProgressMonitor progress)
838 throws IOException {
839 growEntries(baseById.size());
840
841 if (needBaseObjectIds)
842 baseObjectIds = new ObjectIdSubclassMap<>();
843
844 final List<DeltaChain> missing = new ArrayList<>(64);
845 for (DeltaChain baseId : baseById) {
846 if (baseId.head == null)
847 continue;
848
849 if (needBaseObjectIds)
850 baseObjectIds.add(baseId);
851
852 final ObjectLoader ldr;
853 try {
854 ldr = readCurs.open(baseId);
855 } catch (MissingObjectException notFound) {
856 missing.add(baseId);
857 continue;
858 }
859
860 final DeltaVisit visit = new DeltaVisit();
861 visit.data = ldr.getCachedBytes(Integer.MAX_VALUE);
862 visit.id = baseId;
863 final int typeCode = ldr.getType();
864 final PackedObjectInfo oe = newInfo(baseId, null, null);
865 oe.setType(typeCode);
866 oe.setFullSize(ldr.getSize());
867 if (onAppendBase(typeCode, visit.data, oe))
868 entries[entryCount++] = oe;
869 visit.nextChild = firstChildOf(oe);
870 resolveDeltas(visit.next(), typeCode,
871 new ObjectTypeAndSize(), progress);
872
873 if (progress.isCancelled())
874 throw new IOException(
875 JGitText.get().downloadCancelledDuringIndexing);
876 }
877
878 for (DeltaChain base : missing) {
879 if (base.head != null)
880 throw new MissingObjectException(base, "delta base");
881 }
882
883 onEndThinPack();
884 }
885
886 private void growEntries(int extraObjects) {
887 final PackedObjectInfo[] ne;
888
889 ne = new PackedObjectInfo[(int) expectedObjectCount + extraObjects];
890 System.arraycopy(entries, 0, ne, 0, entryCount);
891 entries = ne;
892 }
893
894 private void readPackHeader() throws IOException {
895 if (expectDataAfterPackFooter) {
896 if (!in.markSupported())
897 throw new IOException(
898 JGitText.get().inputStreamMustSupportMark);
899 in.mark(buf.length);
900 }
901
902 final int hdrln = Constants.PACK_SIGNATURE.length + 4 + 4;
903 final int p = fill(Source.INPUT, hdrln);
904 for (int k = 0; k < Constants.PACK_SIGNATURE.length; k++)
905 if (buf[p + k] != Constants.PACK_SIGNATURE[k])
906 throw new IOException(JGitText.get().notAPACKFile);
907
908 final long vers = NB.decodeUInt32(buf, p + 4);
909 if (vers != 2 && vers != 3)
910 throw new IOException(MessageFormat.format(
911 JGitText.get().unsupportedPackVersion, Long.valueOf(vers)));
912 final long objectCount = NB.decodeUInt32(buf, p + 8);
913 use(hdrln);
914 setExpectedObjectCount(objectCount);
915 onPackHeader(objectCount);
916 }
917
918 private void readPackFooter() throws IOException {
919 sync();
920 final byte[] actHash = packDigest.digest();
921
922 final int c = fill(Source.INPUT, 20);
923 final byte[] srcHash = new byte[20];
924 System.arraycopy(buf, c, srcHash, 0, 20);
925 use(20);
926
927 if (bAvail != 0 && !expectDataAfterPackFooter)
928 throw new CorruptObjectException(MessageFormat.format(
929 JGitText.get().expectedEOFReceived,
930 "\\x" + Integer.toHexString(buf[bOffset] & 0xff)));
931 if (isCheckEofAfterPackFooter()) {
932 int eof = in.read();
933 if (0 <= eof)
934 throw new CorruptObjectException(MessageFormat.format(
935 JGitText.get().expectedEOFReceived,
936 "\\x" + Integer.toHexString(eof)));
937 } else if (bAvail > 0 && expectDataAfterPackFooter) {
938 in.reset();
939 IO.skipFully(in, bOffset);
940 }
941
942 if (!Arrays.equals(actHash, srcHash))
943 throw new CorruptObjectException(
944 JGitText.get().corruptObjectPackfileChecksumIncorrect);
945
946 onPackFooter(srcHash);
947 }
948
949
950 private void endInput() {
951 stats.setNumBytesRead(streamPosition());
952 in = null;
953 }
954
955
956 private void indexOneObject() throws IOException {
957 final long streamPosition = streamPosition();
958
959 int hdrPtr = 0;
960 int c = readFrom(Source.INPUT);
961 hdrBuf[hdrPtr++] = (byte) c;
962
963 final int typeCode = (c >> 4) & 7;
964 long sz = c & 15;
965 int shift = 4;
966 while ((c & 0x80) != 0) {
967 c = readFrom(Source.INPUT);
968 hdrBuf[hdrPtr++] = (byte) c;
969 sz += ((long) (c & 0x7f)) << shift;
970 shift += 7;
971 }
972
973 checkIfTooLarge(typeCode, sz);
974
975 switch (typeCode) {
976 case Constants.OBJ_COMMIT:
977 case Constants.OBJ_TREE:
978 case Constants.OBJ_BLOB:
979 case Constants.OBJ_TAG:
980 stats.addWholeObject(typeCode);
981 onBeginWholeObject(streamPosition, typeCode, sz);
982 onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
983 whole(streamPosition, typeCode, sz);
984 break;
985
986 case Constants.OBJ_OFS_DELTA: {
987 stats.addOffsetDelta();
988 c = readFrom(Source.INPUT);
989 hdrBuf[hdrPtr++] = (byte) c;
990 long ofs = c & 127;
991 while ((c & 128) != 0) {
992 ofs += 1;
993 c = readFrom(Source.INPUT);
994 hdrBuf[hdrPtr++] = (byte) c;
995 ofs <<= 7;
996 ofs += (c & 127);
997 }
998 final long base = streamPosition - ofs;
999 onBeginOfsDelta(streamPosition, base, sz);
1000 onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
1001 inflateAndSkip(Source.INPUT, sz);
1002 UnresolvedDelta n = onEndDelta();
1003 n.position = streamPosition;
1004 n.next = baseByPos.put(base, n);
1005 n.sizeBeforeInflating = streamPosition() - streamPosition;
1006 deltaCount++;
1007 break;
1008 }
1009
1010 case Constants.OBJ_REF_DELTA: {
1011 stats.addRefDelta();
1012 c = fill(Source.INPUT, 20);
1013 final ObjectId base = ObjectId.fromRaw(buf, c);
1014 System.arraycopy(buf, c, hdrBuf, hdrPtr, 20);
1015 hdrPtr += 20;
1016 use(20);
1017 DeltaChain r = baseById.get(base);
1018 if (r == null) {
1019 r = new DeltaChain(base);
1020 baseById.add(r);
1021 }
1022 onBeginRefDelta(streamPosition, base, sz);
1023 onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
1024 inflateAndSkip(Source.INPUT, sz);
1025 UnresolvedDelta n = onEndDelta();
1026 n.position = streamPosition;
1027 n.sizeBeforeInflating = streamPosition() - streamPosition;
1028 r.add(n);
1029 deltaCount++;
1030 break;
1031 }
1032
1033 default:
1034 throw new IOException(
1035 MessageFormat.format(JGitText.get().unknownObjectType,
1036 Integer.valueOf(typeCode)));
1037 }
1038 }
1039
1040 private void whole(long pos, int type, long sz)
1041 throws IOException {
1042 SHA1 objectDigest = objectHasher.reset();
1043 objectDigest.update(Constants.encodedTypeString(type));
1044 objectDigest.update((byte) ' ');
1045 objectDigest.update(Constants.encodeASCII(sz));
1046 objectDigest.update((byte) 0);
1047
1048 final byte[] data;
1049 if (type == Constants.OBJ_BLOB) {
1050 byte[] readBuffer = buffer();
1051 BlobObjectChecker checker = null;
1052 if (objCheck != null) {
1053 checker = objCheck.newBlobObjectChecker();
1054 }
1055 if (checker == null) {
1056 checker = BlobObjectChecker.NULL_CHECKER;
1057 }
1058 long cnt = 0;
1059 try (InputStream inf = inflate(Source.INPUT, sz)) {
1060 while (cnt < sz) {
1061 int r = inf.read(readBuffer);
1062 if (r <= 0)
1063 break;
1064 objectDigest.update(readBuffer, 0, r);
1065 checker.update(readBuffer, 0, r);
1066 cnt += r;
1067 }
1068 }
1069 objectDigest.digest(tempObjectId);
1070 checker.endBlob(tempObjectId);
1071 data = null;
1072 } else {
1073 data = inflateAndReturn(Source.INPUT, sz);
1074 objectDigest.update(data);
1075 objectDigest.digest(tempObjectId);
1076 verifySafeObject(tempObjectId, type, data);
1077 }
1078
1079 long sizeBeforeInflating = streamPosition() - pos;
1080 PackedObjectInfo obj = newInfo(tempObjectId, null, null);
1081 obj.setOffset(pos);
1082 obj.setType(type);
1083 obj.setSize(sizeBeforeInflating);
1084 obj.setFullSize(sz);
1085 onEndWholeObject(obj);
1086 if (data != null)
1087 onInflatedObjectData(obj, type, data);
1088 addObjectAndTrack(obj);
1089
1090 if (isCheckObjectCollisions()) {
1091 collisionCheckObjs.add(obj);
1092 }
1093 }
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107 protected void verifySafeObject(final AnyObjectId id, final int type,
1108 final byte[] data) throws CorruptObjectException {
1109 if (objCheck != null) {
1110 try {
1111 objCheck.check(id, type, data);
1112 } catch (CorruptObjectException e) {
1113 if (e.getErrorType() != null) {
1114 throw e;
1115 }
1116 throw new CorruptObjectException(
1117 MessageFormat.format(JGitText.get().invalidObject,
1118 Constants.typeString(type), id.name(),
1119 e.getMessage()),
1120 e);
1121 }
1122 }
1123 }
1124
1125 private void checkObjectCollision() throws IOException {
1126 for (PackedObjectInfo obj : collisionCheckObjs) {
1127 if (!readCurs.has(obj)) {
1128 continue;
1129 }
1130 checkObjectCollision(obj);
1131 }
1132 }
1133
1134 private void checkObjectCollision(PackedObjectInfo obj)
1135 throws IOException {
1136 ObjectTypeAndSize info = openDatabase(obj, new ObjectTypeAndSize());
1137 final byte[] readBuffer = buffer();
1138 final byte[] curBuffer = new byte[readBuffer.length];
1139 long sz = info.size;
1140 try (ObjectStream cur = readCurs.open(obj, info.type).openStream()) {
1141 if (cur.getSize() != sz) {
1142 throw new IOException(MessageFormat.format(
1143 JGitText.get().collisionOn, obj.name()));
1144 }
1145 try (InputStream pck = inflate(Source.DATABASE, sz)) {
1146 while (0 < sz) {
1147 int n = (int) Math.min(readBuffer.length, sz);
1148 IO.readFully(cur, curBuffer, 0, n);
1149 IO.readFully(pck, readBuffer, 0, n);
1150 for (int i = 0; i < n; i++) {
1151 if (curBuffer[i] != readBuffer[i]) {
1152 throw new IOException(MessageFormat.format(
1153 JGitText.get().collisionOn, obj.name()));
1154 }
1155 }
1156 sz -= n;
1157 }
1158 }
1159 stats.incrementObjectsDuplicated();
1160 stats.incrementNumBytesDuplicated(obj.getSize());
1161 } catch (MissingObjectException notLocal) {
1162
1163
1164
1165 }
1166 }
1167
1168 private void checkObjectCollision(AnyObjectId obj, int type, byte[] data,
1169 long sizeBeforeInflating) throws IOException {
1170 try {
1171 final ObjectLoader ldr = readCurs.open(obj, type);
1172 final byte[] existingData = ldr.getCachedBytes(data.length);
1173 if (!Arrays.equals(data, existingData)) {
1174 throw new IOException(MessageFormat
1175 .format(JGitText.get().collisionOn, obj.name()));
1176 }
1177 stats.incrementObjectsDuplicated();
1178 stats.incrementNumBytesDuplicated(sizeBeforeInflating);
1179 } catch (MissingObjectException notLocal) {
1180
1181
1182
1183 }
1184 }
1185
1186
1187 private long streamPosition() {
1188 return bBase + bOffset;
1189 }
1190
1191 private ObjectTypeAndSize openDatabase(PackedObjectInfo obj,
1192 ObjectTypeAndSize info) throws IOException {
1193 bOffset = 0;
1194 bAvail = 0;
1195 return seekDatabase(obj, info);
1196 }
1197
1198 private ObjectTypeAndSize openDatabase(UnresolvedDelta delta,
1199 ObjectTypeAndSize info) throws IOException {
1200 bOffset = 0;
1201 bAvail = 0;
1202 return seekDatabase(delta, info);
1203 }
1204
1205
1206 private int readFrom(Source src) throws IOException {
1207 if (bAvail == 0)
1208 fill(src, 1);
1209 bAvail--;
1210 return buf[bOffset++] & 0xff;
1211 }
1212
1213
1214 void use(int cnt) {
1215 bOffset += cnt;
1216 bAvail -= cnt;
1217 }
1218
1219
1220 int fill(Source src, int need) throws IOException {
1221 while (bAvail < need) {
1222 int next = bOffset + bAvail;
1223 int free = buf.length - next;
1224 if (free + bAvail < need) {
1225 switch (src) {
1226 case INPUT:
1227 sync();
1228 break;
1229 case DATABASE:
1230 if (bAvail > 0)
1231 System.arraycopy(buf, bOffset, buf, 0, bAvail);
1232 bOffset = 0;
1233 break;
1234 }
1235 next = bAvail;
1236 free = buf.length - next;
1237 }
1238 switch (src) {
1239 case INPUT:
1240 next = in.read(buf, next, free);
1241 break;
1242 case DATABASE:
1243 next = readDatabase(buf, next, free);
1244 break;
1245 }
1246 if (next <= 0)
1247 throw new EOFException(
1248 JGitText.get().packfileIsTruncatedNoParam);
1249 bAvail += next;
1250 }
1251 return bOffset;
1252 }
1253
1254
1255 private void sync() throws IOException {
1256 packDigest.update(buf, 0, bOffset);
1257 onStoreStream(buf, 0, bOffset);
1258 if (expectDataAfterPackFooter) {
1259 if (bAvail > 0) {
1260 in.reset();
1261 IO.skipFully(in, bOffset);
1262 bAvail = 0;
1263 }
1264 in.mark(buf.length);
1265 } else if (bAvail > 0)
1266 System.arraycopy(buf, bOffset, buf, 0, bAvail);
1267 bBase += bOffset;
1268 bOffset = 0;
1269 }
1270
1271
1272
1273
1274
1275
1276 protected byte[] buffer() {
1277 return tempBuffer;
1278 }
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294 protected PackedObjectInfo newInfo(AnyObjectId id, UnresolvedDelta delta,
1295 ObjectId deltaBase) {
1296 PackedObjectInfo oe = new PackedObjectInfo(id);
1297 if (delta != null)
1298 oe.setCRC(delta.crc);
1299 return oe;
1300 }
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315 protected void setExpectedObjectCount(long expectedObjectCount) {
1316 this.expectedObjectCount = expectedObjectCount;
1317 }
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340 protected abstract void onStoreStream(byte[] raw, int pos, int len)
1341 throws IOException;
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360 protected abstract void onObjectHeader(Source src, byte[] raw, int pos,
1361 int len) throws IOException;
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383 protected abstract void onObjectData(Source src, byte[] raw, int pos,
1384 int len) throws IOException;
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398 protected abstract void onInflatedObjectData(PackedObjectInfo obj,
1399 int typeCode, byte[] data) throws IOException;
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409 protected abstract void onPackHeader(long objCnt) throws IOException;
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420 protected abstract void onPackFooter(byte[] hash) throws IOException;
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443 protected abstract boolean onAppendBase(int typeCode, byte[] data,
1444 PackedObjectInfo info) throws IOException;
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456 protected abstract void onEndThinPack() throws IOException;
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473 protected abstract ObjectTypeAndSize seekDatabase(PackedObjectInfo obj,
1474 ObjectTypeAndSize info) throws IOException;
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491 protected abstract ObjectTypeAndSize seekDatabase(UnresolvedDelta delta,
1492 ObjectTypeAndSize info) throws IOException;
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508 protected abstract int readDatabase(byte[] dst, int pos, int cnt)
1509 throws IOException;
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527 protected abstract boolean checkCRC(int oldCRC);
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546 protected abstract void onBeginWholeObject(long streamPosition, int type,
1547 long inflatedSize) throws IOException;
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557 protected abstract void onEndWholeObject(PackedObjectInfo info)
1558 throws IOException;
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576 protected abstract void onBeginOfsDelta(long deltaStreamPosition,
1577 long baseStreamPosition, long inflatedSize) throws IOException;
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594 protected abstract void onBeginRefDelta(long deltaStreamPosition,
1595 AnyObjectId baseId, long inflatedSize) throws IOException;
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605 protected UnresolvedDelta onEndDelta() throws IOException {
1606 return new UnresolvedDelta();
1607 }
1608
1609
1610 public static class ObjectTypeAndSize {
1611
1612 public int type;
1613
1614
1615 public long size;
1616 }
1617
1618 private void inflateAndSkip(Source src, long inflatedSize)
1619 throws IOException {
1620 try (InputStream inf = inflate(src, inflatedSize)) {
1621 IO.skipFully(inf, inflatedSize);
1622 }
1623 }
1624
1625 private byte[] inflateAndReturn(Source src, long inflatedSize)
1626 throws IOException {
1627 final byte[] dst = new byte[(int) inflatedSize];
1628 try (InputStream inf = inflate(src, inflatedSize)) {
1629 IO.readFully(inf, dst, 0, dst.length);
1630 }
1631 return dst;
1632 }
1633
1634 private InputStream inflate(Source src, long inflatedSize)
1635 throws IOException {
1636 inflater.open(src, inflatedSize);
1637 return inflater;
1638 }
1639
1640 private static class DeltaChain extends ObjectIdOwnerMap.Entry {
1641 UnresolvedDelta head;
1642
1643 DeltaChain(AnyObjectId id) {
1644 super(id);
1645 }
1646
1647 UnresolvedDelta remove() {
1648 final UnresolvedDelta r = head;
1649 if (r != null)
1650 head = null;
1651 return r;
1652 }
1653
1654 void add(UnresolvedDelta d) {
1655 d.next = head;
1656 head = d;
1657 }
1658 }
1659
1660
1661 public static class UnresolvedDelta {
1662 long position;
1663
1664 int crc;
1665
1666 UnresolvedDelta next;
1667
1668 long sizeBeforeInflating;
1669
1670
1671 public long getOffset() {
1672 return position;
1673 }
1674
1675
1676 public int getCRC() {
1677 return crc;
1678 }
1679
1680
1681
1682
1683
1684 public void setCRC(int crc32) {
1685 crc = crc32;
1686 }
1687 }
1688
1689 private static class DeltaVisit {
1690 final UnresolvedDelta delta;
1691
1692 ObjectId id;
1693
1694 byte[] data;
1695
1696 DeltaVisit parent;
1697
1698 UnresolvedDelta nextChild;
1699
1700 DeltaVisit() {
1701 this.delta = null;
1702 }
1703
1704 DeltaVisit(DeltaVisit parent) {
1705 this.parent = parent;
1706 this.delta = parent.nextChild;
1707 parent.nextChild = delta.next;
1708 }
1709
1710 DeltaVisit next() {
1711
1712 if (parent != null && parent.nextChild == null) {
1713 parent.data = null;
1714 parent = parent.parent;
1715 }
1716
1717 if (nextChild != null)
1718 return new DeltaVisit(this);
1719
1720
1721
1722 if (parent != null)
1723 return new DeltaVisit(parent);
1724 return null;
1725 }
1726 }
1727
1728 private void addObjectAndTrack(PackedObjectInfo oe) {
1729 entries[entryCount++] = oe;
1730 if (needNewObjectIds())
1731 newObjectIds.add(oe);
1732 }
1733
1734 private class InflaterStream extends InputStream {
1735 private final Inflater inf;
1736
1737 private final byte[] skipBuffer;
1738
1739 private Source src;
1740
1741 private long expectedSize;
1742
1743 private long actualSize;
1744
1745 private int p;
1746
1747 InflaterStream() {
1748 inf = InflaterCache.get();
1749 skipBuffer = new byte[512];
1750 }
1751
1752 void release() {
1753 inf.reset();
1754 InflaterCache.release(inf);
1755 }
1756
1757 void open(Source source, long inflatedSize) throws IOException {
1758 src = source;
1759 expectedSize = inflatedSize;
1760 actualSize = 0;
1761
1762 p = fill(src, 1);
1763 inf.setInput(buf, p, bAvail);
1764 }
1765
1766 @Override
1767 public long skip(long toSkip) throws IOException {
1768 long n = 0;
1769 while (n < toSkip) {
1770 final int cnt = (int) Math.min(skipBuffer.length, toSkip - n);
1771 final int r = read(skipBuffer, 0, cnt);
1772 if (r <= 0)
1773 break;
1774 n += r;
1775 }
1776 return n;
1777 }
1778
1779 @Override
1780 public int read() throws IOException {
1781 int n = read(skipBuffer, 0, 1);
1782 return n == 1 ? skipBuffer[0] & 0xff : -1;
1783 }
1784
1785 @Override
1786 public int read(byte[] dst, int pos, int cnt) throws IOException {
1787 try {
1788 int n = 0;
1789 while (n < cnt) {
1790 int r = inf.inflate(dst, pos + n, cnt - n);
1791 n += r;
1792 if (inf.finished())
1793 break;
1794 if (inf.needsInput()) {
1795 onObjectData(src, buf, p, bAvail);
1796 use(bAvail);
1797
1798 p = fill(src, 1);
1799 inf.setInput(buf, p, bAvail);
1800 } else if (r == 0) {
1801 throw new CorruptObjectException(MessageFormat.format(
1802 JGitText.get().packfileCorruptionDetected,
1803 JGitText.get().unknownZlibError));
1804 }
1805 }
1806 actualSize += n;
1807 return 0 < n ? n : -1;
1808 } catch (DataFormatException dfe) {
1809 throw new CorruptObjectException(MessageFormat.format(JGitText
1810 .get().packfileCorruptionDetected, dfe.getMessage()));
1811 }
1812 }
1813
1814 @Override
1815 public void close() throws IOException {
1816
1817
1818
1819
1820 if (read(skipBuffer) != -1 || actualSize != expectedSize) {
1821 throw new CorruptObjectException(MessageFormat.format(JGitText
1822 .get().packfileCorruptionDetected,
1823 JGitText.get().wrongDecompressedLength));
1824 }
1825
1826 int used = bAvail - inf.getRemaining();
1827 if (0 < used) {
1828 onObjectData(src, buf, p, used);
1829 use(used);
1830 }
1831
1832 inf.reset();
1833 }
1834 }
1835 }