1
2
3
4
5
6
7
8
9
10
11
12
13 package org.eclipse.jgit.transport;
14
15 import java.io.EOFException;
16 import java.io.IOException;
17 import java.io.InputStream;
18 import java.security.MessageDigest;
19 import java.text.MessageFormat;
20 import java.util.ArrayList;
21 import java.util.Arrays;
22 import java.util.Comparator;
23 import java.util.List;
24 import java.util.concurrent.TimeUnit;
25 import java.util.zip.DataFormatException;
26 import java.util.zip.Inflater;
27
28 import org.eclipse.jgit.errors.CorruptObjectException;
29 import org.eclipse.jgit.errors.MissingObjectException;
30 import org.eclipse.jgit.errors.TooLargeObjectInPackException;
31 import org.eclipse.jgit.internal.JGitText;
32 import org.eclipse.jgit.internal.storage.file.PackLock;
33 import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
34 import org.eclipse.jgit.lib.AnyObjectId;
35 import org.eclipse.jgit.lib.BatchingProgressMonitor;
36 import org.eclipse.jgit.lib.BlobObjectChecker;
37 import org.eclipse.jgit.lib.Constants;
38 import org.eclipse.jgit.lib.InflaterCache;
39 import org.eclipse.jgit.lib.MutableObjectId;
40 import org.eclipse.jgit.lib.NullProgressMonitor;
41 import org.eclipse.jgit.lib.ObjectChecker;
42 import org.eclipse.jgit.lib.ObjectDatabase;
43 import org.eclipse.jgit.lib.ObjectId;
44 import org.eclipse.jgit.lib.ObjectIdOwnerMap;
45 import org.eclipse.jgit.lib.ObjectIdSubclassMap;
46 import org.eclipse.jgit.lib.ObjectLoader;
47 import org.eclipse.jgit.lib.ObjectReader;
48 import org.eclipse.jgit.lib.ObjectStream;
49 import org.eclipse.jgit.lib.ProgressMonitor;
50 import org.eclipse.jgit.util.BlockList;
51 import org.eclipse.jgit.util.IO;
52 import org.eclipse.jgit.util.LongMap;
53 import org.eclipse.jgit.util.NB;
54 import org.eclipse.jgit.util.sha1.SHA1;
55
56
57
58
59
60
61
62
63
64
65
66
67
68 public abstract class PackParser {
69
70 private static final int BUFFER_SIZE = 8192;
71
72
73 public enum Source {
74
75 INPUT,
76
77
78 DATABASE;
79 }
80
81
82 private final ObjectDatabase objectDatabase;
83
84 private InflaterStream inflater;
85
86 private byte[] tempBuffer;
87
88 private byte[] hdrBuf;
89
90 private final SHA1 objectHasher = SHA1.newInstance();
91 private final MutableObjectId tempObjectId;
92
93 private InputStream in;
94
95 byte[] buf;
96
97
98 private long bBase;
99
100 private int bOffset;
101
102 int bAvail;
103
104 private ObjectChecker objCheck;
105
106 private boolean allowThin;
107
108 private boolean checkObjectCollisions;
109
110 private boolean needBaseObjectIds;
111
112 private boolean checkEofAfterPackFooter;
113
114 private boolean expectDataAfterPackFooter;
115
116 private long expectedObjectCount;
117
118 private PackedObjectInfo[] entries;
119
120
121
122
123
124
125
126
127 private ObjectIdSubclassMap<ObjectId> newObjectIds;
128
129 private int deltaCount;
130
131 private int entryCount;
132
133 private ObjectIdOwnerMap<DeltaChain> baseById;
134
135
136
137
138
139
140
141
142 private ObjectIdSubclassMap<ObjectId> baseObjectIds;
143
144 private LongMap<UnresolvedDelta> baseByPos;
145
146
147 private BlockList<PackedObjectInfo> collisionCheckObjs;
148
149 private MessageDigest packDigest;
150
151 private ObjectReader readCurs;
152
153
154 private String lockMessage;
155
156
157 private long maxObjectSizeLimit;
158
159 private final ReceivedPackStatistics.Builder stats =
160 new ReceivedPackStatistics.Builder();
161
162
163
164
165
166
167
168
169
170 protected PackParser(ObjectDatabase odb, InputStream src) {
171 objectDatabase = odb.newCachedDatabase();
172 in = src;
173
174 inflater = new InflaterStream();
175 readCurs = objectDatabase.newReader();
176 buf = new byte[BUFFER_SIZE];
177 tempBuffer = new byte[BUFFER_SIZE];
178 hdrBuf = new byte[64];
179 tempObjectId = new MutableObjectId();
180 packDigest = Constants.newMessageDigest();
181 checkObjectCollisions = true;
182 }
183
184
185
186
187
188
189 public boolean isAllowThin() {
190 return allowThin;
191 }
192
193
194
195
196
197
198
199
200
201
202 public void setAllowThin(boolean allow) {
203 allowThin = allow;
204 }
205
206
207
208
209
210
211
212 protected boolean isCheckObjectCollisions() {
213 return checkObjectCollisions;
214 }
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237 protected void setCheckObjectCollisions(boolean check) {
238 checkObjectCollisions = check;
239 }
240
241
242
243
244
245
246
247
248
249
250
251 public void setNeedNewObjectIds(boolean b) {
252 if (b)
253 newObjectIds = new ObjectIdSubclassMap<>();
254 else
255 newObjectIds = null;
256 }
257
258 private boolean needNewObjectIds() {
259 return newObjectIds != null;
260 }
261
262
263
264
265
266
267
268
269
270
271
272
273 public void setNeedBaseObjectIds(boolean b) {
274 this.needBaseObjectIds = b;
275 }
276
277
278
279
280
281
282 public boolean isCheckEofAfterPackFooter() {
283 return checkEofAfterPackFooter;
284 }
285
286
287
288
289
290
291
292 public void setCheckEofAfterPackFooter(boolean b) {
293 checkEofAfterPackFooter = b;
294 }
295
296
297
298
299
300
301 public boolean isExpectDataAfterPackFooter() {
302 return expectDataAfterPackFooter;
303 }
304
305
306
307
308
309
310
311
312
313 public void setExpectDataAfterPackFooter(boolean e) {
314 expectDataAfterPackFooter = e;
315 }
316
317
318
319
320
321
322 public ObjectIdSubclassMap<ObjectId> getNewObjectIds() {
323 if (newObjectIds != null)
324 return newObjectIds;
325 return new ObjectIdSubclassMap<>();
326 }
327
328
329
330
331
332
333 public ObjectIdSubclassMap<ObjectId> getBaseObjectIds() {
334 if (baseObjectIds != null)
335 return baseObjectIds;
336 return new ObjectIdSubclassMap<>();
337 }
338
339
340
341
342
343
344
345
346
347
348
349 public void setObjectChecker(ObjectChecker oc) {
350 objCheck = oc;
351 }
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369 public void setObjectChecking(boolean on) {
370 setObjectChecker(on ? new ObjectChecker() : null);
371 }
372
373
374
375
376
377
378 public String getLockMessage() {
379 return lockMessage;
380 }
381
382
383
384
385
386
387
388
389 public void setLockMessage(String msg) {
390 lockMessage = msg;
391 }
392
393
394
395
396
397
398
399
400
401
402 public void setMaxObjectSizeLimit(long limit) {
403 maxObjectSizeLimit = limit;
404 }
405
406
407
408
409
410
411
412
413
414
415 public int getObjectCount() {
416 return entryCount;
417 }
418
419
420
421
422
423
424
425
426
427
428
429
430 public PackedObjectInfo getObject(int nth) {
431 return entries[nth];
432 }
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448 public List<PackedObjectInfo> getSortedObjectList(
449 Comparator<PackedObjectInfo> cmp) {
450 Arrays.sort(entries, 0, entryCount, cmp);
451 List<PackedObjectInfo> list = Arrays.asList(entries);
452 if (entryCount < entries.length)
453 list = list.subList(0, entryCount);
454 return list;
455 }
456
457
458
459
460
461
462
463
464
465
466
467 public long getPackSize() {
468 return -1;
469 }
470
471
472
473
474
475
476
477
478
479 public ReceivedPackStatistics getReceivedPackStatistics() {
480 return stats.build();
481 }
482
483
484
485
486
487
488
489
490
491
492
493
494
495 public final PackLock parse(ProgressMonitor progress) throws IOException {
496 return parse(progress, progress);
497 }
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514 public PackLock parse(ProgressMonitor./org/eclipse/jgit/lib/ProgressMonitor.html#ProgressMonitor">ProgressMonitor receiving, ProgressMonitor resolving)
515 throws IOException {
516 if (receiving == null)
517 receiving = NullProgressMonitor.INSTANCE;
518 if (resolving == null)
519 resolving = NullProgressMonitor.INSTANCE;
520
521 if (receiving == resolving)
522 receiving.start(2 );
523 try {
524 readPackHeader();
525
526 entries = new PackedObjectInfo[(int) expectedObjectCount];
527 baseById = new ObjectIdOwnerMap<>();
528 baseByPos = new LongMap<>();
529 collisionCheckObjs = new BlockList<>();
530
531 receiving.beginTask(JGitText.get().receivingObjects,
532 (int) expectedObjectCount);
533 try {
534 for (int done = 0; done < expectedObjectCount; done++) {
535 indexOneObject();
536 receiving.update(1);
537 if (receiving.isCancelled())
538 throw new IOException(JGitText.get().downloadCancelled);
539 }
540 readPackFooter();
541 endInput();
542 } finally {
543 receiving.endTask();
544 }
545
546 if (!collisionCheckObjs.isEmpty()) {
547 checkObjectCollision();
548 }
549
550 if (deltaCount > 0) {
551 processDeltas(resolving);
552 }
553
554 packDigest = null;
555 baseById = null;
556 baseByPos = null;
557 } finally {
558 try {
559 if (readCurs != null)
560 readCurs.close();
561 } finally {
562 readCurs = null;
563 }
564
565 try {
566 inflater.release();
567 } finally {
568 inflater = null;
569 }
570 }
571 return null;
572 }
573
574 private void processDeltas(ProgressMonitor resolving) throws IOException {
575 if (resolving instanceof BatchingProgressMonitor) {
576 ((BatchingProgressMonitor) resolving).setDelayStart(1000,
577 TimeUnit.MILLISECONDS);
578 }
579 resolving.beginTask(JGitText.get().resolvingDeltas, deltaCount);
580 resolveDeltas(resolving);
581 if (entryCount < expectedObjectCount) {
582 if (!isAllowThin()) {
583 throw new IOException(MessageFormat.format(
584 JGitText.get().packHasUnresolvedDeltas,
585 Long.valueOf(expectedObjectCount - entryCount)));
586 }
587
588 resolveDeltasWithExternalBases(resolving);
589
590 if (entryCount < expectedObjectCount) {
591 throw new IOException(MessageFormat.format(
592 JGitText.get().packHasUnresolvedDeltas,
593 Long.valueOf(expectedObjectCount - entryCount)));
594 }
595 }
596 resolving.endTask();
597 }
598
599 private void resolveDeltas(ProgressMonitor progress)
600 throws IOException {
601 final int last = entryCount;
602 for (int i = 0; i < last; i++) {
603 resolveDeltas(entries[i], progress);
604 if (progress.isCancelled())
605 throw new IOException(
606 JGitText.get().downloadCancelledDuringIndexing);
607 }
608 }
609
610 private void resolveDeltas(final PackedObjectInfo oe,
611 ProgressMonitor progress) throws IOException {
612 UnresolvedDelta children = firstChildOf(oe);
613 if (children == null)
614 return;
615
616 DeltaVisit visit = new DeltaVisit();
617 visit.nextChild = children;
618
619 ObjectTypeAndSize info = openDatabase(oe, new ObjectTypeAndSize());
620 switch (info.type) {
621 case Constants.OBJ_COMMIT:
622 case Constants.OBJ_TREE:
623 case Constants.OBJ_BLOB:
624 case Constants.OBJ_TAG:
625 visit.data = inflateAndReturn(Source.DATABASE, info.size);
626 visit.id = oe;
627 break;
628 default:
629 throw new IOException(MessageFormat.format(
630 JGitText.get().unknownObjectType,
631 Integer.valueOf(info.type)));
632 }
633
634 if (!checkCRC(oe.getCRC())) {
635 throw new IOException(MessageFormat.format(
636 JGitText.get().corruptionDetectedReReadingAt,
637 Long.valueOf(oe.getOffset())));
638 }
639
640 resolveDeltas(visit.next(), info.type, info, progress);
641 }
642
643 private void resolveDeltas(DeltaVisit visit, final int type,
644 ObjectTypeAndSize info, ProgressMonitor progress)
645 throws IOException {
646 stats.addDeltaObject(type);
647 do {
648 progress.update(1);
649 info = openDatabase(visit.delta, info);
650 switch (info.type) {
651 case Constants.OBJ_OFS_DELTA:
652 case Constants.OBJ_REF_DELTA:
653 break;
654
655 default:
656 throw new IOException(MessageFormat.format(
657 JGitText.get().unknownObjectType,
658 Integer.valueOf(info.type)));
659 }
660
661 byte[] delta = inflateAndReturn(Source.DATABASE, info.size);
662 checkIfTooLarge(type, BinaryDelta.getResultSize(delta));
663
664 visit.data = BinaryDelta.apply(visit.parent.data, delta);
665 delta = null;
666
667 if (!checkCRC(visit.delta.crc))
668 throw new IOException(MessageFormat.format(
669 JGitText.get().corruptionDetectedReReadingAt,
670 Long.valueOf(visit.delta.position)));
671
672 SHA1 objectDigest = objectHasher.reset();
673 objectDigest.update(Constants.encodedTypeString(type));
674 objectDigest.update((byte) ' ');
675 objectDigest.update(Constants.encodeASCII(visit.data.length));
676 objectDigest.update((byte) 0);
677 objectDigest.update(visit.data);
678 objectDigest.digest(tempObjectId);
679
680 verifySafeObject(tempObjectId, type, visit.data);
681 if (isCheckObjectCollisions() && readCurs.has(tempObjectId)) {
682 checkObjectCollision(tempObjectId, type, visit.data);
683 }
684
685 PackedObjectInfo oe;
686 oe = newInfo(tempObjectId, visit.delta, visit.parent.id);
687 oe.setOffset(visit.delta.position);
688 oe.setType(type);
689 onInflatedObjectData(oe, type, visit.data);
690 addObjectAndTrack(oe);
691 visit.id = oe;
692
693 visit.nextChild = firstChildOf(oe);
694 visit = visit.next();
695 } while (visit != null);
696 }
697
698 private final void checkIfTooLarge(int typeCode, long size)
699 throws IOException {
700 if (0 < maxObjectSizeLimit && maxObjectSizeLimit < size) {
701 switch (typeCode) {
702 case Constants.OBJ_COMMIT:
703 case Constants.OBJ_TREE:
704 case Constants.OBJ_BLOB:
705 case Constants.OBJ_TAG:
706 throw new TooLargeObjectInPackException(size, maxObjectSizeLimit);
707
708 case Constants.OBJ_OFS_DELTA:
709 case Constants.OBJ_REF_DELTA:
710 throw new TooLargeObjectInPackException(size, maxObjectSizeLimit);
711
712 default:
713 throw new IOException(MessageFormat.format(
714 JGitText.get().unknownObjectType,
715 Integer.valueOf(typeCode)));
716 }
717 }
718 if (size > Integer.MAX_VALUE - 8) {
719 throw new TooLargeObjectInPackException(size, Integer.MAX_VALUE - 8);
720 }
721 }
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739 protected ObjectTypeAndSize readObjectHeader(ObjectTypeAndSize info)
740 throws IOException {
741 int hdrPtr = 0;
742 int c = readFrom(Source.DATABASE);
743 hdrBuf[hdrPtr++] = (byte) c;
744
745 info.type = (c >> 4) & 7;
746 long sz = c & 15;
747 int shift = 4;
748 while ((c & 0x80) != 0) {
749 c = readFrom(Source.DATABASE);
750 hdrBuf[hdrPtr++] = (byte) c;
751 sz += ((long) (c & 0x7f)) << shift;
752 shift += 7;
753 }
754 info.size = sz;
755
756 switch (info.type) {
757 case Constants.OBJ_COMMIT:
758 case Constants.OBJ_TREE:
759 case Constants.OBJ_BLOB:
760 case Constants.OBJ_TAG:
761 onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
762 break;
763
764 case Constants.OBJ_OFS_DELTA:
765 c = readFrom(Source.DATABASE);
766 hdrBuf[hdrPtr++] = (byte) c;
767 while ((c & 128) != 0) {
768 c = readFrom(Source.DATABASE);
769 hdrBuf[hdrPtr++] = (byte) c;
770 }
771 onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
772 break;
773
774 case Constants.OBJ_REF_DELTA:
775 System.arraycopy(buf, fill(Source.DATABASE, 20), hdrBuf, hdrPtr, 20);
776 hdrPtr += 20;
777 use(20);
778 onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
779 break;
780
781 default:
782 throw new IOException(MessageFormat.format(
783 JGitText.get().unknownObjectType,
784 Integer.valueOf(info.type)));
785 }
786 return info;
787 }
788
789 private UnresolvedDelta removeBaseById(AnyObjectId id) {
790 final DeltaChain d = baseById.get(id);
791 return d != null ? d.remove() : null;
792 }
793
794 private static UnresolvedDelta reverse(UnresolvedDelta c) {
795 UnresolvedDelta tail = null;
796 while (c != null) {
797 final UnresolvedDelta n = c.next;
798 c.next = tail;
799 tail = c;
800 c = n;
801 }
802 return tail;
803 }
804
805 private UnresolvedDelta firstChildOf(PackedObjectInfo oe) {
806 UnresolvedDelta a = reverse(removeBaseById(oe));
807 UnresolvedDelta b = reverse(baseByPos.remove(oe.getOffset()));
808
809 if (a == null)
810 return b;
811 if (b == null)
812 return a;
813
814 UnresolvedDelta first = null;
815 UnresolvedDelta last = null;
816 while (a != null || b != null) {
817 UnresolvedDelta curr;
818 if (b == null || (a != null && a.position < b.position)) {
819 curr = a;
820 a = a.next;
821 } else {
822 curr = b;
823 b = b.next;
824 }
825 if (last != null)
826 last.next = curr;
827 else
828 first = curr;
829 last = curr;
830 curr.next = null;
831 }
832 return first;
833 }
834
835 private void resolveDeltasWithExternalBases(ProgressMonitor progress)
836 throws IOException {
837 growEntries(baseById.size());
838
839 if (needBaseObjectIds)
840 baseObjectIds = new ObjectIdSubclassMap<>();
841
842 final List<DeltaChain> missing = new ArrayList<>(64);
843 for (DeltaChain baseId : baseById) {
844 if (baseId.head == null)
845 continue;
846
847 if (needBaseObjectIds)
848 baseObjectIds.add(baseId);
849
850 final ObjectLoader ldr;
851 try {
852 ldr = readCurs.open(baseId);
853 } catch (MissingObjectException notFound) {
854 missing.add(baseId);
855 continue;
856 }
857
858 final DeltaVisit visit = new DeltaVisit();
859 visit.data = ldr.getCachedBytes(Integer.MAX_VALUE);
860 visit.id = baseId;
861 final int typeCode = ldr.getType();
862 final PackedObjectInfo oe = newInfo(baseId, null, null);
863 oe.setType(typeCode);
864 if (onAppendBase(typeCode, visit.data, oe))
865 entries[entryCount++] = oe;
866 visit.nextChild = firstChildOf(oe);
867 resolveDeltas(visit.next(), typeCode,
868 new ObjectTypeAndSize(), progress);
869
870 if (progress.isCancelled())
871 throw new IOException(
872 JGitText.get().downloadCancelledDuringIndexing);
873 }
874
875 for (DeltaChain base : missing) {
876 if (base.head != null)
877 throw new MissingObjectException(base, "delta base");
878 }
879
880 onEndThinPack();
881 }
882
883 private void growEntries(int extraObjects) {
884 final PackedObjectInfo[] ne;
885
886 ne = new PackedObjectInfo[(int) expectedObjectCount + extraObjects];
887 System.arraycopy(entries, 0, ne, 0, entryCount);
888 entries = ne;
889 }
890
891 private void readPackHeader() throws IOException {
892 if (expectDataAfterPackFooter) {
893 if (!in.markSupported())
894 throw new IOException(
895 JGitText.get().inputStreamMustSupportMark);
896 in.mark(buf.length);
897 }
898
899 final int hdrln = Constants.PACK_SIGNATURE.length + 4 + 4;
900 final int p = fill(Source.INPUT, hdrln);
901 for (int k = 0; k < Constants.PACK_SIGNATURE.length; k++)
902 if (buf[p + k] != Constants.PACK_SIGNATURE[k])
903 throw new IOException(JGitText.get().notAPACKFile);
904
905 final long vers = NB.decodeUInt32(buf, p + 4);
906 if (vers != 2 && vers != 3)
907 throw new IOException(MessageFormat.format(
908 JGitText.get().unsupportedPackVersion, Long.valueOf(vers)));
909 final long objectCount = NB.decodeUInt32(buf, p + 8);
910 use(hdrln);
911 setExpectedObjectCount(objectCount);
912 onPackHeader(objectCount);
913 }
914
915 private void readPackFooter() throws IOException {
916 sync();
917 final byte[] actHash = packDigest.digest();
918
919 final int c = fill(Source.INPUT, 20);
920 final byte[] srcHash = new byte[20];
921 System.arraycopy(buf, c, srcHash, 0, 20);
922 use(20);
923
924 if (bAvail != 0 && !expectDataAfterPackFooter)
925 throw new CorruptObjectException(MessageFormat.format(
926 JGitText.get().expectedEOFReceived,
927 "\\x" + Integer.toHexString(buf[bOffset] & 0xff)));
928 if (isCheckEofAfterPackFooter()) {
929 int eof = in.read();
930 if (0 <= eof)
931 throw new CorruptObjectException(MessageFormat.format(
932 JGitText.get().expectedEOFReceived,
933 "\\x" + Integer.toHexString(eof)));
934 } else if (bAvail > 0 && expectDataAfterPackFooter) {
935 in.reset();
936 IO.skipFully(in, bOffset);
937 }
938
939 if (!Arrays.equals(actHash, srcHash))
940 throw new CorruptObjectException(
941 JGitText.get().corruptObjectPackfileChecksumIncorrect);
942
943 onPackFooter(srcHash);
944 }
945
946
947 private void endInput() {
948 stats.setNumBytesRead(streamPosition());
949 in = null;
950 }
951
952
953 private void indexOneObject() throws IOException {
954 final long streamPosition = streamPosition();
955
956 int hdrPtr = 0;
957 int c = readFrom(Source.INPUT);
958 hdrBuf[hdrPtr++] = (byte) c;
959
960 final int typeCode = (c >> 4) & 7;
961 long sz = c & 15;
962 int shift = 4;
963 while ((c & 0x80) != 0) {
964 c = readFrom(Source.INPUT);
965 hdrBuf[hdrPtr++] = (byte) c;
966 sz += ((long) (c & 0x7f)) << shift;
967 shift += 7;
968 }
969
970 checkIfTooLarge(typeCode, sz);
971
972 switch (typeCode) {
973 case Constants.OBJ_COMMIT:
974 case Constants.OBJ_TREE:
975 case Constants.OBJ_BLOB:
976 case Constants.OBJ_TAG:
977 stats.addWholeObject(typeCode);
978 onBeginWholeObject(streamPosition, typeCode, sz);
979 onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
980 whole(streamPosition, typeCode, sz);
981 break;
982
983 case Constants.OBJ_OFS_DELTA: {
984 stats.addOffsetDelta();
985 c = readFrom(Source.INPUT);
986 hdrBuf[hdrPtr++] = (byte) c;
987 long ofs = c & 127;
988 while ((c & 128) != 0) {
989 ofs += 1;
990 c = readFrom(Source.INPUT);
991 hdrBuf[hdrPtr++] = (byte) c;
992 ofs <<= 7;
993 ofs += (c & 127);
994 }
995 final long base = streamPosition - ofs;
996 onBeginOfsDelta(streamPosition, base, sz);
997 onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
998 inflateAndSkip(Source.INPUT, sz);
999 UnresolvedDelta n = onEndDelta();
1000 n.position = streamPosition;
1001 n.next = baseByPos.put(base, n);
1002 deltaCount++;
1003 break;
1004 }
1005
1006 case Constants.OBJ_REF_DELTA: {
1007 stats.addRefDelta();
1008 c = fill(Source.INPUT, 20);
1009 final ObjectId base = ObjectId.fromRaw(buf, c);
1010 System.arraycopy(buf, c, hdrBuf, hdrPtr, 20);
1011 hdrPtr += 20;
1012 use(20);
1013 DeltaChain r = baseById.get(base);
1014 if (r == null) {
1015 r = new DeltaChain(base);
1016 baseById.add(r);
1017 }
1018 onBeginRefDelta(streamPosition, base, sz);
1019 onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
1020 inflateAndSkip(Source.INPUT, sz);
1021 UnresolvedDelta n = onEndDelta();
1022 n.position = streamPosition;
1023 r.add(n);
1024 deltaCount++;
1025 break;
1026 }
1027
1028 default:
1029 throw new IOException(
1030 MessageFormat.format(JGitText.get().unknownObjectType,
1031 Integer.valueOf(typeCode)));
1032 }
1033 }
1034
1035 private void whole(long pos, int type, long sz)
1036 throws IOException {
1037 SHA1 objectDigest = objectHasher.reset();
1038 objectDigest.update(Constants.encodedTypeString(type));
1039 objectDigest.update((byte) ' ');
1040 objectDigest.update(Constants.encodeASCII(sz));
1041 objectDigest.update((byte) 0);
1042
1043 final byte[] data;
1044 if (type == Constants.OBJ_BLOB) {
1045 byte[] readBuffer = buffer();
1046 BlobObjectChecker checker = null;
1047 if (objCheck != null) {
1048 checker = objCheck.newBlobObjectChecker();
1049 }
1050 if (checker == null) {
1051 checker = BlobObjectChecker.NULL_CHECKER;
1052 }
1053 long cnt = 0;
1054 try (InputStream inf = inflate(Source.INPUT, sz)) {
1055 while (cnt < sz) {
1056 int r = inf.read(readBuffer);
1057 if (r <= 0)
1058 break;
1059 objectDigest.update(readBuffer, 0, r);
1060 checker.update(readBuffer, 0, r);
1061 cnt += r;
1062 }
1063 }
1064 objectDigest.digest(tempObjectId);
1065 checker.endBlob(tempObjectId);
1066 data = null;
1067 } else {
1068 data = inflateAndReturn(Source.INPUT, sz);
1069 objectDigest.update(data);
1070 objectDigest.digest(tempObjectId);
1071 verifySafeObject(tempObjectId, type, data);
1072 }
1073
1074 PackedObjectInfo obj = newInfo(tempObjectId, null, null);
1075 obj.setOffset(pos);
1076 obj.setType(type);
1077 onEndWholeObject(obj);
1078 if (data != null)
1079 onInflatedObjectData(obj, type, data);
1080 addObjectAndTrack(obj);
1081
1082 if (isCheckObjectCollisions()) {
1083 collisionCheckObjs.add(obj);
1084 }
1085 }
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099 protected void verifySafeObject(final AnyObjectId id, final int type,
1100 final byte[] data) throws CorruptObjectException {
1101 if (objCheck != null) {
1102 try {
1103 objCheck.check(id, type, data);
1104 } catch (CorruptObjectException e) {
1105 if (e.getErrorType() != null) {
1106 throw e;
1107 }
1108 throw new CorruptObjectException(
1109 MessageFormat.format(JGitText.get().invalidObject,
1110 Constants.typeString(type), id.name(),
1111 e.getMessage()),
1112 e);
1113 }
1114 }
1115 }
1116
1117 private void checkObjectCollision() throws IOException {
1118 for (PackedObjectInfo obj : collisionCheckObjs) {
1119 if (!readCurs.has(obj)) {
1120 continue;
1121 }
1122 checkObjectCollision(obj);
1123 }
1124 }
1125
1126 private void checkObjectCollision(PackedObjectInfo obj)
1127 throws IOException {
1128 ObjectTypeAndSize info = openDatabase(obj, new ObjectTypeAndSize());
1129 final byte[] readBuffer = buffer();
1130 final byte[] curBuffer = new byte[readBuffer.length];
1131 long sz = info.size;
1132 try (ObjectStream cur = readCurs.open(obj, info.type).openStream()) {
1133 if (cur.getSize() != sz) {
1134 throw new IOException(MessageFormat.format(
1135 JGitText.get().collisionOn, obj.name()));
1136 }
1137 try (InputStream pck = inflate(Source.DATABASE, sz)) {
1138 while (0 < sz) {
1139 int n = (int) Math.min(readBuffer.length, sz);
1140 IO.readFully(cur, curBuffer, 0, n);
1141 IO.readFully(pck, readBuffer, 0, n);
1142 for (int i = 0; i < n; i++) {
1143 if (curBuffer[i] != readBuffer[i]) {
1144 throw new IOException(MessageFormat.format(
1145 JGitText.get().collisionOn, obj.name()));
1146 }
1147 }
1148 sz -= n;
1149 }
1150 }
1151 } catch (MissingObjectException notLocal) {
1152
1153
1154
1155 }
1156 }
1157
1158 private void checkObjectCollision(AnyObjectId obj, int type, byte[] data)
1159 throws IOException {
1160 try {
1161 final ObjectLoader ldr = readCurs.open(obj, type);
1162 final byte[] existingData = ldr.getCachedBytes(data.length);
1163 if (!Arrays.equals(data, existingData)) {
1164 throw new IOException(MessageFormat.format(
1165 JGitText.get().collisionOn, obj.name()));
1166 }
1167 } catch (MissingObjectException notLocal) {
1168
1169
1170
1171 }
1172 }
1173
1174
1175 private long streamPosition() {
1176 return bBase + bOffset;
1177 }
1178
1179 private ObjectTypeAndSize openDatabase(PackedObjectInfo obj,
1180 ObjectTypeAndSize info) throws IOException {
1181 bOffset = 0;
1182 bAvail = 0;
1183 return seekDatabase(obj, info);
1184 }
1185
1186 private ObjectTypeAndSize openDatabase(UnresolvedDelta delta,
1187 ObjectTypeAndSize info) throws IOException {
1188 bOffset = 0;
1189 bAvail = 0;
1190 return seekDatabase(delta, info);
1191 }
1192
1193
1194 private int readFrom(Source src) throws IOException {
1195 if (bAvail == 0)
1196 fill(src, 1);
1197 bAvail--;
1198 return buf[bOffset++] & 0xff;
1199 }
1200
1201
1202 void use(int cnt) {
1203 bOffset += cnt;
1204 bAvail -= cnt;
1205 }
1206
1207
1208 int fill(Source src, int need) throws IOException {
1209 while (bAvail < need) {
1210 int next = bOffset + bAvail;
1211 int free = buf.length - next;
1212 if (free + bAvail < need) {
1213 switch (src) {
1214 case INPUT:
1215 sync();
1216 break;
1217 case DATABASE:
1218 if (bAvail > 0)
1219 System.arraycopy(buf, bOffset, buf, 0, bAvail);
1220 bOffset = 0;
1221 break;
1222 }
1223 next = bAvail;
1224 free = buf.length - next;
1225 }
1226 switch (src) {
1227 case INPUT:
1228 next = in.read(buf, next, free);
1229 break;
1230 case DATABASE:
1231 next = readDatabase(buf, next, free);
1232 break;
1233 }
1234 if (next <= 0)
1235 throw new EOFException(
1236 JGitText.get().packfileIsTruncatedNoParam);
1237 bAvail += next;
1238 }
1239 return bOffset;
1240 }
1241
1242
1243 private void sync() throws IOException {
1244 packDigest.update(buf, 0, bOffset);
1245 onStoreStream(buf, 0, bOffset);
1246 if (expectDataAfterPackFooter) {
1247 if (bAvail > 0) {
1248 in.reset();
1249 IO.skipFully(in, bOffset);
1250 bAvail = 0;
1251 }
1252 in.mark(buf.length);
1253 } else if (bAvail > 0)
1254 System.arraycopy(buf, bOffset, buf, 0, bAvail);
1255 bBase += bOffset;
1256 bOffset = 0;
1257 }
1258
1259
1260
1261
1262
1263
1264 protected byte[] buffer() {
1265 return tempBuffer;
1266 }
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282 protected PackedObjectInfo newInfo(AnyObjectId id, UnresolvedDelta delta,
1283 ObjectId deltaBase) {
1284 PackedObjectInfo oe = new PackedObjectInfo(id);
1285 if (delta != null)
1286 oe.setCRC(delta.crc);
1287 return oe;
1288 }
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303 protected void setExpectedObjectCount(long expectedObjectCount) {
1304 this.expectedObjectCount = expectedObjectCount;
1305 }
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328 protected abstract void onStoreStream(byte[] raw, int pos, int len)
1329 throws IOException;
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348 protected abstract void onObjectHeader(Source src, byte[] raw, int pos,
1349 int len) throws IOException;
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371 protected abstract void onObjectData(Source src, byte[] raw, int pos,
1372 int len) throws IOException;
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386 protected abstract void onInflatedObjectData(PackedObjectInfo obj,
1387 int typeCode, byte[] data) throws IOException;
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397 protected abstract void onPackHeader(long objCnt) throws IOException;
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408 protected abstract void onPackFooter(byte[] hash) throws IOException;
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431 protected abstract boolean onAppendBase(int typeCode, byte[] data,
1432 PackedObjectInfo info) throws IOException;
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444 protected abstract void onEndThinPack() throws IOException;
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461 protected abstract ObjectTypeAndSize seekDatabase(PackedObjectInfo obj,
1462 ObjectTypeAndSize info) throws IOException;
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479 protected abstract ObjectTypeAndSize seekDatabase(UnresolvedDelta delta,
1480 ObjectTypeAndSize info) throws IOException;
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496 protected abstract int readDatabase(byte[] dst, int pos, int cnt)
1497 throws IOException;
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515 protected abstract boolean checkCRC(int oldCRC);
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534 protected abstract void onBeginWholeObject(long streamPosition, int type,
1535 long inflatedSize) throws IOException;
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545 protected abstract void onEndWholeObject(PackedObjectInfo info)
1546 throws IOException;
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564 protected abstract void onBeginOfsDelta(long deltaStreamPosition,
1565 long baseStreamPosition, long inflatedSize) throws IOException;
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582 protected abstract void onBeginRefDelta(long deltaStreamPosition,
1583 AnyObjectId baseId, long inflatedSize) throws IOException;
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593 protected UnresolvedDelta onEndDelta() throws IOException {
1594 return new UnresolvedDelta();
1595 }
1596
1597
1598 public static class ObjectTypeAndSize {
1599
1600 public int type;
1601
1602
1603 public long size;
1604 }
1605
1606 private void inflateAndSkip(Source src, long inflatedSize)
1607 throws IOException {
1608 try (InputStream inf = inflate(src, inflatedSize)) {
1609 IO.skipFully(inf, inflatedSize);
1610 }
1611 }
1612
1613 private byte[] inflateAndReturn(Source src, long inflatedSize)
1614 throws IOException {
1615 final byte[] dst = new byte[(int) inflatedSize];
1616 try (InputStream inf = inflate(src, inflatedSize)) {
1617 IO.readFully(inf, dst, 0, dst.length);
1618 }
1619 return dst;
1620 }
1621
1622 private InputStream inflate(Source src, long inflatedSize)
1623 throws IOException {
1624 inflater.open(src, inflatedSize);
1625 return inflater;
1626 }
1627
1628 private static class DeltaChain extends ObjectIdOwnerMap.Entry {
1629 UnresolvedDelta head;
1630
1631 DeltaChain(AnyObjectId id) {
1632 super(id);
1633 }
1634
1635 UnresolvedDelta remove() {
1636 final UnresolvedDelta r = head;
1637 if (r != null)
1638 head = null;
1639 return r;
1640 }
1641
1642 void add(UnresolvedDelta d) {
1643 d.next = head;
1644 head = d;
1645 }
1646 }
1647
1648
1649 public static class UnresolvedDelta {
1650 long position;
1651
1652 int crc;
1653
1654 UnresolvedDelta next;
1655
1656
1657 public long getOffset() {
1658 return position;
1659 }
1660
1661
1662 public int getCRC() {
1663 return crc;
1664 }
1665
1666
1667
1668
1669
1670 public void setCRC(int crc32) {
1671 crc = crc32;
1672 }
1673 }
1674
1675 private static class DeltaVisit {
1676 final UnresolvedDelta delta;
1677
1678 ObjectId id;
1679
1680 byte[] data;
1681
1682 DeltaVisit parent;
1683
1684 UnresolvedDelta nextChild;
1685
1686 DeltaVisit() {
1687 this.delta = null;
1688 }
1689
1690 DeltaVisit(DeltaVisit parent) {
1691 this.parent = parent;
1692 this.delta = parent.nextChild;
1693 parent.nextChild = delta.next;
1694 }
1695
1696 DeltaVisit next() {
1697
1698 if (parent != null && parent.nextChild == null) {
1699 parent.data = null;
1700 parent = parent.parent;
1701 }
1702
1703 if (nextChild != null)
1704 return new DeltaVisit(this);
1705
1706
1707
1708 if (parent != null)
1709 return new DeltaVisit(parent);
1710 return null;
1711 }
1712 }
1713
1714 private void addObjectAndTrack(PackedObjectInfo oe) {
1715 entries[entryCount++] = oe;
1716 if (needNewObjectIds())
1717 newObjectIds.add(oe);
1718 }
1719
1720 private class InflaterStream extends InputStream {
1721 private final Inflater inf;
1722
1723 private final byte[] skipBuffer;
1724
1725 private Source src;
1726
1727 private long expectedSize;
1728
1729 private long actualSize;
1730
1731 private int p;
1732
1733 InflaterStream() {
1734 inf = InflaterCache.get();
1735 skipBuffer = new byte[512];
1736 }
1737
1738 void release() {
1739 inf.reset();
1740 InflaterCache.release(inf);
1741 }
1742
1743 void open(Source source, long inflatedSize) throws IOException {
1744 src = source;
1745 expectedSize = inflatedSize;
1746 actualSize = 0;
1747
1748 p = fill(src, 1);
1749 inf.setInput(buf, p, bAvail);
1750 }
1751
1752 @Override
1753 public long skip(long toSkip) throws IOException {
1754 long n = 0;
1755 while (n < toSkip) {
1756 final int cnt = (int) Math.min(skipBuffer.length, toSkip - n);
1757 final int r = read(skipBuffer, 0, cnt);
1758 if (r <= 0)
1759 break;
1760 n += r;
1761 }
1762 return n;
1763 }
1764
1765 @Override
1766 public int read() throws IOException {
1767 int n = read(skipBuffer, 0, 1);
1768 return n == 1 ? skipBuffer[0] & 0xff : -1;
1769 }
1770
1771 @Override
1772 public int read(byte[] dst, int pos, int cnt) throws IOException {
1773 try {
1774 int n = 0;
1775 while (n < cnt) {
1776 int r = inf.inflate(dst, pos + n, cnt - n);
1777 n += r;
1778 if (inf.finished())
1779 break;
1780 if (inf.needsInput()) {
1781 onObjectData(src, buf, p, bAvail);
1782 use(bAvail);
1783
1784 p = fill(src, 1);
1785 inf.setInput(buf, p, bAvail);
1786 } else if (r == 0) {
1787 throw new CorruptObjectException(MessageFormat.format(
1788 JGitText.get().packfileCorruptionDetected,
1789 JGitText.get().unknownZlibError));
1790 }
1791 }
1792 actualSize += n;
1793 return 0 < n ? n : -1;
1794 } catch (DataFormatException dfe) {
1795 throw new CorruptObjectException(MessageFormat.format(JGitText
1796 .get().packfileCorruptionDetected, dfe.getMessage()));
1797 }
1798 }
1799
1800 @Override
1801 public void close() throws IOException {
1802
1803
1804
1805
1806 if (read(skipBuffer) != -1 || actualSize != expectedSize) {
1807 throw new CorruptObjectException(MessageFormat.format(JGitText
1808 .get().packfileCorruptionDetected,
1809 JGitText.get().wrongDecompressedLength));
1810 }
1811
1812 int used = bAvail - inf.getRemaining();
1813 if (0 < used) {
1814 onObjectData(src, buf, p, used);
1815 use(used);
1816 }
1817
1818 inf.reset();
1819 }
1820 }
1821 }