1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46 package org.eclipse.jgit.internal.storage.dfs;
47
48 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
49 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
50 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
51 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
52
53 import java.io.BufferedInputStream;
54 import java.io.EOFException;
55 import java.io.IOException;
56 import java.io.InputStream;
57 import java.nio.ByteBuffer;
58 import java.nio.channels.Channels;
59 import java.text.MessageFormat;
60 import java.util.Set;
61 import java.util.zip.CRC32;
62 import java.util.zip.DataFormatException;
63 import java.util.zip.Inflater;
64
65 import org.eclipse.jgit.errors.CorruptObjectException;
66 import org.eclipse.jgit.errors.LargeObjectException;
67 import org.eclipse.jgit.errors.MissingObjectException;
68 import org.eclipse.jgit.errors.PackInvalidException;
69 import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
70 import org.eclipse.jgit.internal.JGitText;
71 import org.eclipse.jgit.internal.storage.file.PackBitmapIndex;
72 import org.eclipse.jgit.internal.storage.file.PackIndex;
73 import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
74 import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
75 import org.eclipse.jgit.internal.storage.pack.PackExt;
76 import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
77 import org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation;
78 import org.eclipse.jgit.lib.AbbreviatedObjectId;
79 import org.eclipse.jgit.lib.AnyObjectId;
80 import org.eclipse.jgit.lib.Constants;
81 import org.eclipse.jgit.lib.ObjectId;
82 import org.eclipse.jgit.lib.ObjectLoader;
83 import org.eclipse.jgit.lib.Repository;
84 import org.eclipse.jgit.util.LongList;
85
86
87
88
89
90
91 public final class DfsPackFile {
92
93
94
95
96
97
98
99
100
101 private static final long POS_INDEX = -1;
102
103
104 private static final long POS_REVERSE_INDEX = -2;
105
106
107 private static final long POS_BITMAP_INDEX = -3;
108
109
110 private final DfsBlockCache cache;
111
112
113 private final DfsPackDescription packDesc;
114
115
116 final DfsPackKey key;
117
118
119
120
121
122
123 volatile long length;
124
125
126
127
128
129
130
131
132
133 private volatile int blockSize;
134
135
136 private volatile boolean invalid;
137
138
139
140
141
142
143 private final Object initLock = new Object();
144
145
146 private volatile DfsBlockCache.Ref<PackIndex> index;
147
148
149 private volatile DfsBlockCache.Ref<PackReverseIndex> reverseIndex;
150
151
152 private volatile DfsBlockCache.Ref<PackBitmapIndex> bitmapIndex;
153
154
155
156
157
158
159
160
161 private volatile LongList corruptObjects;
162
163
164
165
166
167
168
169
170
171
172
173 DfsPackFile(DfsBlockCache cache, DfsPackDescription desc, DfsPackKey key) {
174 this.cache = cache;
175 this.packDesc = desc;
176 this.key = key;
177
178 length = desc.getFileSize(PACK);
179 if (length <= 0)
180 length = -1;
181 }
182
183
184 public DfsPackDescription getPackDescription() {
185 return packDesc;
186 }
187
188
189
190
191 public boolean isIndexLoaded() {
192 DfsBlockCache.Ref<PackIndex> idxref = index;
193 return idxref != null && idxref.has();
194 }
195
196
197 public long getCachedSize() {
198 return key.cachedSize.get();
199 }
200
201 String getPackName() {
202 return packDesc.getFileName(PACK);
203 }
204
205 void setBlockSize(int newSize) {
206 blockSize = newSize;
207 }
208
209 void setPackIndex(PackIndex idx) {
210 long objCnt = idx.getObjectCount();
211 int recSize = Constants.OBJECT_ID_LENGTH + 8;
212 int sz = (int) Math.min(objCnt * recSize, Integer.MAX_VALUE);
213 index = cache.put(key, POS_INDEX, sz, idx);
214 }
215
216
217
218
219
220
221
222
223
224
225
226 public PackIndex getPackIndex(DfsReader ctx) throws IOException {
227 return idx(ctx);
228 }
229
230 private PackIndex idx(DfsReader ctx) throws IOException {
231 DfsBlockCache.Ref<PackIndex> idxref = index;
232 if (idxref != null) {
233 PackIndex idx = idxref.get();
234 if (idx != null)
235 return idx;
236 }
237
238 if (invalid)
239 throw new PackInvalidException(getPackName());
240
241 Repository.getGlobalListenerList()
242 .dispatch(new BeforeDfsPackIndexLoadedEvent(this));
243
244 synchronized (initLock) {
245 idxref = index;
246 if (idxref != null) {
247 PackIndex idx = idxref.get();
248 if (idx != null)
249 return idx;
250 }
251
252 PackIndex idx;
253 try {
254 ctx.stats.readIdx++;
255 long start = System.nanoTime();
256 ReadableChannel rc = ctx.db.openFile(packDesc, INDEX);
257 try {
258 InputStream in = Channels.newInputStream(rc);
259 int wantSize = 8192;
260 int bs = rc.blockSize();
261 if (0 < bs && bs < wantSize)
262 bs = (wantSize / bs) * bs;
263 else if (bs <= 0)
264 bs = wantSize;
265 idx = PackIndex.read(new BufferedInputStream(in, bs));
266 ctx.stats.readIdxBytes += rc.position();
267 } finally {
268 rc.close();
269 ctx.stats.readIdxMicros += elapsedMicros(start);
270 }
271 } catch (EOFException e) {
272 invalid = true;
273 IOException e2 = new IOException(MessageFormat.format(
274 DfsText.get().shortReadOfIndex,
275 packDesc.getFileName(INDEX)));
276 e2.initCause(e);
277 throw e2;
278 } catch (IOException e) {
279 invalid = true;
280 IOException e2 = new IOException(MessageFormat.format(
281 DfsText.get().cannotReadIndex,
282 packDesc.getFileName(INDEX)));
283 e2.initCause(e);
284 throw e2;
285 }
286
287 setPackIndex(idx);
288 return idx;
289 }
290 }
291
292 private static long elapsedMicros(long start) {
293 return (System.nanoTime() - start) / 1000L;
294 }
295
296 final boolean isGarbage() {
297 return packDesc.getPackSource() == UNREACHABLE_GARBAGE;
298 }
299
300 PackBitmapIndex getBitmapIndex(DfsReader ctx) throws IOException {
301 if (invalid || isGarbage())
302 return null;
303 DfsBlockCache.Ref<PackBitmapIndex> idxref = bitmapIndex;
304 if (idxref != null) {
305 PackBitmapIndex idx = idxref.get();
306 if (idx != null)
307 return idx;
308 }
309
310 if (!packDesc.hasFileExt(PackExt.BITMAP_INDEX))
311 return null;
312
313 synchronized (initLock) {
314 idxref = bitmapIndex;
315 if (idxref != null) {
316 PackBitmapIndex idx = idxref.get();
317 if (idx != null)
318 return idx;
319 }
320
321 long size;
322 PackBitmapIndex idx;
323 try {
324 ctx.stats.readBitmap++;
325 long start = System.nanoTime();
326 ReadableChannel rc = ctx.db.openFile(packDesc, BITMAP_INDEX);
327 try {
328 InputStream in = Channels.newInputStream(rc);
329 int wantSize = 8192;
330 int bs = rc.blockSize();
331 if (0 < bs && bs < wantSize)
332 bs = (wantSize / bs) * bs;
333 else if (bs <= 0)
334 bs = wantSize;
335 in = new BufferedInputStream(in, bs);
336 idx = PackBitmapIndex.read(
337 in, idx(ctx), getReverseIdx(ctx));
338 } finally {
339 size = rc.position();
340 rc.close();
341 ctx.stats.readIdxBytes += size;
342 ctx.stats.readIdxMicros += elapsedMicros(start);
343 }
344 } catch (EOFException e) {
345 IOException e2 = new IOException(MessageFormat.format(
346 DfsText.get().shortReadOfIndex,
347 packDesc.getFileName(BITMAP_INDEX)));
348 e2.initCause(e);
349 throw e2;
350 } catch (IOException e) {
351 IOException e2 = new IOException(MessageFormat.format(
352 DfsText.get().cannotReadIndex,
353 packDesc.getFileName(BITMAP_INDEX)));
354 e2.initCause(e);
355 throw e2;
356 }
357
358 bitmapIndex = cache.put(key, POS_BITMAP_INDEX,
359 (int) Math.min(size, Integer.MAX_VALUE), idx);
360 return idx;
361 }
362 }
363
364 PackReverseIndex getReverseIdx(DfsReader ctx) throws IOException {
365 DfsBlockCache.Ref<PackReverseIndex> revref = reverseIndex;
366 if (revref != null) {
367 PackReverseIndex revidx = revref.get();
368 if (revidx != null)
369 return revidx;
370 }
371
372 synchronized (initLock) {
373 revref = reverseIndex;
374 if (revref != null) {
375 PackReverseIndex revidx = revref.get();
376 if (revidx != null)
377 return revidx;
378 }
379
380 PackIndex idx = idx(ctx);
381 PackReverseIndex revidx = new PackReverseIndex(idx);
382 int sz = (int) Math.min(
383 idx.getObjectCount() * 8, Integer.MAX_VALUE);
384 reverseIndex = cache.put(key, POS_REVERSE_INDEX, sz, revidx);
385 return revidx;
386 }
387 }
388
389
390
391
392
393
394
395
396
397
398
399
400
401 public boolean hasObject(DfsReader ctx, AnyObjectId id) throws IOException {
402 final long offset = idx(ctx).findOffset(id);
403 return 0 < offset && !isCorrupt(offset);
404 }
405
406
407
408
409
410
411
412
413
414
415
416
417
418 ObjectLoader get(DfsReader ctx, AnyObjectId id)
419 throws IOException {
420 long offset = idx(ctx).findOffset(id);
421 return 0 < offset && !isCorrupt(offset) ? load(ctx, offset) : null;
422 }
423
424 long findOffset(DfsReader ctx, AnyObjectId id) throws IOException {
425 return idx(ctx).findOffset(id);
426 }
427
428 void resolve(DfsReader ctx, Set<ObjectId> matches, AbbreviatedObjectId id,
429 int matchLimit) throws IOException {
430 idx(ctx).resolve(matches, id, matchLimit);
431 }
432
433
434 public void close() {
435 cache.remove(this);
436 index = null;
437 reverseIndex = null;
438 }
439
440
441
442
443
444
445
446
447
448
449
450 long getObjectCount(DfsReader ctx) throws IOException {
451 return idx(ctx).getObjectCount();
452 }
453
454 private byte[] decompress(long position, int sz, DfsReader ctx)
455 throws IOException, DataFormatException {
456 byte[] dstbuf;
457 try {
458 dstbuf = new byte[sz];
459 } catch (OutOfMemoryError noMemory) {
460
461
462
463
464
465
466
467 return null;
468 }
469
470 if (ctx.inflate(this, position, dstbuf, false) != sz)
471 throw new EOFException(MessageFormat.format(
472 JGitText.get().shortCompressedStreamAt,
473 Long.valueOf(position)));
474 return dstbuf;
475 }
476
477 void copyPackAsIs(PackOutputStream out, DfsReader ctx)
478 throws IOException {
479
480 if (length == -1) {
481 ctx.pin(this, 0);
482 ctx.unpin();
483 }
484 if (cache.shouldCopyThroughCache(length))
485 copyPackThroughCache(out, ctx);
486 else
487 copyPackBypassCache(out, ctx);
488 }
489
490 private void copyPackThroughCache(PackOutputStream out, DfsReader ctx)
491 throws IOException {
492 long position = 12;
493 long remaining = length - (12 + 20);
494 while (0 < remaining) {
495 DfsBlock b = cache.getOrLoad(this, position, ctx);
496 int ptr = (int) (position - b.start);
497 int n = (int) Math.min(b.size() - ptr, remaining);
498 b.write(out, position, n);
499 position += n;
500 remaining -= n;
501 }
502 }
503
504 private long copyPackBypassCache(PackOutputStream out, DfsReader ctx)
505 throws IOException {
506 try (ReadableChannel rc = ctx.db.openFile(packDesc, PACK)) {
507 ByteBuffer buf = newCopyBuffer(out, rc);
508 if (ctx.getOptions().getStreamPackBufferSize() > 0)
509 rc.setReadAheadBytes(ctx.getOptions().getStreamPackBufferSize());
510 long position = 12;
511 long remaining = length - (12 + 20);
512 boolean packHeadSkipped = false;
513 while (0 < remaining) {
514 DfsBlock b = cache.get(key, alignToBlock(position));
515 if (b != null) {
516 int ptr = (int) (position - b.start);
517 int n = (int) Math.min(b.size() - ptr, remaining);
518 b.write(out, position, n);
519 position += n;
520 remaining -= n;
521 rc.position(position);
522 packHeadSkipped = true;
523 continue;
524 }
525
526 buf.position(0);
527 int n = read(rc, buf);
528 if (n <= 0)
529 throw packfileIsTruncated();
530 else if (n > remaining)
531 n = (int) remaining;
532
533 if (!packHeadSkipped) {
534
535 out.write(buf.array(), 12, n - 12);
536 packHeadSkipped = true;
537 } else {
538 out.write(buf.array(), 0, n);
539 }
540 position += n;
541 remaining -= n;
542 }
543 return position;
544 }
545 }
546
547 private ByteBuffer newCopyBuffer(PackOutputStream out, ReadableChannel rc) {
548 int bs = blockSize(rc);
549 byte[] copyBuf = out.getCopyBuffer();
550 if (bs > copyBuf.length)
551 copyBuf = new byte[bs];
552 return ByteBuffer.wrap(copyBuf, 0, bs);
553 }
554
555 void copyAsIs(PackOutputStream out, DfsObjectToPack src,
556 boolean validate, DfsReader ctx) throws IOException,
557 StoredObjectRepresentationNotAvailableException {
558 final CRC32 crc1 = validate ? new CRC32() : null;
559 final CRC32 crc2 = validate ? new CRC32() : null;
560 final byte[] buf = out.getCopyBuffer();
561
562
563
564 try {
565 readFully(src.offset, buf, 0, 20, ctx);
566 } catch (IOException ioError) {
567 StoredObjectRepresentationNotAvailableException gone;
568 gone = new StoredObjectRepresentationNotAvailableException(src);
569 gone.initCause(ioError);
570 throw gone;
571 }
572 int c = buf[0] & 0xff;
573 final int typeCode = (c >> 4) & 7;
574 long inflatedLength = c & 15;
575 int shift = 4;
576 int headerCnt = 1;
577 while ((c & 0x80) != 0) {
578 c = buf[headerCnt++] & 0xff;
579 inflatedLength += ((long) (c & 0x7f)) << shift;
580 shift += 7;
581 }
582
583 if (typeCode == Constants.OBJ_OFS_DELTA) {
584 do {
585 c = buf[headerCnt++] & 0xff;
586 } while ((c & 128) != 0);
587 if (validate) {
588 assert(crc1 != null && crc2 != null);
589 crc1.update(buf, 0, headerCnt);
590 crc2.update(buf, 0, headerCnt);
591 }
592 } else if (typeCode == Constants.OBJ_REF_DELTA) {
593 if (validate) {
594 assert(crc1 != null && crc2 != null);
595 crc1.update(buf, 0, headerCnt);
596 crc2.update(buf, 0, headerCnt);
597 }
598
599 readFully(src.offset + headerCnt, buf, 0, 20, ctx);
600 if (validate) {
601 assert(crc1 != null && crc2 != null);
602 crc1.update(buf, 0, 20);
603 crc2.update(buf, 0, 20);
604 }
605 headerCnt += 20;
606 } else if (validate) {
607 assert(crc1 != null && crc2 != null);
608 crc1.update(buf, 0, headerCnt);
609 crc2.update(buf, 0, headerCnt);
610 }
611
612 final long dataOffset = src.offset + headerCnt;
613 final long dataLength = src.length;
614 final long expectedCRC;
615 final DfsBlock quickCopy;
616
617
618
619
620 try {
621 quickCopy = ctx.quickCopy(this, dataOffset, dataLength);
622
623 if (validate && idx(ctx).hasCRC32Support()) {
624 assert(crc1 != null);
625
626
627 expectedCRC = idx(ctx).findCRC32(src);
628 if (quickCopy != null) {
629 quickCopy.crc32(crc1, dataOffset, (int) dataLength);
630 } else {
631 long pos = dataOffset;
632 long cnt = dataLength;
633 while (cnt > 0) {
634 final int n = (int) Math.min(cnt, buf.length);
635 readFully(pos, buf, 0, n, ctx);
636 crc1.update(buf, 0, n);
637 pos += n;
638 cnt -= n;
639 }
640 }
641 if (crc1.getValue() != expectedCRC) {
642 setCorrupt(src.offset);
643 throw new CorruptObjectException(MessageFormat.format(
644 JGitText.get().objectAtHasBadZlibStream,
645 Long.valueOf(src.offset), getPackName()));
646 }
647 } else if (validate) {
648 assert(crc1 != null);
649
650
651
652
653 Inflater inf = ctx.inflater();
654 byte[] tmp = new byte[1024];
655 if (quickCopy != null) {
656 quickCopy.check(inf, tmp, dataOffset, (int) dataLength);
657 } else {
658 long pos = dataOffset;
659 long cnt = dataLength;
660 while (cnt > 0) {
661 final int n = (int) Math.min(cnt, buf.length);
662 readFully(pos, buf, 0, n, ctx);
663 crc1.update(buf, 0, n);
664 inf.setInput(buf, 0, n);
665 while (inf.inflate(tmp, 0, tmp.length) > 0)
666 continue;
667 pos += n;
668 cnt -= n;
669 }
670 }
671 if (!inf.finished() || inf.getBytesRead() != dataLength) {
672 setCorrupt(src.offset);
673 throw new EOFException(MessageFormat.format(
674 JGitText.get().shortCompressedStreamAt,
675 Long.valueOf(src.offset)));
676 }
677 expectedCRC = crc1.getValue();
678 } else {
679 expectedCRC = -1;
680 }
681 } catch (DataFormatException dataFormat) {
682 setCorrupt(src.offset);
683
684 CorruptObjectException corruptObject = new CorruptObjectException(
685 MessageFormat.format(
686 JGitText.get().objectAtHasBadZlibStream,
687 Long.valueOf(src.offset), getPackName()));
688 corruptObject.initCause(dataFormat);
689
690 StoredObjectRepresentationNotAvailableException gone;
691 gone = new StoredObjectRepresentationNotAvailableException(src);
692 gone.initCause(corruptObject);
693 throw gone;
694
695 } catch (IOException ioError) {
696 StoredObjectRepresentationNotAvailableException gone;
697 gone = new StoredObjectRepresentationNotAvailableException(src);
698 gone.initCause(ioError);
699 throw gone;
700 }
701
702 if (quickCopy != null) {
703
704
705
706 out.writeHeader(src, inflatedLength);
707 quickCopy.write(out, dataOffset, (int) dataLength);
708
709 } else if (dataLength <= buf.length) {
710
711
712
713 if (!validate) {
714 long pos = dataOffset;
715 long cnt = dataLength;
716 while (cnt > 0) {
717 final int n = (int) Math.min(cnt, buf.length);
718 readFully(pos, buf, 0, n, ctx);
719 pos += n;
720 cnt -= n;
721 }
722 }
723 out.writeHeader(src, inflatedLength);
724 out.write(buf, 0, (int) dataLength);
725 } else {
726
727
728
729
730 out.writeHeader(src, inflatedLength);
731 long pos = dataOffset;
732 long cnt = dataLength;
733 while (cnt > 0) {
734 final int n = (int) Math.min(cnt, buf.length);
735 readFully(pos, buf, 0, n, ctx);
736 if (validate) {
737 assert(crc2 != null);
738 crc2.update(buf, 0, n);
739 }
740 out.write(buf, 0, n);
741 pos += n;
742 cnt -= n;
743 }
744 if (validate) {
745 assert(crc2 != null);
746 if (crc2.getValue() != expectedCRC) {
747 throw new CorruptObjectException(MessageFormat.format(
748 JGitText.get().objectAtHasBadZlibStream,
749 Long.valueOf(src.offset), getPackName()));
750 }
751 }
752 }
753 }
754
755 boolean invalid() {
756 return invalid;
757 }
758
759 void setInvalid() {
760 invalid = true;
761 }
762
763 private IOException packfileIsTruncated() {
764 invalid = true;
765 return new IOException(MessageFormat.format(
766 JGitText.get().packfileIsTruncated, getPackName()));
767 }
768
769 private void readFully(long position, byte[] dstbuf, int dstoff, int cnt,
770 DfsReader ctx) throws IOException {
771 if (ctx.copy(this, position, dstbuf, dstoff, cnt) != cnt)
772 throw new EOFException();
773 }
774
775 long alignToBlock(long pos) {
776 int size = blockSize;
777 if (size == 0)
778 size = cache.getBlockSize();
779 return (pos / size) * size;
780 }
781
782 DfsBlock getOrLoadBlock(long pos, DfsReader ctx) throws IOException {
783 return cache.getOrLoad(this, pos, ctx);
784 }
785
786 DfsBlock readOneBlock(long pos, DfsReader ctx)
787 throws IOException {
788 if (invalid)
789 throw new PackInvalidException(getPackName());
790
791 ctx.stats.readBlock++;
792 long start = System.nanoTime();
793 ReadableChannel rc = ctx.db.openFile(packDesc, PACK);
794 try {
795 int size = blockSize(rc);
796 pos = (pos / size) * size;
797
798
799
800
801
802 long len = length;
803 if (len < 0) {
804 len = rc.size();
805 if (0 <= len)
806 length = len;
807 }
808
809 if (0 <= len && len < pos + size)
810 size = (int) (len - pos);
811 if (size <= 0)
812 throw new EOFException(MessageFormat.format(
813 DfsText.get().shortReadOfBlock, Long.valueOf(pos),
814 getPackName(), Long.valueOf(0), Long.valueOf(0)));
815
816 byte[] buf = new byte[size];
817 rc.position(pos);
818 int cnt = read(rc, ByteBuffer.wrap(buf, 0, size));
819 ctx.stats.readBlockBytes += cnt;
820 if (cnt != size) {
821 if (0 <= len) {
822 throw new EOFException(MessageFormat.format(
823 DfsText.get().shortReadOfBlock,
824 Long.valueOf(pos),
825 getPackName(),
826 Integer.valueOf(size),
827 Integer.valueOf(cnt)));
828 }
829
830
831
832 byte[] n = new byte[cnt];
833 System.arraycopy(buf, 0, n, 0, n.length);
834 buf = n;
835 } else if (len < 0) {
836
837
838 length = len = rc.size();
839 }
840
841 return new DfsBlock(key, pos, buf);
842 } finally {
843 rc.close();
844 ctx.stats.readBlockMicros += elapsedMicros(start);
845 }
846 }
847
848 private int blockSize(ReadableChannel rc) {
849
850
851 int size = blockSize;
852 if (size == 0) {
853 size = rc.blockSize();
854 if (size <= 0)
855 size = cache.getBlockSize();
856 else if (size < cache.getBlockSize())
857 size = (cache.getBlockSize() / size) * size;
858 blockSize = size;
859 }
860 return size;
861 }
862
863 private static int read(ReadableChannel rc, ByteBuffer buf)
864 throws IOException {
865 int n;
866 do {
867 n = rc.read(buf);
868 } while (0 < n && buf.hasRemaining());
869 return buf.position();
870 }
871
872 ObjectLoader load(DfsReader ctx, long pos)
873 throws IOException {
874 try {
875 final byte[] ib = ctx.tempId;
876 Delta delta = null;
877 byte[] data = null;
878 int type = Constants.OBJ_BAD;
879 boolean cached = false;
880
881 SEARCH: for (;;) {
882 readFully(pos, ib, 0, 20, ctx);
883 int c = ib[0] & 0xff;
884 final int typeCode = (c >> 4) & 7;
885 long sz = c & 15;
886 int shift = 4;
887 int p = 1;
888 while ((c & 0x80) != 0) {
889 c = ib[p++] & 0xff;
890 sz += ((long) (c & 0x7f)) << shift;
891 shift += 7;
892 }
893
894 switch (typeCode) {
895 case Constants.OBJ_COMMIT:
896 case Constants.OBJ_TREE:
897 case Constants.OBJ_BLOB:
898 case Constants.OBJ_TAG: {
899 if (delta != null) {
900 data = decompress(pos + p, (int) sz, ctx);
901 type = typeCode;
902 break SEARCH;
903 }
904
905 if (sz < ctx.getStreamFileThreshold()) {
906 data = decompress(pos + p, (int) sz, ctx);
907 if (data != null)
908 return new ObjectLoader.SmallObject(typeCode, data);
909 }
910 return new LargePackedWholeObject(typeCode, sz, pos, p, this, ctx.db);
911 }
912
913 case Constants.OBJ_OFS_DELTA: {
914 c = ib[p++] & 0xff;
915 long base = c & 127;
916 while ((c & 128) != 0) {
917 base += 1;
918 c = ib[p++] & 0xff;
919 base <<= 7;
920 base += (c & 127);
921 }
922 base = pos - base;
923 delta = new Delta(delta, pos, (int) sz, p, base);
924 if (sz != delta.deltaSize)
925 break SEARCH;
926
927 DeltaBaseCache.Entry e = ctx.getDeltaBaseCache().get(key, base);
928 if (e != null) {
929 type = e.type;
930 data = e.data;
931 cached = true;
932 break SEARCH;
933 }
934 pos = base;
935 continue SEARCH;
936 }
937
938 case Constants.OBJ_REF_DELTA: {
939 readFully(pos + p, ib, 0, 20, ctx);
940 long base = findDeltaBase(ctx, ObjectId.fromRaw(ib));
941 delta = new Delta(delta, pos, (int) sz, p + 20, base);
942 if (sz != delta.deltaSize)
943 break SEARCH;
944
945 DeltaBaseCache.Entry e = ctx.getDeltaBaseCache().get(key, base);
946 if (e != null) {
947 type = e.type;
948 data = e.data;
949 cached = true;
950 break SEARCH;
951 }
952 pos = base;
953 continue SEARCH;
954 }
955
956 default:
957 throw new IOException(MessageFormat.format(
958 JGitText.get().unknownObjectType, Integer.valueOf(typeCode)));
959 }
960 }
961
962
963
964
965 if (data == null)
966 throw new LargeObjectException();
967
968 assert(delta != null);
969 do {
970
971 if (cached)
972 cached = false;
973 else if (delta.next == null)
974 ctx.getDeltaBaseCache().put(key, delta.basePos, type, data);
975
976 pos = delta.deltaPos;
977
978 byte[] cmds = decompress(pos + delta.hdrLen, delta.deltaSize, ctx);
979 if (cmds == null) {
980 data = null;
981 throw new LargeObjectException();
982 }
983
984 final long sz = BinaryDelta.getResultSize(cmds);
985 if (Integer.MAX_VALUE <= sz)
986 throw new LargeObjectException.ExceedsByteArrayLimit();
987
988 final byte[] result;
989 try {
990 result = new byte[(int) sz];
991 } catch (OutOfMemoryError tooBig) {
992 data = null;
993 cmds = null;
994 throw new LargeObjectException.OutOfMemory(tooBig);
995 }
996
997 BinaryDelta.apply(data, cmds, result);
998 data = result;
999 delta = delta.next;
1000 } while (delta != null);
1001
1002 return new ObjectLoader.SmallObject(type, data);
1003
1004 } catch (DataFormatException dfe) {
1005 CorruptObjectException coe = new CorruptObjectException(
1006 MessageFormat.format(
1007 JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
1008 getPackName()));
1009 coe.initCause(dfe);
1010 throw coe;
1011 }
1012 }
1013
1014 private long findDeltaBase(DfsReader ctx, ObjectId baseId)
1015 throws IOException, MissingObjectException {
1016 long ofs = idx(ctx).findOffset(baseId);
1017 if (ofs < 0)
1018 throw new MissingObjectException(baseId,
1019 JGitText.get().missingDeltaBase);
1020 return ofs;
1021 }
1022
1023 private static class Delta {
1024
1025 final Delta next;
1026
1027
1028 final long deltaPos;
1029
1030
1031 final int deltaSize;
1032
1033
1034 final int hdrLen;
1035
1036
1037 final long basePos;
1038
1039 Delta(Delta next, long ofs, int sz, int hdrLen, long baseOffset) {
1040 this.next = next;
1041 this.deltaPos = ofs;
1042 this.deltaSize = sz;
1043 this.hdrLen = hdrLen;
1044 this.basePos = baseOffset;
1045 }
1046 }
1047
1048 byte[] getDeltaHeader(DfsReader wc, long pos)
1049 throws IOException, DataFormatException {
1050
1051
1052
1053
1054
1055 final byte[] hdr = new byte[32];
1056 wc.inflate(this, pos, hdr, true );
1057 return hdr;
1058 }
1059
1060 int getObjectType(DfsReader ctx, long pos) throws IOException {
1061 final byte[] ib = ctx.tempId;
1062 for (;;) {
1063 readFully(pos, ib, 0, 20, ctx);
1064 int c = ib[0] & 0xff;
1065 final int type = (c >> 4) & 7;
1066
1067 switch (type) {
1068 case Constants.OBJ_COMMIT:
1069 case Constants.OBJ_TREE:
1070 case Constants.OBJ_BLOB:
1071 case Constants.OBJ_TAG:
1072 return type;
1073
1074 case Constants.OBJ_OFS_DELTA: {
1075 int p = 1;
1076 while ((c & 0x80) != 0)
1077 c = ib[p++] & 0xff;
1078 c = ib[p++] & 0xff;
1079 long ofs = c & 127;
1080 while ((c & 128) != 0) {
1081 ofs += 1;
1082 c = ib[p++] & 0xff;
1083 ofs <<= 7;
1084 ofs += (c & 127);
1085 }
1086 pos = pos - ofs;
1087 continue;
1088 }
1089
1090 case Constants.OBJ_REF_DELTA: {
1091 int p = 1;
1092 while ((c & 0x80) != 0)
1093 c = ib[p++] & 0xff;
1094 readFully(pos + p, ib, 0, 20, ctx);
1095 pos = findDeltaBase(ctx, ObjectId.fromRaw(ib));
1096 continue;
1097 }
1098
1099 default:
1100 throw new IOException(MessageFormat.format(
1101 JGitText.get().unknownObjectType, Integer.valueOf(type)));
1102 }
1103 }
1104 }
1105
1106 long getObjectSize(DfsReader ctx, AnyObjectId id) throws IOException {
1107 final long offset = idx(ctx).findOffset(id);
1108 return 0 < offset ? getObjectSize(ctx, offset) : -1;
1109 }
1110
1111 long getObjectSize(DfsReader ctx, long pos)
1112 throws IOException {
1113 final byte[] ib = ctx.tempId;
1114 readFully(pos, ib, 0, 20, ctx);
1115 int c = ib[0] & 0xff;
1116 final int type = (c >> 4) & 7;
1117 long sz = c & 15;
1118 int shift = 4;
1119 int p = 1;
1120 while ((c & 0x80) != 0) {
1121 c = ib[p++] & 0xff;
1122 sz += ((long) (c & 0x7f)) << shift;
1123 shift += 7;
1124 }
1125
1126 long deltaAt;
1127 switch (type) {
1128 case Constants.OBJ_COMMIT:
1129 case Constants.OBJ_TREE:
1130 case Constants.OBJ_BLOB:
1131 case Constants.OBJ_TAG:
1132 return sz;
1133
1134 case Constants.OBJ_OFS_DELTA:
1135 c = ib[p++] & 0xff;
1136 while ((c & 128) != 0)
1137 c = ib[p++] & 0xff;
1138 deltaAt = pos + p;
1139 break;
1140
1141 case Constants.OBJ_REF_DELTA:
1142 deltaAt = pos + p + 20;
1143 break;
1144
1145 default:
1146 throw new IOException(MessageFormat.format(
1147 JGitText.get().unknownObjectType, Integer.valueOf(type)));
1148 }
1149
1150 try {
1151 return BinaryDelta.getResultSize(getDeltaHeader(ctx, deltaAt));
1152 } catch (DataFormatException dfe) {
1153 CorruptObjectException coe = new CorruptObjectException(
1154 MessageFormat.format(
1155 JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
1156 getPackName()));
1157 coe.initCause(dfe);
1158 throw coe;
1159 }
1160 }
1161
1162 void representation(DfsObjectRepresentation r, final long pos,
1163 DfsReader ctx, PackReverseIndex rev)
1164 throws IOException {
1165 r.offset = pos;
1166 final byte[] ib = ctx.tempId;
1167 readFully(pos, ib, 0, 20, ctx);
1168 int c = ib[0] & 0xff;
1169 int p = 1;
1170 final int typeCode = (c >> 4) & 7;
1171 while ((c & 0x80) != 0)
1172 c = ib[p++] & 0xff;
1173
1174 long len = rev.findNextOffset(pos, length - 20) - pos;
1175 switch (typeCode) {
1176 case Constants.OBJ_COMMIT:
1177 case Constants.OBJ_TREE:
1178 case Constants.OBJ_BLOB:
1179 case Constants.OBJ_TAG:
1180 r.format = StoredObjectRepresentation.PACK_WHOLE;
1181 r.baseId = null;
1182 r.length = len - p;
1183 return;
1184
1185 case Constants.OBJ_OFS_DELTA: {
1186 c = ib[p++] & 0xff;
1187 long ofs = c & 127;
1188 while ((c & 128) != 0) {
1189 ofs += 1;
1190 c = ib[p++] & 0xff;
1191 ofs <<= 7;
1192 ofs += (c & 127);
1193 }
1194 r.format = StoredObjectRepresentation.PACK_DELTA;
1195 r.baseId = rev.findObject(pos - ofs);
1196 r.length = len - p;
1197 return;
1198 }
1199
1200 case Constants.OBJ_REF_DELTA: {
1201 readFully(pos + p, ib, 0, 20, ctx);
1202 r.format = StoredObjectRepresentation.PACK_DELTA;
1203 r.baseId = ObjectId.fromRaw(ib);
1204 r.length = len - p - 20;
1205 return;
1206 }
1207
1208 default:
1209 throw new IOException(MessageFormat.format(
1210 JGitText.get().unknownObjectType, Integer.valueOf(typeCode)));
1211 }
1212 }
1213
1214 boolean isCorrupt(long offset) {
1215 LongList list = corruptObjects;
1216 if (list == null)
1217 return false;
1218 synchronized (list) {
1219 return list.contains(offset);
1220 }
1221 }
1222
1223 private void setCorrupt(long offset) {
1224 LongList list = corruptObjects;
1225 if (list == null) {
1226 synchronized (initLock) {
1227 list = corruptObjects;
1228 if (list == null) {
1229 list = new LongList();
1230 corruptObjects = list;
1231 }
1232 }
1233 }
1234 synchronized (list) {
1235 list.add(offset);
1236 }
1237 }
1238 }