1
2
3
4
5
6
7
8
9
10
11
12
13 package org.eclipse.jgit.internal.storage.dfs;
14
15 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
16 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
17 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
18 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
19
20 import java.io.BufferedInputStream;
21 import java.io.EOFException;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.nio.ByteBuffer;
25 import java.nio.channels.Channels;
26 import java.text.MessageFormat;
27 import java.util.Set;
28 import java.util.concurrent.atomic.AtomicBoolean;
29 import java.util.zip.CRC32;
30 import java.util.zip.DataFormatException;
31 import java.util.zip.Inflater;
32
33 import org.eclipse.jgit.errors.CorruptObjectException;
34 import org.eclipse.jgit.errors.LargeObjectException;
35 import org.eclipse.jgit.errors.MissingObjectException;
36 import org.eclipse.jgit.errors.PackInvalidException;
37 import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
38 import org.eclipse.jgit.internal.JGitText;
39 import org.eclipse.jgit.internal.storage.file.PackBitmapIndex;
40 import org.eclipse.jgit.internal.storage.file.PackIndex;
41 import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
42 import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
43 import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
44 import org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation;
45 import org.eclipse.jgit.lib.AbbreviatedObjectId;
46 import org.eclipse.jgit.lib.AnyObjectId;
47 import org.eclipse.jgit.lib.Constants;
48 import org.eclipse.jgit.lib.ObjectId;
49 import org.eclipse.jgit.lib.ObjectLoader;
50 import org.eclipse.jgit.lib.Repository;
51 import org.eclipse.jgit.util.LongList;
52
53
54
55
56
57
58 public final class DfsPackFile extends BlockBasedFile {
59 private static final int REC_SIZE = Constants.OBJECT_ID_LENGTH + 8;
60 private static final long REF_POSITION = 0;
61
62
63 private volatile PackIndex index;
64
65
66 private volatile PackReverseIndex reverseIndex;
67
68
69 private volatile PackBitmapIndex bitmapIndex;
70
71
72
73
74
75
76
77
78 private volatile LongList corruptObjects;
79
80
81 private final Object corruptObjectsLock = new Object();
82
83
84
85
86
87
88
89
90
91 DfsPackFile(DfsBlockCache cache, DfsPackDescription desc) {
92 super(cache, desc, PACK);
93
94 int bs = desc.getBlockSize(PACK);
95 if (bs > 0) {
96 setBlockSize(bs);
97 }
98
99 long sz = desc.getFileSize(PACK);
100 length = sz > 0 ? sz : -1;
101 }
102
103
104
105
106
107
108 public DfsPackDescription getPackDescription() {
109 return desc;
110 }
111
112
113
114
115
116
117 public boolean isIndexLoaded() {
118 return index != null;
119 }
120
121 void setPackIndex(PackIndex idx) {
122 long objCnt = idx.getObjectCount();
123 int recSize = Constants.OBJECT_ID_LENGTH + 8;
124 long sz = objCnt * recSize;
125 cache.putRef(desc.getStreamKey(INDEX), sz, idx);
126 index = idx;
127 }
128
129
130
131
132
133
134
135
136
137
138
139 public PackIndex getPackIndex(DfsReader ctx) throws IOException {
140 return idx(ctx);
141 }
142
143 private PackIndex idx(DfsReader ctx) throws IOException {
144 if (index != null) {
145 return index;
146 }
147
148 if (invalid) {
149 throw new PackInvalidException(getFileName(), invalidatingCause);
150 }
151
152 Repository.getGlobalListenerList()
153 .dispatch(new BeforeDfsPackIndexLoadedEvent(this));
154 try {
155 DfsStreamKey idxKey = desc.getStreamKey(INDEX);
156 AtomicBoolean cacheHit = new AtomicBoolean(true);
157 DfsBlockCache.Ref<PackIndex> idxref = cache.getOrLoadRef(idxKey,
158 REF_POSITION, () -> {
159 cacheHit.set(false);
160 return loadPackIndex(ctx, idxKey);
161 });
162 if (cacheHit.get()) {
163 ctx.stats.idxCacheHit++;
164 }
165 PackIndex idx = idxref.get();
166 if (index == null && idx != null) {
167 index = idx;
168 }
169 return index;
170 } catch (IOException e) {
171 invalid = true;
172 invalidatingCause = e;
173 throw e;
174 }
175 }
176
177 final boolean isGarbage() {
178 return desc.getPackSource() == UNREACHABLE_GARBAGE;
179 }
180
181
182
183
184
185
186
187
188
189
190
191 public PackBitmapIndex getBitmapIndex(DfsReader ctx) throws IOException {
192 if (invalid || isGarbage() || !desc.hasFileExt(BITMAP_INDEX)) {
193 return null;
194 }
195
196 if (bitmapIndex != null) {
197 return bitmapIndex;
198 }
199
200 DfsStreamKey bitmapKey = desc.getStreamKey(BITMAP_INDEX);
201 AtomicBoolean cacheHit = new AtomicBoolean(true);
202 DfsBlockCache.Ref<PackBitmapIndex> idxref = cache
203 .getOrLoadRef(bitmapKey, REF_POSITION, () -> {
204 cacheHit.set(false);
205 return loadBitmapIndex(ctx, bitmapKey);
206 });
207 if (cacheHit.get()) {
208 ctx.stats.bitmapCacheHit++;
209 }
210 PackBitmapIndex bmidx = idxref.get();
211 if (bitmapIndex == null && bmidx != null) {
212 bitmapIndex = bmidx;
213 }
214 return bitmapIndex;
215 }
216
217 PackReverseIndex getReverseIdx(DfsReader ctx) throws IOException {
218 if (reverseIndex != null) {
219 return reverseIndex;
220 }
221
222 PackIndex idx = idx(ctx);
223 DfsStreamKey revKey = new DfsStreamKey.ForReverseIndex(
224 desc.getStreamKey(INDEX));
225 AtomicBoolean cacheHit = new AtomicBoolean(true);
226 DfsBlockCache.Ref<PackReverseIndex> revref = cache.getOrLoadRef(revKey,
227 REF_POSITION, () -> {
228 cacheHit.set(false);
229 return loadReverseIdx(ctx, revKey, idx);
230 });
231 if (cacheHit.get()) {
232 ctx.stats.ridxCacheHit++;
233 }
234 PackReverseIndex revidx = revref.get();
235 if (reverseIndex == null && revidx != null) {
236 reverseIndex = revidx;
237 }
238 return reverseIndex;
239 }
240
241
242
243
244
245
246
247
248
249
250
251
252
253 public boolean hasObject(DfsReader ctx, AnyObjectId id) throws IOException {
254 final long offset = idx(ctx).findOffset(id);
255 return 0 < offset && !isCorrupt(offset);
256 }
257
258
259
260
261
262
263
264
265
266
267
268
269
270 ObjectLoader get(DfsReader ctx, AnyObjectId id)
271 throws IOException {
272 long offset = idx(ctx).findOffset(id);
273 return 0 < offset && !isCorrupt(offset) ? load(ctx, offset) : null;
274 }
275
276 long findOffset(DfsReader ctx, AnyObjectId id) throws IOException {
277 return idx(ctx).findOffset(id);
278 }
279
280 void resolve(DfsReader ctx, Set<ObjectId> matches, AbbreviatedObjectId id,
281 int matchLimit) throws IOException {
282 idx(ctx).resolve(matches, id, matchLimit);
283 }
284
285
286
287
288
289
290
291
292
293
294
295 long getObjectCount(DfsReader ctx) throws IOException {
296 return idx(ctx).getObjectCount();
297 }
298
299 private byte[] decompress(long position, int sz, DfsReader ctx)
300 throws IOException, DataFormatException {
301 byte[] dstbuf;
302 try {
303 dstbuf = new byte[sz];
304 } catch (OutOfMemoryError noMemory) {
305
306
307
308
309
310
311
312 return null;
313 }
314
315 if (ctx.inflate(this, position, dstbuf, false) != sz) {
316 throw new EOFException(MessageFormat.format(
317 JGitText.get().shortCompressedStreamAt,
318 Long.valueOf(position)));
319 }
320 return dstbuf;
321 }
322
323 void copyPackAsIs(PackOutputStream out, DfsReader ctx) throws IOException {
324
325 if (length == -1) {
326 ctx.pin(this, 0);
327 ctx.unpin();
328 }
329 try (ReadableChannel rc = ctx.db.openFile(desc, PACK)) {
330 int sz = ctx.getOptions().getStreamPackBufferSize();
331 if (sz > 0) {
332 rc.setReadAheadBytes(sz);
333 }
334 if (cache.shouldCopyThroughCache(length)) {
335 copyPackThroughCache(out, ctx, rc);
336 } else {
337 copyPackBypassCache(out, rc);
338 }
339 }
340 }
341
342 private void copyPackThroughCache(PackOutputStream out, DfsReader ctx,
343 ReadableChannel rc) throws IOException {
344 long position = 12;
345 long remaining = length - (12 + 20);
346 while (0 < remaining) {
347 DfsBlock b = cache.getOrLoad(this, position, ctx, () -> rc);
348 int ptr = (int) (position - b.start);
349 if (b.size() <= ptr) {
350 throw packfileIsTruncated();
351 }
352 int n = (int) Math.min(b.size() - ptr, remaining);
353 b.write(out, position, n);
354 position += n;
355 remaining -= n;
356 }
357 }
358
359 private long copyPackBypassCache(PackOutputStream out, ReadableChannel rc)
360 throws IOException {
361 ByteBuffer buf = newCopyBuffer(out, rc);
362 long position = 12;
363 long remaining = length - (12 + 20);
364 boolean packHeadSkipped = false;
365 while (0 < remaining) {
366 DfsBlock b = cache.get(key, alignToBlock(position));
367 if (b != null) {
368 int ptr = (int) (position - b.start);
369 if (b.size() <= ptr) {
370 throw packfileIsTruncated();
371 }
372 int n = (int) Math.min(b.size() - ptr, remaining);
373 b.write(out, position, n);
374 position += n;
375 remaining -= n;
376 rc.position(position);
377 packHeadSkipped = true;
378 continue;
379 }
380
381
382 int ptr = packHeadSkipped ? 0 : 12;
383 buf.position(0);
384 int bufLen = read(rc, buf);
385 if (bufLen <= ptr) {
386 throw packfileIsTruncated();
387 }
388 int n = (int) Math.min(bufLen - ptr, remaining);
389 out.write(buf.array(), ptr, n);
390 position += n;
391 remaining -= n;
392 packHeadSkipped = true;
393 }
394 return position;
395 }
396
397 private ByteBuffer newCopyBuffer(PackOutputStream out, ReadableChannel rc) {
398 int bs = blockSize(rc);
399 byte[] copyBuf = out.getCopyBuffer();
400 if (bs > copyBuf.length) {
401 copyBuf = new byte[bs];
402 }
403 return ByteBuffer.wrap(copyBuf, 0, bs);
404 }
405
406 void copyAsIs(PackOutputStream out, DfsObjectToPack src,
407 boolean validate, DfsReader ctx) throws IOException,
408 StoredObjectRepresentationNotAvailableException {
409 final CRC32 crc1 = validate ? new CRC32() : null;
410 final CRC32 crc2 = validate ? new CRC32() : null;
411 final byte[] buf = out.getCopyBuffer();
412
413
414
415 try {
416 readFully(src.offset, buf, 0, 20, ctx);
417 } catch (IOException ioError) {
418 throw new StoredObjectRepresentationNotAvailableException(ioError);
419 }
420 int c = buf[0] & 0xff;
421 final int typeCode = (c >> 4) & 7;
422 long inflatedLength = c & 15;
423 int shift = 4;
424 int headerCnt = 1;
425 while ((c & 0x80) != 0) {
426 c = buf[headerCnt++] & 0xff;
427 inflatedLength += ((long) (c & 0x7f)) << shift;
428 shift += 7;
429 }
430
431 if (typeCode == Constants.OBJ_OFS_DELTA) {
432 do {
433 c = buf[headerCnt++] & 0xff;
434 } while ((c & 128) != 0);
435 if (validate) {
436 assert(crc1 != null && crc2 != null);
437 crc1.update(buf, 0, headerCnt);
438 crc2.update(buf, 0, headerCnt);
439 }
440 } else if (typeCode == Constants.OBJ_REF_DELTA) {
441 if (validate) {
442 assert(crc1 != null && crc2 != null);
443 crc1.update(buf, 0, headerCnt);
444 crc2.update(buf, 0, headerCnt);
445 }
446
447 readFully(src.offset + headerCnt, buf, 0, 20, ctx);
448 if (validate) {
449 assert(crc1 != null && crc2 != null);
450 crc1.update(buf, 0, 20);
451 crc2.update(buf, 0, 20);
452 }
453 headerCnt += 20;
454 } else if (validate) {
455 assert(crc1 != null && crc2 != null);
456 crc1.update(buf, 0, headerCnt);
457 crc2.update(buf, 0, headerCnt);
458 }
459
460 final long dataOffset = src.offset + headerCnt;
461 final long dataLength = src.length;
462 final long expectedCRC;
463 final DfsBlock quickCopy;
464
465
466
467
468 try {
469 quickCopy = ctx.quickCopy(this, dataOffset, dataLength);
470
471 if (validate && idx(ctx).hasCRC32Support()) {
472 assert(crc1 != null);
473
474
475 expectedCRC = idx(ctx).findCRC32(src);
476 if (quickCopy != null) {
477 quickCopy.crc32(crc1, dataOffset, (int) dataLength);
478 } else {
479 long pos = dataOffset;
480 long cnt = dataLength;
481 while (cnt > 0) {
482 final int n = (int) Math.min(cnt, buf.length);
483 readFully(pos, buf, 0, n, ctx);
484 crc1.update(buf, 0, n);
485 pos += n;
486 cnt -= n;
487 }
488 }
489 if (crc1.getValue() != expectedCRC) {
490 setCorrupt(src.offset);
491 throw new CorruptObjectException(MessageFormat.format(
492 JGitText.get().objectAtHasBadZlibStream,
493 Long.valueOf(src.offset), getFileName()));
494 }
495 } else if (validate) {
496 assert(crc1 != null);
497
498
499
500
501 Inflater inf = ctx.inflater();
502 byte[] tmp = new byte[1024];
503 if (quickCopy != null) {
504 quickCopy.check(inf, tmp, dataOffset, (int) dataLength);
505 } else {
506 long pos = dataOffset;
507 long cnt = dataLength;
508 while (cnt > 0) {
509 final int n = (int) Math.min(cnt, buf.length);
510 readFully(pos, buf, 0, n, ctx);
511 crc1.update(buf, 0, n);
512 inf.setInput(buf, 0, n);
513 while (inf.inflate(tmp, 0, tmp.length) > 0) {
514 continue;
515 }
516 pos += n;
517 cnt -= n;
518 }
519 }
520 if (!inf.finished() || inf.getBytesRead() != dataLength) {
521 setCorrupt(src.offset);
522 throw new EOFException(MessageFormat.format(
523 JGitText.get().shortCompressedStreamAt,
524 Long.valueOf(src.offset)));
525 }
526 expectedCRC = crc1.getValue();
527 } else {
528 expectedCRC = -1;
529 }
530 } catch (DataFormatException dataFormat) {
531 setCorrupt(src.offset);
532
533 CorruptObjectException corruptObject = new CorruptObjectException(
534 MessageFormat.format(
535 JGitText.get().objectAtHasBadZlibStream,
536 Long.valueOf(src.offset), getFileName()),
537 dataFormat);
538
539 throw new StoredObjectRepresentationNotAvailableException(
540 corruptObject);
541
542 } catch (IOException ioError) {
543 throw new StoredObjectRepresentationNotAvailableException(ioError);
544 }
545
546 if (quickCopy != null) {
547
548
549
550 out.writeHeader(src, inflatedLength);
551 quickCopy.write(out, dataOffset, (int) dataLength);
552
553 } else if (dataLength <= buf.length) {
554
555
556
557 if (!validate) {
558 long pos = dataOffset;
559 long cnt = dataLength;
560 while (cnt > 0) {
561 final int n = (int) Math.min(cnt, buf.length);
562 readFully(pos, buf, 0, n, ctx);
563 pos += n;
564 cnt -= n;
565 }
566 }
567 out.writeHeader(src, inflatedLength);
568 out.write(buf, 0, (int) dataLength);
569 } else {
570
571
572
573
574 out.writeHeader(src, inflatedLength);
575 long pos = dataOffset;
576 long cnt = dataLength;
577 while (cnt > 0) {
578 final int n = (int) Math.min(cnt, buf.length);
579 readFully(pos, buf, 0, n, ctx);
580 if (validate) {
581 assert(crc2 != null);
582 crc2.update(buf, 0, n);
583 }
584 out.write(buf, 0, n);
585 pos += n;
586 cnt -= n;
587 }
588 if (validate) {
589 assert(crc2 != null);
590 if (crc2.getValue() != expectedCRC) {
591 throw new CorruptObjectException(MessageFormat.format(
592 JGitText.get().objectAtHasBadZlibStream,
593 Long.valueOf(src.offset), getFileName()));
594 }
595 }
596 }
597 }
598
599 private IOException packfileIsTruncated() {
600 invalid = true;
601 IOException exc = new IOException(MessageFormat.format(
602 JGitText.get().packfileIsTruncated, getFileName()));
603 invalidatingCause = exc;
604 return exc;
605 }
606
607 private void readFully(long position, byte[] dstbuf, int dstoff, int cnt,
608 DfsReader ctx) throws IOException {
609 while (cnt > 0) {
610 int copied = ctx.copy(this, position, dstbuf, dstoff, cnt);
611 if (copied == 0) {
612 throw new EOFException();
613 }
614 position += copied;
615 dstoff += copied;
616 cnt -= copied;
617 }
618 }
619
620 ObjectLoader load(DfsReader ctx, long pos)
621 throws IOException {
622 try {
623 final byte[] ib = ctx.tempId;
624 Delta delta = null;
625 byte[] data = null;
626 int type = Constants.OBJ_BAD;
627 boolean cached = false;
628
629 SEARCH: for (;;) {
630 readFully(pos, ib, 0, 20, ctx);
631 int c = ib[0] & 0xff;
632 final int typeCode = (c >> 4) & 7;
633 long sz = c & 15;
634 int shift = 4;
635 int p = 1;
636 while ((c & 0x80) != 0) {
637 c = ib[p++] & 0xff;
638 sz += ((long) (c & 0x7f)) << shift;
639 shift += 7;
640 }
641
642 switch (typeCode) {
643 case Constants.OBJ_COMMIT:
644 case Constants.OBJ_TREE:
645 case Constants.OBJ_BLOB:
646 case Constants.OBJ_TAG: {
647 if (delta != null) {
648 data = decompress(pos + p, (int) sz, ctx);
649 type = typeCode;
650 break SEARCH;
651 }
652
653 if (sz < ctx.getStreamFileThreshold()) {
654 data = decompress(pos + p, (int) sz, ctx);
655 if (data != null) {
656 return new ObjectLoader.SmallObject(typeCode, data);
657 }
658 }
659 return new LargePackedWholeObject(typeCode, sz, pos, p, this, ctx.db);
660 }
661
662 case Constants.OBJ_OFS_DELTA: {
663 c = ib[p++] & 0xff;
664 long base = c & 127;
665 while ((c & 128) != 0) {
666 base += 1;
667 c = ib[p++] & 0xff;
668 base <<= 7;
669 base += (c & 127);
670 }
671 base = pos - base;
672 delta = new Delta(delta, pos, (int) sz, p, base);
673 if (sz != delta.deltaSize) {
674 break SEARCH;
675 }
676
677 DeltaBaseCache.Entry e = ctx.getDeltaBaseCache().get(key, base);
678 if (e != null) {
679 type = e.type;
680 data = e.data;
681 cached = true;
682 break SEARCH;
683 }
684 pos = base;
685 continue SEARCH;
686 }
687
688 case Constants.OBJ_REF_DELTA: {
689 readFully(pos + p, ib, 0, 20, ctx);
690 long base = findDeltaBase(ctx, ObjectId.fromRaw(ib));
691 delta = new Delta(delta, pos, (int) sz, p + 20, base);
692 if (sz != delta.deltaSize) {
693 break SEARCH;
694 }
695
696 DeltaBaseCache.Entry e = ctx.getDeltaBaseCache().get(key, base);
697 if (e != null) {
698 type = e.type;
699 data = e.data;
700 cached = true;
701 break SEARCH;
702 }
703 pos = base;
704 continue SEARCH;
705 }
706
707 default:
708 throw new IOException(MessageFormat.format(
709 JGitText.get().unknownObjectType, Integer.valueOf(typeCode)));
710 }
711 }
712
713
714
715
716 if (data == null)
717 throw new LargeObjectException();
718
719 assert(delta != null);
720 do {
721
722 if (cached) {
723 cached = false;
724 } else if (delta.next == null) {
725 ctx.getDeltaBaseCache().put(key, delta.basePos, type, data);
726 }
727
728 pos = delta.deltaPos;
729
730 byte[] cmds = decompress(pos + delta.hdrLen, delta.deltaSize, ctx);
731 if (cmds == null) {
732 data = null;
733 throw new LargeObjectException();
734 }
735
736 final long sz = BinaryDelta.getResultSize(cmds);
737 if (Integer.MAX_VALUE <= sz) {
738 throw new LargeObjectException.ExceedsByteArrayLimit();
739 }
740
741 final byte[] result;
742 try {
743 result = new byte[(int) sz];
744 } catch (OutOfMemoryError tooBig) {
745 data = null;
746 cmds = null;
747 throw new LargeObjectException.OutOfMemory(tooBig);
748 }
749
750 BinaryDelta.apply(data, cmds, result);
751 data = result;
752 delta = delta.next;
753 } while (delta != null);
754
755 return new ObjectLoader.SmallObject(type, data);
756
757 } catch (DataFormatException dfe) {
758 throw new CorruptObjectException(
759 MessageFormat.format(
760 JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
761 getFileName()),
762 dfe);
763 }
764 }
765
766 private long findDeltaBase(DfsReader ctx, ObjectId baseId)
767 throws IOException, MissingObjectException {
768 long ofs = idx(ctx).findOffset(baseId);
769 if (ofs < 0) {
770 throw new MissingObjectException(baseId,
771 JGitText.get().missingDeltaBase);
772 }
773 return ofs;
774 }
775
776 private static class Delta {
777
778 final Delta next;
779
780
781 final long deltaPos;
782
783
784 final int deltaSize;
785
786
787 final int hdrLen;
788
789
790 final long basePos;
791
792 Delta(Delta next, long ofs, int sz, int hdrLen, long baseOffset) {
793 this.next = next;
794 this.deltaPos = ofs;
795 this.deltaSize = sz;
796 this.hdrLen = hdrLen;
797 this.basePos = baseOffset;
798 }
799 }
800
801 byte[] getDeltaHeader(DfsReader wc, long pos)
802 throws IOException, DataFormatException {
803
804
805
806
807
808 final byte[] hdr = new byte[32];
809 wc.inflate(this, pos, hdr, true );
810 return hdr;
811 }
812
813 int getObjectType(DfsReader ctx, long pos) throws IOException {
814 final byte[] ib = ctx.tempId;
815 for (;;) {
816 readFully(pos, ib, 0, 20, ctx);
817 int c = ib[0] & 0xff;
818 final int type = (c >> 4) & 7;
819
820 switch (type) {
821 case Constants.OBJ_COMMIT:
822 case Constants.OBJ_TREE:
823 case Constants.OBJ_BLOB:
824 case Constants.OBJ_TAG:
825 return type;
826
827 case Constants.OBJ_OFS_DELTA: {
828 int p = 1;
829 while ((c & 0x80) != 0) {
830 c = ib[p++] & 0xff;
831 }
832 c = ib[p++] & 0xff;
833 long ofs = c & 127;
834 while ((c & 128) != 0) {
835 ofs += 1;
836 c = ib[p++] & 0xff;
837 ofs <<= 7;
838 ofs += (c & 127);
839 }
840 pos = pos - ofs;
841 continue;
842 }
843
844 case Constants.OBJ_REF_DELTA: {
845 int p = 1;
846 while ((c & 0x80) != 0) {
847 c = ib[p++] & 0xff;
848 }
849 readFully(pos + p, ib, 0, 20, ctx);
850 pos = findDeltaBase(ctx, ObjectId.fromRaw(ib));
851 continue;
852 }
853
854 default:
855 throw new IOException(MessageFormat.format(
856 JGitText.get().unknownObjectType, Integer.valueOf(type)));
857 }
858 }
859 }
860
861 long getObjectSize(DfsReader ctx, AnyObjectId id) throws IOException {
862 final long offset = idx(ctx).findOffset(id);
863 return 0 < offset ? getObjectSize(ctx, offset) : -1;
864 }
865
866 long getObjectSize(DfsReader ctx, long pos)
867 throws IOException {
868 final byte[] ib = ctx.tempId;
869 readFully(pos, ib, 0, 20, ctx);
870 int c = ib[0] & 0xff;
871 final int type = (c >> 4) & 7;
872 long sz = c & 15;
873 int shift = 4;
874 int p = 1;
875 while ((c & 0x80) != 0) {
876 c = ib[p++] & 0xff;
877 sz += ((long) (c & 0x7f)) << shift;
878 shift += 7;
879 }
880
881 long deltaAt;
882 switch (type) {
883 case Constants.OBJ_COMMIT:
884 case Constants.OBJ_TREE:
885 case Constants.OBJ_BLOB:
886 case Constants.OBJ_TAG:
887 return sz;
888
889 case Constants.OBJ_OFS_DELTA:
890 c = ib[p++] & 0xff;
891 while ((c & 128) != 0) {
892 c = ib[p++] & 0xff;
893 }
894 deltaAt = pos + p;
895 break;
896
897 case Constants.OBJ_REF_DELTA:
898 deltaAt = pos + p + 20;
899 break;
900
901 default:
902 throw new IOException(MessageFormat.format(
903 JGitText.get().unknownObjectType, Integer.valueOf(type)));
904 }
905
906 try {
907 return BinaryDelta.getResultSize(getDeltaHeader(ctx, deltaAt));
908 } catch (DataFormatException dfe) {
909 throw new CorruptObjectException(
910 MessageFormat.format(
911 JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
912 getFileName()),
913 dfe);
914 }
915 }
916
917 void representation(DfsObjectRepresentation r, final long pos,
918 DfsReader ctx, PackReverseIndex rev)
919 throws IOException {
920 r.offset = pos;
921 final byte[] ib = ctx.tempId;
922 readFully(pos, ib, 0, 20, ctx);
923 int c = ib[0] & 0xff;
924 int p = 1;
925 final int typeCode = (c >> 4) & 7;
926 while ((c & 0x80) != 0) {
927 c = ib[p++] & 0xff;
928 }
929
930 long len = rev.findNextOffset(pos, length - 20) - pos;
931 switch (typeCode) {
932 case Constants.OBJ_COMMIT:
933 case Constants.OBJ_TREE:
934 case Constants.OBJ_BLOB:
935 case Constants.OBJ_TAG:
936 r.format = StoredObjectRepresentation.PACK_WHOLE;
937 r.baseId = null;
938 r.length = len - p;
939 return;
940
941 case Constants.OBJ_OFS_DELTA: {
942 c = ib[p++] & 0xff;
943 long ofs = c & 127;
944 while ((c & 128) != 0) {
945 ofs += 1;
946 c = ib[p++] & 0xff;
947 ofs <<= 7;
948 ofs += (c & 127);
949 }
950 r.format = StoredObjectRepresentation.PACK_DELTA;
951 r.baseId = rev.findObject(pos - ofs);
952 r.length = len - p;
953 return;
954 }
955
956 case Constants.OBJ_REF_DELTA: {
957 readFully(pos + p, ib, 0, 20, ctx);
958 r.format = StoredObjectRepresentation.PACK_DELTA;
959 r.baseId = ObjectId.fromRaw(ib);
960 r.length = len - p - 20;
961 return;
962 }
963
964 default:
965 throw new IOException(MessageFormat.format(
966 JGitText.get().unknownObjectType, Integer.valueOf(typeCode)));
967 }
968 }
969
970 boolean isCorrupt(long offset) {
971 LongList list = corruptObjects;
972 if (list == null) {
973 return false;
974 }
975 synchronized (list) {
976 return list.contains(offset);
977 }
978 }
979
980 private void setCorrupt(long offset) {
981 LongList list = corruptObjects;
982 if (list == null) {
983 synchronized (corruptObjectsLock) {
984 list = corruptObjects;
985 if (list == null) {
986 list = new LongList();
987 corruptObjects = list;
988 }
989 }
990 }
991 synchronized (list) {
992 list.add(offset);
993 }
994 }
995
996 private DfsBlockCache.Ref<PackIndex> loadPackIndex(
997 DfsReader ctx, DfsStreamKey idxKey) throws IOException {
998 try {
999 ctx.stats.readIdx++;
1000 long start = System.nanoTime();
1001 try (ReadableChannel rc = ctx.db.openFile(desc, INDEX)) {
1002 InputStream in = Channels.newInputStream(rc);
1003 int wantSize = 8192;
1004 int bs = rc.blockSize();
1005 if (0 < bs && bs < wantSize) {
1006 bs = (wantSize / bs) * bs;
1007 } else if (bs <= 0) {
1008 bs = wantSize;
1009 }
1010 PackIndex idx = PackIndex.read(new BufferedInputStream(in, bs));
1011 ctx.stats.readIdxBytes += rc.position();
1012 index = idx;
1013 return new DfsBlockCache.Ref<>(
1014 idxKey,
1015 REF_POSITION,
1016 idx.getObjectCount() * REC_SIZE,
1017 idx);
1018 } finally {
1019 ctx.stats.readIdxMicros += elapsedMicros(start);
1020 }
1021 } catch (EOFException e) {
1022 throw new IOException(MessageFormat.format(
1023 DfsText.get().shortReadOfIndex,
1024 desc.getFileName(INDEX)), e);
1025 } catch (IOException e) {
1026 throw new IOException(MessageFormat.format(
1027 DfsText.get().cannotReadIndex,
1028 desc.getFileName(INDEX)), e);
1029 }
1030 }
1031
1032 private DfsBlockCache.Ref<PackReverseIndex> loadReverseIdx(
1033 DfsReader ctx, DfsStreamKey revKey, PackIndex idx) {
1034 ctx.stats.readReverseIdx++;
1035 long start = System.nanoTime();
1036 PackReverseIndex revidx = new PackReverseIndex(idx);
1037 reverseIndex = revidx;
1038 ctx.stats.readReverseIdxMicros += elapsedMicros(start);
1039 return new DfsBlockCache.Ref<>(
1040 revKey,
1041 REF_POSITION,
1042 idx.getObjectCount() * 8,
1043 revidx);
1044 }
1045
1046 private DfsBlockCache.Ref<PackBitmapIndex> loadBitmapIndex(DfsReader ctx,
1047 DfsStreamKey bitmapKey) throws IOException {
1048 ctx.stats.readBitmap++;
1049 long start = System.nanoTime();
1050 try (ReadableChannel rc = ctx.db.openFile(desc, BITMAP_INDEX)) {
1051 long size;
1052 PackBitmapIndex bmidx;
1053 try {
1054 InputStream in = Channels.newInputStream(rc);
1055 int wantSize = 8192;
1056 int bs = rc.blockSize();
1057 if (0 < bs && bs < wantSize) {
1058 bs = (wantSize / bs) * bs;
1059 } else if (bs <= 0) {
1060 bs = wantSize;
1061 }
1062 in = new BufferedInputStream(in, bs);
1063 bmidx = PackBitmapIndex.read(in, () -> idx(ctx),
1064 () -> getReverseIdx(ctx),
1065 ctx.getOptions().shouldLoadRevIndexInParallel());
1066 } finally {
1067 size = rc.position();
1068 ctx.stats.readBitmapIdxBytes += size;
1069 ctx.stats.readBitmapIdxMicros += elapsedMicros(start);
1070 }
1071 bitmapIndex = bmidx;
1072 return new DfsBlockCache.Ref<>(
1073 bitmapKey, REF_POSITION, size, bmidx);
1074 } catch (EOFException e) {
1075 throw new IOException(MessageFormat.format(
1076 DfsText.get().shortReadOfIndex,
1077 desc.getFileName(BITMAP_INDEX)), e);
1078 } catch (IOException e) {
1079 throw new IOException(MessageFormat.format(
1080 DfsText.get().cannotReadIndex,
1081 desc.getFileName(BITMAP_INDEX)), e);
1082 }
1083 }
1084 }