1
2
3
4
5
6
7
8
9
10
11
12
13 package org.eclipse.jgit.internal.storage.dfs;
14
15 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
16 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
17 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
18 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
19
20 import java.io.BufferedInputStream;
21 import java.io.EOFException;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.nio.ByteBuffer;
25 import java.nio.channels.Channels;
26 import java.text.MessageFormat;
27 import java.util.Set;
28 import java.util.zip.CRC32;
29 import java.util.zip.DataFormatException;
30 import java.util.zip.Inflater;
31
32 import org.eclipse.jgit.errors.CorruptObjectException;
33 import org.eclipse.jgit.errors.LargeObjectException;
34 import org.eclipse.jgit.errors.MissingObjectException;
35 import org.eclipse.jgit.errors.PackInvalidException;
36 import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
37 import org.eclipse.jgit.internal.JGitText;
38 import org.eclipse.jgit.internal.storage.file.PackBitmapIndex;
39 import org.eclipse.jgit.internal.storage.file.PackIndex;
40 import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
41 import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
42 import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
43 import org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation;
44 import org.eclipse.jgit.lib.AbbreviatedObjectId;
45 import org.eclipse.jgit.lib.AnyObjectId;
46 import org.eclipse.jgit.lib.Constants;
47 import org.eclipse.jgit.lib.ObjectId;
48 import org.eclipse.jgit.lib.ObjectLoader;
49 import org.eclipse.jgit.lib.Repository;
50 import org.eclipse.jgit.util.LongList;
51
52
53
54
55
56
57 public final class DfsPackFile extends BlockBasedFile {
58 private static final int REC_SIZE = Constants.OBJECT_ID_LENGTH + 8;
59 private static final long REF_POSITION = 0;
60
61
62
63
64
65
66 private final Object initLock = new Object();
67
68
69 private volatile PackIndex index;
70
71
72 private volatile PackReverseIndex reverseIndex;
73
74
75 private volatile PackBitmapIndex bitmapIndex;
76
77
78
79
80
81
82
83
84 private volatile LongList corruptObjects;
85
86
87
88
89
90
91
92
93
94 DfsPackFile(DfsBlockCache cache, DfsPackDescription desc) {
95 super(cache, desc, PACK);
96
97 int bs = desc.getBlockSize(PACK);
98 if (bs > 0) {
99 setBlockSize(bs);
100 }
101
102 long sz = desc.getFileSize(PACK);
103 length = sz > 0 ? sz : -1;
104 }
105
106
107
108
109
110
111 public DfsPackDescription getPackDescription() {
112 return desc;
113 }
114
115
116
117
118
119
120 public boolean isIndexLoaded() {
121 return index != null;
122 }
123
124 void setPackIndex(PackIndex idx) {
125 long objCnt = idx.getObjectCount();
126 int recSize = Constants.OBJECT_ID_LENGTH + 8;
127 long sz = objCnt * recSize;
128 cache.putRef(desc.getStreamKey(INDEX), sz, idx);
129 index = idx;
130 }
131
132
133
134
135
136
137
138
139
140
141
142 public PackIndex getPackIndex(DfsReader ctx) throws IOException {
143 return idx(ctx);
144 }
145
146 private PackIndex idx(DfsReader ctx) throws IOException {
147 if (index != null) {
148 return index;
149 }
150
151 if (invalid) {
152 throw new PackInvalidException(getFileName(), invalidatingCause);
153 }
154
155 Repository.getGlobalListenerList()
156 .dispatch(new BeforeDfsPackIndexLoadedEvent(this));
157
158 synchronized (initLock) {
159 if (index != null) {
160 return index;
161 }
162
163 try {
164 DfsStreamKey idxKey = desc.getStreamKey(INDEX);
165 DfsBlockCache.Ref<PackIndex> idxref = cache.getOrLoadRef(
166 idxKey,
167 REF_POSITION,
168 () -> loadPackIndex(ctx, idxKey));
169 PackIndex idx = idxref.get();
170 if (index == null && idx != null) {
171 index = idx;
172 }
173 return index;
174 } catch (IOException e) {
175 invalid = true;
176 invalidatingCause = e;
177 throw e;
178 }
179 }
180 }
181
182 final boolean isGarbage() {
183 return desc.getPackSource() == UNREACHABLE_GARBAGE;
184 }
185
186 PackBitmapIndex getBitmapIndex(DfsReader ctx) throws IOException {
187 if (invalid || isGarbage() || !desc.hasFileExt(BITMAP_INDEX)) {
188 return null;
189 }
190
191 if (bitmapIndex != null) {
192 return bitmapIndex;
193 }
194
195 synchronized (initLock) {
196 if (bitmapIndex != null) {
197 return bitmapIndex;
198 }
199
200 PackIndex idx = idx(ctx);
201 PackReverseIndex revidx = getReverseIdx(ctx);
202 DfsStreamKey bitmapKey = desc.getStreamKey(BITMAP_INDEX);
203 DfsBlockCache.Ref<PackBitmapIndex> idxref = cache.getOrLoadRef(
204 bitmapKey,
205 REF_POSITION,
206 () -> loadBitmapIndex(ctx, bitmapKey, idx, revidx));
207 PackBitmapIndex bmidx = idxref.get();
208 if (bitmapIndex == null && bmidx != null) {
209 bitmapIndex = bmidx;
210 }
211 return bitmapIndex;
212 }
213 }
214
215 PackReverseIndex getReverseIdx(DfsReader ctx) throws IOException {
216 if (reverseIndex != null) {
217 return reverseIndex;
218 }
219
220 synchronized (initLock) {
221 if (reverseIndex != null) {
222 return reverseIndex;
223 }
224
225 PackIndex idx = idx(ctx);
226 DfsStreamKey revKey = new DfsStreamKey.ForReverseIndex(
227 desc.getStreamKey(INDEX));
228 DfsBlockCache.Ref<PackReverseIndex> revref = cache.getOrLoadRef(
229 revKey,
230 REF_POSITION,
231 () -> loadReverseIdx(revKey, idx));
232 PackReverseIndex revidx = revref.get();
233 if (reverseIndex == null && revidx != null) {
234 reverseIndex = revidx;
235 }
236 return reverseIndex;
237 }
238 }
239
240
241
242
243
244
245
246
247
248
249
250
251
252 public boolean hasObject(DfsReader ctx, AnyObjectId id) throws IOException {
253 final long offset = idx(ctx).findOffset(id);
254 return 0 < offset && !isCorrupt(offset);
255 }
256
257
258
259
260
261
262
263
264
265
266
267
268
269 ObjectLoader get(DfsReader ctx, AnyObjectId id)
270 throws IOException {
271 long offset = idx(ctx).findOffset(id);
272 return 0 < offset && !isCorrupt(offset) ? load(ctx, offset) : null;
273 }
274
275 long findOffset(DfsReader ctx, AnyObjectId id) throws IOException {
276 return idx(ctx).findOffset(id);
277 }
278
279 void resolve(DfsReader ctx, Set<ObjectId> matches, AbbreviatedObjectId id,
280 int matchLimit) throws IOException {
281 idx(ctx).resolve(matches, id, matchLimit);
282 }
283
284
285
286
287
288
289
290
291
292
293
294 long getObjectCount(DfsReader ctx) throws IOException {
295 return idx(ctx).getObjectCount();
296 }
297
298 private byte[] decompress(long position, int sz, DfsReader ctx)
299 throws IOException, DataFormatException {
300 byte[] dstbuf;
301 try {
302 dstbuf = new byte[sz];
303 } catch (OutOfMemoryError noMemory) {
304
305
306
307
308
309
310
311 return null;
312 }
313
314 if (ctx.inflate(this, position, dstbuf, false) != sz) {
315 throw new EOFException(MessageFormat.format(
316 JGitText.get().shortCompressedStreamAt,
317 Long.valueOf(position)));
318 }
319 return dstbuf;
320 }
321
322 void copyPackAsIs(PackOutputStream out, DfsReader ctx) throws IOException {
323
324 if (length == -1) {
325 ctx.pin(this, 0);
326 ctx.unpin();
327 }
328 try (ReadableChannel rc = ctx.db.openFile(desc, PACK)) {
329 int sz = ctx.getOptions().getStreamPackBufferSize();
330 if (sz > 0) {
331 rc.setReadAheadBytes(sz);
332 }
333 if (cache.shouldCopyThroughCache(length)) {
334 copyPackThroughCache(out, ctx, rc);
335 } else {
336 copyPackBypassCache(out, rc);
337 }
338 }
339 }
340
341 private void copyPackThroughCache(PackOutputStream out, DfsReader ctx,
342 ReadableChannel rc) throws IOException {
343 long position = 12;
344 long remaining = length - (12 + 20);
345 while (0 < remaining) {
346 DfsBlock b = cache.getOrLoad(this, position, ctx, () -> rc);
347 int ptr = (int) (position - b.start);
348 if (b.size() <= ptr) {
349 throw packfileIsTruncated();
350 }
351 int n = (int) Math.min(b.size() - ptr, remaining);
352 b.write(out, position, n);
353 position += n;
354 remaining -= n;
355 }
356 }
357
358 private long copyPackBypassCache(PackOutputStream out, ReadableChannel rc)
359 throws IOException {
360 ByteBuffer buf = newCopyBuffer(out, rc);
361 long position = 12;
362 long remaining = length - (12 + 20);
363 boolean packHeadSkipped = false;
364 while (0 < remaining) {
365 DfsBlock b = cache.get(key, alignToBlock(position));
366 if (b != null) {
367 int ptr = (int) (position - b.start);
368 if (b.size() <= ptr) {
369 throw packfileIsTruncated();
370 }
371 int n = (int) Math.min(b.size() - ptr, remaining);
372 b.write(out, position, n);
373 position += n;
374 remaining -= n;
375 rc.position(position);
376 packHeadSkipped = true;
377 continue;
378 }
379
380
381 int ptr = packHeadSkipped ? 0 : 12;
382 buf.position(0);
383 int bufLen = read(rc, buf);
384 if (bufLen <= ptr) {
385 throw packfileIsTruncated();
386 }
387 int n = (int) Math.min(bufLen - ptr, remaining);
388 out.write(buf.array(), ptr, n);
389 position += n;
390 remaining -= n;
391 packHeadSkipped = true;
392 }
393 return position;
394 }
395
396 private ByteBuffer newCopyBuffer(PackOutputStream out, ReadableChannel rc) {
397 int bs = blockSize(rc);
398 byte[] copyBuf = out.getCopyBuffer();
399 if (bs > copyBuf.length) {
400 copyBuf = new byte[bs];
401 }
402 return ByteBuffer.wrap(copyBuf, 0, bs);
403 }
404
405 void copyAsIs(PackOutputStream out, DfsObjectToPack src,
406 boolean validate, DfsReader ctx) throws IOException,
407 StoredObjectRepresentationNotAvailableException {
408 final CRC32 crc1 = validate ? new CRC32() : null;
409 final CRC32 crc2 = validate ? new CRC32() : null;
410 final byte[] buf = out.getCopyBuffer();
411
412
413
414 try {
415 readFully(src.offset, buf, 0, 20, ctx);
416 } catch (IOException ioError) {
417 throw new StoredObjectRepresentationNotAvailableException(src,
418 ioError);
419 }
420 int c = buf[0] & 0xff;
421 final int typeCode = (c >> 4) & 7;
422 long inflatedLength = c & 15;
423 int shift = 4;
424 int headerCnt = 1;
425 while ((c & 0x80) != 0) {
426 c = buf[headerCnt++] & 0xff;
427 inflatedLength += ((long) (c & 0x7f)) << shift;
428 shift += 7;
429 }
430
431 if (typeCode == Constants.OBJ_OFS_DELTA) {
432 do {
433 c = buf[headerCnt++] & 0xff;
434 } while ((c & 128) != 0);
435 if (validate) {
436 assert(crc1 != null && crc2 != null);
437 crc1.update(buf, 0, headerCnt);
438 crc2.update(buf, 0, headerCnt);
439 }
440 } else if (typeCode == Constants.OBJ_REF_DELTA) {
441 if (validate) {
442 assert(crc1 != null && crc2 != null);
443 crc1.update(buf, 0, headerCnt);
444 crc2.update(buf, 0, headerCnt);
445 }
446
447 readFully(src.offset + headerCnt, buf, 0, 20, ctx);
448 if (validate) {
449 assert(crc1 != null && crc2 != null);
450 crc1.update(buf, 0, 20);
451 crc2.update(buf, 0, 20);
452 }
453 headerCnt += 20;
454 } else if (validate) {
455 assert(crc1 != null && crc2 != null);
456 crc1.update(buf, 0, headerCnt);
457 crc2.update(buf, 0, headerCnt);
458 }
459
460 final long dataOffset = src.offset + headerCnt;
461 final long dataLength = src.length;
462 final long expectedCRC;
463 final DfsBlock quickCopy;
464
465
466
467
468 try {
469 quickCopy = ctx.quickCopy(this, dataOffset, dataLength);
470
471 if (validate && idx(ctx).hasCRC32Support()) {
472 assert(crc1 != null);
473
474
475 expectedCRC = idx(ctx).findCRC32(src);
476 if (quickCopy != null) {
477 quickCopy.crc32(crc1, dataOffset, (int) dataLength);
478 } else {
479 long pos = dataOffset;
480 long cnt = dataLength;
481 while (cnt > 0) {
482 final int n = (int) Math.min(cnt, buf.length);
483 readFully(pos, buf, 0, n, ctx);
484 crc1.update(buf, 0, n);
485 pos += n;
486 cnt -= n;
487 }
488 }
489 if (crc1.getValue() != expectedCRC) {
490 setCorrupt(src.offset);
491 throw new CorruptObjectException(MessageFormat.format(
492 JGitText.get().objectAtHasBadZlibStream,
493 Long.valueOf(src.offset), getFileName()));
494 }
495 } else if (validate) {
496 assert(crc1 != null);
497
498
499
500
501 Inflater inf = ctx.inflater();
502 byte[] tmp = new byte[1024];
503 if (quickCopy != null) {
504 quickCopy.check(inf, tmp, dataOffset, (int) dataLength);
505 } else {
506 long pos = dataOffset;
507 long cnt = dataLength;
508 while (cnt > 0) {
509 final int n = (int) Math.min(cnt, buf.length);
510 readFully(pos, buf, 0, n, ctx);
511 crc1.update(buf, 0, n);
512 inf.setInput(buf, 0, n);
513 while (inf.inflate(tmp, 0, tmp.length) > 0) {
514 continue;
515 }
516 pos += n;
517 cnt -= n;
518 }
519 }
520 if (!inf.finished() || inf.getBytesRead() != dataLength) {
521 setCorrupt(src.offset);
522 throw new EOFException(MessageFormat.format(
523 JGitText.get().shortCompressedStreamAt,
524 Long.valueOf(src.offset)));
525 }
526 expectedCRC = crc1.getValue();
527 } else {
528 expectedCRC = -1;
529 }
530 } catch (DataFormatException dataFormat) {
531 setCorrupt(src.offset);
532
533 CorruptObjectException corruptObject = new CorruptObjectException(
534 MessageFormat.format(
535 JGitText.get().objectAtHasBadZlibStream,
536 Long.valueOf(src.offset), getFileName()),
537 dataFormat);
538
539 throw new StoredObjectRepresentationNotAvailableException(src,
540 corruptObject);
541
542 } catch (IOException ioError) {
543 throw new StoredObjectRepresentationNotAvailableException(src,
544 ioError);
545 }
546
547 if (quickCopy != null) {
548
549
550
551 out.writeHeader(src, inflatedLength);
552 quickCopy.write(out, dataOffset, (int) dataLength);
553
554 } else if (dataLength <= buf.length) {
555
556
557
558 if (!validate) {
559 long pos = dataOffset;
560 long cnt = dataLength;
561 while (cnt > 0) {
562 final int n = (int) Math.min(cnt, buf.length);
563 readFully(pos, buf, 0, n, ctx);
564 pos += n;
565 cnt -= n;
566 }
567 }
568 out.writeHeader(src, inflatedLength);
569 out.write(buf, 0, (int) dataLength);
570 } else {
571
572
573
574
575 out.writeHeader(src, inflatedLength);
576 long pos = dataOffset;
577 long cnt = dataLength;
578 while (cnt > 0) {
579 final int n = (int) Math.min(cnt, buf.length);
580 readFully(pos, buf, 0, n, ctx);
581 if (validate) {
582 assert(crc2 != null);
583 crc2.update(buf, 0, n);
584 }
585 out.write(buf, 0, n);
586 pos += n;
587 cnt -= n;
588 }
589 if (validate) {
590 assert(crc2 != null);
591 if (crc2.getValue() != expectedCRC) {
592 throw new CorruptObjectException(MessageFormat.format(
593 JGitText.get().objectAtHasBadZlibStream,
594 Long.valueOf(src.offset), getFileName()));
595 }
596 }
597 }
598 }
599
600 private IOException packfileIsTruncated() {
601 invalid = true;
602 IOException exc = new IOException(MessageFormat.format(
603 JGitText.get().packfileIsTruncated, getFileName()));
604 invalidatingCause = exc;
605 return exc;
606 }
607
608 private void readFully(long position, byte[] dstbuf, int dstoff, int cnt,
609 DfsReader ctx) throws IOException {
610 if (ctx.copy(this, position, dstbuf, dstoff, cnt) != cnt)
611 throw new EOFException();
612 }
613
614 ObjectLoader load(DfsReader ctx, long pos)
615 throws IOException {
616 try {
617 final byte[] ib = ctx.tempId;
618 Delta delta = null;
619 byte[] data = null;
620 int type = Constants.OBJ_BAD;
621 boolean cached = false;
622
623 SEARCH: for (;;) {
624 readFully(pos, ib, 0, 20, ctx);
625 int c = ib[0] & 0xff;
626 final int typeCode = (c >> 4) & 7;
627 long sz = c & 15;
628 int shift = 4;
629 int p = 1;
630 while ((c & 0x80) != 0) {
631 c = ib[p++] & 0xff;
632 sz += ((long) (c & 0x7f)) << shift;
633 shift += 7;
634 }
635
636 switch (typeCode) {
637 case Constants.OBJ_COMMIT:
638 case Constants.OBJ_TREE:
639 case Constants.OBJ_BLOB:
640 case Constants.OBJ_TAG: {
641 if (delta != null) {
642 data = decompress(pos + p, (int) sz, ctx);
643 type = typeCode;
644 break SEARCH;
645 }
646
647 if (sz < ctx.getStreamFileThreshold()) {
648 data = decompress(pos + p, (int) sz, ctx);
649 if (data != null) {
650 return new ObjectLoader.SmallObject(typeCode, data);
651 }
652 }
653 return new LargePackedWholeObject(typeCode, sz, pos, p, this, ctx.db);
654 }
655
656 case Constants.OBJ_OFS_DELTA: {
657 c = ib[p++] & 0xff;
658 long base = c & 127;
659 while ((c & 128) != 0) {
660 base += 1;
661 c = ib[p++] & 0xff;
662 base <<= 7;
663 base += (c & 127);
664 }
665 base = pos - base;
666 delta = new Delta(delta, pos, (int) sz, p, base);
667 if (sz != delta.deltaSize) {
668 break SEARCH;
669 }
670
671 DeltaBaseCache.Entry e = ctx.getDeltaBaseCache().get(key, base);
672 if (e != null) {
673 type = e.type;
674 data = e.data;
675 cached = true;
676 break SEARCH;
677 }
678 pos = base;
679 continue SEARCH;
680 }
681
682 case Constants.OBJ_REF_DELTA: {
683 readFully(pos + p, ib, 0, 20, ctx);
684 long base = findDeltaBase(ctx, ObjectId.fromRaw(ib));
685 delta = new Delta(delta, pos, (int) sz, p + 20, base);
686 if (sz != delta.deltaSize) {
687 break SEARCH;
688 }
689
690 DeltaBaseCache.Entry e = ctx.getDeltaBaseCache().get(key, base);
691 if (e != null) {
692 type = e.type;
693 data = e.data;
694 cached = true;
695 break SEARCH;
696 }
697 pos = base;
698 continue SEARCH;
699 }
700
701 default:
702 throw new IOException(MessageFormat.format(
703 JGitText.get().unknownObjectType, Integer.valueOf(typeCode)));
704 }
705 }
706
707
708
709
710 if (data == null)
711 throw new LargeObjectException();
712
713 assert(delta != null);
714 do {
715
716 if (cached) {
717 cached = false;
718 } else if (delta.next == null) {
719 ctx.getDeltaBaseCache().put(key, delta.basePos, type, data);
720 }
721
722 pos = delta.deltaPos;
723
724 byte[] cmds = decompress(pos + delta.hdrLen, delta.deltaSize, ctx);
725 if (cmds == null) {
726 data = null;
727 throw new LargeObjectException();
728 }
729
730 final long sz = BinaryDelta.getResultSize(cmds);
731 if (Integer.MAX_VALUE <= sz) {
732 throw new LargeObjectException.ExceedsByteArrayLimit();
733 }
734
735 final byte[] result;
736 try {
737 result = new byte[(int) sz];
738 } catch (OutOfMemoryError tooBig) {
739 data = null;
740 cmds = null;
741 throw new LargeObjectException.OutOfMemory(tooBig);
742 }
743
744 BinaryDelta.apply(data, cmds, result);
745 data = result;
746 delta = delta.next;
747 } while (delta != null);
748
749 return new ObjectLoader.SmallObject(type, data);
750
751 } catch (DataFormatException dfe) {
752 throw new CorruptObjectException(
753 MessageFormat.format(
754 JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
755 getFileName()),
756 dfe);
757 }
758 }
759
760 private long findDeltaBase(DfsReader ctx, ObjectId baseId)
761 throws IOException, MissingObjectException {
762 long ofs = idx(ctx).findOffset(baseId);
763 if (ofs < 0) {
764 throw new MissingObjectException(baseId,
765 JGitText.get().missingDeltaBase);
766 }
767 return ofs;
768 }
769
770 private static class Delta {
771
772 final Delta next;
773
774
775 final long deltaPos;
776
777
778 final int deltaSize;
779
780
781 final int hdrLen;
782
783
784 final long basePos;
785
786 Delta(Delta next, long ofs, int sz, int hdrLen, long baseOffset) {
787 this.next = next;
788 this.deltaPos = ofs;
789 this.deltaSize = sz;
790 this.hdrLen = hdrLen;
791 this.basePos = baseOffset;
792 }
793 }
794
795 byte[] getDeltaHeader(DfsReader wc, long pos)
796 throws IOException, DataFormatException {
797
798
799
800
801
802 final byte[] hdr = new byte[32];
803 wc.inflate(this, pos, hdr, true );
804 return hdr;
805 }
806
807 int getObjectType(DfsReader ctx, long pos) throws IOException {
808 final byte[] ib = ctx.tempId;
809 for (;;) {
810 readFully(pos, ib, 0, 20, ctx);
811 int c = ib[0] & 0xff;
812 final int type = (c >> 4) & 7;
813
814 switch (type) {
815 case Constants.OBJ_COMMIT:
816 case Constants.OBJ_TREE:
817 case Constants.OBJ_BLOB:
818 case Constants.OBJ_TAG:
819 return type;
820
821 case Constants.OBJ_OFS_DELTA: {
822 int p = 1;
823 while ((c & 0x80) != 0) {
824 c = ib[p++] & 0xff;
825 }
826 c = ib[p++] & 0xff;
827 long ofs = c & 127;
828 while ((c & 128) != 0) {
829 ofs += 1;
830 c = ib[p++] & 0xff;
831 ofs <<= 7;
832 ofs += (c & 127);
833 }
834 pos = pos - ofs;
835 continue;
836 }
837
838 case Constants.OBJ_REF_DELTA: {
839 int p = 1;
840 while ((c & 0x80) != 0) {
841 c = ib[p++] & 0xff;
842 }
843 readFully(pos + p, ib, 0, 20, ctx);
844 pos = findDeltaBase(ctx, ObjectId.fromRaw(ib));
845 continue;
846 }
847
848 default:
849 throw new IOException(MessageFormat.format(
850 JGitText.get().unknownObjectType, Integer.valueOf(type)));
851 }
852 }
853 }
854
855 long getObjectSize(DfsReader ctx, AnyObjectId id) throws IOException {
856 final long offset = idx(ctx).findOffset(id);
857 return 0 < offset ? getObjectSize(ctx, offset) : -1;
858 }
859
860 long getObjectSize(DfsReader ctx, long pos)
861 throws IOException {
862 final byte[] ib = ctx.tempId;
863 readFully(pos, ib, 0, 20, ctx);
864 int c = ib[0] & 0xff;
865 final int type = (c >> 4) & 7;
866 long sz = c & 15;
867 int shift = 4;
868 int p = 1;
869 while ((c & 0x80) != 0) {
870 c = ib[p++] & 0xff;
871 sz += ((long) (c & 0x7f)) << shift;
872 shift += 7;
873 }
874
875 long deltaAt;
876 switch (type) {
877 case Constants.OBJ_COMMIT:
878 case Constants.OBJ_TREE:
879 case Constants.OBJ_BLOB:
880 case Constants.OBJ_TAG:
881 return sz;
882
883 case Constants.OBJ_OFS_DELTA:
884 c = ib[p++] & 0xff;
885 while ((c & 128) != 0) {
886 c = ib[p++] & 0xff;
887 }
888 deltaAt = pos + p;
889 break;
890
891 case Constants.OBJ_REF_DELTA:
892 deltaAt = pos + p + 20;
893 break;
894
895 default:
896 throw new IOException(MessageFormat.format(
897 JGitText.get().unknownObjectType, Integer.valueOf(type)));
898 }
899
900 try {
901 return BinaryDelta.getResultSize(getDeltaHeader(ctx, deltaAt));
902 } catch (DataFormatException dfe) {
903 throw new CorruptObjectException(
904 MessageFormat.format(
905 JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
906 getFileName()),
907 dfe);
908 }
909 }
910
911 void representation(DfsObjectRepresentation r, final long pos,
912 DfsReader ctx, PackReverseIndex rev)
913 throws IOException {
914 r.offset = pos;
915 final byte[] ib = ctx.tempId;
916 readFully(pos, ib, 0, 20, ctx);
917 int c = ib[0] & 0xff;
918 int p = 1;
919 final int typeCode = (c >> 4) & 7;
920 while ((c & 0x80) != 0) {
921 c = ib[p++] & 0xff;
922 }
923
924 long len = rev.findNextOffset(pos, length - 20) - pos;
925 switch (typeCode) {
926 case Constants.OBJ_COMMIT:
927 case Constants.OBJ_TREE:
928 case Constants.OBJ_BLOB:
929 case Constants.OBJ_TAG:
930 r.format = StoredObjectRepresentation.PACK_WHOLE;
931 r.baseId = null;
932 r.length = len - p;
933 return;
934
935 case Constants.OBJ_OFS_DELTA: {
936 c = ib[p++] & 0xff;
937 long ofs = c & 127;
938 while ((c & 128) != 0) {
939 ofs += 1;
940 c = ib[p++] & 0xff;
941 ofs <<= 7;
942 ofs += (c & 127);
943 }
944 r.format = StoredObjectRepresentation.PACK_DELTA;
945 r.baseId = rev.findObject(pos - ofs);
946 r.length = len - p;
947 return;
948 }
949
950 case Constants.OBJ_REF_DELTA: {
951 readFully(pos + p, ib, 0, 20, ctx);
952 r.format = StoredObjectRepresentation.PACK_DELTA;
953 r.baseId = ObjectId.fromRaw(ib);
954 r.length = len - p - 20;
955 return;
956 }
957
958 default:
959 throw new IOException(MessageFormat.format(
960 JGitText.get().unknownObjectType, Integer.valueOf(typeCode)));
961 }
962 }
963
964 boolean isCorrupt(long offset) {
965 LongList list = corruptObjects;
966 if (list == null) {
967 return false;
968 }
969 synchronized (list) {
970 return list.contains(offset);
971 }
972 }
973
974 private void setCorrupt(long offset) {
975 LongList list = corruptObjects;
976 if (list == null) {
977 synchronized (initLock) {
978 list = corruptObjects;
979 if (list == null) {
980 list = new LongList();
981 corruptObjects = list;
982 }
983 }
984 }
985 synchronized (list) {
986 list.add(offset);
987 }
988 }
989
990 private DfsBlockCache.Ref<PackIndex> loadPackIndex(
991 DfsReader ctx, DfsStreamKey idxKey) throws IOException {
992 try {
993 ctx.stats.readIdx++;
994 long start = System.nanoTime();
995 try (ReadableChannel rc = ctx.db.openFile(desc, INDEX)) {
996 InputStream in = Channels.newInputStream(rc);
997 int wantSize = 8192;
998 int bs = rc.blockSize();
999 if (0 < bs && bs < wantSize) {
1000 bs = (wantSize / bs) * bs;
1001 } else if (bs <= 0) {
1002 bs = wantSize;
1003 }
1004 PackIndex idx = PackIndex.read(new BufferedInputStream(in, bs));
1005 ctx.stats.readIdxBytes += rc.position();
1006 index = idx;
1007 return new DfsBlockCache.Ref<>(
1008 idxKey,
1009 REF_POSITION,
1010 idx.getObjectCount() * REC_SIZE,
1011 idx);
1012 } finally {
1013 ctx.stats.readIdxMicros += elapsedMicros(start);
1014 }
1015 } catch (EOFException e) {
1016 throw new IOException(MessageFormat.format(
1017 DfsText.get().shortReadOfIndex,
1018 desc.getFileName(INDEX)), e);
1019 } catch (IOException e) {
1020 throw new IOException(MessageFormat.format(
1021 DfsText.get().cannotReadIndex,
1022 desc.getFileName(INDEX)), e);
1023 }
1024 }
1025
1026 private DfsBlockCache.Ref<PackReverseIndex> loadReverseIdx(
1027 DfsStreamKey revKey, PackIndex idx) {
1028 PackReverseIndex revidx = new PackReverseIndex(idx);
1029 reverseIndex = revidx;
1030 return new DfsBlockCache.Ref<>(
1031 revKey,
1032 REF_POSITION,
1033 idx.getObjectCount() * 8,
1034 revidx);
1035 }
1036
1037 private DfsBlockCache.Ref<PackBitmapIndex> loadBitmapIndex(
1038 DfsReader ctx,
1039 DfsStreamKey bitmapKey,
1040 PackIndex idx,
1041 PackReverseIndex revidx) throws IOException {
1042 ctx.stats.readBitmap++;
1043 long start = System.nanoTime();
1044 try (ReadableChannel rc = ctx.db.openFile(desc, BITMAP_INDEX)) {
1045 long size;
1046 PackBitmapIndex bmidx;
1047 try {
1048 InputStream in = Channels.newInputStream(rc);
1049 int wantSize = 8192;
1050 int bs = rc.blockSize();
1051 if (0 < bs && bs < wantSize) {
1052 bs = (wantSize / bs) * bs;
1053 } else if (bs <= 0) {
1054 bs = wantSize;
1055 }
1056 in = new BufferedInputStream(in, bs);
1057 bmidx = PackBitmapIndex.read(in, idx, revidx);
1058 } finally {
1059 size = rc.position();
1060 ctx.stats.readIdxBytes += size;
1061 ctx.stats.readIdxMicros += elapsedMicros(start);
1062 }
1063 bitmapIndex = bmidx;
1064 return new DfsBlockCache.Ref<>(
1065 bitmapKey, REF_POSITION, size, bmidx);
1066 } catch (EOFException e) {
1067 throw new IOException(MessageFormat.format(
1068 DfsText.get().shortReadOfIndex,
1069 desc.getFileName(BITMAP_INDEX)), e);
1070 } catch (IOException e) {
1071 throw new IOException(MessageFormat.format(
1072 DfsText.get().cannotReadIndex,
1073 desc.getFileName(BITMAP_INDEX)), e);
1074 }
1075 }
1076 }