1
2
3
4
5
6
7
8
9
10
11
12
13 package org.eclipse.jgit.internal.storage.dfs;
14
15 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
16 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
17 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
18 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
19
20 import java.io.BufferedInputStream;
21 import java.io.EOFException;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.nio.ByteBuffer;
25 import java.nio.channels.Channels;
26 import java.text.MessageFormat;
27 import java.util.Set;
28 import java.util.zip.CRC32;
29 import java.util.zip.DataFormatException;
30 import java.util.zip.Inflater;
31
32 import org.eclipse.jgit.errors.CorruptObjectException;
33 import org.eclipse.jgit.errors.LargeObjectException;
34 import org.eclipse.jgit.errors.MissingObjectException;
35 import org.eclipse.jgit.errors.PackInvalidException;
36 import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
37 import org.eclipse.jgit.internal.JGitText;
38 import org.eclipse.jgit.internal.storage.file.PackBitmapIndex;
39 import org.eclipse.jgit.internal.storage.file.PackIndex;
40 import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
41 import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
42 import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
43 import org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation;
44 import org.eclipse.jgit.lib.AbbreviatedObjectId;
45 import org.eclipse.jgit.lib.AnyObjectId;
46 import org.eclipse.jgit.lib.Constants;
47 import org.eclipse.jgit.lib.ObjectId;
48 import org.eclipse.jgit.lib.ObjectLoader;
49 import org.eclipse.jgit.lib.Repository;
50 import org.eclipse.jgit.util.LongList;
51
52
53
54
55
56
57 public final class DfsPackFile extends BlockBasedFile {
58 private static final int REC_SIZE = Constants.OBJECT_ID_LENGTH + 8;
59 private static final long REF_POSITION = 0;
60
61
62
63
64
65
66 private final Object initLock = new Object();
67
68
69 private volatile PackIndex index;
70
71
72 private volatile PackReverseIndex reverseIndex;
73
74
75 private volatile PackBitmapIndex bitmapIndex;
76
77
78
79
80
81
82
83
84 private volatile LongList corruptObjects;
85
86
87
88
89
90
91
92
93
94 DfsPackFile(DfsBlockCache cache, DfsPackDescription desc) {
95 super(cache, desc, PACK);
96
97 int bs = desc.getBlockSize(PACK);
98 if (bs > 0) {
99 setBlockSize(bs);
100 }
101
102 long sz = desc.getFileSize(PACK);
103 length = sz > 0 ? sz : -1;
104 }
105
106
107
108
109
110
111 public DfsPackDescription getPackDescription() {
112 return desc;
113 }
114
115
116
117
118
119
120 public boolean isIndexLoaded() {
121 return index != null;
122 }
123
124 void setPackIndex(PackIndex idx) {
125 long objCnt = idx.getObjectCount();
126 int recSize = Constants.OBJECT_ID_LENGTH + 8;
127 long sz = objCnt * recSize;
128 cache.putRef(desc.getStreamKey(INDEX), sz, idx);
129 index = idx;
130 }
131
132
133
134
135
136
137
138
139
140
141
142 public PackIndex getPackIndex(DfsReader ctx) throws IOException {
143 return idx(ctx);
144 }
145
146 private PackIndex idx(DfsReader ctx) throws IOException {
147 if (index != null) {
148 return index;
149 }
150
151 if (invalid) {
152 throw new PackInvalidException(getFileName(), invalidatingCause);
153 }
154
155 Repository.getGlobalListenerList()
156 .dispatch(new BeforeDfsPackIndexLoadedEvent(this));
157
158 synchronized (initLock) {
159 if (index != null) {
160 return index;
161 }
162
163 try {
164 DfsStreamKey idxKey = desc.getStreamKey(INDEX);
165 DfsBlockCache.Ref<PackIndex> idxref = cache.getOrLoadRef(
166 idxKey,
167 REF_POSITION,
168 () -> loadPackIndex(ctx, idxKey));
169 PackIndex idx = idxref.get();
170 if (index == null && idx != null) {
171 index = idx;
172 }
173 return index;
174 } catch (IOException e) {
175 invalid = true;
176 invalidatingCause = e;
177 throw e;
178 }
179 }
180 }
181
182 final boolean isGarbage() {
183 return desc.getPackSource() == UNREACHABLE_GARBAGE;
184 }
185
186 PackBitmapIndex getBitmapIndex(DfsReader ctx) throws IOException {
187 if (invalid || isGarbage() || !desc.hasFileExt(BITMAP_INDEX)) {
188 return null;
189 }
190
191 if (bitmapIndex != null) {
192 return bitmapIndex;
193 }
194
195 synchronized (initLock) {
196 if (bitmapIndex != null) {
197 return bitmapIndex;
198 }
199
200 PackIndex idx = idx(ctx);
201 PackReverseIndex revidx = getReverseIdx(ctx);
202 DfsStreamKey bitmapKey = desc.getStreamKey(BITMAP_INDEX);
203 DfsBlockCache.Ref<PackBitmapIndex> idxref = cache.getOrLoadRef(
204 bitmapKey,
205 REF_POSITION,
206 () -> loadBitmapIndex(ctx, bitmapKey, idx, revidx));
207 PackBitmapIndex bmidx = idxref.get();
208 if (bitmapIndex == null && bmidx != null) {
209 bitmapIndex = bmidx;
210 }
211 return bitmapIndex;
212 }
213 }
214
215 PackReverseIndex getReverseIdx(DfsReader ctx) throws IOException {
216 if (reverseIndex != null) {
217 return reverseIndex;
218 }
219
220 synchronized (initLock) {
221 if (reverseIndex != null) {
222 return reverseIndex;
223 }
224
225 PackIndex idx = idx(ctx);
226 DfsStreamKey revKey = new DfsStreamKey.ForReverseIndex(
227 desc.getStreamKey(INDEX));
228 DfsBlockCache.Ref<PackReverseIndex> revref = cache.getOrLoadRef(
229 revKey,
230 REF_POSITION,
231 () -> loadReverseIdx(revKey, idx));
232 PackReverseIndex revidx = revref.get();
233 if (reverseIndex == null && revidx != null) {
234 reverseIndex = revidx;
235 }
236 return reverseIndex;
237 }
238 }
239
240
241
242
243
244
245
246
247
248
249
250
251
252 public boolean hasObject(DfsReader ctx, AnyObjectId id) throws IOException {
253 final long offset = idx(ctx).findOffset(id);
254 return 0 < offset && !isCorrupt(offset);
255 }
256
257
258
259
260
261
262
263
264
265
266
267
268
269 ObjectLoader get(DfsReader ctx, AnyObjectId id)
270 throws IOException {
271 long offset = idx(ctx).findOffset(id);
272 return 0 < offset && !isCorrupt(offset) ? load(ctx, offset) : null;
273 }
274
275 long findOffset(DfsReader ctx, AnyObjectId id) throws IOException {
276 return idx(ctx).findOffset(id);
277 }
278
279 void resolve(DfsReader ctx, Set<ObjectId> matches, AbbreviatedObjectId id,
280 int matchLimit) throws IOException {
281 idx(ctx).resolve(matches, id, matchLimit);
282 }
283
284
285
286
287
288
289
290
291
292
293
294 long getObjectCount(DfsReader ctx) throws IOException {
295 return idx(ctx).getObjectCount();
296 }
297
298 private byte[] decompress(long position, int sz, DfsReader ctx)
299 throws IOException, DataFormatException {
300 byte[] dstbuf;
301 try {
302 dstbuf = new byte[sz];
303 } catch (OutOfMemoryError noMemory) {
304
305
306
307
308
309
310
311 return null;
312 }
313
314 if (ctx.inflate(this, position, dstbuf, false) != sz) {
315 throw new EOFException(MessageFormat.format(
316 JGitText.get().shortCompressedStreamAt,
317 Long.valueOf(position)));
318 }
319 return dstbuf;
320 }
321
322 void copyPackAsIs(PackOutputStream out, DfsReader ctx) throws IOException {
323
324 if (length == -1) {
325 ctx.pin(this, 0);
326 ctx.unpin();
327 }
328 try (ReadableChannel rc = ctx.db.openFile(desc, PACK)) {
329 int sz = ctx.getOptions().getStreamPackBufferSize();
330 if (sz > 0) {
331 rc.setReadAheadBytes(sz);
332 }
333 if (cache.shouldCopyThroughCache(length)) {
334 copyPackThroughCache(out, ctx, rc);
335 } else {
336 copyPackBypassCache(out, rc);
337 }
338 }
339 }
340
341 private void copyPackThroughCache(PackOutputStream out, DfsReader ctx,
342 ReadableChannel rc) throws IOException {
343 long position = 12;
344 long remaining = length - (12 + 20);
345 while (0 < remaining) {
346 DfsBlock b = cache.getOrLoad(this, position, ctx, () -> rc);
347 int ptr = (int) (position - b.start);
348 if (b.size() <= ptr) {
349 throw packfileIsTruncated();
350 }
351 int n = (int) Math.min(b.size() - ptr, remaining);
352 b.write(out, position, n);
353 position += n;
354 remaining -= n;
355 }
356 }
357
358 private long copyPackBypassCache(PackOutputStream out, ReadableChannel rc)
359 throws IOException {
360 ByteBuffer buf = newCopyBuffer(out, rc);
361 long position = 12;
362 long remaining = length - (12 + 20);
363 boolean packHeadSkipped = false;
364 while (0 < remaining) {
365 DfsBlock b = cache.get(key, alignToBlock(position));
366 if (b != null) {
367 int ptr = (int) (position - b.start);
368 if (b.size() <= ptr) {
369 throw packfileIsTruncated();
370 }
371 int n = (int) Math.min(b.size() - ptr, remaining);
372 b.write(out, position, n);
373 position += n;
374 remaining -= n;
375 rc.position(position);
376 packHeadSkipped = true;
377 continue;
378 }
379
380
381 int ptr = packHeadSkipped ? 0 : 12;
382 buf.position(0);
383 int bufLen = read(rc, buf);
384 if (bufLen <= ptr) {
385 throw packfileIsTruncated();
386 }
387 int n = (int) Math.min(bufLen - ptr, remaining);
388 out.write(buf.array(), ptr, n);
389 position += n;
390 remaining -= n;
391 packHeadSkipped = true;
392 }
393 return position;
394 }
395
396 private ByteBuffer newCopyBuffer(PackOutputStream out, ReadableChannel rc) {
397 int bs = blockSize(rc);
398 byte[] copyBuf = out.getCopyBuffer();
399 if (bs > copyBuf.length) {
400 copyBuf = new byte[bs];
401 }
402 return ByteBuffer.wrap(copyBuf, 0, bs);
403 }
404
405 void copyAsIs(PackOutputStream out, DfsObjectToPack src,
406 boolean validate, DfsReader ctx) throws IOException,
407 StoredObjectRepresentationNotAvailableException {
408 final CRC32 crc1 = validate ? new CRC32() : null;
409 final CRC32 crc2 = validate ? new CRC32() : null;
410 final byte[] buf = out.getCopyBuffer();
411
412
413
414 try {
415 readFully(src.offset, buf, 0, 20, ctx);
416 } catch (IOException ioError) {
417 throw new StoredObjectRepresentationNotAvailableException(src,
418 ioError);
419 }
420 int c = buf[0] & 0xff;
421 final int typeCode = (c >> 4) & 7;
422 long inflatedLength = c & 15;
423 int shift = 4;
424 int headerCnt = 1;
425 while ((c & 0x80) != 0) {
426 c = buf[headerCnt++] & 0xff;
427 inflatedLength += ((long) (c & 0x7f)) << shift;
428 shift += 7;
429 }
430
431 if (typeCode == Constants.OBJ_OFS_DELTA) {
432 do {
433 c = buf[headerCnt++] & 0xff;
434 } while ((c & 128) != 0);
435 if (validate) {
436 assert(crc1 != null && crc2 != null);
437 crc1.update(buf, 0, headerCnt);
438 crc2.update(buf, 0, headerCnt);
439 }
440 } else if (typeCode == Constants.OBJ_REF_DELTA) {
441 if (validate) {
442 assert(crc1 != null && crc2 != null);
443 crc1.update(buf, 0, headerCnt);
444 crc2.update(buf, 0, headerCnt);
445 }
446
447 readFully(src.offset + headerCnt, buf, 0, 20, ctx);
448 if (validate) {
449 assert(crc1 != null && crc2 != null);
450 crc1.update(buf, 0, 20);
451 crc2.update(buf, 0, 20);
452 }
453 headerCnt += 20;
454 } else if (validate) {
455 assert(crc1 != null && crc2 != null);
456 crc1.update(buf, 0, headerCnt);
457 crc2.update(buf, 0, headerCnt);
458 }
459
460 final long dataOffset = src.offset + headerCnt;
461 final long dataLength = src.length;
462 final long expectedCRC;
463 final DfsBlock quickCopy;
464
465
466
467
468 try {
469 quickCopy = ctx.quickCopy(this, dataOffset, dataLength);
470
471 if (validate && idx(ctx).hasCRC32Support()) {
472 assert(crc1 != null);
473
474
475 expectedCRC = idx(ctx).findCRC32(src);
476 if (quickCopy != null) {
477 quickCopy.crc32(crc1, dataOffset, (int) dataLength);
478 } else {
479 long pos = dataOffset;
480 long cnt = dataLength;
481 while (cnt > 0) {
482 final int n = (int) Math.min(cnt, buf.length);
483 readFully(pos, buf, 0, n, ctx);
484 crc1.update(buf, 0, n);
485 pos += n;
486 cnt -= n;
487 }
488 }
489 if (crc1.getValue() != expectedCRC) {
490 setCorrupt(src.offset);
491 throw new CorruptObjectException(MessageFormat.format(
492 JGitText.get().objectAtHasBadZlibStream,
493 Long.valueOf(src.offset), getFileName()));
494 }
495 } else if (validate) {
496 assert(crc1 != null);
497
498
499
500
501 Inflater inf = ctx.inflater();
502 byte[] tmp = new byte[1024];
503 if (quickCopy != null) {
504 quickCopy.check(inf, tmp, dataOffset, (int) dataLength);
505 } else {
506 long pos = dataOffset;
507 long cnt = dataLength;
508 while (cnt > 0) {
509 final int n = (int) Math.min(cnt, buf.length);
510 readFully(pos, buf, 0, n, ctx);
511 crc1.update(buf, 0, n);
512 inf.setInput(buf, 0, n);
513 while (inf.inflate(tmp, 0, tmp.length) > 0) {
514 continue;
515 }
516 pos += n;
517 cnt -= n;
518 }
519 }
520 if (!inf.finished() || inf.getBytesRead() != dataLength) {
521 setCorrupt(src.offset);
522 throw new EOFException(MessageFormat.format(
523 JGitText.get().shortCompressedStreamAt,
524 Long.valueOf(src.offset)));
525 }
526 expectedCRC = crc1.getValue();
527 } else {
528 expectedCRC = -1;
529 }
530 } catch (DataFormatException dataFormat) {
531 setCorrupt(src.offset);
532
533 CorruptObjectException corruptObject = new CorruptObjectException(
534 MessageFormat.format(
535 JGitText.get().objectAtHasBadZlibStream,
536 Long.valueOf(src.offset), getFileName()),
537 dataFormat);
538
539 throw new StoredObjectRepresentationNotAvailableException(src,
540 corruptObject);
541
542 } catch (IOException ioError) {
543 throw new StoredObjectRepresentationNotAvailableException(src,
544 ioError);
545 }
546
547 if (quickCopy != null) {
548
549
550
551 out.writeHeader(src, inflatedLength);
552 quickCopy.write(out, dataOffset, (int) dataLength);
553
554 } else if (dataLength <= buf.length) {
555
556
557
558 if (!validate) {
559 long pos = dataOffset;
560 long cnt = dataLength;
561 while (cnt > 0) {
562 final int n = (int) Math.min(cnt, buf.length);
563 readFully(pos, buf, 0, n, ctx);
564 pos += n;
565 cnt -= n;
566 }
567 }
568 out.writeHeader(src, inflatedLength);
569 out.write(buf, 0, (int) dataLength);
570 } else {
571
572
573
574
575 out.writeHeader(src, inflatedLength);
576 long pos = dataOffset;
577 long cnt = dataLength;
578 while (cnt > 0) {
579 final int n = (int) Math.min(cnt, buf.length);
580 readFully(pos, buf, 0, n, ctx);
581 if (validate) {
582 assert(crc2 != null);
583 crc2.update(buf, 0, n);
584 }
585 out.write(buf, 0, n);
586 pos += n;
587 cnt -= n;
588 }
589 if (validate) {
590 assert(crc2 != null);
591 if (crc2.getValue() != expectedCRC) {
592 throw new CorruptObjectException(MessageFormat.format(
593 JGitText.get().objectAtHasBadZlibStream,
594 Long.valueOf(src.offset), getFileName()));
595 }
596 }
597 }
598 }
599
600 private IOException packfileIsTruncated() {
601 invalid = true;
602 IOException exc = new IOException(MessageFormat.format(
603 JGitText.get().packfileIsTruncated, getFileName()));
604 invalidatingCause = exc;
605 return exc;
606 }
607
608 private void readFully(long position, byte[] dstbuf, int dstoff, int cnt,
609 DfsReader ctx) throws IOException {
610 while (cnt > 0) {
611 int copied = ctx.copy(this, position, dstbuf, dstoff, cnt);
612 if (copied == 0) {
613 throw new EOFException();
614 }
615 position += copied;
616 dstoff += copied;
617 cnt -= copied;
618 }
619 }
620
621 ObjectLoader load(DfsReader ctx, long pos)
622 throws IOException {
623 try {
624 final byte[] ib = ctx.tempId;
625 Delta delta = null;
626 byte[] data = null;
627 int type = Constants.OBJ_BAD;
628 boolean cached = false;
629
630 SEARCH: for (;;) {
631 readFully(pos, ib, 0, 20, ctx);
632 int c = ib[0] & 0xff;
633 final int typeCode = (c >> 4) & 7;
634 long sz = c & 15;
635 int shift = 4;
636 int p = 1;
637 while ((c & 0x80) != 0) {
638 c = ib[p++] & 0xff;
639 sz += ((long) (c & 0x7f)) << shift;
640 shift += 7;
641 }
642
643 switch (typeCode) {
644 case Constants.OBJ_COMMIT:
645 case Constants.OBJ_TREE:
646 case Constants.OBJ_BLOB:
647 case Constants.OBJ_TAG: {
648 if (delta != null) {
649 data = decompress(pos + p, (int) sz, ctx);
650 type = typeCode;
651 break SEARCH;
652 }
653
654 if (sz < ctx.getStreamFileThreshold()) {
655 data = decompress(pos + p, (int) sz, ctx);
656 if (data != null) {
657 return new ObjectLoader.SmallObject(typeCode, data);
658 }
659 }
660 return new LargePackedWholeObject(typeCode, sz, pos, p, this, ctx.db);
661 }
662
663 case Constants.OBJ_OFS_DELTA: {
664 c = ib[p++] & 0xff;
665 long base = c & 127;
666 while ((c & 128) != 0) {
667 base += 1;
668 c = ib[p++] & 0xff;
669 base <<= 7;
670 base += (c & 127);
671 }
672 base = pos - base;
673 delta = new Delta(delta, pos, (int) sz, p, base);
674 if (sz != delta.deltaSize) {
675 break SEARCH;
676 }
677
678 DeltaBaseCache.Entry e = ctx.getDeltaBaseCache().get(key, base);
679 if (e != null) {
680 type = e.type;
681 data = e.data;
682 cached = true;
683 break SEARCH;
684 }
685 pos = base;
686 continue SEARCH;
687 }
688
689 case Constants.OBJ_REF_DELTA: {
690 readFully(pos + p, ib, 0, 20, ctx);
691 long base = findDeltaBase(ctx, ObjectId.fromRaw(ib));
692 delta = new Delta(delta, pos, (int) sz, p + 20, base);
693 if (sz != delta.deltaSize) {
694 break SEARCH;
695 }
696
697 DeltaBaseCache.Entry e = ctx.getDeltaBaseCache().get(key, base);
698 if (e != null) {
699 type = e.type;
700 data = e.data;
701 cached = true;
702 break SEARCH;
703 }
704 pos = base;
705 continue SEARCH;
706 }
707
708 default:
709 throw new IOException(MessageFormat.format(
710 JGitText.get().unknownObjectType, Integer.valueOf(typeCode)));
711 }
712 }
713
714
715
716
717 if (data == null)
718 throw new LargeObjectException();
719
720 assert(delta != null);
721 do {
722
723 if (cached) {
724 cached = false;
725 } else if (delta.next == null) {
726 ctx.getDeltaBaseCache().put(key, delta.basePos, type, data);
727 }
728
729 pos = delta.deltaPos;
730
731 byte[] cmds = decompress(pos + delta.hdrLen, delta.deltaSize, ctx);
732 if (cmds == null) {
733 data = null;
734 throw new LargeObjectException();
735 }
736
737 final long sz = BinaryDelta.getResultSize(cmds);
738 if (Integer.MAX_VALUE <= sz) {
739 throw new LargeObjectException.ExceedsByteArrayLimit();
740 }
741
742 final byte[] result;
743 try {
744 result = new byte[(int) sz];
745 } catch (OutOfMemoryError tooBig) {
746 data = null;
747 cmds = null;
748 throw new LargeObjectException.OutOfMemory(tooBig);
749 }
750
751 BinaryDelta.apply(data, cmds, result);
752 data = result;
753 delta = delta.next;
754 } while (delta != null);
755
756 return new ObjectLoader.SmallObject(type, data);
757
758 } catch (DataFormatException dfe) {
759 throw new CorruptObjectException(
760 MessageFormat.format(
761 JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
762 getFileName()),
763 dfe);
764 }
765 }
766
767 private long findDeltaBase(DfsReader ctx, ObjectId baseId)
768 throws IOException, MissingObjectException {
769 long ofs = idx(ctx).findOffset(baseId);
770 if (ofs < 0) {
771 throw new MissingObjectException(baseId,
772 JGitText.get().missingDeltaBase);
773 }
774 return ofs;
775 }
776
777 private static class Delta {
778
779 final Delta next;
780
781
782 final long deltaPos;
783
784
785 final int deltaSize;
786
787
788 final int hdrLen;
789
790
791 final long basePos;
792
793 Delta(Delta next, long ofs, int sz, int hdrLen, long baseOffset) {
794 this.next = next;
795 this.deltaPos = ofs;
796 this.deltaSize = sz;
797 this.hdrLen = hdrLen;
798 this.basePos = baseOffset;
799 }
800 }
801
802 byte[] getDeltaHeader(DfsReader wc, long pos)
803 throws IOException, DataFormatException {
804
805
806
807
808
809 final byte[] hdr = new byte[32];
810 wc.inflate(this, pos, hdr, true );
811 return hdr;
812 }
813
814 int getObjectType(DfsReader ctx, long pos) throws IOException {
815 final byte[] ib = ctx.tempId;
816 for (;;) {
817 readFully(pos, ib, 0, 20, ctx);
818 int c = ib[0] & 0xff;
819 final int type = (c >> 4) & 7;
820
821 switch (type) {
822 case Constants.OBJ_COMMIT:
823 case Constants.OBJ_TREE:
824 case Constants.OBJ_BLOB:
825 case Constants.OBJ_TAG:
826 return type;
827
828 case Constants.OBJ_OFS_DELTA: {
829 int p = 1;
830 while ((c & 0x80) != 0) {
831 c = ib[p++] & 0xff;
832 }
833 c = ib[p++] & 0xff;
834 long ofs = c & 127;
835 while ((c & 128) != 0) {
836 ofs += 1;
837 c = ib[p++] & 0xff;
838 ofs <<= 7;
839 ofs += (c & 127);
840 }
841 pos = pos - ofs;
842 continue;
843 }
844
845 case Constants.OBJ_REF_DELTA: {
846 int p = 1;
847 while ((c & 0x80) != 0) {
848 c = ib[p++] & 0xff;
849 }
850 readFully(pos + p, ib, 0, 20, ctx);
851 pos = findDeltaBase(ctx, ObjectId.fromRaw(ib));
852 continue;
853 }
854
855 default:
856 throw new IOException(MessageFormat.format(
857 JGitText.get().unknownObjectType, Integer.valueOf(type)));
858 }
859 }
860 }
861
862 long getObjectSize(DfsReader ctx, AnyObjectId id) throws IOException {
863 final long offset = idx(ctx).findOffset(id);
864 return 0 < offset ? getObjectSize(ctx, offset) : -1;
865 }
866
867 long getObjectSize(DfsReader ctx, long pos)
868 throws IOException {
869 final byte[] ib = ctx.tempId;
870 readFully(pos, ib, 0, 20, ctx);
871 int c = ib[0] & 0xff;
872 final int type = (c >> 4) & 7;
873 long sz = c & 15;
874 int shift = 4;
875 int p = 1;
876 while ((c & 0x80) != 0) {
877 c = ib[p++] & 0xff;
878 sz += ((long) (c & 0x7f)) << shift;
879 shift += 7;
880 }
881
882 long deltaAt;
883 switch (type) {
884 case Constants.OBJ_COMMIT:
885 case Constants.OBJ_TREE:
886 case Constants.OBJ_BLOB:
887 case Constants.OBJ_TAG:
888 return sz;
889
890 case Constants.OBJ_OFS_DELTA:
891 c = ib[p++] & 0xff;
892 while ((c & 128) != 0) {
893 c = ib[p++] & 0xff;
894 }
895 deltaAt = pos + p;
896 break;
897
898 case Constants.OBJ_REF_DELTA:
899 deltaAt = pos + p + 20;
900 break;
901
902 default:
903 throw new IOException(MessageFormat.format(
904 JGitText.get().unknownObjectType, Integer.valueOf(type)));
905 }
906
907 try {
908 return BinaryDelta.getResultSize(getDeltaHeader(ctx, deltaAt));
909 } catch (DataFormatException dfe) {
910 throw new CorruptObjectException(
911 MessageFormat.format(
912 JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
913 getFileName()),
914 dfe);
915 }
916 }
917
918 void representation(DfsObjectRepresentation r, final long pos,
919 DfsReader ctx, PackReverseIndex rev)
920 throws IOException {
921 r.offset = pos;
922 final byte[] ib = ctx.tempId;
923 readFully(pos, ib, 0, 20, ctx);
924 int c = ib[0] & 0xff;
925 int p = 1;
926 final int typeCode = (c >> 4) & 7;
927 while ((c & 0x80) != 0) {
928 c = ib[p++] & 0xff;
929 }
930
931 long len = rev.findNextOffset(pos, length - 20) - pos;
932 switch (typeCode) {
933 case Constants.OBJ_COMMIT:
934 case Constants.OBJ_TREE:
935 case Constants.OBJ_BLOB:
936 case Constants.OBJ_TAG:
937 r.format = StoredObjectRepresentation.PACK_WHOLE;
938 r.baseId = null;
939 r.length = len - p;
940 return;
941
942 case Constants.OBJ_OFS_DELTA: {
943 c = ib[p++] & 0xff;
944 long ofs = c & 127;
945 while ((c & 128) != 0) {
946 ofs += 1;
947 c = ib[p++] & 0xff;
948 ofs <<= 7;
949 ofs += (c & 127);
950 }
951 r.format = StoredObjectRepresentation.PACK_DELTA;
952 r.baseId = rev.findObject(pos - ofs);
953 r.length = len - p;
954 return;
955 }
956
957 case Constants.OBJ_REF_DELTA: {
958 readFully(pos + p, ib, 0, 20, ctx);
959 r.format = StoredObjectRepresentation.PACK_DELTA;
960 r.baseId = ObjectId.fromRaw(ib);
961 r.length = len - p - 20;
962 return;
963 }
964
965 default:
966 throw new IOException(MessageFormat.format(
967 JGitText.get().unknownObjectType, Integer.valueOf(typeCode)));
968 }
969 }
970
971 boolean isCorrupt(long offset) {
972 LongList list = corruptObjects;
973 if (list == null) {
974 return false;
975 }
976 synchronized (list) {
977 return list.contains(offset);
978 }
979 }
980
981 private void setCorrupt(long offset) {
982 LongList list = corruptObjects;
983 if (list == null) {
984 synchronized (initLock) {
985 list = corruptObjects;
986 if (list == null) {
987 list = new LongList();
988 corruptObjects = list;
989 }
990 }
991 }
992 synchronized (list) {
993 list.add(offset);
994 }
995 }
996
997 private DfsBlockCache.Ref<PackIndex> loadPackIndex(
998 DfsReader ctx, DfsStreamKey idxKey) throws IOException {
999 try {
1000 ctx.stats.readIdx++;
1001 long start = System.nanoTime();
1002 try (ReadableChannel rc = ctx.db.openFile(desc, INDEX)) {
1003 InputStream in = Channels.newInputStream(rc);
1004 int wantSize = 8192;
1005 int bs = rc.blockSize();
1006 if (0 < bs && bs < wantSize) {
1007 bs = (wantSize / bs) * bs;
1008 } else if (bs <= 0) {
1009 bs = wantSize;
1010 }
1011 PackIndex idx = PackIndex.read(new BufferedInputStream(in, bs));
1012 ctx.stats.readIdxBytes += rc.position();
1013 index = idx;
1014 return new DfsBlockCache.Ref<>(
1015 idxKey,
1016 REF_POSITION,
1017 idx.getObjectCount() * REC_SIZE,
1018 idx);
1019 } finally {
1020 ctx.stats.readIdxMicros += elapsedMicros(start);
1021 }
1022 } catch (EOFException e) {
1023 throw new IOException(MessageFormat.format(
1024 DfsText.get().shortReadOfIndex,
1025 desc.getFileName(INDEX)), e);
1026 } catch (IOException e) {
1027 throw new IOException(MessageFormat.format(
1028 DfsText.get().cannotReadIndex,
1029 desc.getFileName(INDEX)), e);
1030 }
1031 }
1032
1033 private DfsBlockCache.Ref<PackReverseIndex> loadReverseIdx(
1034 DfsStreamKey revKey, PackIndex idx) {
1035 PackReverseIndex revidx = new PackReverseIndex(idx);
1036 reverseIndex = revidx;
1037 return new DfsBlockCache.Ref<>(
1038 revKey,
1039 REF_POSITION,
1040 idx.getObjectCount() * 8,
1041 revidx);
1042 }
1043
1044 private DfsBlockCache.Ref<PackBitmapIndex> loadBitmapIndex(
1045 DfsReader ctx,
1046 DfsStreamKey bitmapKey,
1047 PackIndex idx,
1048 PackReverseIndex revidx) throws IOException {
1049 ctx.stats.readBitmap++;
1050 long start = System.nanoTime();
1051 try (ReadableChannel rc = ctx.db.openFile(desc, BITMAP_INDEX)) {
1052 long size;
1053 PackBitmapIndex bmidx;
1054 try {
1055 InputStream in = Channels.newInputStream(rc);
1056 int wantSize = 8192;
1057 int bs = rc.blockSize();
1058 if (0 < bs && bs < wantSize) {
1059 bs = (wantSize / bs) * bs;
1060 } else if (bs <= 0) {
1061 bs = wantSize;
1062 }
1063 in = new BufferedInputStream(in, bs);
1064 bmidx = PackBitmapIndex.read(in, idx, revidx);
1065 } finally {
1066 size = rc.position();
1067 ctx.stats.readIdxBytes += size;
1068 ctx.stats.readIdxMicros += elapsedMicros(start);
1069 }
1070 bitmapIndex = bmidx;
1071 return new DfsBlockCache.Ref<>(
1072 bitmapKey, REF_POSITION, size, bmidx);
1073 } catch (EOFException e) {
1074 throw new IOException(MessageFormat.format(
1075 DfsText.get().shortReadOfIndex,
1076 desc.getFileName(BITMAP_INDEX)), e);
1077 } catch (IOException e) {
1078 throw new IOException(MessageFormat.format(
1079 DfsText.get().cannotReadIndex,
1080 desc.getFileName(BITMAP_INDEX)), e);
1081 }
1082 }
1083 }