1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46 package org.eclipse.jgit.internal.storage.dfs;
47
48 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
49 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
50 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
51 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
52
53 import java.io.BufferedInputStream;
54 import java.io.EOFException;
55 import java.io.IOException;
56 import java.io.InputStream;
57 import java.nio.ByteBuffer;
58 import java.nio.channels.Channels;
59 import java.text.MessageFormat;
60 import java.util.Set;
61 import java.util.zip.CRC32;
62 import java.util.zip.DataFormatException;
63 import java.util.zip.Inflater;
64
65 import org.eclipse.jgit.errors.CorruptObjectException;
66 import org.eclipse.jgit.errors.LargeObjectException;
67 import org.eclipse.jgit.errors.MissingObjectException;
68 import org.eclipse.jgit.errors.PackInvalidException;
69 import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
70 import org.eclipse.jgit.internal.JGitText;
71 import org.eclipse.jgit.internal.storage.file.PackBitmapIndex;
72 import org.eclipse.jgit.internal.storage.file.PackIndex;
73 import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
74 import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
75 import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
76 import org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation;
77 import org.eclipse.jgit.lib.AbbreviatedObjectId;
78 import org.eclipse.jgit.lib.AnyObjectId;
79 import org.eclipse.jgit.lib.Constants;
80 import org.eclipse.jgit.lib.ObjectId;
81 import org.eclipse.jgit.lib.ObjectLoader;
82 import org.eclipse.jgit.lib.Repository;
83 import org.eclipse.jgit.util.LongList;
84
85
86
87
88
89
90 public final class DfsPackFile extends BlockBasedFile {
91 private static final int REC_SIZE = Constants.OBJECT_ID_LENGTH + 8;
92 private static final long REF_POSITION = 0;
93
94
95
96
97
98
99 private final Object initLock = new Object();
100
101
102 private volatile PackIndex index;
103
104
105 private volatile PackReverseIndex reverseIndex;
106
107
108 private volatile PackBitmapIndex bitmapIndex;
109
110
111
112
113
114
115
116
117 private volatile LongList corruptObjects;
118
119
120
121
122
123
124
125
126
127 DfsPackFile(DfsBlockCache cache, DfsPackDescription desc) {
128 super(cache, desc, PACK);
129
130 int bs = desc.getBlockSize(PACK);
131 if (bs > 0) {
132 setBlockSize(bs);
133 }
134
135 long sz = desc.getFileSize(PACK);
136 length = sz > 0 ? sz : -1;
137 }
138
139
140
141
142
143
144 public DfsPackDescription getPackDescription() {
145 return desc;
146 }
147
148
149
150
151
152
153 public boolean isIndexLoaded() {
154 return index != null;
155 }
156
157 void setPackIndex(PackIndex idx) {
158 long objCnt = idx.getObjectCount();
159 int recSize = Constants.OBJECT_ID_LENGTH + 8;
160 long sz = objCnt * recSize;
161 cache.putRef(desc.getStreamKey(INDEX), sz, idx);
162 index = idx;
163 }
164
165
166
167
168
169
170
171
172
173
174
175 public PackIndex getPackIndex(DfsReader ctx) throws IOException {
176 return idx(ctx);
177 }
178
179 private PackIndex idx(DfsReader ctx) throws IOException {
180 if (index != null) {
181 return index;
182 }
183
184 if (invalid) {
185 throw new PackInvalidException(getFileName(), invalidatingCause);
186 }
187
188 Repository.getGlobalListenerList()
189 .dispatch(new BeforeDfsPackIndexLoadedEvent(this));
190
191 synchronized (initLock) {
192 if (index != null) {
193 return index;
194 }
195
196 try {
197 DfsStreamKey idxKey = desc.getStreamKey(INDEX);
198 DfsBlockCache.Ref<PackIndex> idxref = cache.getOrLoadRef(
199 idxKey,
200 REF_POSITION,
201 () -> loadPackIndex(ctx, idxKey));
202 PackIndex idx = idxref.get();
203 if (index == null && idx != null) {
204 index = idx;
205 }
206 return index;
207 } catch (IOException e) {
208 invalid = true;
209 invalidatingCause = e;
210 throw e;
211 }
212 }
213 }
214
215 final boolean isGarbage() {
216 return desc.getPackSource() == UNREACHABLE_GARBAGE;
217 }
218
219 PackBitmapIndex getBitmapIndex(DfsReader ctx) throws IOException {
220 if (invalid || isGarbage() || !desc.hasFileExt(BITMAP_INDEX)) {
221 return null;
222 }
223
224 if (bitmapIndex != null) {
225 return bitmapIndex;
226 }
227
228 synchronized (initLock) {
229 if (bitmapIndex != null) {
230 return bitmapIndex;
231 }
232
233 PackIndex idx = idx(ctx);
234 PackReverseIndex revidx = getReverseIdx(ctx);
235 DfsStreamKey bitmapKey = desc.getStreamKey(BITMAP_INDEX);
236 DfsBlockCache.Ref<PackBitmapIndex> idxref = cache.getOrLoadRef(
237 bitmapKey,
238 REF_POSITION,
239 () -> loadBitmapIndex(ctx, bitmapKey, idx, revidx));
240 PackBitmapIndex bmidx = idxref.get();
241 if (bitmapIndex == null && bmidx != null) {
242 bitmapIndex = bmidx;
243 }
244 return bitmapIndex;
245 }
246 }
247
248 PackReverseIndex getReverseIdx(DfsReader ctx) throws IOException {
249 if (reverseIndex != null) {
250 return reverseIndex;
251 }
252
253 synchronized (initLock) {
254 if (reverseIndex != null) {
255 return reverseIndex;
256 }
257
258 PackIndex idx = idx(ctx);
259 DfsStreamKey revKey = new DfsStreamKey.ForReverseIndex(
260 desc.getStreamKey(INDEX));
261 DfsBlockCache.Ref<PackReverseIndex> revref = cache.getOrLoadRef(
262 revKey,
263 REF_POSITION,
264 () -> loadReverseIdx(revKey, idx));
265 PackReverseIndex revidx = revref.get();
266 if (reverseIndex == null && revidx != null) {
267 reverseIndex = revidx;
268 }
269 return reverseIndex;
270 }
271 }
272
273
274
275
276
277
278
279
280
281
282
283
284
285 public boolean hasObject(DfsReader ctx, AnyObjectId id) throws IOException {
286 final long offset = idx(ctx).findOffset(id);
287 return 0 < offset && !isCorrupt(offset);
288 }
289
290
291
292
293
294
295
296
297
298
299
300
301
302 ObjectLoader get(DfsReader ctx, AnyObjectId id)
303 throws IOException {
304 long offset = idx(ctx).findOffset(id);
305 return 0 < offset && !isCorrupt(offset) ? load(ctx, offset) : null;
306 }
307
308 long findOffset(DfsReader ctx, AnyObjectId id) throws IOException {
309 return idx(ctx).findOffset(id);
310 }
311
312 void resolve(DfsReader ctx, Set<ObjectId> matches, AbbreviatedObjectId id,
313 int matchLimit) throws IOException {
314 idx(ctx).resolve(matches, id, matchLimit);
315 }
316
317
318
319
320
321
322
323
324
325
326
327 long getObjectCount(DfsReader ctx) throws IOException {
328 return idx(ctx).getObjectCount();
329 }
330
331 private byte[] decompress(long position, int sz, DfsReader ctx)
332 throws IOException, DataFormatException {
333 byte[] dstbuf;
334 try {
335 dstbuf = new byte[sz];
336 } catch (OutOfMemoryError noMemory) {
337
338
339
340
341
342
343
344 return null;
345 }
346
347 if (ctx.inflate(this, position, dstbuf, false) != sz) {
348 throw new EOFException(MessageFormat.format(
349 JGitText.get().shortCompressedStreamAt,
350 Long.valueOf(position)));
351 }
352 return dstbuf;
353 }
354
355 void copyPackAsIs(PackOutputStream out, DfsReader ctx) throws IOException {
356
357 if (length == -1) {
358 ctx.pin(this, 0);
359 ctx.unpin();
360 }
361 try (ReadableChannel rc = ctx.db.openFile(desc, PACK)) {
362 int sz = ctx.getOptions().getStreamPackBufferSize();
363 if (sz > 0) {
364 rc.setReadAheadBytes(sz);
365 }
366 if (cache.shouldCopyThroughCache(length)) {
367 copyPackThroughCache(out, ctx, rc);
368 } else {
369 copyPackBypassCache(out, rc);
370 }
371 }
372 }
373
374 private void copyPackThroughCache(PackOutputStream out, DfsReader ctx,
375 ReadableChannel rc) throws IOException {
376 long position = 12;
377 long remaining = length - (12 + 20);
378 while (0 < remaining) {
379 DfsBlock b = cache.getOrLoad(this, position, ctx, () -> rc);
380 int ptr = (int) (position - b.start);
381 if (b.size() <= ptr) {
382 throw packfileIsTruncated();
383 }
384 int n = (int) Math.min(b.size() - ptr, remaining);
385 b.write(out, position, n);
386 position += n;
387 remaining -= n;
388 }
389 }
390
391 private long copyPackBypassCache(PackOutputStream out, ReadableChannel rc)
392 throws IOException {
393 ByteBuffer buf = newCopyBuffer(out, rc);
394 long position = 12;
395 long remaining = length - (12 + 20);
396 boolean packHeadSkipped = false;
397 while (0 < remaining) {
398 DfsBlock b = cache.get(key, alignToBlock(position));
399 if (b != null) {
400 int ptr = (int) (position - b.start);
401 if (b.size() <= ptr) {
402 throw packfileIsTruncated();
403 }
404 int n = (int) Math.min(b.size() - ptr, remaining);
405 b.write(out, position, n);
406 position += n;
407 remaining -= n;
408 rc.position(position);
409 packHeadSkipped = true;
410 continue;
411 }
412
413
414 int ptr = packHeadSkipped ? 0 : 12;
415 buf.position(0);
416 int bufLen = read(rc, buf);
417 if (bufLen <= ptr) {
418 throw packfileIsTruncated();
419 }
420 int n = (int) Math.min(bufLen - ptr, remaining);
421 out.write(buf.array(), ptr, n);
422 position += n;
423 remaining -= n;
424 packHeadSkipped = true;
425 }
426 return position;
427 }
428
429 private ByteBuffer newCopyBuffer(PackOutputStream out, ReadableChannel rc) {
430 int bs = blockSize(rc);
431 byte[] copyBuf = out.getCopyBuffer();
432 if (bs > copyBuf.length) {
433 copyBuf = new byte[bs];
434 }
435 return ByteBuffer.wrap(copyBuf, 0, bs);
436 }
437
438 void copyAsIs(PackOutputStream out, DfsObjectToPack src,
439 boolean validate, DfsReader ctx) throws IOException,
440 StoredObjectRepresentationNotAvailableException {
441 final CRC32 crc1 = validate ? new CRC32() : null;
442 final CRC32 crc2 = validate ? new CRC32() : null;
443 final byte[] buf = out.getCopyBuffer();
444
445
446
447 try {
448 readFully(src.offset, buf, 0, 20, ctx);
449 } catch (IOException ioError) {
450 throw new StoredObjectRepresentationNotAvailableException(src,
451 ioError);
452 }
453 int c = buf[0] & 0xff;
454 final int typeCode = (c >> 4) & 7;
455 long inflatedLength = c & 15;
456 int shift = 4;
457 int headerCnt = 1;
458 while ((c & 0x80) != 0) {
459 c = buf[headerCnt++] & 0xff;
460 inflatedLength += ((long) (c & 0x7f)) << shift;
461 shift += 7;
462 }
463
464 if (typeCode == Constants.OBJ_OFS_DELTA) {
465 do {
466 c = buf[headerCnt++] & 0xff;
467 } while ((c & 128) != 0);
468 if (validate) {
469 assert(crc1 != null && crc2 != null);
470 crc1.update(buf, 0, headerCnt);
471 crc2.update(buf, 0, headerCnt);
472 }
473 } else if (typeCode == Constants.OBJ_REF_DELTA) {
474 if (validate) {
475 assert(crc1 != null && crc2 != null);
476 crc1.update(buf, 0, headerCnt);
477 crc2.update(buf, 0, headerCnt);
478 }
479
480 readFully(src.offset + headerCnt, buf, 0, 20, ctx);
481 if (validate) {
482 assert(crc1 != null && crc2 != null);
483 crc1.update(buf, 0, 20);
484 crc2.update(buf, 0, 20);
485 }
486 headerCnt += 20;
487 } else if (validate) {
488 assert(crc1 != null && crc2 != null);
489 crc1.update(buf, 0, headerCnt);
490 crc2.update(buf, 0, headerCnt);
491 }
492
493 final long dataOffset = src.offset + headerCnt;
494 final long dataLength = src.length;
495 final long expectedCRC;
496 final DfsBlock quickCopy;
497
498
499
500
501 try {
502 quickCopy = ctx.quickCopy(this, dataOffset, dataLength);
503
504 if (validate && idx(ctx).hasCRC32Support()) {
505 assert(crc1 != null);
506
507
508 expectedCRC = idx(ctx).findCRC32(src);
509 if (quickCopy != null) {
510 quickCopy.crc32(crc1, dataOffset, (int) dataLength);
511 } else {
512 long pos = dataOffset;
513 long cnt = dataLength;
514 while (cnt > 0) {
515 final int n = (int) Math.min(cnt, buf.length);
516 readFully(pos, buf, 0, n, ctx);
517 crc1.update(buf, 0, n);
518 pos += n;
519 cnt -= n;
520 }
521 }
522 if (crc1.getValue() != expectedCRC) {
523 setCorrupt(src.offset);
524 throw new CorruptObjectException(MessageFormat.format(
525 JGitText.get().objectAtHasBadZlibStream,
526 Long.valueOf(src.offset), getFileName()));
527 }
528 } else if (validate) {
529 assert(crc1 != null);
530
531
532
533
534 Inflater inf = ctx.inflater();
535 byte[] tmp = new byte[1024];
536 if (quickCopy != null) {
537 quickCopy.check(inf, tmp, dataOffset, (int) dataLength);
538 } else {
539 long pos = dataOffset;
540 long cnt = dataLength;
541 while (cnt > 0) {
542 final int n = (int) Math.min(cnt, buf.length);
543 readFully(pos, buf, 0, n, ctx);
544 crc1.update(buf, 0, n);
545 inf.setInput(buf, 0, n);
546 while (inf.inflate(tmp, 0, tmp.length) > 0) {
547 continue;
548 }
549 pos += n;
550 cnt -= n;
551 }
552 }
553 if (!inf.finished() || inf.getBytesRead() != dataLength) {
554 setCorrupt(src.offset);
555 throw new EOFException(MessageFormat.format(
556 JGitText.get().shortCompressedStreamAt,
557 Long.valueOf(src.offset)));
558 }
559 expectedCRC = crc1.getValue();
560 } else {
561 expectedCRC = -1;
562 }
563 } catch (DataFormatException dataFormat) {
564 setCorrupt(src.offset);
565
566 CorruptObjectException corruptObject = new CorruptObjectException(
567 MessageFormat.format(
568 JGitText.get().objectAtHasBadZlibStream,
569 Long.valueOf(src.offset), getFileName()),
570 dataFormat);
571
572 throw new StoredObjectRepresentationNotAvailableException(src,
573 corruptObject);
574
575 } catch (IOException ioError) {
576 throw new StoredObjectRepresentationNotAvailableException(src,
577 ioError);
578 }
579
580 if (quickCopy != null) {
581
582
583
584 out.writeHeader(src, inflatedLength);
585 quickCopy.write(out, dataOffset, (int) dataLength);
586
587 } else if (dataLength <= buf.length) {
588
589
590
591 if (!validate) {
592 long pos = dataOffset;
593 long cnt = dataLength;
594 while (cnt > 0) {
595 final int n = (int) Math.min(cnt, buf.length);
596 readFully(pos, buf, 0, n, ctx);
597 pos += n;
598 cnt -= n;
599 }
600 }
601 out.writeHeader(src, inflatedLength);
602 out.write(buf, 0, (int) dataLength);
603 } else {
604
605
606
607
608 out.writeHeader(src, inflatedLength);
609 long pos = dataOffset;
610 long cnt = dataLength;
611 while (cnt > 0) {
612 final int n = (int) Math.min(cnt, buf.length);
613 readFully(pos, buf, 0, n, ctx);
614 if (validate) {
615 assert(crc2 != null);
616 crc2.update(buf, 0, n);
617 }
618 out.write(buf, 0, n);
619 pos += n;
620 cnt -= n;
621 }
622 if (validate) {
623 assert(crc2 != null);
624 if (crc2.getValue() != expectedCRC) {
625 throw new CorruptObjectException(MessageFormat.format(
626 JGitText.get().objectAtHasBadZlibStream,
627 Long.valueOf(src.offset), getFileName()));
628 }
629 }
630 }
631 }
632
633 private IOException packfileIsTruncated() {
634 invalid = true;
635 IOException exc = new IOException(MessageFormat.format(
636 JGitText.get().packfileIsTruncated, getFileName()));
637 invalidatingCause = exc;
638 return exc;
639 }
640
641 private void readFully(long position, byte[] dstbuf, int dstoff, int cnt,
642 DfsReader ctx) throws IOException {
643 if (ctx.copy(this, position, dstbuf, dstoff, cnt) != cnt)
644 throw new EOFException();
645 }
646
647 ObjectLoader load(DfsReader ctx, long pos)
648 throws IOException {
649 try {
650 final byte[] ib = ctx.tempId;
651 Delta delta = null;
652 byte[] data = null;
653 int type = Constants.OBJ_BAD;
654 boolean cached = false;
655
656 SEARCH: for (;;) {
657 readFully(pos, ib, 0, 20, ctx);
658 int c = ib[0] & 0xff;
659 final int typeCode = (c >> 4) & 7;
660 long sz = c & 15;
661 int shift = 4;
662 int p = 1;
663 while ((c & 0x80) != 0) {
664 c = ib[p++] & 0xff;
665 sz += ((long) (c & 0x7f)) << shift;
666 shift += 7;
667 }
668
669 switch (typeCode) {
670 case Constants.OBJ_COMMIT:
671 case Constants.OBJ_TREE:
672 case Constants.OBJ_BLOB:
673 case Constants.OBJ_TAG: {
674 if (delta != null) {
675 data = decompress(pos + p, (int) sz, ctx);
676 type = typeCode;
677 break SEARCH;
678 }
679
680 if (sz < ctx.getStreamFileThreshold()) {
681 data = decompress(pos + p, (int) sz, ctx);
682 if (data != null) {
683 return new ObjectLoader.SmallObject(typeCode, data);
684 }
685 }
686 return new LargePackedWholeObject(typeCode, sz, pos, p, this, ctx.db);
687 }
688
689 case Constants.OBJ_OFS_DELTA: {
690 c = ib[p++] & 0xff;
691 long base = c & 127;
692 while ((c & 128) != 0) {
693 base += 1;
694 c = ib[p++] & 0xff;
695 base <<= 7;
696 base += (c & 127);
697 }
698 base = pos - base;
699 delta = new Delta(delta, pos, (int) sz, p, base);
700 if (sz != delta.deltaSize) {
701 break SEARCH;
702 }
703
704 DeltaBaseCache.Entry e = ctx.getDeltaBaseCache().get(key, base);
705 if (e != null) {
706 type = e.type;
707 data = e.data;
708 cached = true;
709 break SEARCH;
710 }
711 pos = base;
712 continue SEARCH;
713 }
714
715 case Constants.OBJ_REF_DELTA: {
716 readFully(pos + p, ib, 0, 20, ctx);
717 long base = findDeltaBase(ctx, ObjectId.fromRaw(ib));
718 delta = new Delta(delta, pos, (int) sz, p + 20, base);
719 if (sz != delta.deltaSize) {
720 break SEARCH;
721 }
722
723 DeltaBaseCache.Entry e = ctx.getDeltaBaseCache().get(key, base);
724 if (e != null) {
725 type = e.type;
726 data = e.data;
727 cached = true;
728 break SEARCH;
729 }
730 pos = base;
731 continue SEARCH;
732 }
733
734 default:
735 throw new IOException(MessageFormat.format(
736 JGitText.get().unknownObjectType, Integer.valueOf(typeCode)));
737 }
738 }
739
740
741
742
743 if (data == null)
744 throw new LargeObjectException();
745
746 assert(delta != null);
747 do {
748
749 if (cached) {
750 cached = false;
751 } else if (delta.next == null) {
752 ctx.getDeltaBaseCache().put(key, delta.basePos, type, data);
753 }
754
755 pos = delta.deltaPos;
756
757 byte[] cmds = decompress(pos + delta.hdrLen, delta.deltaSize, ctx);
758 if (cmds == null) {
759 data = null;
760 throw new LargeObjectException();
761 }
762
763 final long sz = BinaryDelta.getResultSize(cmds);
764 if (Integer.MAX_VALUE <= sz) {
765 throw new LargeObjectException.ExceedsByteArrayLimit();
766 }
767
768 final byte[] result;
769 try {
770 result = new byte[(int) sz];
771 } catch (OutOfMemoryError tooBig) {
772 data = null;
773 cmds = null;
774 throw new LargeObjectException.OutOfMemory(tooBig);
775 }
776
777 BinaryDelta.apply(data, cmds, result);
778 data = result;
779 delta = delta.next;
780 } while (delta != null);
781
782 return new ObjectLoader.SmallObject(type, data);
783
784 } catch (DataFormatException dfe) {
785 throw new CorruptObjectException(
786 MessageFormat.format(
787 JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
788 getFileName()),
789 dfe);
790 }
791 }
792
793 private long findDeltaBase(DfsReader ctx, ObjectId baseId)
794 throws IOException, MissingObjectException {
795 long ofs = idx(ctx).findOffset(baseId);
796 if (ofs < 0) {
797 throw new MissingObjectException(baseId,
798 JGitText.get().missingDeltaBase);
799 }
800 return ofs;
801 }
802
803 private static class Delta {
804
805 final Delta next;
806
807
808 final long deltaPos;
809
810
811 final int deltaSize;
812
813
814 final int hdrLen;
815
816
817 final long basePos;
818
819 Delta(Delta next, long ofs, int sz, int hdrLen, long baseOffset) {
820 this.next = next;
821 this.deltaPos = ofs;
822 this.deltaSize = sz;
823 this.hdrLen = hdrLen;
824 this.basePos = baseOffset;
825 }
826 }
827
828 byte[] getDeltaHeader(DfsReader wc, long pos)
829 throws IOException, DataFormatException {
830
831
832
833
834
835 final byte[] hdr = new byte[32];
836 wc.inflate(this, pos, hdr, true );
837 return hdr;
838 }
839
840 int getObjectType(DfsReader ctx, long pos) throws IOException {
841 final byte[] ib = ctx.tempId;
842 for (;;) {
843 readFully(pos, ib, 0, 20, ctx);
844 int c = ib[0] & 0xff;
845 final int type = (c >> 4) & 7;
846
847 switch (type) {
848 case Constants.OBJ_COMMIT:
849 case Constants.OBJ_TREE:
850 case Constants.OBJ_BLOB:
851 case Constants.OBJ_TAG:
852 return type;
853
854 case Constants.OBJ_OFS_DELTA: {
855 int p = 1;
856 while ((c & 0x80) != 0) {
857 c = ib[p++] & 0xff;
858 }
859 c = ib[p++] & 0xff;
860 long ofs = c & 127;
861 while ((c & 128) != 0) {
862 ofs += 1;
863 c = ib[p++] & 0xff;
864 ofs <<= 7;
865 ofs += (c & 127);
866 }
867 pos = pos - ofs;
868 continue;
869 }
870
871 case Constants.OBJ_REF_DELTA: {
872 int p = 1;
873 while ((c & 0x80) != 0) {
874 c = ib[p++] & 0xff;
875 }
876 readFully(pos + p, ib, 0, 20, ctx);
877 pos = findDeltaBase(ctx, ObjectId.fromRaw(ib));
878 continue;
879 }
880
881 default:
882 throw new IOException(MessageFormat.format(
883 JGitText.get().unknownObjectType, Integer.valueOf(type)));
884 }
885 }
886 }
887
888 long getObjectSize(DfsReader ctx, AnyObjectId id) throws IOException {
889 final long offset = idx(ctx).findOffset(id);
890 return 0 < offset ? getObjectSize(ctx, offset) : -1;
891 }
892
893 long getObjectSize(DfsReader ctx, long pos)
894 throws IOException {
895 final byte[] ib = ctx.tempId;
896 readFully(pos, ib, 0, 20, ctx);
897 int c = ib[0] & 0xff;
898 final int type = (c >> 4) & 7;
899 long sz = c & 15;
900 int shift = 4;
901 int p = 1;
902 while ((c & 0x80) != 0) {
903 c = ib[p++] & 0xff;
904 sz += ((long) (c & 0x7f)) << shift;
905 shift += 7;
906 }
907
908 long deltaAt;
909 switch (type) {
910 case Constants.OBJ_COMMIT:
911 case Constants.OBJ_TREE:
912 case Constants.OBJ_BLOB:
913 case Constants.OBJ_TAG:
914 return sz;
915
916 case Constants.OBJ_OFS_DELTA:
917 c = ib[p++] & 0xff;
918 while ((c & 128) != 0) {
919 c = ib[p++] & 0xff;
920 }
921 deltaAt = pos + p;
922 break;
923
924 case Constants.OBJ_REF_DELTA:
925 deltaAt = pos + p + 20;
926 break;
927
928 default:
929 throw new IOException(MessageFormat.format(
930 JGitText.get().unknownObjectType, Integer.valueOf(type)));
931 }
932
933 try {
934 return BinaryDelta.getResultSize(getDeltaHeader(ctx, deltaAt));
935 } catch (DataFormatException dfe) {
936 throw new CorruptObjectException(
937 MessageFormat.format(
938 JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
939 getFileName()),
940 dfe);
941 }
942 }
943
944 void representation(DfsObjectRepresentation r, final long pos,
945 DfsReader ctx, PackReverseIndex rev)
946 throws IOException {
947 r.offset = pos;
948 final byte[] ib = ctx.tempId;
949 readFully(pos, ib, 0, 20, ctx);
950 int c = ib[0] & 0xff;
951 int p = 1;
952 final int typeCode = (c >> 4) & 7;
953 while ((c & 0x80) != 0) {
954 c = ib[p++] & 0xff;
955 }
956
957 long len = rev.findNextOffset(pos, length - 20) - pos;
958 switch (typeCode) {
959 case Constants.OBJ_COMMIT:
960 case Constants.OBJ_TREE:
961 case Constants.OBJ_BLOB:
962 case Constants.OBJ_TAG:
963 r.format = StoredObjectRepresentation.PACK_WHOLE;
964 r.baseId = null;
965 r.length = len - p;
966 return;
967
968 case Constants.OBJ_OFS_DELTA: {
969 c = ib[p++] & 0xff;
970 long ofs = c & 127;
971 while ((c & 128) != 0) {
972 ofs += 1;
973 c = ib[p++] & 0xff;
974 ofs <<= 7;
975 ofs += (c & 127);
976 }
977 r.format = StoredObjectRepresentation.PACK_DELTA;
978 r.baseId = rev.findObject(pos - ofs);
979 r.length = len - p;
980 return;
981 }
982
983 case Constants.OBJ_REF_DELTA: {
984 readFully(pos + p, ib, 0, 20, ctx);
985 r.format = StoredObjectRepresentation.PACK_DELTA;
986 r.baseId = ObjectId.fromRaw(ib);
987 r.length = len - p - 20;
988 return;
989 }
990
991 default:
992 throw new IOException(MessageFormat.format(
993 JGitText.get().unknownObjectType, Integer.valueOf(typeCode)));
994 }
995 }
996
997 boolean isCorrupt(long offset) {
998 LongList list = corruptObjects;
999 if (list == null) {
1000 return false;
1001 }
1002 synchronized (list) {
1003 return list.contains(offset);
1004 }
1005 }
1006
1007 private void setCorrupt(long offset) {
1008 LongList list = corruptObjects;
1009 if (list == null) {
1010 synchronized (initLock) {
1011 list = corruptObjects;
1012 if (list == null) {
1013 list = new LongList();
1014 corruptObjects = list;
1015 }
1016 }
1017 }
1018 synchronized (list) {
1019 list.add(offset);
1020 }
1021 }
1022
1023 private DfsBlockCache.Ref<PackIndex> loadPackIndex(
1024 DfsReader ctx, DfsStreamKey idxKey) throws IOException {
1025 try {
1026 ctx.stats.readIdx++;
1027 long start = System.nanoTime();
1028 try (ReadableChannel rc = ctx.db.openFile(desc, INDEX)) {
1029 InputStream in = Channels.newInputStream(rc);
1030 int wantSize = 8192;
1031 int bs = rc.blockSize();
1032 if (0 < bs && bs < wantSize) {
1033 bs = (wantSize / bs) * bs;
1034 } else if (bs <= 0) {
1035 bs = wantSize;
1036 }
1037 PackIndex idx = PackIndex.read(new BufferedInputStream(in, bs));
1038 ctx.stats.readIdxBytes += rc.position();
1039 index = idx;
1040 return new DfsBlockCache.Ref<>(
1041 idxKey,
1042 REF_POSITION,
1043 idx.getObjectCount() * REC_SIZE,
1044 idx);
1045 } finally {
1046 ctx.stats.readIdxMicros += elapsedMicros(start);
1047 }
1048 } catch (EOFException e) {
1049 throw new IOException(MessageFormat.format(
1050 DfsText.get().shortReadOfIndex,
1051 desc.getFileName(INDEX)), e);
1052 } catch (IOException e) {
1053 throw new IOException(MessageFormat.format(
1054 DfsText.get().cannotReadIndex,
1055 desc.getFileName(INDEX)), e);
1056 }
1057 }
1058
1059 private DfsBlockCache.Ref<PackReverseIndex> loadReverseIdx(
1060 DfsStreamKey revKey, PackIndex idx) {
1061 PackReverseIndex revidx = new PackReverseIndex(idx);
1062 reverseIndex = revidx;
1063 return new DfsBlockCache.Ref<>(
1064 revKey,
1065 REF_POSITION,
1066 idx.getObjectCount() * 8,
1067 revidx);
1068 }
1069
1070 private DfsBlockCache.Ref<PackBitmapIndex> loadBitmapIndex(
1071 DfsReader ctx,
1072 DfsStreamKey bitmapKey,
1073 PackIndex idx,
1074 PackReverseIndex revidx) throws IOException {
1075 ctx.stats.readBitmap++;
1076 long start = System.nanoTime();
1077 try (ReadableChannel rc = ctx.db.openFile(desc, BITMAP_INDEX)) {
1078 long size;
1079 PackBitmapIndex bmidx;
1080 try {
1081 InputStream in = Channels.newInputStream(rc);
1082 int wantSize = 8192;
1083 int bs = rc.blockSize();
1084 if (0 < bs && bs < wantSize) {
1085 bs = (wantSize / bs) * bs;
1086 } else if (bs <= 0) {
1087 bs = wantSize;
1088 }
1089 in = new BufferedInputStream(in, bs);
1090 bmidx = PackBitmapIndex.read(in, idx, revidx);
1091 } finally {
1092 size = rc.position();
1093 ctx.stats.readIdxBytes += size;
1094 ctx.stats.readIdxMicros += elapsedMicros(start);
1095 }
1096 bitmapIndex = bmidx;
1097 return new DfsBlockCache.Ref<>(
1098 bitmapKey, REF_POSITION, size, bmidx);
1099 } catch (EOFException e) {
1100 throw new IOException(MessageFormat.format(
1101 DfsText.get().shortReadOfIndex,
1102 desc.getFileName(BITMAP_INDEX)), e);
1103 } catch (IOException e) {
1104 throw new IOException(MessageFormat.format(
1105 DfsText.get().cannotReadIndex,
1106 desc.getFileName(BITMAP_INDEX)), e);
1107 }
1108 }
1109 }