1
2
3
4
5
6
7
8
9
10
11
12
13 package org.eclipse.jgit.internal.storage.dfs;
14
15 import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
16 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
17 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
18 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
19
20 import java.io.BufferedInputStream;
21 import java.io.EOFException;
22 import java.io.IOException;
23 import java.io.InputStream;
24 import java.nio.ByteBuffer;
25 import java.nio.channels.Channels;
26 import java.text.MessageFormat;
27 import java.util.Set;
28 import java.util.concurrent.atomic.AtomicBoolean;
29 import java.util.zip.CRC32;
30 import java.util.zip.DataFormatException;
31 import java.util.zip.Inflater;
32
33 import org.eclipse.jgit.errors.CorruptObjectException;
34 import org.eclipse.jgit.errors.LargeObjectException;
35 import org.eclipse.jgit.errors.MissingObjectException;
36 import org.eclipse.jgit.errors.PackInvalidException;
37 import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
38 import org.eclipse.jgit.internal.JGitText;
39 import org.eclipse.jgit.internal.storage.file.PackBitmapIndex;
40 import org.eclipse.jgit.internal.storage.file.PackIndex;
41 import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
42 import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
43 import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
44 import org.eclipse.jgit.internal.storage.pack.StoredObjectRepresentation;
45 import org.eclipse.jgit.lib.AbbreviatedObjectId;
46 import org.eclipse.jgit.lib.AnyObjectId;
47 import org.eclipse.jgit.lib.Constants;
48 import org.eclipse.jgit.lib.ObjectId;
49 import org.eclipse.jgit.lib.ObjectLoader;
50 import org.eclipse.jgit.lib.Repository;
51 import org.eclipse.jgit.util.LongList;
52
53
54
55
56
57
58 public final class DfsPackFile extends BlockBasedFile {
59 private static final int REC_SIZE = Constants.OBJECT_ID_LENGTH + 8;
60 private static final long REF_POSITION = 0;
61
62
63
64
65
66
67 private final Object initLock = new Object();
68
69
70 private volatile PackIndex index;
71
72
73 private volatile PackReverseIndex reverseIndex;
74
75
76 private volatile PackBitmapIndex bitmapIndex;
77
78
79
80
81
82
83
84
85 private volatile LongList corruptObjects;
86
87
88
89
90
91
92
93
94
95 DfsPackFile(DfsBlockCache cache, DfsPackDescription desc) {
96 super(cache, desc, PACK);
97
98 int bs = desc.getBlockSize(PACK);
99 if (bs > 0) {
100 setBlockSize(bs);
101 }
102
103 long sz = desc.getFileSize(PACK);
104 length = sz > 0 ? sz : -1;
105 }
106
107
108
109
110
111
112 public DfsPackDescription getPackDescription() {
113 return desc;
114 }
115
116
117
118
119
120
121 public boolean isIndexLoaded() {
122 return index != null;
123 }
124
125 void setPackIndex(PackIndex idx) {
126 long objCnt = idx.getObjectCount();
127 int recSize = Constants.OBJECT_ID_LENGTH + 8;
128 long sz = objCnt * recSize;
129 cache.putRef(desc.getStreamKey(INDEX), sz, idx);
130 index = idx;
131 }
132
133
134
135
136
137
138
139
140
141
142
143 public PackIndex getPackIndex(DfsReader ctx) throws IOException {
144 return idx(ctx);
145 }
146
147 private PackIndex idx(DfsReader ctx) throws IOException {
148 if (index != null) {
149 return index;
150 }
151
152 if (invalid) {
153 throw new PackInvalidException(getFileName(), invalidatingCause);
154 }
155
156 Repository.getGlobalListenerList()
157 .dispatch(new BeforeDfsPackIndexLoadedEvent(this));
158
159 synchronized (initLock) {
160 if (index != null) {
161 return index;
162 }
163
164 try {
165 DfsStreamKey idxKey = desc.getStreamKey(INDEX);
166 AtomicBoolean cacheHit = new AtomicBoolean(true);
167 DfsBlockCache.Ref<PackIndex> idxref = cache.getOrLoadRef(
168 idxKey,
169 REF_POSITION,
170 () -> {
171 cacheHit.set(false);
172 return loadPackIndex(ctx, idxKey);
173 });
174 if (cacheHit.get()) {
175 ctx.stats.idxCacheHit++;
176 }
177 PackIndex idx = idxref.get();
178 if (index == null && idx != null) {
179 index = idx;
180 }
181 return index;
182 } catch (IOException e) {
183 invalid = true;
184 invalidatingCause = e;
185 throw e;
186 }
187 }
188 }
189
190 final boolean isGarbage() {
191 return desc.getPackSource() == UNREACHABLE_GARBAGE;
192 }
193
194 PackBitmapIndex getBitmapIndex(DfsReader ctx) throws IOException {
195 if (invalid || isGarbage() || !desc.hasFileExt(BITMAP_INDEX)) {
196 return null;
197 }
198
199 if (bitmapIndex != null) {
200 return bitmapIndex;
201 }
202
203 synchronized (initLock) {
204 if (bitmapIndex != null) {
205 return bitmapIndex;
206 }
207
208 PackIndex idx = idx(ctx);
209 PackReverseIndex revidx = getReverseIdx(ctx);
210 DfsStreamKey bitmapKey = desc.getStreamKey(BITMAP_INDEX);
211 AtomicBoolean cacheHit = new AtomicBoolean(true);
212 DfsBlockCache.Ref<PackBitmapIndex> idxref = cache.getOrLoadRef(
213 bitmapKey,
214 REF_POSITION,
215 () -> {
216 cacheHit.set(false);
217 return loadBitmapIndex(ctx, bitmapKey, idx, revidx);
218 });
219 if (cacheHit.get()) {
220 ctx.stats.bitmapCacheHit++;
221 }
222 PackBitmapIndex bmidx = idxref.get();
223 if (bitmapIndex == null && bmidx != null) {
224 bitmapIndex = bmidx;
225 }
226 return bitmapIndex;
227 }
228 }
229
230 PackReverseIndex getReverseIdx(DfsReader ctx) throws IOException {
231 if (reverseIndex != null) {
232 return reverseIndex;
233 }
234
235 synchronized (initLock) {
236 if (reverseIndex != null) {
237 return reverseIndex;
238 }
239
240 PackIndex idx = idx(ctx);
241 DfsStreamKey revKey = new DfsStreamKey.ForReverseIndex(
242 desc.getStreamKey(INDEX));
243 AtomicBoolean cacheHit = new AtomicBoolean(true);
244 DfsBlockCache.Ref<PackReverseIndex> revref = cache.getOrLoadRef(
245 revKey,
246 REF_POSITION,
247 () -> {
248 cacheHit.set(false);
249 return loadReverseIdx(ctx, revKey, idx);
250 });
251 if (cacheHit.get()) {
252 ctx.stats.ridxCacheHit++;
253 }
254 PackReverseIndex revidx = revref.get();
255 if (reverseIndex == null && revidx != null) {
256 reverseIndex = revidx;
257 }
258 return reverseIndex;
259 }
260 }
261
262
263
264
265
266
267
268
269
270
271
272
273
274 public boolean hasObject(DfsReader ctx, AnyObjectId id) throws IOException {
275 final long offset = idx(ctx).findOffset(id);
276 return 0 < offset && !isCorrupt(offset);
277 }
278
279
280
281
282
283
284
285
286
287
288
289
290
291 ObjectLoader get(DfsReader ctx, AnyObjectId id)
292 throws IOException {
293 long offset = idx(ctx).findOffset(id);
294 return 0 < offset && !isCorrupt(offset) ? load(ctx, offset) : null;
295 }
296
297 long findOffset(DfsReader ctx, AnyObjectId id) throws IOException {
298 return idx(ctx).findOffset(id);
299 }
300
301 void resolve(DfsReader ctx, Set<ObjectId> matches, AbbreviatedObjectId id,
302 int matchLimit) throws IOException {
303 idx(ctx).resolve(matches, id, matchLimit);
304 }
305
306
307
308
309
310
311
312
313
314
315
316 long getObjectCount(DfsReader ctx) throws IOException {
317 return idx(ctx).getObjectCount();
318 }
319
320 private byte[] decompress(long position, int sz, DfsReader ctx)
321 throws IOException, DataFormatException {
322 byte[] dstbuf;
323 try {
324 dstbuf = new byte[sz];
325 } catch (OutOfMemoryError noMemory) {
326
327
328
329
330
331
332
333 return null;
334 }
335
336 if (ctx.inflate(this, position, dstbuf, false) != sz) {
337 throw new EOFException(MessageFormat.format(
338 JGitText.get().shortCompressedStreamAt,
339 Long.valueOf(position)));
340 }
341 return dstbuf;
342 }
343
344 void copyPackAsIs(PackOutputStream out, DfsReader ctx) throws IOException {
345
346 if (length == -1) {
347 ctx.pin(this, 0);
348 ctx.unpin();
349 }
350 try (ReadableChannel rc = ctx.db.openFile(desc, PACK)) {
351 int sz = ctx.getOptions().getStreamPackBufferSize();
352 if (sz > 0) {
353 rc.setReadAheadBytes(sz);
354 }
355 if (cache.shouldCopyThroughCache(length)) {
356 copyPackThroughCache(out, ctx, rc);
357 } else {
358 copyPackBypassCache(out, rc);
359 }
360 }
361 }
362
363 private void copyPackThroughCache(PackOutputStream out, DfsReader ctx,
364 ReadableChannel rc) throws IOException {
365 long position = 12;
366 long remaining = length - (12 + 20);
367 while (0 < remaining) {
368 DfsBlock b = cache.getOrLoad(this, position, ctx, () -> rc);
369 int ptr = (int) (position - b.start);
370 if (b.size() <= ptr) {
371 throw packfileIsTruncated();
372 }
373 int n = (int) Math.min(b.size() - ptr, remaining);
374 b.write(out, position, n);
375 position += n;
376 remaining -= n;
377 }
378 }
379
380 private long copyPackBypassCache(PackOutputStream out, ReadableChannel rc)
381 throws IOException {
382 ByteBuffer buf = newCopyBuffer(out, rc);
383 long position = 12;
384 long remaining = length - (12 + 20);
385 boolean packHeadSkipped = false;
386 while (0 < remaining) {
387 DfsBlock b = cache.get(key, alignToBlock(position));
388 if (b != null) {
389 int ptr = (int) (position - b.start);
390 if (b.size() <= ptr) {
391 throw packfileIsTruncated();
392 }
393 int n = (int) Math.min(b.size() - ptr, remaining);
394 b.write(out, position, n);
395 position += n;
396 remaining -= n;
397 rc.position(position);
398 packHeadSkipped = true;
399 continue;
400 }
401
402
403 int ptr = packHeadSkipped ? 0 : 12;
404 buf.position(0);
405 int bufLen = read(rc, buf);
406 if (bufLen <= ptr) {
407 throw packfileIsTruncated();
408 }
409 int n = (int) Math.min(bufLen - ptr, remaining);
410 out.write(buf.array(), ptr, n);
411 position += n;
412 remaining -= n;
413 packHeadSkipped = true;
414 }
415 return position;
416 }
417
418 private ByteBuffer newCopyBuffer(PackOutputStream out, ReadableChannel rc) {
419 int bs = blockSize(rc);
420 byte[] copyBuf = out.getCopyBuffer();
421 if (bs > copyBuf.length) {
422 copyBuf = new byte[bs];
423 }
424 return ByteBuffer.wrap(copyBuf, 0, bs);
425 }
426
427 void copyAsIs(PackOutputStream out, DfsObjectToPack src,
428 boolean validate, DfsReader ctx) throws IOException,
429 StoredObjectRepresentationNotAvailableException {
430 final CRC32 crc1 = validate ? new CRC32() : null;
431 final CRC32 crc2 = validate ? new CRC32() : null;
432 final byte[] buf = out.getCopyBuffer();
433
434
435
436 try {
437 readFully(src.offset, buf, 0, 20, ctx);
438 } catch (IOException ioError) {
439 throw new StoredObjectRepresentationNotAvailableException(src,
440 ioError);
441 }
442 int c = buf[0] & 0xff;
443 final int typeCode = (c >> 4) & 7;
444 long inflatedLength = c & 15;
445 int shift = 4;
446 int headerCnt = 1;
447 while ((c & 0x80) != 0) {
448 c = buf[headerCnt++] & 0xff;
449 inflatedLength += ((long) (c & 0x7f)) << shift;
450 shift += 7;
451 }
452
453 if (typeCode == Constants.OBJ_OFS_DELTA) {
454 do {
455 c = buf[headerCnt++] & 0xff;
456 } while ((c & 128) != 0);
457 if (validate) {
458 assert(crc1 != null && crc2 != null);
459 crc1.update(buf, 0, headerCnt);
460 crc2.update(buf, 0, headerCnt);
461 }
462 } else if (typeCode == Constants.OBJ_REF_DELTA) {
463 if (validate) {
464 assert(crc1 != null && crc2 != null);
465 crc1.update(buf, 0, headerCnt);
466 crc2.update(buf, 0, headerCnt);
467 }
468
469 readFully(src.offset + headerCnt, buf, 0, 20, ctx);
470 if (validate) {
471 assert(crc1 != null && crc2 != null);
472 crc1.update(buf, 0, 20);
473 crc2.update(buf, 0, 20);
474 }
475 headerCnt += 20;
476 } else if (validate) {
477 assert(crc1 != null && crc2 != null);
478 crc1.update(buf, 0, headerCnt);
479 crc2.update(buf, 0, headerCnt);
480 }
481
482 final long dataOffset = src.offset + headerCnt;
483 final long dataLength = src.length;
484 final long expectedCRC;
485 final DfsBlock quickCopy;
486
487
488
489
490 try {
491 quickCopy = ctx.quickCopy(this, dataOffset, dataLength);
492
493 if (validate && idx(ctx).hasCRC32Support()) {
494 assert(crc1 != null);
495
496
497 expectedCRC = idx(ctx).findCRC32(src);
498 if (quickCopy != null) {
499 quickCopy.crc32(crc1, dataOffset, (int) dataLength);
500 } else {
501 long pos = dataOffset;
502 long cnt = dataLength;
503 while (cnt > 0) {
504 final int n = (int) Math.min(cnt, buf.length);
505 readFully(pos, buf, 0, n, ctx);
506 crc1.update(buf, 0, n);
507 pos += n;
508 cnt -= n;
509 }
510 }
511 if (crc1.getValue() != expectedCRC) {
512 setCorrupt(src.offset);
513 throw new CorruptObjectException(MessageFormat.format(
514 JGitText.get().objectAtHasBadZlibStream,
515 Long.valueOf(src.offset), getFileName()));
516 }
517 } else if (validate) {
518 assert(crc1 != null);
519
520
521
522
523 Inflater inf = ctx.inflater();
524 byte[] tmp = new byte[1024];
525 if (quickCopy != null) {
526 quickCopy.check(inf, tmp, dataOffset, (int) dataLength);
527 } else {
528 long pos = dataOffset;
529 long cnt = dataLength;
530 while (cnt > 0) {
531 final int n = (int) Math.min(cnt, buf.length);
532 readFully(pos, buf, 0, n, ctx);
533 crc1.update(buf, 0, n);
534 inf.setInput(buf, 0, n);
535 while (inf.inflate(tmp, 0, tmp.length) > 0) {
536 continue;
537 }
538 pos += n;
539 cnt -= n;
540 }
541 }
542 if (!inf.finished() || inf.getBytesRead() != dataLength) {
543 setCorrupt(src.offset);
544 throw new EOFException(MessageFormat.format(
545 JGitText.get().shortCompressedStreamAt,
546 Long.valueOf(src.offset)));
547 }
548 expectedCRC = crc1.getValue();
549 } else {
550 expectedCRC = -1;
551 }
552 } catch (DataFormatException dataFormat) {
553 setCorrupt(src.offset);
554
555 CorruptObjectException corruptObject = new CorruptObjectException(
556 MessageFormat.format(
557 JGitText.get().objectAtHasBadZlibStream,
558 Long.valueOf(src.offset), getFileName()),
559 dataFormat);
560
561 throw new StoredObjectRepresentationNotAvailableException(src,
562 corruptObject);
563
564 } catch (IOException ioError) {
565 throw new StoredObjectRepresentationNotAvailableException(src,
566 ioError);
567 }
568
569 if (quickCopy != null) {
570
571
572
573 out.writeHeader(src, inflatedLength);
574 quickCopy.write(out, dataOffset, (int) dataLength);
575
576 } else if (dataLength <= buf.length) {
577
578
579
580 if (!validate) {
581 long pos = dataOffset;
582 long cnt = dataLength;
583 while (cnt > 0) {
584 final int n = (int) Math.min(cnt, buf.length);
585 readFully(pos, buf, 0, n, ctx);
586 pos += n;
587 cnt -= n;
588 }
589 }
590 out.writeHeader(src, inflatedLength);
591 out.write(buf, 0, (int) dataLength);
592 } else {
593
594
595
596
597 out.writeHeader(src, inflatedLength);
598 long pos = dataOffset;
599 long cnt = dataLength;
600 while (cnt > 0) {
601 final int n = (int) Math.min(cnt, buf.length);
602 readFully(pos, buf, 0, n, ctx);
603 if (validate) {
604 assert(crc2 != null);
605 crc2.update(buf, 0, n);
606 }
607 out.write(buf, 0, n);
608 pos += n;
609 cnt -= n;
610 }
611 if (validate) {
612 assert(crc2 != null);
613 if (crc2.getValue() != expectedCRC) {
614 throw new CorruptObjectException(MessageFormat.format(
615 JGitText.get().objectAtHasBadZlibStream,
616 Long.valueOf(src.offset), getFileName()));
617 }
618 }
619 }
620 }
621
622 private IOException packfileIsTruncated() {
623 invalid = true;
624 IOException exc = new IOException(MessageFormat.format(
625 JGitText.get().packfileIsTruncated, getFileName()));
626 invalidatingCause = exc;
627 return exc;
628 }
629
630 private void readFully(long position, byte[] dstbuf, int dstoff, int cnt,
631 DfsReader ctx) throws IOException {
632 while (cnt > 0) {
633 int copied = ctx.copy(this, position, dstbuf, dstoff, cnt);
634 if (copied == 0) {
635 throw new EOFException();
636 }
637 position += copied;
638 dstoff += copied;
639 cnt -= copied;
640 }
641 }
642
643 ObjectLoader load(DfsReader ctx, long pos)
644 throws IOException {
645 try {
646 final byte[] ib = ctx.tempId;
647 Delta delta = null;
648 byte[] data = null;
649 int type = Constants.OBJ_BAD;
650 boolean cached = false;
651
652 SEARCH: for (;;) {
653 readFully(pos, ib, 0, 20, ctx);
654 int c = ib[0] & 0xff;
655 final int typeCode = (c >> 4) & 7;
656 long sz = c & 15;
657 int shift = 4;
658 int p = 1;
659 while ((c & 0x80) != 0) {
660 c = ib[p++] & 0xff;
661 sz += ((long) (c & 0x7f)) << shift;
662 shift += 7;
663 }
664
665 switch (typeCode) {
666 case Constants.OBJ_COMMIT:
667 case Constants.OBJ_TREE:
668 case Constants.OBJ_BLOB:
669 case Constants.OBJ_TAG: {
670 if (delta != null) {
671 data = decompress(pos + p, (int) sz, ctx);
672 type = typeCode;
673 break SEARCH;
674 }
675
676 if (sz < ctx.getStreamFileThreshold()) {
677 data = decompress(pos + p, (int) sz, ctx);
678 if (data != null) {
679 return new ObjectLoader.SmallObject(typeCode, data);
680 }
681 }
682 return new LargePackedWholeObject(typeCode, sz, pos, p, this, ctx.db);
683 }
684
685 case Constants.OBJ_OFS_DELTA: {
686 c = ib[p++] & 0xff;
687 long base = c & 127;
688 while ((c & 128) != 0) {
689 base += 1;
690 c = ib[p++] & 0xff;
691 base <<= 7;
692 base += (c & 127);
693 }
694 base = pos - base;
695 delta = new Delta(delta, pos, (int) sz, p, base);
696 if (sz != delta.deltaSize) {
697 break SEARCH;
698 }
699
700 DeltaBaseCache.Entry e = ctx.getDeltaBaseCache().get(key, base);
701 if (e != null) {
702 type = e.type;
703 data = e.data;
704 cached = true;
705 break SEARCH;
706 }
707 pos = base;
708 continue SEARCH;
709 }
710
711 case Constants.OBJ_REF_DELTA: {
712 readFully(pos + p, ib, 0, 20, ctx);
713 long base = findDeltaBase(ctx, ObjectId.fromRaw(ib));
714 delta = new Delta(delta, pos, (int) sz, p + 20, base);
715 if (sz != delta.deltaSize) {
716 break SEARCH;
717 }
718
719 DeltaBaseCache.Entry e = ctx.getDeltaBaseCache().get(key, base);
720 if (e != null) {
721 type = e.type;
722 data = e.data;
723 cached = true;
724 break SEARCH;
725 }
726 pos = base;
727 continue SEARCH;
728 }
729
730 default:
731 throw new IOException(MessageFormat.format(
732 JGitText.get().unknownObjectType, Integer.valueOf(typeCode)));
733 }
734 }
735
736
737
738
739 if (data == null)
740 throw new LargeObjectException();
741
742 assert(delta != null);
743 do {
744
745 if (cached) {
746 cached = false;
747 } else if (delta.next == null) {
748 ctx.getDeltaBaseCache().put(key, delta.basePos, type, data);
749 }
750
751 pos = delta.deltaPos;
752
753 byte[] cmds = decompress(pos + delta.hdrLen, delta.deltaSize, ctx);
754 if (cmds == null) {
755 data = null;
756 throw new LargeObjectException();
757 }
758
759 final long sz = BinaryDelta.getResultSize(cmds);
760 if (Integer.MAX_VALUE <= sz) {
761 throw new LargeObjectException.ExceedsByteArrayLimit();
762 }
763
764 final byte[] result;
765 try {
766 result = new byte[(int) sz];
767 } catch (OutOfMemoryError tooBig) {
768 data = null;
769 cmds = null;
770 throw new LargeObjectException.OutOfMemory(tooBig);
771 }
772
773 BinaryDelta.apply(data, cmds, result);
774 data = result;
775 delta = delta.next;
776 } while (delta != null);
777
778 return new ObjectLoader.SmallObject(type, data);
779
780 } catch (DataFormatException dfe) {
781 throw new CorruptObjectException(
782 MessageFormat.format(
783 JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
784 getFileName()),
785 dfe);
786 }
787 }
788
789 private long findDeltaBase(DfsReader ctx, ObjectId baseId)
790 throws IOException, MissingObjectException {
791 long ofs = idx(ctx).findOffset(baseId);
792 if (ofs < 0) {
793 throw new MissingObjectException(baseId,
794 JGitText.get().missingDeltaBase);
795 }
796 return ofs;
797 }
798
799 private static class Delta {
800
801 final Delta next;
802
803
804 final long deltaPos;
805
806
807 final int deltaSize;
808
809
810 final int hdrLen;
811
812
813 final long basePos;
814
815 Delta(Delta next, long ofs, int sz, int hdrLen, long baseOffset) {
816 this.next = next;
817 this.deltaPos = ofs;
818 this.deltaSize = sz;
819 this.hdrLen = hdrLen;
820 this.basePos = baseOffset;
821 }
822 }
823
824 byte[] getDeltaHeader(DfsReader wc, long pos)
825 throws IOException, DataFormatException {
826
827
828
829
830
831 final byte[] hdr = new byte[32];
832 wc.inflate(this, pos, hdr, true );
833 return hdr;
834 }
835
836 int getObjectType(DfsReader ctx, long pos) throws IOException {
837 final byte[] ib = ctx.tempId;
838 for (;;) {
839 readFully(pos, ib, 0, 20, ctx);
840 int c = ib[0] & 0xff;
841 final int type = (c >> 4) & 7;
842
843 switch (type) {
844 case Constants.OBJ_COMMIT:
845 case Constants.OBJ_TREE:
846 case Constants.OBJ_BLOB:
847 case Constants.OBJ_TAG:
848 return type;
849
850 case Constants.OBJ_OFS_DELTA: {
851 int p = 1;
852 while ((c & 0x80) != 0) {
853 c = ib[p++] & 0xff;
854 }
855 c = ib[p++] & 0xff;
856 long ofs = c & 127;
857 while ((c & 128) != 0) {
858 ofs += 1;
859 c = ib[p++] & 0xff;
860 ofs <<= 7;
861 ofs += (c & 127);
862 }
863 pos = pos - ofs;
864 continue;
865 }
866
867 case Constants.OBJ_REF_DELTA: {
868 int p = 1;
869 while ((c & 0x80) != 0) {
870 c = ib[p++] & 0xff;
871 }
872 readFully(pos + p, ib, 0, 20, ctx);
873 pos = findDeltaBase(ctx, ObjectId.fromRaw(ib));
874 continue;
875 }
876
877 default:
878 throw new IOException(MessageFormat.format(
879 JGitText.get().unknownObjectType, Integer.valueOf(type)));
880 }
881 }
882 }
883
884 long getObjectSize(DfsReader ctx, AnyObjectId id) throws IOException {
885 final long offset = idx(ctx).findOffset(id);
886 return 0 < offset ? getObjectSize(ctx, offset) : -1;
887 }
888
889 long getObjectSize(DfsReader ctx, long pos)
890 throws IOException {
891 final byte[] ib = ctx.tempId;
892 readFully(pos, ib, 0, 20, ctx);
893 int c = ib[0] & 0xff;
894 final int type = (c >> 4) & 7;
895 long sz = c & 15;
896 int shift = 4;
897 int p = 1;
898 while ((c & 0x80) != 0) {
899 c = ib[p++] & 0xff;
900 sz += ((long) (c & 0x7f)) << shift;
901 shift += 7;
902 }
903
904 long deltaAt;
905 switch (type) {
906 case Constants.OBJ_COMMIT:
907 case Constants.OBJ_TREE:
908 case Constants.OBJ_BLOB:
909 case Constants.OBJ_TAG:
910 return sz;
911
912 case Constants.OBJ_OFS_DELTA:
913 c = ib[p++] & 0xff;
914 while ((c & 128) != 0) {
915 c = ib[p++] & 0xff;
916 }
917 deltaAt = pos + p;
918 break;
919
920 case Constants.OBJ_REF_DELTA:
921 deltaAt = pos + p + 20;
922 break;
923
924 default:
925 throw new IOException(MessageFormat.format(
926 JGitText.get().unknownObjectType, Integer.valueOf(type)));
927 }
928
929 try {
930 return BinaryDelta.getResultSize(getDeltaHeader(ctx, deltaAt));
931 } catch (DataFormatException dfe) {
932 throw new CorruptObjectException(
933 MessageFormat.format(
934 JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
935 getFileName()),
936 dfe);
937 }
938 }
939
940 void representation(DfsObjectRepresentation r, final long pos,
941 DfsReader ctx, PackReverseIndex rev)
942 throws IOException {
943 r.offset = pos;
944 final byte[] ib = ctx.tempId;
945 readFully(pos, ib, 0, 20, ctx);
946 int c = ib[0] & 0xff;
947 int p = 1;
948 final int typeCode = (c >> 4) & 7;
949 while ((c & 0x80) != 0) {
950 c = ib[p++] & 0xff;
951 }
952
953 long len = rev.findNextOffset(pos, length - 20) - pos;
954 switch (typeCode) {
955 case Constants.OBJ_COMMIT:
956 case Constants.OBJ_TREE:
957 case Constants.OBJ_BLOB:
958 case Constants.OBJ_TAG:
959 r.format = StoredObjectRepresentation.PACK_WHOLE;
960 r.baseId = null;
961 r.length = len - p;
962 return;
963
964 case Constants.OBJ_OFS_DELTA: {
965 c = ib[p++] & 0xff;
966 long ofs = c & 127;
967 while ((c & 128) != 0) {
968 ofs += 1;
969 c = ib[p++] & 0xff;
970 ofs <<= 7;
971 ofs += (c & 127);
972 }
973 r.format = StoredObjectRepresentation.PACK_DELTA;
974 r.baseId = rev.findObject(pos - ofs);
975 r.length = len - p;
976 return;
977 }
978
979 case Constants.OBJ_REF_DELTA: {
980 readFully(pos + p, ib, 0, 20, ctx);
981 r.format = StoredObjectRepresentation.PACK_DELTA;
982 r.baseId = ObjectId.fromRaw(ib);
983 r.length = len - p - 20;
984 return;
985 }
986
987 default:
988 throw new IOException(MessageFormat.format(
989 JGitText.get().unknownObjectType, Integer.valueOf(typeCode)));
990 }
991 }
992
993 boolean isCorrupt(long offset) {
994 LongList list = corruptObjects;
995 if (list == null) {
996 return false;
997 }
998 synchronized (list) {
999 return list.contains(offset);
1000 }
1001 }
1002
1003 private void setCorrupt(long offset) {
1004 LongList list = corruptObjects;
1005 if (list == null) {
1006 synchronized (initLock) {
1007 list = corruptObjects;
1008 if (list == null) {
1009 list = new LongList();
1010 corruptObjects = list;
1011 }
1012 }
1013 }
1014 synchronized (list) {
1015 list.add(offset);
1016 }
1017 }
1018
1019 private DfsBlockCache.Ref<PackIndex> loadPackIndex(
1020 DfsReader ctx, DfsStreamKey idxKey) throws IOException {
1021 try {
1022 ctx.stats.readIdx++;
1023 long start = System.nanoTime();
1024 try (ReadableChannel rc = ctx.db.openFile(desc, INDEX)) {
1025 InputStream in = Channels.newInputStream(rc);
1026 int wantSize = 8192;
1027 int bs = rc.blockSize();
1028 if (0 < bs && bs < wantSize) {
1029 bs = (wantSize / bs) * bs;
1030 } else if (bs <= 0) {
1031 bs = wantSize;
1032 }
1033 PackIndex idx = PackIndex.read(new BufferedInputStream(in, bs));
1034 ctx.stats.readIdxBytes += rc.position();
1035 index = idx;
1036 return new DfsBlockCache.Ref<>(
1037 idxKey,
1038 REF_POSITION,
1039 idx.getObjectCount() * REC_SIZE,
1040 idx);
1041 } finally {
1042 ctx.stats.readIdxMicros += elapsedMicros(start);
1043 }
1044 } catch (EOFException e) {
1045 throw new IOException(MessageFormat.format(
1046 DfsText.get().shortReadOfIndex,
1047 desc.getFileName(INDEX)), e);
1048 } catch (IOException e) {
1049 throw new IOException(MessageFormat.format(
1050 DfsText.get().cannotReadIndex,
1051 desc.getFileName(INDEX)), e);
1052 }
1053 }
1054
1055 private DfsBlockCache.Ref<PackReverseIndex> loadReverseIdx(
1056 DfsReader ctx, DfsStreamKey revKey, PackIndex idx) {
1057 ctx.stats.readReverseIdx++;
1058 long start = System.nanoTime();
1059 PackReverseIndex revidx = new PackReverseIndex(idx);
1060 reverseIndex = revidx;
1061 ctx.stats.readReverseIdxMicros += elapsedMicros(start);
1062 return new DfsBlockCache.Ref<>(
1063 revKey,
1064 REF_POSITION,
1065 idx.getObjectCount() * 8,
1066 revidx);
1067 }
1068
1069 private DfsBlockCache.Ref<PackBitmapIndex> loadBitmapIndex(
1070 DfsReader ctx,
1071 DfsStreamKey bitmapKey,
1072 PackIndex idx,
1073 PackReverseIndex revidx) throws IOException {
1074 ctx.stats.readBitmap++;
1075 long start = System.nanoTime();
1076 try (ReadableChannel rc = ctx.db.openFile(desc, BITMAP_INDEX)) {
1077 long size;
1078 PackBitmapIndex bmidx;
1079 try {
1080 InputStream in = Channels.newInputStream(rc);
1081 int wantSize = 8192;
1082 int bs = rc.blockSize();
1083 if (0 < bs && bs < wantSize) {
1084 bs = (wantSize / bs) * bs;
1085 } else if (bs <= 0) {
1086 bs = wantSize;
1087 }
1088 in = new BufferedInputStream(in, bs);
1089 bmidx = PackBitmapIndex.read(in, idx, revidx);
1090 } finally {
1091 size = rc.position();
1092 ctx.stats.readBitmapIdxBytes += size;
1093 ctx.stats.readBitmapIdxMicros += elapsedMicros(start);
1094 }
1095 bitmapIndex = bmidx;
1096 return new DfsBlockCache.Ref<>(
1097 bitmapKey, REF_POSITION, size, bmidx);
1098 } catch (EOFException e) {
1099 throw new IOException(MessageFormat.format(
1100 DfsText.get().shortReadOfIndex,
1101 desc.getFileName(BITMAP_INDEX)), e);
1102 } catch (IOException e) {
1103 throw new IOException(MessageFormat.format(
1104 DfsText.get().cannotReadIndex,
1105 desc.getFileName(BITMAP_INDEX)), e);
1106 }
1107 }
1108 }