1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46 package org.eclipse.jgit.internal.storage.file;
47
48 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
49 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
50 import static org.eclipse.jgit.internal.storage.pack.PackExt.KEEP;
51
52 import java.io.EOFException;
53 import java.io.File;
54 import java.io.FileNotFoundException;
55 import java.io.IOException;
56 import java.io.InterruptedIOException;
57 import java.io.RandomAccessFile;
58 import java.nio.MappedByteBuffer;
59 import java.nio.channels.FileChannel.MapMode;
60 import java.nio.file.AccessDeniedException;
61 import java.nio.file.NoSuchFileException;
62 import java.text.MessageFormat;
63 import java.time.Instant;
64 import java.util.Arrays;
65 import java.util.Collections;
66 import java.util.Comparator;
67 import java.util.Iterator;
68 import java.util.Set;
69 import java.util.concurrent.atomic.AtomicInteger;
70 import java.util.zip.CRC32;
71 import java.util.zip.DataFormatException;
72 import java.util.zip.Inflater;
73
74 import org.eclipse.jgit.errors.CorruptObjectException;
75 import org.eclipse.jgit.errors.LargeObjectException;
76 import org.eclipse.jgit.errors.MissingObjectException;
77 import org.eclipse.jgit.errors.NoPackSignatureException;
78 import org.eclipse.jgit.errors.PackInvalidException;
79 import org.eclipse.jgit.errors.PackMismatchException;
80 import org.eclipse.jgit.errors.StoredObjectRepresentationNotAvailableException;
81 import org.eclipse.jgit.errors.UnpackException;
82 import org.eclipse.jgit.errors.UnsupportedPackIndexVersionException;
83 import org.eclipse.jgit.errors.UnsupportedPackVersionException;
84 import org.eclipse.jgit.internal.JGitText;
85 import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
86 import org.eclipse.jgit.internal.storage.pack.ObjectToPack;
87 import org.eclipse.jgit.internal.storage.pack.PackExt;
88 import org.eclipse.jgit.internal.storage.pack.PackOutputStream;
89 import org.eclipse.jgit.lib.AbbreviatedObjectId;
90 import org.eclipse.jgit.lib.AnyObjectId;
91 import org.eclipse.jgit.lib.Constants;
92 import org.eclipse.jgit.lib.ObjectId;
93 import org.eclipse.jgit.lib.ObjectLoader;
94 import org.eclipse.jgit.util.LongList;
95 import org.eclipse.jgit.util.NB;
96 import org.eclipse.jgit.util.RawParseUtils;
97 import org.slf4j.Logger;
98 import org.slf4j.LoggerFactory;
99
100
101
102
103
104
105 public class PackFile implements Iterable<PackIndex.MutableEntry> {
106 private final static Logger LOG = LoggerFactory.getLogger(PackFile.class);
107
108 public static final Comparator<PackFile> SORT = new Comparator<PackFile>() {
109 @Override
110 public int compare(PackFileref="../../../../../../org/eclipse/jgit/internal/storage/file/PackFile.html#PackFile">PackFile a, PackFile b) {
111 return b.packLastModified.compareTo(a.packLastModified);
112 }
113 };
114
115 private final File packFile;
116
117 private final int extensions;
118
119 private File keepFile;
120
121 private volatile String packName;
122
123 final int hash;
124
125 private RandomAccessFile fd;
126
127
128 private final Object readLock = new Object();
129
130 long length;
131
132 private int activeWindows;
133
134 private int activeCopyRawData;
135
136 Instant packLastModified;
137
138 private PackFileSnapshot fileSnapshot;
139
140 private volatile boolean invalid;
141
142 private volatile Exception invalidatingCause;
143
144 private boolean invalidBitmap;
145
146 private AtomicInteger transientErrorCount = new AtomicInteger();
147
148 private byte[] packChecksum;
149
150 private volatile PackIndex loadedIdx;
151
152 private PackReverseIndex reverseIdx;
153
154 private PackBitmapIndex bitmapIdx;
155
156
157
158
159
160
161
162
163 private volatile LongList corruptObjects;
164
165
166
167
168
169
170
171
172
173 public PackFile(File packFile, int extensions) {
174 this.packFile = packFile;
175 this.fileSnapshot = PackFileSnapshot.save(packFile);
176 this.packLastModified = fileSnapshot.lastModifiedInstant();
177 this.extensions = extensions;
178
179
180
181
182 hash = System.identityHashCode(this) * 31;
183 length = Long.MAX_VALUE;
184 }
185
186 private PackIndex idx() throws IOException {
187 PackIndex idx = loadedIdx;
188 if (idx == null) {
189 synchronized (this) {
190 idx = loadedIdx;
191 if (idx == null) {
192 if (invalid) {
193 throw new PackInvalidException(packFile, invalidatingCause);
194 }
195 try {
196 long start = System.currentTimeMillis();
197 idx = PackIndex.open(extFile(INDEX));
198 if (LOG.isDebugEnabled()) {
199 LOG.debug(String.format(
200 "Opening pack index %s, size %.3f MB took %d ms",
201 extFile(INDEX).getAbsolutePath(),
202 Float.valueOf(extFile(INDEX).length()
203 / (1024f * 1024)),
204 Long.valueOf(System.currentTimeMillis()
205 - start)));
206 }
207
208 if (packChecksum == null) {
209 packChecksum = idx.packChecksum;
210 fileSnapshot.setChecksum(
211 ObjectId.fromRaw(packChecksum));
212 } else if (!Arrays.equals(packChecksum,
213 idx.packChecksum)) {
214 throw new PackMismatchException(MessageFormat
215 .format(JGitText.get().packChecksumMismatch,
216 packFile.getPath(),
217 ObjectId.fromRaw(packChecksum)
218 .name(),
219 ObjectId.fromRaw(idx.packChecksum)
220 .name()));
221 }
222 loadedIdx = idx;
223 } catch (InterruptedIOException e) {
224
225
226 throw e;
227 } catch (IOException e) {
228 invalid = true;
229 invalidatingCause = e;
230 throw e;
231 }
232 }
233 }
234 }
235 return idx;
236 }
237
238
239
240
241
242 public File getPackFile() {
243 return packFile;
244 }
245
246
247
248
249
250
251
252 public PackIndex getIndex() throws IOException {
253 return idx();
254 }
255
256
257
258
259
260
261 public String getPackName() {
262 String name = packName;
263 if (name == null) {
264 name = getPackFile().getName();
265 if (name.startsWith("pack-"))
266 name = name.substring("pack-".length());
267 if (name.endsWith(".pack"))
268 name = name.substring(0, name.length() - ".pack".length());
269 packName = name;
270 }
271 return name;
272 }
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287 public boolean hasObject(AnyObjectId id) throws IOException {
288 final long offset = idx().findOffset(id);
289 return 0 < offset && !isCorrupt(offset);
290 }
291
292
293
294
295
296
297 public boolean shouldBeKept() {
298 if (keepFile == null)
299 keepFile = extFile(KEEP);
300 return keepFile.exists();
301 }
302
303
304
305
306
307
308
309
310
311
312
313
314
315 ObjectLoader get(WindowCursor curs, AnyObjectId id)
316 throws IOException {
317 final long offset = idx().findOffset(id);
318 return 0 < offset && !isCorrupt(offset) ? load(curs, offset) : null;
319 }
320
321 void resolve(Set<ObjectId> matches, AbbreviatedObjectId id, int matchLimit)
322 throws IOException {
323 idx().resolve(matches, id, matchLimit);
324 }
325
326
327
328
329 public void close() {
330 WindowCache.purge(this);
331 synchronized (this) {
332 loadedIdx = null;
333 reverseIdx = null;
334 }
335 }
336
337
338
339
340
341
342
343
344
345
346
347
348
349 @Override
350 public Iterator<PackIndex.MutableEntry> iterator() {
351 try {
352 return idx().iterator();
353 } catch (IOException e) {
354 return Collections.<PackIndex.MutableEntry> emptyList().iterator();
355 }
356 }
357
358
359
360
361
362
363
364
365
366 long getObjectCount() throws IOException {
367 return idx().getObjectCount();
368 }
369
370
371
372
373
374
375
376
377
378
379
380 ObjectId findObjectForOffset(long offset) throws IOException {
381 return getReverseIdx().findObject(offset);
382 }
383
384
385
386
387
388
389
390 PackFileSnapshot getFileSnapshot() {
391 return fileSnapshot;
392 }
393
394 AnyObjectId getPackChecksum() {
395 return ObjectId.fromRaw(packChecksum);
396 }
397
398 private final byte[] decompress(final long position, final int sz,
399 final WindowCursor curs) throws IOException, DataFormatException {
400 byte[] dstbuf;
401 try {
402 dstbuf = new byte[sz];
403 } catch (OutOfMemoryError noMemory) {
404
405
406
407
408
409
410
411 return null;
412 }
413
414 if (curs.inflate(this, position, dstbuf, false) != sz)
415 throw new EOFException(MessageFormat.format(
416 JGitText.get().shortCompressedStreamAt,
417 Long.valueOf(position)));
418 return dstbuf;
419 }
420
421 void copyPackAsIs(PackOutputStream out, WindowCursor curs)
422 throws IOException {
423
424 curs.pin(this, 0);
425 curs.copyPackAsIs(this, length, out);
426 }
427
428 final void copyAsIs(PackOutputStream out, LocalObjectToPack src,
429 boolean validate, WindowCursor curs) throws IOException,
430 StoredObjectRepresentationNotAvailableException {
431 beginCopyAsIs(src);
432 try {
433 copyAsIs2(out, src, validate, curs);
434 } finally {
435 endCopyAsIs();
436 }
437 }
438
439 private void copyAsIs2(PackOutputStream out, LocalObjectToPack src,
440 boolean validate, WindowCursor curs) throws IOException,
441 StoredObjectRepresentationNotAvailableException {
442 final CRC32 crc1 = validate ? new CRC32() : null;
443 final CRC32 crc2 = validate ? new CRC32() : null;
444 final byte[] buf = out.getCopyBuffer();
445
446
447
448 readFully(src.offset, buf, 0, 20, curs);
449 int c = buf[0] & 0xff;
450 final int typeCode = (c >> 4) & 7;
451 long inflatedLength = c & 15;
452 int shift = 4;
453 int headerCnt = 1;
454 while ((c & 0x80) != 0) {
455 c = buf[headerCnt++] & 0xff;
456 inflatedLength += ((long) (c & 0x7f)) << shift;
457 shift += 7;
458 }
459
460 if (typeCode == Constants.OBJ_OFS_DELTA) {
461 do {
462 c = buf[headerCnt++] & 0xff;
463 } while ((c & 128) != 0);
464 if (validate) {
465 assert(crc1 != null && crc2 != null);
466 crc1.update(buf, 0, headerCnt);
467 crc2.update(buf, 0, headerCnt);
468 }
469 } else if (typeCode == Constants.OBJ_REF_DELTA) {
470 if (validate) {
471 assert(crc1 != null && crc2 != null);
472 crc1.update(buf, 0, headerCnt);
473 crc2.update(buf, 0, headerCnt);
474 }
475
476 readFully(src.offset + headerCnt, buf, 0, 20, curs);
477 if (validate) {
478 assert(crc1 != null && crc2 != null);
479 crc1.update(buf, 0, 20);
480 crc2.update(buf, 0, 20);
481 }
482 headerCnt += 20;
483 } else if (validate) {
484 assert(crc1 != null && crc2 != null);
485 crc1.update(buf, 0, headerCnt);
486 crc2.update(buf, 0, headerCnt);
487 }
488
489 final long dataOffset = src.offset + headerCnt;
490 final long dataLength = src.length;
491 final long expectedCRC;
492 final ByteArrayWindow quickCopy;
493
494
495
496
497 try {
498 quickCopy = curs.quickCopy(this, dataOffset, dataLength);
499
500 if (validate && idx().hasCRC32Support()) {
501 assert(crc1 != null);
502
503
504 expectedCRC = idx().findCRC32(src);
505 if (quickCopy != null) {
506 quickCopy.crc32(crc1, dataOffset, (int) dataLength);
507 } else {
508 long pos = dataOffset;
509 long cnt = dataLength;
510 while (cnt > 0) {
511 final int n = (int) Math.min(cnt, buf.length);
512 readFully(pos, buf, 0, n, curs);
513 crc1.update(buf, 0, n);
514 pos += n;
515 cnt -= n;
516 }
517 }
518 if (crc1.getValue() != expectedCRC) {
519 setCorrupt(src.offset);
520 throw new CorruptObjectException(MessageFormat.format(
521 JGitText.get().objectAtHasBadZlibStream,
522 Long.valueOf(src.offset), getPackFile()));
523 }
524 } else if (validate) {
525
526
527
528
529 Inflater inf = curs.inflater();
530 byte[] tmp = new byte[1024];
531 if (quickCopy != null) {
532 quickCopy.check(inf, tmp, dataOffset, (int) dataLength);
533 } else {
534 assert(crc1 != null);
535 long pos = dataOffset;
536 long cnt = dataLength;
537 while (cnt > 0) {
538 final int n = (int) Math.min(cnt, buf.length);
539 readFully(pos, buf, 0, n, curs);
540 crc1.update(buf, 0, n);
541 inf.setInput(buf, 0, n);
542 while (inf.inflate(tmp, 0, tmp.length) > 0)
543 continue;
544 pos += n;
545 cnt -= n;
546 }
547 }
548 if (!inf.finished() || inf.getBytesRead() != dataLength) {
549 setCorrupt(src.offset);
550 throw new EOFException(MessageFormat.format(
551 JGitText.get().shortCompressedStreamAt,
552 Long.valueOf(src.offset)));
553 }
554 assert(crc1 != null);
555 expectedCRC = crc1.getValue();
556 } else {
557 expectedCRC = -1;
558 }
559 } catch (DataFormatException dataFormat) {
560 setCorrupt(src.offset);
561
562 CorruptObjectException corruptObject = new CorruptObjectException(
563 MessageFormat.format(
564 JGitText.get().objectAtHasBadZlibStream,
565 Long.valueOf(src.offset), getPackFile()),
566 dataFormat);
567
568 throw new StoredObjectRepresentationNotAvailableException(src,
569 corruptObject);
570
571 } catch (IOException ioError) {
572 throw new StoredObjectRepresentationNotAvailableException(src,
573 ioError);
574 }
575
576 if (quickCopy != null) {
577
578
579
580 out.writeHeader(src, inflatedLength);
581 quickCopy.write(out, dataOffset, (int) dataLength);
582
583 } else if (dataLength <= buf.length) {
584
585
586
587 if (!validate) {
588 long pos = dataOffset;
589 long cnt = dataLength;
590 while (cnt > 0) {
591 final int n = (int) Math.min(cnt, buf.length);
592 readFully(pos, buf, 0, n, curs);
593 pos += n;
594 cnt -= n;
595 }
596 }
597 out.writeHeader(src, inflatedLength);
598 out.write(buf, 0, (int) dataLength);
599 } else {
600
601
602
603
604 out.writeHeader(src, inflatedLength);
605 long pos = dataOffset;
606 long cnt = dataLength;
607 while (cnt > 0) {
608 final int n = (int) Math.min(cnt, buf.length);
609 readFully(pos, buf, 0, n, curs);
610 if (validate) {
611 assert(crc2 != null);
612 crc2.update(buf, 0, n);
613 }
614 out.write(buf, 0, n);
615 pos += n;
616 cnt -= n;
617 }
618 if (validate) {
619 assert(crc2 != null);
620 if (crc2.getValue() != expectedCRC) {
621 throw new CorruptObjectException(MessageFormat.format(
622 JGitText.get().objectAtHasBadZlibStream,
623 Long.valueOf(src.offset), getPackFile()));
624 }
625 }
626 }
627 }
628
629 boolean invalid() {
630 return invalid;
631 }
632
633 void setInvalid() {
634 invalid = true;
635 }
636
637 int incrementTransientErrorCount() {
638 return transientErrorCount.incrementAndGet();
639 }
640
641 void resetTransientErrorCount() {
642 transientErrorCount.set(0);
643 }
644
645 private void readFully(final long position, final byte[] dstbuf,
646 int dstoff, final int cnt, final WindowCursor curs)
647 throws IOException {
648 if (curs.copy(this, position, dstbuf, dstoff, cnt) != cnt)
649 throw new EOFException();
650 }
651
652 private synchronized void beginCopyAsIs(ObjectToPack otp)
653 throws StoredObjectRepresentationNotAvailableException {
654 if (++activeCopyRawData == 1 && activeWindows == 0) {
655 try {
656 doOpen();
657 } catch (IOException thisPackNotValid) {
658 throw new StoredObjectRepresentationNotAvailableException(otp,
659 thisPackNotValid);
660 }
661 }
662 }
663
664 private synchronized void endCopyAsIs() {
665 if (--activeCopyRawData == 0 && activeWindows == 0)
666 doClose();
667 }
668
669 synchronized boolean beginWindowCache() throws IOException {
670 if (++activeWindows == 1) {
671 if (activeCopyRawData == 0)
672 doOpen();
673 return true;
674 }
675 return false;
676 }
677
678 synchronized boolean endWindowCache() {
679 final boolean r = --activeWindows == 0;
680 if (r && activeCopyRawData == 0)
681 doClose();
682 return r;
683 }
684
685 private void doOpen() throws IOException {
686 if (invalid) {
687 throw new PackInvalidException(packFile, invalidatingCause);
688 }
689 try {
690 synchronized (readLock) {
691 fd = new RandomAccessFile(packFile, "r");
692 length = fd.length();
693 onOpenPack();
694 }
695 } catch (InterruptedIOException e) {
696
697 openFail(false, e);
698 throw e;
699 } catch (FileNotFoundException fn) {
700
701
702
703 openFail(!packFile.exists(), fn);
704 throw fn;
705 } catch (EOFException | AccessDeniedException | NoSuchFileException
706 | CorruptObjectException | NoPackSignatureException
707 | PackMismatchException | UnpackException
708 | UnsupportedPackIndexVersionException
709 | UnsupportedPackVersionException pe) {
710
711 openFail(true, pe);
712 throw pe;
713 } catch (IOException | RuntimeException ge) {
714
715
716 openFail(false, ge);
717 throw ge;
718 }
719 }
720
721 private void openFail(boolean invalidate, Exception cause) {
722 activeWindows = 0;
723 activeCopyRawData = 0;
724 invalid = invalidate;
725 invalidatingCause = cause;
726 doClose();
727 }
728
729 private void doClose() {
730 synchronized (readLock) {
731 if (fd != null) {
732 try {
733 fd.close();
734 } catch (IOException err) {
735
736
737
738 }
739 fd = null;
740 }
741 }
742 }
743
744 ByteArrayWindow read(long pos, int size) throws IOException {
745 synchronized (readLock) {
746 if (invalid || fd == null) {
747
748
749
750
751
752 throw new PackInvalidException(packFile, invalidatingCause);
753 }
754 if (length < pos + size)
755 size = (int) (length - pos);
756 final byte[] buf = new byte[size];
757 fd.seek(pos);
758 fd.readFully(buf, 0, size);
759 return new ByteArrayWindow(this, pos, buf);
760 }
761 }
762
763 ByteWindow mmap(long pos, int size) throws IOException {
764 synchronized (readLock) {
765 if (length < pos + size)
766 size = (int) (length - pos);
767
768 MappedByteBuffer map;
769 try {
770 map = fd.getChannel().map(MapMode.READ_ONLY, pos, size);
771 } catch (IOException ioe1) {
772
773
774
775
776 System.gc();
777 System.runFinalization();
778 map = fd.getChannel().map(MapMode.READ_ONLY, pos, size);
779 }
780
781 if (map.hasArray())
782 return new ByteArrayWindow(this, pos, map.array());
783 return new ByteBufferWindow(this, pos, map);
784 }
785 }
786
787 private void onOpenPack() throws IOException {
788 final PackIndex idx = idx();
789 final byte[] buf = new byte[20];
790
791 fd.seek(0);
792 fd.readFully(buf, 0, 12);
793 if (RawParseUtils.match(buf, 0, Constants.PACK_SIGNATURE) != 4) {
794 throw new NoPackSignatureException(JGitText.get().notAPACKFile);
795 }
796 final long vers = NB.decodeUInt32(buf, 4);
797 final long packCnt = NB.decodeUInt32(buf, 8);
798 if (vers != 2 && vers != 3) {
799 throw new UnsupportedPackVersionException(vers);
800 }
801
802 if (packCnt != idx.getObjectCount()) {
803 throw new PackMismatchException(MessageFormat.format(
804 JGitText.get().packObjectCountMismatch,
805 Long.valueOf(packCnt), Long.valueOf(idx.getObjectCount()),
806 getPackFile()));
807 }
808
809 fd.seek(length - 20);
810 fd.readFully(buf, 0, 20);
811 if (!Arrays.equals(buf, packChecksum)) {
812 throw new PackMismatchException(MessageFormat.format(
813 JGitText.get().packChecksumMismatch,
814 getPackFile(),
815 ObjectId.fromRaw(buf).name(),
816 ObjectId.fromRaw(idx.packChecksum).name()));
817 }
818 }
819
820 ObjectLoader load(WindowCursor curs, long pos)
821 throws IOException, LargeObjectException {
822 try {
823 final byte[] ib = curs.tempId;
824 Delta delta = null;
825 byte[] data = null;
826 int type = Constants.OBJ_BAD;
827 boolean cached = false;
828
829 SEARCH: for (;;) {
830 readFully(pos, ib, 0, 20, curs);
831 int c = ib[0] & 0xff;
832 final int typeCode = (c >> 4) & 7;
833 long sz = c & 15;
834 int shift = 4;
835 int p = 1;
836 while ((c & 0x80) != 0) {
837 c = ib[p++] & 0xff;
838 sz += ((long) (c & 0x7f)) << shift;
839 shift += 7;
840 }
841
842 switch (typeCode) {
843 case Constants.OBJ_COMMIT:
844 case Constants.OBJ_TREE:
845 case Constants.OBJ_BLOB:
846 case Constants.OBJ_TAG: {
847 if (delta != null || sz < curs.getStreamFileThreshold())
848 data = decompress(pos + p, (int) sz, curs);
849
850 if (delta != null) {
851 type = typeCode;
852 break SEARCH;
853 }
854
855 if (data != null)
856 return new ObjectLoader.SmallObject(typeCode, data);
857 else
858 return new LargePackedWholeObject(typeCode, sz, pos, p,
859 this, curs.db);
860 }
861
862 case Constants.OBJ_OFS_DELTA: {
863 c = ib[p++] & 0xff;
864 long base = c & 127;
865 while ((c & 128) != 0) {
866 base += 1;
867 c = ib[p++] & 0xff;
868 base <<= 7;
869 base += (c & 127);
870 }
871 base = pos - base;
872 delta = new Delta(delta, pos, (int) sz, p, base);
873 if (sz != delta.deltaSize)
874 break SEARCH;
875
876 DeltaBaseCache.Entry e = curs.getDeltaBaseCache().get(this, base);
877 if (e != null) {
878 type = e.type;
879 data = e.data;
880 cached = true;
881 break SEARCH;
882 }
883 pos = base;
884 continue SEARCH;
885 }
886
887 case Constants.OBJ_REF_DELTA: {
888 readFully(pos + p, ib, 0, 20, curs);
889 long base = findDeltaBase(ObjectId.fromRaw(ib));
890 delta = new Delta(delta, pos, (int) sz, p + 20, base);
891 if (sz != delta.deltaSize)
892 break SEARCH;
893
894 DeltaBaseCache.Entry e = curs.getDeltaBaseCache().get(this, base);
895 if (e != null) {
896 type = e.type;
897 data = e.data;
898 cached = true;
899 break SEARCH;
900 }
901 pos = base;
902 continue SEARCH;
903 }
904
905 default:
906 throw new IOException(MessageFormat.format(
907 JGitText.get().unknownObjectType,
908 Integer.valueOf(typeCode)));
909 }
910 }
911
912
913
914
915 if (data == null)
916 throw new IOException(JGitText.get().inMemoryBufferLimitExceeded);
917
918 assert(delta != null);
919 do {
920
921 if (cached)
922 cached = false;
923 else if (delta.next == null)
924 curs.getDeltaBaseCache().store(this, delta.basePos, data, type);
925
926 pos = delta.deltaPos;
927
928 final byte[] cmds = decompress(pos + delta.hdrLen,
929 delta.deltaSize, curs);
930 if (cmds == null) {
931 data = null;
932 throw new LargeObjectException.OutOfMemory(new OutOfMemoryError());
933 }
934
935 final long sz = BinaryDelta.getResultSize(cmds);
936 if (Integer.MAX_VALUE <= sz)
937 throw new LargeObjectException.ExceedsByteArrayLimit();
938
939 final byte[] result;
940 try {
941 result = new byte[(int) sz];
942 } catch (OutOfMemoryError tooBig) {
943 data = null;
944 throw new LargeObjectException.OutOfMemory(tooBig);
945 }
946
947 BinaryDelta.apply(data, cmds, result);
948 data = result;
949 delta = delta.next;
950 } while (delta != null);
951
952 return new ObjectLoader.SmallObject(type, data);
953
954 } catch (DataFormatException dfe) {
955 throw new CorruptObjectException(
956 MessageFormat.format(
957 JGitText.get().objectAtHasBadZlibStream,
958 Long.valueOf(pos), getPackFile()),
959 dfe);
960 }
961 }
962
963 private long findDeltaBase(ObjectId baseId) throws IOException,
964 MissingObjectException {
965 long ofs = idx().findOffset(baseId);
966 if (ofs < 0)
967 throw new MissingObjectException(baseId,
968 JGitText.get().missingDeltaBase);
969 return ofs;
970 }
971
972 private static class Delta {
973
974 final Delta next;
975
976
977 final long deltaPos;
978
979
980 final int deltaSize;
981
982
983 final int hdrLen;
984
985
986 final long basePos;
987
988 Delta(Delta next, long ofs, int sz, int hdrLen, long baseOffset) {
989 this.next = next;
990 this.deltaPos = ofs;
991 this.deltaSize = sz;
992 this.hdrLen = hdrLen;
993 this.basePos = baseOffset;
994 }
995 }
996
997 byte[] getDeltaHeader(WindowCursor wc, long pos)
998 throws IOException, DataFormatException {
999
1000
1001
1002
1003
1004 final byte[] hdr = new byte[18];
1005 wc.inflate(this, pos, hdr, true );
1006 return hdr;
1007 }
1008
1009 int getObjectType(WindowCursor curs, long pos) throws IOException {
1010 final byte[] ib = curs.tempId;
1011 for (;;) {
1012 readFully(pos, ib, 0, 20, curs);
1013 int c = ib[0] & 0xff;
1014 final int type = (c >> 4) & 7;
1015
1016 switch (type) {
1017 case Constants.OBJ_COMMIT:
1018 case Constants.OBJ_TREE:
1019 case Constants.OBJ_BLOB:
1020 case Constants.OBJ_TAG:
1021 return type;
1022
1023 case Constants.OBJ_OFS_DELTA: {
1024 int p = 1;
1025 while ((c & 0x80) != 0)
1026 c = ib[p++] & 0xff;
1027 c = ib[p++] & 0xff;
1028 long ofs = c & 127;
1029 while ((c & 128) != 0) {
1030 ofs += 1;
1031 c = ib[p++] & 0xff;
1032 ofs <<= 7;
1033 ofs += (c & 127);
1034 }
1035 pos = pos - ofs;
1036 continue;
1037 }
1038
1039 case Constants.OBJ_REF_DELTA: {
1040 int p = 1;
1041 while ((c & 0x80) != 0)
1042 c = ib[p++] & 0xff;
1043 readFully(pos + p, ib, 0, 20, curs);
1044 pos = findDeltaBase(ObjectId.fromRaw(ib));
1045 continue;
1046 }
1047
1048 default:
1049 throw new IOException(
1050 MessageFormat.format(JGitText.get().unknownObjectType,
1051 Integer.valueOf(type)));
1052 }
1053 }
1054 }
1055
1056 long getObjectSize(WindowCursor curs, AnyObjectId id)
1057 throws IOException {
1058 final long offset = idx().findOffset(id);
1059 return 0 < offset ? getObjectSize(curs, offset) : -1;
1060 }
1061
1062 long getObjectSize(WindowCursor curs, long pos)
1063 throws IOException {
1064 final byte[] ib = curs.tempId;
1065 readFully(pos, ib, 0, 20, curs);
1066 int c = ib[0] & 0xff;
1067 final int type = (c >> 4) & 7;
1068 long sz = c & 15;
1069 int shift = 4;
1070 int p = 1;
1071 while ((c & 0x80) != 0) {
1072 c = ib[p++] & 0xff;
1073 sz += ((long) (c & 0x7f)) << shift;
1074 shift += 7;
1075 }
1076
1077 long deltaAt;
1078 switch (type) {
1079 case Constants.OBJ_COMMIT:
1080 case Constants.OBJ_TREE:
1081 case Constants.OBJ_BLOB:
1082 case Constants.OBJ_TAG:
1083 return sz;
1084
1085 case Constants.OBJ_OFS_DELTA:
1086 c = ib[p++] & 0xff;
1087 while ((c & 128) != 0)
1088 c = ib[p++] & 0xff;
1089 deltaAt = pos + p;
1090 break;
1091
1092 case Constants.OBJ_REF_DELTA:
1093 deltaAt = pos + p + 20;
1094 break;
1095
1096 default:
1097 throw new IOException(MessageFormat.format(
1098 JGitText.get().unknownObjectType, Integer.valueOf(type)));
1099 }
1100
1101 try {
1102 return BinaryDelta.getResultSize(getDeltaHeader(curs, deltaAt));
1103 } catch (DataFormatException e) {
1104 throw new CorruptObjectException(MessageFormat.format(
1105 JGitText.get().objectAtHasBadZlibStream, Long.valueOf(pos),
1106 getPackFile()));
1107 }
1108 }
1109
1110 LocalObjectRepresentation representation(final WindowCursor curs,
1111 final AnyObjectId objectId) throws IOException {
1112 final long pos = idx().findOffset(objectId);
1113 if (pos < 0)
1114 return null;
1115
1116 final byte[] ib = curs.tempId;
1117 readFully(pos, ib, 0, 20, curs);
1118 int c = ib[0] & 0xff;
1119 int p = 1;
1120 final int typeCode = (c >> 4) & 7;
1121 while ((c & 0x80) != 0)
1122 c = ib[p++] & 0xff;
1123
1124 long len = (findEndOffset(pos) - pos);
1125 switch (typeCode) {
1126 case Constants.OBJ_COMMIT:
1127 case Constants.OBJ_TREE:
1128 case Constants.OBJ_BLOB:
1129 case Constants.OBJ_TAG:
1130 return LocalObjectRepresentation.newWhole(this, pos, len - p);
1131
1132 case Constants.OBJ_OFS_DELTA: {
1133 c = ib[p++] & 0xff;
1134 long ofs = c & 127;
1135 while ((c & 128) != 0) {
1136 ofs += 1;
1137 c = ib[p++] & 0xff;
1138 ofs <<= 7;
1139 ofs += (c & 127);
1140 }
1141 ofs = pos - ofs;
1142 return LocalObjectRepresentation.newDelta(this, pos, len - p, ofs);
1143 }
1144
1145 case Constants.OBJ_REF_DELTA: {
1146 len -= p;
1147 len -= Constants.OBJECT_ID_LENGTH;
1148 readFully(pos + p, ib, 0, 20, curs);
1149 ObjectId id = ObjectId.fromRaw(ib);
1150 return LocalObjectRepresentation.newDelta(this, pos, len, id);
1151 }
1152
1153 default:
1154 throw new IOException(
1155 MessageFormat.format(JGitText.get().unknownObjectType,
1156 Integer.valueOf(typeCode)));
1157 }
1158 }
1159
1160 private long findEndOffset(long startOffset)
1161 throws IOException, CorruptObjectException {
1162 final long maxOffset = length - 20;
1163 return getReverseIdx().findNextOffset(startOffset, maxOffset);
1164 }
1165
1166 synchronized PackBitmapIndex getBitmapIndex() throws IOException {
1167 if (invalid || invalidBitmap)
1168 return null;
1169 if (bitmapIdx == null && hasExt(BITMAP_INDEX)) {
1170 final PackBitmapIndex idx;
1171 try {
1172 idx = PackBitmapIndex.open(extFile(BITMAP_INDEX), idx(),
1173 getReverseIdx());
1174 } catch (FileNotFoundException e) {
1175
1176
1177
1178 invalidBitmap = true;
1179 return null;
1180 }
1181
1182
1183 if (Arrays.equals(packChecksum, idx.packChecksum))
1184 bitmapIdx = idx;
1185 else
1186 invalidBitmap = true;
1187 }
1188 return bitmapIdx;
1189 }
1190
1191 private synchronized PackReverseIndex getReverseIdx() throws IOException {
1192 if (reverseIdx == null)
1193 reverseIdx = new PackReverseIndex(idx());
1194 return reverseIdx;
1195 }
1196
1197 private boolean isCorrupt(long offset) {
1198 LongList list = corruptObjects;
1199 if (list == null)
1200 return false;
1201 synchronized (list) {
1202 return list.contains(offset);
1203 }
1204 }
1205
1206 private void setCorrupt(long offset) {
1207 LongList list = corruptObjects;
1208 if (list == null) {
1209 synchronized (readLock) {
1210 list = corruptObjects;
1211 if (list == null) {
1212 list = new LongList();
1213 corruptObjects = list;
1214 }
1215 }
1216 }
1217 synchronized (list) {
1218 list.add(offset);
1219 }
1220 }
1221
1222 private File extFile(PackExt ext) {
1223 String p = packFile.getName();
1224 int dot = p.lastIndexOf('.');
1225 String b = (dot < 0) ? p : p.substring(0, dot);
1226 return new File(packFile.getParentFile(), b + '.' + ext.getExtension());
1227 }
1228
1229 private boolean hasExt(PackExt ext) {
1230 return (extensions & ext.getBit()) != 0;
1231 }
1232
1233 @SuppressWarnings("nls")
1234 @Override
1235 public String toString() {
1236 return "PackFile [packFileName=" + packFile.getName() + ", length="
1237 + packFile.length() + ", packChecksum="
1238 + ObjectId.fromRaw(packChecksum).name() + "]";
1239 }
1240 }