1
2
3
4
5
6
7
8
9
10
11 package org.eclipse.jgit.internal.storage.file;
12
13 import static java.nio.charset.StandardCharsets.UTF_8;
14 import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
15 import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
16 import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
17
18 import java.io.BufferedReader;
19 import java.io.File;
20 import java.io.FileNotFoundException;
21 import java.io.IOException;
22 import java.io.OutputStream;
23 import java.nio.file.Files;
24 import java.text.MessageFormat;
25 import java.util.ArrayList;
26 import java.util.Collection;
27 import java.util.Collections;
28 import java.util.HashSet;
29 import java.util.List;
30 import java.util.Objects;
31 import java.util.Set;
32 import java.util.concurrent.atomic.AtomicReference;
33
34 import org.eclipse.jgit.internal.JGitText;
35 import org.eclipse.jgit.internal.storage.pack.ObjectToPack;
36 import org.eclipse.jgit.internal.storage.pack.PackExt;
37 import org.eclipse.jgit.internal.storage.pack.PackWriter;
38 import org.eclipse.jgit.lib.AbbreviatedObjectId;
39 import org.eclipse.jgit.lib.AnyObjectId;
40 import org.eclipse.jgit.lib.Config;
41 import org.eclipse.jgit.lib.Constants;
42 import org.eclipse.jgit.lib.ObjectDatabase;
43 import org.eclipse.jgit.lib.ObjectId;
44 import org.eclipse.jgit.lib.ObjectLoader;
45 import org.eclipse.jgit.lib.RepositoryCache;
46 import org.eclipse.jgit.lib.RepositoryCache.FileKey;
47 import org.eclipse.jgit.util.FS;
48 import org.eclipse.jgit.util.FileUtils;
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68 public class ObjectDirectory extends FileObjectDatabase {
69
70 private static final int RESOLVE_ABBREV_LIMIT = 256;
71
72 private final AlternateHandle handle = new AlternateHandle(this);
73
74 private final Config config;
75
76 private final File objects;
77
78 private final File infoDirectory;
79
80 private final LooseObjects loose;
81
82 private final PackDirectory packed;
83
84 private final PackDirectory preserved;
85
86 private final File alternatesFile;
87
88 private final FS fs;
89
90 private final AtomicReference<AlternateHandle[]> alternates;
91
92 private final File shallowFile;
93
94 private FileSnapshot shallowFileSnapshot = FileSnapshot.DIRTY;
95
96 private Set<ObjectId> shallowCommitsIds;
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116 public ObjectDirectory(final Config cfg, final File dir,
117 File[] alternatePaths, FS fs, File shallowFile) throws IOException {
118 config = cfg;
119 objects = dir;
120 infoDirectory = new File(objects, "info");
121 File packDirectory = new File(objects, "pack");
122 File preservedDirectory = new File(packDirectory, "preserved");
123 alternatesFile = new File(objects, Constants.INFO_ALTERNATES);
124 loose = new LooseObjects(objects);
125 packed = new PackDirectory(config, packDirectory);
126 preserved = new PackDirectory(config, preservedDirectory);
127 this.fs = fs;
128 this.shallowFile = shallowFile;
129
130 alternates = new AtomicReference<>();
131 if (alternatePaths != null) {
132 AlternateHandle[] alt;
133
134 alt = new AlternateHandle[alternatePaths.length];
135 for (int i = 0; i < alternatePaths.length; i++)
136 alt[i] = openAlternate(alternatePaths[i]);
137 alternates.set(alt);
138 }
139 }
140
141
142 @Override
143 public final File getDirectory() {
144 return loose.getDirectory();
145 }
146
147
148
149
150
151
152 public final File getPackDirectory() {
153 return packed.getDirectory();
154 }
155
156
157
158
159
160
161 public final File getPreservedDirectory() {
162 return preserved.getDirectory();
163 }
164
165
166 @Override
167 public boolean exists() {
168 return fs.exists(objects);
169 }
170
171
172 @Override
173 public void create() throws IOException {
174 loose.create();
175 FileUtils.mkdir(infoDirectory);
176 packed.create();
177 }
178
179
180 @Override
181 public ObjectDirectoryInserter newInserter() {
182 return new ObjectDirectoryInserter(this, config);
183 }
184
185
186
187
188
189
190
191 public PackInserter newPackInserter() {
192 return new PackInserter(this);
193 }
194
195
196 @Override
197 public void close() {
198 loose.close();
199
200 packed.close();
201
202
203 AlternateHandle[] alt = alternates.get();
204 if (alt != null && alternates.compareAndSet(alt, null)) {
205 for(AlternateHandle od : alt)
206 od.close();
207 }
208 }
209
210
211 @Override
212 public Collection<Pack> getPacks() {
213 return packed.getPacks();
214 }
215
216
217 @Override
218 public long getApproximateObjectCount() {
219 long count = 0;
220 for (Pack p : getPacks()) {
221 try {
222 count += p.getIndex().getObjectCount();
223 } catch (IOException e) {
224 return -1;
225 }
226 }
227 return count;
228 }
229
230
231
232
233
234
235 @Override
236 public Pack openPack(File pack) throws IOException {
237 PackFile pf;
238 try {
239 pf = new PackFile(pack);
240 } catch (IllegalArgumentException e) {
241 throw new IOException(
242 MessageFormat.format(JGitText.get().notAValidPack, pack),
243 e);
244 }
245
246 String p = pf.getName();
247
248 if (p.length() != 50 || !p.startsWith("pack-")
249 || !pf.getPackExt().equals(PACK)) {
250 throw new IOException(
251 MessageFormat.format(JGitText.get().notAValidPack, pack));
252 }
253
254 PackFile bitmapIdx = pf.create(BITMAP_INDEX);
255 Pack res = new Pack(pack, bitmapIdx.exists() ? bitmapIdx : null);
256 packed.insert(res);
257 return res;
258 }
259
260
261 @Override
262 public String toString() {
263 return "ObjectDirectory[" + getDirectory() + "]";
264 }
265
266
267 @Override
268 public boolean has(AnyObjectId objectId) {
269 return loose.hasCached(objectId)
270 || hasPackedOrLooseInSelfOrAlternate(objectId)
271 || (restoreFromSelfOrAlternate(objectId, null)
272 && hasPackedOrLooseInSelfOrAlternate(objectId));
273 }
274
275 private boolean hasPackedOrLooseInSelfOrAlternate(AnyObjectId objectId) {
276 return hasPackedInSelfOrAlternate(objectId, null)
277 || hasLooseInSelfOrAlternate(objectId, null);
278 }
279
280 private boolean hasPackedInSelfOrAlternate(AnyObjectId objectId,
281 Set<AlternateHandle.Id> skips) {
282 if (hasPackedObject(objectId)) {
283 return true;
284 }
285 skips = addMe(skips);
286 for (AlternateHandle alt : myAlternates()) {
287 if (!skips.contains(alt.getId())) {
288 if (alt.db.hasPackedInSelfOrAlternate(objectId, skips)) {
289 return true;
290 }
291 }
292 }
293 return false;
294 }
295
296 private boolean hasLooseInSelfOrAlternate(AnyObjectId objectId,
297 Set<AlternateHandle.Id> skips) {
298 if (loose.has(objectId)) {
299 return true;
300 }
301 skips = addMe(skips);
302 for (AlternateHandle alt : myAlternates()) {
303 if (!skips.contains(alt.getId())) {
304 if (alt.db.hasLooseInSelfOrAlternate(objectId, skips)) {
305 return true;
306 }
307 }
308 }
309 return false;
310 }
311
312 boolean hasPackedObject(AnyObjectId objectId) {
313 return packed.has(objectId);
314 }
315
316 @Override
317 void resolve(Set<ObjectId> matches, AbbreviatedObjectId id)
318 throws IOException {
319 resolve(matches, id, null);
320 }
321
322 private void resolve(Set<ObjectId> matches, AbbreviatedObjectId id,
323 Set<AlternateHandle.Id> skips)
324 throws IOException {
325 if (!packed.resolve(matches, id, RESOLVE_ABBREV_LIMIT))
326 return;
327
328 if (!loose.resolve(matches, id, RESOLVE_ABBREV_LIMIT))
329 return;
330
331 skips = addMe(skips);
332 for (AlternateHandle alt : myAlternates()) {
333 if (!skips.contains(alt.getId())) {
334 alt.db.resolve(matches, id, skips);
335 if (matches.size() > RESOLVE_ABBREV_LIMIT) {
336 return;
337 }
338 }
339 }
340 }
341
342 @Override
343 ObjectLoader openObject(WindowCursor curs, AnyObjectId objectId)
344 throws IOException {
345 ObjectLoader ldr = openObjectWithoutRestoring(curs, objectId);
346 if (ldr == null && restoreFromSelfOrAlternate(objectId, null)) {
347 ldr = openObjectWithoutRestoring(curs, objectId);
348 }
349 return ldr;
350 }
351
352 private ObjectLoader openObjectWithoutRestoring(WindowCursor curs, AnyObjectId objectId)
353 throws IOException {
354 if (loose.hasCached(objectId)) {
355 ObjectLoader ldr = openLooseObject(curs, objectId);
356 if (ldr != null) {
357 return ldr;
358 }
359 }
360 ObjectLoader ldr = openPackedFromSelfOrAlternate(curs, objectId, null);
361 if (ldr != null) {
362 return ldr;
363 }
364 return openLooseFromSelfOrAlternate(curs, objectId, null);
365 }
366
367 private ObjectLoader openPackedFromSelfOrAlternate(WindowCursor curs,
368 AnyObjectId objectId, Set<AlternateHandle.Id> skips) {
369 ObjectLoader ldr = openPackedObject(curs, objectId);
370 if (ldr != null) {
371 return ldr;
372 }
373 skips = addMe(skips);
374 for (AlternateHandle alt : myAlternates()) {
375 if (!skips.contains(alt.getId())) {
376 ldr = alt.db.openPackedFromSelfOrAlternate(curs, objectId, skips);
377 if (ldr != null) {
378 return ldr;
379 }
380 }
381 }
382 return null;
383 }
384
385 private ObjectLoader openLooseFromSelfOrAlternate(WindowCursor curs,
386 AnyObjectId objectId, Set<AlternateHandle.Id> skips)
387 throws IOException {
388 ObjectLoader ldr = openLooseObject(curs, objectId);
389 if (ldr != null) {
390 return ldr;
391 }
392 skips = addMe(skips);
393 for (AlternateHandle alt : myAlternates()) {
394 if (!skips.contains(alt.getId())) {
395 ldr = alt.db.openLooseFromSelfOrAlternate(curs, objectId, skips);
396 if (ldr != null) {
397 return ldr;
398 }
399 }
400 }
401 return null;
402 }
403
404 ObjectLoader openPackedObject(WindowCursor curs, AnyObjectId objectId) {
405 return packed.open(curs, objectId);
406 }
407
408 @Override
409 ObjectLoader openLooseObject(WindowCursor curs, AnyObjectId id)
410 throws IOException {
411 return loose.open(curs, id);
412 }
413
414 @Override
415 long getObjectSize(WindowCursor curs, AnyObjectId id) throws IOException {
416 long sz = getObjectSizeWithoutRestoring(curs, id);
417 if (0 > sz && restoreFromSelfOrAlternate(id, null)) {
418 sz = getObjectSizeWithoutRestoring(curs, id);
419 }
420 return sz;
421 }
422
423 private long getObjectSizeWithoutRestoring(WindowCursor curs,
424 AnyObjectId id) throws IOException {
425 if (loose.hasCached(id)) {
426 long len = loose.getSize(curs, id);
427 if (0 <= len) {
428 return len;
429 }
430 }
431 long len = getPackedSizeFromSelfOrAlternate(curs, id, null);
432 if (0 <= len) {
433 return len;
434 }
435 return getLooseSizeFromSelfOrAlternate(curs, id, null);
436 }
437
438 private long getPackedSizeFromSelfOrAlternate(WindowCursor curs,
439 AnyObjectId id, Set<AlternateHandle.Id> skips) {
440 long len = packed.getSize(curs, id);
441 if (0 <= len) {
442 return len;
443 }
444 skips = addMe(skips);
445 for (AlternateHandle alt : myAlternates()) {
446 if (!skips.contains(alt.getId())) {
447 len = alt.db.getPackedSizeFromSelfOrAlternate(curs, id, skips);
448 if (0 <= len) {
449 return len;
450 }
451 }
452 }
453 return -1;
454 }
455
456 private long getLooseSizeFromSelfOrAlternate(WindowCursor curs,
457 AnyObjectId id, Set<AlternateHandle.Id> skips) throws IOException {
458 long len = loose.getSize(curs, id);
459 if (0 <= len) {
460 return len;
461 }
462 skips = addMe(skips);
463 for (AlternateHandle alt : myAlternates()) {
464 if (!skips.contains(alt.getId())) {
465 len = alt.db.getLooseSizeFromSelfOrAlternate(curs, id, skips);
466 if (0 <= len) {
467 return len;
468 }
469 }
470 }
471 return -1;
472 }
473
474 @Override
475 void selectObjectRepresentation(PackWriter packer, ObjectToPack otp,
476 WindowCursor curs) throws IOException {
477 selectObjectRepresentation(packer, otp, curs, null);
478 }
479
480 private void selectObjectRepresentation(PackWriter packer, ObjectToPack otp,
481 WindowCursor curs, Set<AlternateHandle.Id> skips) throws IOException {
482 packed.selectRepresentation(packer, otp, curs);
483
484 skips = addMe(skips);
485 for (AlternateHandle h : myAlternates()) {
486 if (!skips.contains(h.getId())) {
487 h.db.selectObjectRepresentation(packer, otp, curs, skips);
488 }
489 }
490 }
491
492 private boolean restoreFromSelfOrAlternate(AnyObjectId objectId,
493 Set<AlternateHandle.Id> skips) {
494 if (restoreFromSelf(objectId)) {
495 return true;
496 }
497
498 skips = addMe(skips);
499 for (AlternateHandle alt : myAlternates()) {
500 if (!skips.contains(alt.getId())) {
501 if (alt.db.restoreFromSelfOrAlternate(objectId, skips)) {
502 return true;
503 }
504 }
505 }
506 return false;
507 }
508
509 private boolean restoreFromSelf(AnyObjectId objectId) {
510 Pack preservedPack = preserved.getPack(objectId);
511 if (preservedPack == null) {
512 return false;
513 }
514 PackFile preservedFile = new PackFile(preservedPack.getPackFile());
515
516
517 for (PackExt ext : PackExt.values()) {
518 if (!INDEX.equals(ext)) {
519 restore(preservedFile.create(ext));
520 }
521 }
522 restore(preservedFile.create(INDEX));
523 return true;
524 }
525
526 private boolean restore(PackFile preservedPack) {
527 PackFile restored = preservedPack
528 .createForDirectory(packed.getDirectory());
529 try {
530 Files.createLink(restored.toPath(), preservedPack.toPath());
531 } catch (IOException e) {
532 return false;
533 }
534 return true;
535 }
536
537 @Override
538 InsertLooseObjectResult insertUnpackedObject(File tmp, ObjectId id,
539 boolean createDuplicate) throws IOException {
540
541
542 if (loose.hasCached(id)) {
543 FileUtils.delete(tmp, FileUtils.RETRY);
544 return InsertLooseObjectResult.EXISTS_LOOSE;
545 }
546 if (!createDuplicate && has(id)) {
547 FileUtils.delete(tmp, FileUtils.RETRY);
548 return InsertLooseObjectResult.EXISTS_PACKED;
549 }
550 return loose.insert(tmp, id);
551 }
552
553 @Override
554 Config getConfig() {
555 return config;
556 }
557
558 @Override
559 FS getFS() {
560 return fs;
561 }
562
563 @Override
564 public Set<ObjectId> getShallowCommits() throws IOException {
565 if (shallowFile == null || !shallowFile.isFile())
566 return Collections.emptySet();
567
568 if (shallowFileSnapshot == null
569 || shallowFileSnapshot.isModified(shallowFile)) {
570 try {
571 shallowCommitsIds = FileUtils.readWithRetries(shallowFile,
572 f -> {
573 FileSnapshot newSnapshot = FileSnapshot.save(f);
574 HashSet<ObjectId> result = new HashSet<>();
575 try (BufferedReader reader = open(f)) {
576 String line;
577 while ((line = reader.readLine()) != null) {
578 if (!ObjectId.isId(line)) {
579 throw new IOException(
580 MessageFormat.format(JGitText
581 .get().badShallowLine,
582 f.getAbsolutePath(),
583 line));
584
585 }
586 result.add(ObjectId.fromString(line));
587 }
588 }
589 shallowFileSnapshot = newSnapshot;
590 return result;
591 });
592 } catch (IOException e) {
593 throw e;
594 } catch (Exception e) {
595 throw new IOException(
596 MessageFormat.format(JGitText.get().readShallowFailed,
597 shallowFile.getAbsolutePath()),
598 e);
599 }
600 }
601
602 return shallowCommitsIds;
603 }
604
605 @Override
606 public void setShallowCommits(Set<ObjectId> shallowCommits) throws IOException {
607 this.shallowCommitsIds = shallowCommits;
608 LockFile lock = new LockFile(shallowFile);
609 if (!lock.lock()) {
610 throw new IOException(MessageFormat.format(JGitText.get().lockError,
611 shallowFile.getAbsolutePath()));
612 }
613
614 try {
615 if (shallowCommits.isEmpty()) {
616 if (shallowFile.isFile()) {
617 shallowFile.delete();
618 }
619 } else {
620 try (OutputStream out = lock.getOutputStream()) {
621 for (ObjectId shallowCommit : shallowCommits) {
622 byte[] buf = new byte[Constants.OBJECT_ID_STRING_LENGTH + 1];
623 shallowCommit.copyTo(buf, 0);
624 buf[Constants.OBJECT_ID_STRING_LENGTH] = '\n';
625 out.write(buf);
626 }
627 } finally {
628 lock.commit();
629 }
630 }
631 } finally {
632 lock.unlock();
633 }
634
635 if (shallowCommits.isEmpty()) {
636 shallowFileSnapshot = FileSnapshot.DIRTY;
637 } else {
638 shallowFileSnapshot = FileSnapshot.save(shallowFile);
639 }
640 }
641
642 void closeAllPackHandles(File packFile) {
643
644
645
646
647 if (packFile.exists()) {
648 for (Pack p : packed.getPacks()) {
649 if (packFile.getPath().equals(p.getPackFile().getPath())) {
650 p.close();
651 break;
652 }
653 }
654 }
655 }
656
657 AlternateHandle[] myAlternates() {
658 AlternateHandle[] alt = alternates.get();
659 if (alt == null) {
660 synchronized (alternates) {
661 alt = alternates.get();
662 if (alt == null) {
663 try {
664 alt = loadAlternates();
665 } catch (IOException e) {
666 alt = new AlternateHandle[0];
667 }
668 alternates.set(alt);
669 }
670 }
671 }
672 return alt;
673 }
674
675 Set<AlternateHandle.Id> addMe(Set<AlternateHandle.Id> skips) {
676 if (skips == null) {
677 skips = new HashSet<>();
678 }
679 skips.add(handle.getId());
680 return skips;
681 }
682
683 private AlternateHandle[] loadAlternates() throws IOException {
684 final List<AlternateHandle> l = new ArrayList<>(4);
685 try (BufferedReader br = open(alternatesFile)) {
686 String line;
687 while ((line = br.readLine()) != null) {
688 l.add(openAlternate(line));
689 }
690 }
691 return l.toArray(new AlternateHandle[0]);
692 }
693
694 private static BufferedReader open(File f)
695 throws IOException, FileNotFoundException {
696 return Files.newBufferedReader(f.toPath(), UTF_8);
697 }
698
699 private AlternateHandle openAlternate(String location)
700 throws IOException {
701 final File objdir = fs.resolve(objects, location);
702 return openAlternate(objdir);
703 }
704
705 private AlternateHandle openAlternate(File objdir) throws IOException {
706 final File parent = objdir.getParentFile();
707 if (FileKey.isGitRepository(parent, fs)) {
708 FileKey key = FileKey.exact(parent, fs);
709 FileRepository db = (FileRepository) RepositoryCache.open(key);
710 return new AlternateRepository(db);
711 }
712
713 ObjectDirectory db = new ObjectDirectory(config, objdir, null, fs, null);
714 return new AlternateHandle(db);
715 }
716
717
718
719
720 @Override
721 public File fileFor(AnyObjectId objectId) {
722 return loose.fileFor(objectId);
723 }
724
725 static class AlternateHandle {
726 static class Id {
727 private final String alternateId;
728
729 public Id(File object) {
730 String id = null;
731 try {
732
733 id = object.toPath().toRealPath().normalize().toString();
734 } catch (Exception ignored) {
735
736 }
737 this.alternateId = id;
738 }
739
740 @Override
741 public boolean equals(Object o) {
742 if (o == this) {
743 return true;
744 }
745 if (o == null || !(o instanceof Id)) {
746 return false;
747 }
748 Id aId = (Id) o;
749 return Objects.equals(alternateId, aId.alternateId);
750 }
751
752 @Override
753 public int hashCode() {
754 if (alternateId == null) {
755 return 1;
756 }
757 return alternateId.hashCode();
758 }
759 }
760
761 final ObjectDirectory db;
762
763 private AlternateHandle.Id id;
764
765 AlternateHandle(ObjectDirectory db) {
766 this.db = db;
767 }
768
769 void close() {
770 db.close();
771 }
772
773 public synchronized Id getId() {
774 if (id == null) {
775 id = new AlternateHandle.Id(db.objects);
776 }
777 return id;
778 }
779 }
780
781 static class AlternateRepository extends AlternateHandle {
782 final FileRepository repository;
783
784 AlternateRepository(FileRepository r) {
785 super(r.getObjectDatabase());
786 repository = r;
787 }
788
789 @Override
790 void close() {
791 repository.close();
792 }
793 }
794
795
796 @Override
797 public ObjectDatabase newCachedDatabase() {
798 return newCachedFileObjectDatabase();
799 }
800
801 CachedObjectDirectory newCachedFileObjectDatabase() {
802 return new CachedObjectDirectory(this);
803 }
804
805 AlternateHandle.Id getAlternateId() {
806 return handle.getId();
807 }
808 }