1
2
3
4
5
6
7
8
9
10
11 package org.eclipse.jgit.internal.storage.dfs;
12
13 import static java.util.stream.Collectors.joining;
14
15 import java.io.FileNotFoundException;
16 import java.io.IOException;
17 import java.util.ArrayList;
18 import java.util.Arrays;
19 import java.util.Collection;
20 import java.util.Collections;
21 import java.util.Comparator;
22 import java.util.HashMap;
23 import java.util.HashSet;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.Set;
27 import java.util.concurrent.atomic.AtomicReference;
28
29 import org.eclipse.jgit.internal.storage.pack.PackExt;
30 import org.eclipse.jgit.lib.AnyObjectId;
31 import org.eclipse.jgit.lib.ObjectDatabase;
32 import org.eclipse.jgit.lib.ObjectInserter;
33 import org.eclipse.jgit.lib.ObjectReader;
34
35
36
37
38
39
40 public abstract class DfsObjDatabase extends ObjectDatabase {
41 private static final PackList NO_PACKS = new PackList(
42 new DfsPackFile[0],
43 new DfsReftable[0]) {
44 @Override
45 boolean dirty() {
46 return true;
47 }
48
49 @Override
50 void clearDirty() {
51
52 }
53
54 @Override
55 public void markDirty() {
56
57 }
58 };
59
60
61
62
63
64
65
66
67 public enum PackSource {
68
69 INSERT,
70
71
72
73
74
75
76
77
78
79
80 RECEIVE,
81
82
83
84
85
86
87
88
89
90
91 COMPACT,
92
93
94
95
96
97
98
99
100
101
102 GC,
103
104
105 GC_REST,
106
107
108
109
110
111
112 GC_TXN,
113
114
115
116
117
118
119
120
121 UNREACHABLE_GARBAGE;
122
123
124
125
126
127
128
129
130 public static final Comparator<PackSource> DEFAULT_COMPARATOR =
131 new ComparatorBuilder()
132 .add(INSERT, RECEIVE)
133 .add(COMPACT)
134 .add(GC)
135 .add(GC_REST)
136 .add(GC_TXN)
137 .add(UNREACHABLE_GARBAGE)
138 .build();
139
140
141
142
143
144 public static class ComparatorBuilder {
145 private final Map<PackSource, Integer> ranks = new HashMap<>();
146 private int counter;
147
148
149
150
151
152
153
154
155
156
157
158 public ComparatorBuilder add(PackSource... sources) {
159 for (PackSource s : sources) {
160 ranks.put(s, Integer.valueOf(counter));
161 }
162 counter++;
163 return this;
164 }
165
166
167
168
169
170
171
172
173
174 public Comparator<PackSource> build() {
175 return new PackSourceComparator(ranks);
176 }
177 }
178
179 private static class PackSourceComparator implements Comparator<PackSource> {
180 private final Map<PackSource, Integer> ranks;
181
182 private PackSourceComparator(Map<PackSource, Integer> ranks) {
183 if (!ranks.keySet().equals(
184 new HashSet<>(Arrays.asList(PackSource.values())))) {
185 throw new IllegalArgumentException();
186 }
187 this.ranks = new HashMap<>(ranks);
188 }
189
190 @Override
191 public int compare(PackSource a, PackSource b) {
192 return ranks.get(a).compareTo(ranks.get(b));
193 }
194
195 @Override
196 public String toString() {
197 return Arrays.stream(PackSource.values())
198 .map(s -> s + "=" + ranks.get(s))
199 .collect(joining(", ", getClass().getSimpleName() + "{", "}"));
200 }
201 }
202 }
203
204 private final AtomicReference<PackList> packList;
205
206 private final DfsRepository repository;
207
208 private DfsReaderOptions readerOptions;
209
210 private Comparator<DfsPackDescription> packComparator;
211
212
213
214
215
216
217
218
219
220 protected DfsObjDatabase(DfsRepository repository,
221 DfsReaderOptions options) {
222 this.repository = repository;
223 this.packList = new AtomicReference<>(NO_PACKS);
224 this.readerOptions = options;
225 this.packComparator = DfsPackDescription.objectLookupComparator();
226 }
227
228
229
230
231
232
233 public DfsReaderOptions getReaderOptions() {
234 return readerOptions;
235 }
236
237
238
239
240
241
242
243
244
245
246
247
248 public void setPackComparator(Comparator<DfsPackDescription> packComparator) {
249 this.packComparator = packComparator;
250 }
251
252
253 @Override
254 public DfsReader newReader() {
255 return new DfsReader(this);
256 }
257
258
259 @Override
260 public ObjectInserter newInserter() {
261 return new DfsInserter(this);
262 }
263
264
265
266
267
268
269
270
271
272 public DfsPackFile[] getPacks() throws IOException {
273 return getPackList().packs;
274 }
275
276
277
278
279
280
281
282
283
284 public DfsReftable[] getReftables() throws IOException {
285 return getPackList().reftables;
286 }
287
288
289
290
291
292
293
294
295
296
297 public PackList getPackList() throws IOException {
298 return scanPacks(NO_PACKS);
299 }
300
301
302
303
304
305
306 protected DfsRepository getRepository() {
307 return repository;
308 }
309
310
311
312
313
314
315
316 public DfsPackFile[] getCurrentPacks() {
317 return getCurrentPackList().packs;
318 }
319
320
321
322
323
324
325
326 public DfsReftable[] getCurrentReftables() {
327 return getCurrentPackList().reftables;
328 }
329
330
331
332
333
334
335
336
337 public PackList getCurrentPackList() {
338 return packList.get();
339 }
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355 public boolean has(AnyObjectId objectId, boolean avoidUnreachableObjects)
356 throws IOException {
357 try (ObjectReader or = newReader()) {
358 or.setAvoidUnreachableObjects(avoidUnreachableObjects);
359 return or.has(objectId);
360 }
361 }
362
363
364
365
366
367
368
369
370
371
372
373 protected abstract DfsPackDescription newPack(PackSource source)
374 throws IOException;
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396 protected DfsPackDescription newPack(PackSource source,
397 long estimatedPackSize) throws IOException {
398 DfsPackDescription pack = newPack(source);
399 pack.setEstimatedPackSize(estimatedPackSize);
400 return pack;
401 }
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428 protected void commitPack(Collection<DfsPackDescription> desc,
429 Collection<DfsPackDescription> replaces) throws IOException {
430 commitPackImpl(desc, replaces);
431 getRepository().fireEvent(new DfsPacksChangedEvent());
432 }
433
434
435
436
437
438
439
440
441
442
443
444
445 protected abstract void commitPackImpl(Collection<DfsPackDescription> desc,
446 Collection<DfsPackDescription> replaces) throws IOException;
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463 protected abstract void rollbackPack(Collection<DfsPackDescription> desc);
464
465
466
467
468
469
470
471
472
473
474
475
476
477 protected abstract List<DfsPackDescription> listPacks() throws IOException;
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495 protected abstract ReadableChannel openFile(
496 DfsPackDescription desc, PackExt ext)
497 throws FileNotFoundException, IOException;
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512 protected abstract DfsOutputStream writeFile(
513 DfsPackDescription desc, PackExt ext) throws IOException;
514
515 void addPack(DfsPackFile newPack) throws IOException {
516 PackList o, n;
517 do {
518 o = packList.get();
519 if (o == NO_PACKS) {
520
521
522
523
524
525 o = scanPacks(o);
526
527
528
529
530 for (DfsPackFile p : o.packs) {
531 if (p.key.equals(newPack.key)) {
532 return;
533 }
534 }
535 }
536
537 DfsPackFile[] packs = new DfsPackFile[1 + o.packs.length];
538 packs[0] = newPack;
539 System.arraycopy(o.packs, 0, packs, 1, o.packs.length);
540 n = new PackListImpl(packs, o.reftables);
541 } while (!packList.compareAndSet(o, n));
542 }
543
544 void addReftable(DfsPackDescription add, Set<DfsPackDescription> remove)
545 throws IOException {
546 PackList o, n;
547 do {
548 o = packList.get();
549 if (o == NO_PACKS) {
550 o = scanPacks(o);
551 for (DfsReftable t : o.reftables) {
552 if (t.getPackDescription().equals(add)) {
553 return;
554 }
555 }
556 }
557
558 List<DfsReftable> tables = new ArrayList<>(1 + o.reftables.length);
559 for (DfsReftable t : o.reftables) {
560 if (!remove.contains(t.getPackDescription())) {
561 tables.add(t);
562 }
563 }
564 tables.add(new DfsReftable(add));
565 n = new PackListImpl(o.packs, tables.toArray(new DfsReftable[0]));
566 } while (!packList.compareAndSet(o, n));
567 }
568
569 PackList scanPacks(PackList original) throws IOException {
570 PackList o, n;
571 synchronized (packList) {
572 do {
573 o = packList.get();
574 if (o != original) {
575
576
577
578 return o;
579 }
580 n = scanPacksImpl(o);
581 if (n == o)
582 return n;
583 } while (!packList.compareAndSet(o, n));
584 }
585 getRepository().fireEvent(new DfsPacksChangedEvent());
586 return n;
587 }
588
589 private PackList scanPacksImpl(PackList old) throws IOException {
590 DfsBlockCache cache = DfsBlockCache.getInstance();
591 Map<DfsPackDescription, DfsPackFile> packs = packMap(old);
592 Map<DfsPackDescription, DfsReftable> reftables = reftableMap(old);
593
594 List<DfsPackDescription> scanned = listPacks();
595 Collections.sort(scanned, packComparator);
596
597 List<DfsPackFile> newPacks = new ArrayList<>(scanned.size());
598 List<DfsReftable> newReftables = new ArrayList<>(scanned.size());
599 boolean foundNew = false;
600 for (DfsPackDescription dsc : scanned) {
601 DfsPackFile oldPack = packs.remove(dsc);
602 if (oldPack != null) {
603 newPacks.add(oldPack);
604 } else if (dsc.hasFileExt(PackExt.PACK)) {
605 newPacks.add(new DfsPackFile(cache, dsc));
606 foundNew = true;
607 }
608
609 DfsReftable oldReftable = reftables.remove(dsc);
610 if (oldReftable != null) {
611 newReftables.add(oldReftable);
612 } else if (dsc.hasFileExt(PackExt.REFTABLE)) {
613 newReftables.add(new DfsReftable(cache, dsc));
614 foundNew = true;
615 }
616 }
617
618 if (newPacks.isEmpty() && newReftables.isEmpty())
619 return new PackListImpl(NO_PACKS.packs, NO_PACKS.reftables);
620 if (!foundNew) {
621 old.clearDirty();
622 return old;
623 }
624 Collections.sort(newReftables, reftableComparator());
625 return new PackListImpl(
626 newPacks.toArray(new DfsPackFile[0]),
627 newReftables.toArray(new DfsReftable[0]));
628 }
629
630 private static Map<DfsPackDescription, DfsPackFile> packMap(PackList old) {
631 Map<DfsPackDescription, DfsPackFile> forReuse = new HashMap<>();
632 for (DfsPackFile p : old.packs) {
633 if (!p.invalid()) {
634 forReuse.put(p.desc, p);
635 }
636 }
637 return forReuse;
638 }
639
640 private static Map<DfsPackDescription, DfsReftable> reftableMap(PackList old) {
641 Map<DfsPackDescription, DfsReftable> forReuse = new HashMap<>();
642 for (DfsReftable p : old.reftables) {
643 if (!p.invalid()) {
644 forReuse.put(p.desc, p);
645 }
646 }
647 return forReuse;
648 }
649
650
651
652
653
654
655 protected Comparator<DfsReftable> reftableComparator() {
656 return Comparator.comparing(
657 DfsReftable::getPackDescription,
658 DfsPackDescription.reftableComparator());
659 }
660
661
662
663
664 protected void clearCache() {
665 packList.set(NO_PACKS);
666 }
667
668
669 @Override
670 public void close() {
671 packList.set(NO_PACKS);
672 }
673
674
675 public abstract static class PackList {
676
677 public final DfsPackFile[] packs;
678
679
680 public final DfsReftable[] reftables;
681
682 private long lastModified = -1;
683
684 PackList(DfsPackFile[] packs, DfsReftable[] reftables) {
685 this.packs = packs;
686 this.reftables = reftables;
687 }
688
689
690 public long getLastModified() {
691 if (lastModified < 0) {
692 long max = 0;
693 for (DfsPackFile pack : packs) {
694 max = Math.max(max, pack.getPackDescription().getLastModified());
695 }
696 lastModified = max;
697 }
698 return lastModified;
699 }
700
701 abstract boolean dirty();
702 abstract void clearDirty();
703
704
705
706
707
708
709
710
711 public abstract void markDirty();
712 }
713
714 private static final class PackListImpl extends PackList {
715 private volatile boolean dirty;
716
717 PackListImpl(DfsPackFile[] packs, DfsReftable[] reftables) {
718 super(packs, reftables);
719 }
720
721 @Override
722 boolean dirty() {
723 return dirty;
724 }
725
726 @Override
727 void clearDirty() {
728 dirty = false;
729 }
730
731 @Override
732 public void markDirty() {
733 dirty = true;
734 }
735 }
736 }