1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44 package org.eclipse.jgit.internal.storage.file;
45
46 import static java.nio.charset.StandardCharsets.UTF_8;
47
48 import java.io.BufferedReader;
49 import java.io.File;
50 import java.io.FileInputStream;
51 import java.io.FileNotFoundException;
52 import java.io.FileOutputStream;
53 import java.io.IOException;
54 import java.io.InputStreamReader;
55 import java.nio.file.Files;
56 import java.nio.file.StandardCopyOption;
57 import java.util.ArrayList;
58 import java.util.Comparator;
59 import java.util.List;
60 import java.util.Map;
61 import java.util.Optional;
62 import java.util.function.Supplier;
63 import java.util.stream.Collectors;
64
65 import org.eclipse.jgit.annotations.Nullable;
66 import org.eclipse.jgit.errors.LockFailedException;
67 import org.eclipse.jgit.internal.storage.io.BlockSource;
68 import org.eclipse.jgit.internal.storage.reftable.MergedReftable;
69 import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor;
70 import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
71 import org.eclipse.jgit.internal.storage.reftable.ReftableReader;
72 import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
73 import org.eclipse.jgit.lib.Config;
74 import org.eclipse.jgit.util.FileUtils;
75
76
77
78
79
80
81 public class FileReftableStack implements AutoCloseable {
82 private static class StackEntry {
83
84 String name;
85
86 ReftableReader reftableReader;
87 }
88
89 private MergedReftable mergedReftable;
90
91 private List<StackEntry> stack;
92
93 private long lastNextUpdateIndex;
94
95 private final File stackPath;
96
97 private final File reftableDir;
98
99 private final Runnable onChange;
100
101 private final Supplier<Config> configSupplier;
102
103
104 static class CompactionStats {
105
106 long tables;
107
108 long bytes;
109
110 int attempted;
111
112 int failed;
113
114 long refCount;
115
116 long logCount;
117
118 CompactionStats() {
119 tables = 0;
120 bytes = 0;
121 attempted = 0;
122 failed = 0;
123 logCount = 0;
124 refCount = 0;
125 }
126 }
127
128 private final CompactionStats stats;
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144 public FileReftableStack(File stackPath, File reftableDir,
145 @Nullable Runnable onChange, Supplier<Config> configSupplier)
146 throws IOException {
147 this.stackPath = stackPath;
148 this.reftableDir = reftableDir;
149 this.stack = new ArrayList<>();
150 this.configSupplier = configSupplier;
151 this.onChange = onChange;
152
153
154 lastNextUpdateIndex = 0;
155 reload();
156
157 stats = new CompactionStats();
158 }
159
160 CompactionStats getStats() {
161 return stats;
162 }
163
164
165 public static class ReftableNumbersNotIncreasingException
166 extends RuntimeException {
167 private static final long serialVersionUID = 1L;
168
169 String name;
170
171 long lastMax;
172
173 long min;
174
175 ReftableNumbersNotIncreasingException(String name, long lastMax,
176 long min) {
177 this.name = name;
178 this.lastMax = lastMax;
179 this.min = min;
180 }
181 }
182
183
184
185
186
187
188
189
190
191
192
193 private void reloadOnce(List<String> names)
194 throws IOException, FileNotFoundException {
195 Map<String, ReftableReader> current = stack.stream()
196 .collect(Collectors.toMap(e -> e.name, e -> e.reftableReader));
197
198 List<ReftableReader> newTables = new ArrayList<>();
199 List<StackEntry> newStack = new ArrayList<>(stack.size() + 1);
200 try {
201 ReftableReader last = null;
202 for (String name : names) {
203 StackEntry entry = new StackEntry();
204 entry.name = name;
205
206 ReftableReader t = null;
207 if (current.containsKey(name)) {
208 t = current.remove(name);
209 } else {
210 File subtable = new File(reftableDir, name);
211 FileInputStream is;
212
213 is = new FileInputStream(subtable);
214
215 t = new ReftableReader(BlockSource.from(is));
216 newTables.add(t);
217 }
218
219 if (last != null) {
220
221 if (last.maxUpdateIndex() >= t.minUpdateIndex()) {
222 throw new ReftableNumbersNotIncreasingException(name,
223 last.maxUpdateIndex(), t.minUpdateIndex());
224 }
225 }
226 last = t;
227
228 entry.reftableReader = t;
229 newStack.add(entry);
230 }
231
232
233 stack = newStack;
234 newTables.clear();
235
236 current.values().forEach(r -> {
237 try {
238 r.close();
239 } catch (IOException e) {
240 throw new AssertionError(e);
241 }
242 });
243 } finally {
244 newTables.forEach(t -> {
245 try {
246 t.close();
247 } catch (IOException ioe) {
248
249 throw new AssertionError(ioe);
250 }
251 });
252 }
253 }
254
255 void reload() throws IOException {
256
257 long deadline = System.currentTimeMillis() + 2500;
258
259
260
261 long min = 1;
262 long max = 1000;
263 long delay = 0;
264 boolean success = false;
265 while (System.currentTimeMillis() < deadline) {
266 List<String> names = readTableNames();
267 try {
268 reloadOnce(names);
269 success = true;
270 break;
271 } catch (FileNotFoundException e) {
272 List<String> changed = readTableNames();
273 if (changed.equals(names)) {
274 throw e;
275 }
276 }
277
278 delay = FileUtils.delay(delay, min, max);
279 try {
280 Thread.sleep(delay);
281 } catch (InterruptedException e) {
282 Thread.currentThread().interrupt();
283 throw new RuntimeException(e);
284 }
285 }
286
287 if (!success) {
288
289
290
291 throw new LockFailedException(stackPath);
292 }
293
294 mergedReftable = new MergedReftable(stack.stream()
295 .map(x -> x.reftableReader).collect(Collectors.toList()));
296 long curr = nextUpdateIndex();
297 if (lastNextUpdateIndex > 0 && lastNextUpdateIndex != curr
298 && onChange != null) {
299 onChange.run();
300 }
301 lastNextUpdateIndex = curr;
302 }
303
304
305
306
307 public MergedReftable getMergedReftable() {
308 return mergedReftable;
309 }
310
311
312
313
314
315
316 public interface Writer {
317
318
319
320
321
322
323
324 void call(ReftableWriter w) throws IOException;
325 }
326
327 private List<String> readTableNames() throws IOException {
328 List<String> names = new ArrayList<>(stack.size() + 1);
329
330 try (BufferedReader br = new BufferedReader(
331 new InputStreamReader(new FileInputStream(stackPath), UTF_8))) {
332 String line;
333 while ((line = br.readLine()) != null) {
334 if (!line.isEmpty()) {
335 names.add(line);
336 }
337 }
338 } catch (FileNotFoundException e) {
339
340 }
341 return names;
342 }
343
344
345
346
347
348
349 boolean isUpToDate() throws IOException {
350
351
352 try {
353 List<String> names = readTableNames();
354 if (names.size() != stack.size()) {
355 return false;
356 }
357 for (int i = 0; i < names.size(); i++) {
358 if (!names.get(i).equals(stack.get(i).name)) {
359 return false;
360 }
361 }
362 } catch (FileNotFoundException e) {
363 return stack.isEmpty();
364 }
365 return true;
366 }
367
368
369
370
371 @Override
372 public void close() {
373 for (StackEntry entry : stack) {
374 try {
375 entry.reftableReader.close();
376 } catch (Exception e) {
377
378 throw new AssertionError(e);
379 }
380 }
381 }
382
383 private long nextUpdateIndex() throws IOException {
384 return stack.size() > 0
385 ? stack.get(stack.size() - 1).reftableReader.maxUpdateIndex()
386 + 1
387 : 1;
388 }
389
390 private String filename(long low, long high) {
391 return String.format("%012x-%012x",
392 Long.valueOf(low), Long.valueOf(high));
393 }
394
395
396
397
398
399
400
401
402
403
404
405
406 @SuppressWarnings("nls")
407 public boolean addReftable(Writer w) throws IOException {
408 LockFile lock = new LockFile(stackPath);
409 try {
410 if (!lock.lockForAppend()) {
411 return false;
412 }
413 if (!isUpToDate()) {
414 return false;
415 }
416
417 String fn = filename(nextUpdateIndex(), nextUpdateIndex());
418
419 File tmpTable = File.createTempFile(fn + "_", ".ref",
420 stackPath.getParentFile());
421
422 ReftableWriter.Stats s;
423 try (FileOutputStream fos = new FileOutputStream(tmpTable)) {
424 ReftableWriter rw = new ReftableWriter(reftableConfig(), fos);
425 w.call(rw);
426 rw.finish();
427 s = rw.getStats();
428 }
429
430 if (s.minUpdateIndex() < nextUpdateIndex()) {
431 return false;
432 }
433
434
435
436 fn += s.refCount() > 0 ? ".ref" : ".log";
437 File dest = new File(reftableDir, fn);
438
439 FileUtils.rename(tmpTable, dest, StandardCopyOption.ATOMIC_MOVE);
440 lock.write((fn + "\n").getBytes(UTF_8));
441 if (!lock.commit()) {
442 FileUtils.delete(dest);
443 return false;
444 }
445
446 reload();
447
448 autoCompact();
449 } finally {
450 lock.unlock();
451 }
452 return true;
453 }
454
455 private ReftableConfig reftableConfig() {
456 return new ReftableConfig(configSupplier.get());
457 }
458
459
460
461
462
463
464
465
466
467
468
469
470 private File compactLocked(int first, int last) throws IOException {
471 String fn = filename(first, last);
472
473 File tmpTable = File.createTempFile(fn + "_", ".ref",
474 stackPath.getParentFile());
475 try (FileOutputStream fos = new FileOutputStream(tmpTable)) {
476 ReftableCompactor c = new ReftableCompactor(fos)
477 .setConfig(reftableConfig())
478 .setMinUpdateIndex(
479 stack.get(first).reftableReader.minUpdateIndex())
480 .setMaxUpdateIndex(
481 stack.get(last).reftableReader.maxUpdateIndex())
482 .setIncludeDeletes(first > 0);
483
484 List<ReftableReader> compactMe = new ArrayList<>();
485 long totalBytes = 0;
486 for (int i = first; i <= last; i++) {
487 compactMe.add(stack.get(i).reftableReader);
488 totalBytes += stack.get(i).reftableReader.size();
489 }
490 c.addAll(compactMe);
491
492 c.compact();
493
494
495
496 stats.bytes += totalBytes;
497 stats.tables += first - last + 1;
498 stats.attempted++;
499 stats.refCount += c.getStats().refCount();
500 stats.logCount += c.getStats().logCount();
501 }
502
503 return tmpTable;
504 }
505
506
507
508
509
510
511
512
513
514
515
516
517
518 boolean compactRange(int first, int last) throws IOException {
519 if (first >= last) {
520 return true;
521 }
522 LockFile lock = new LockFile(stackPath);
523
524 File tmpTable = null;
525 List<LockFile> subtableLocks = new ArrayList<>();
526
527 try {
528 if (!lock.lock()) {
529 return false;
530 }
531 if (!isUpToDate()) {
532 return false;
533 }
534
535 List<File> deleteOnSuccess = new ArrayList<>();
536 for (int i = first; i <= last; i++) {
537 File f = new File(reftableDir, stack.get(i).name);
538 LockFile lf = new LockFile(f);
539 if (!lf.lock()) {
540 return false;
541 }
542 subtableLocks.add(lf);
543 deleteOnSuccess.add(f);
544 }
545
546 lock.unlock();
547 lock = null;
548
549 tmpTable = compactLocked(first, last);
550
551 lock = new LockFile(stackPath);
552 if (!lock.lock()) {
553 return false;
554 }
555 if (!isUpToDate()) {
556 return false;
557 }
558
559 String fn = filename(
560 stack.get(first).reftableReader.minUpdateIndex(),
561 stack.get(last).reftableReader.maxUpdateIndex());
562
563
564
565
566
567
568 fn += ".ref";
569 File dest = new File(reftableDir, fn);
570
571 FileUtils.rename(tmpTable, dest, StandardCopyOption.ATOMIC_MOVE);
572 tmpTable = null;
573
574 StringBuilder sb = new StringBuilder();
575
576 for (int i = 0; i < first; i++) {
577 sb.append(stack.get(i).name + "\n");
578 }
579 sb.append(fn + "\n");
580 for (int i = last + 1; i < stack.size(); i++) {
581 sb.append(stack.get(i).name + "\n");
582 }
583
584 lock.write(sb.toString().getBytes(UTF_8));
585 if (!lock.commit()) {
586 dest.delete();
587 return false;
588 }
589
590 for (File f : deleteOnSuccess) {
591 Files.delete(f.toPath());
592 }
593
594 reload();
595 return true;
596 } finally {
597 if (tmpTable != null) {
598 tmpTable.delete();
599 }
600 for (LockFile lf : subtableLocks) {
601 lf.unlock();
602 }
603 if (lock != null) {
604 lock.unlock();
605 }
606 }
607 }
608
609
610
611
612
613
614
615 static int log(long sz) {
616 long base = 2;
617 if (sz <= 0) {
618 throw new IllegalArgumentException("log2 negative");
619 }
620 int l = 0;
621 while (sz > 0) {
622 l++;
623 sz /= base;
624 }
625
626 return l - 1;
627 }
628
629
630
631
632
633 static class Segment {
634
635 int log;
636
637
638 long bytes;
639
640 int start;
641
642 int end;
643
644 int size() {
645 return end - start;
646 }
647
648 Segment(int start, int end, int log, long bytes) {
649 this.log = log;
650 this.start = start;
651 this.end = end;
652 this.bytes = bytes;
653 }
654
655 Segment() {
656 this(0, 0, 0, 0);
657 }
658
659 @Override
660 public int hashCode() {
661 return 0;
662 }
663
664 @Override
665 public boolean equals(Object other) {
666 Segment o = (Segment) other;
667 return o.bytes == bytes && o.log == log && o.start == start
668 && o.end == end;
669 }
670
671 @SuppressWarnings("boxing")
672 @Override
673 public String toString() {
674 return String.format("{ [%d,%d) l=%d sz=%d }", start, end, log,
675 bytes);
676 }
677 }
678
679 static List<Segment> segmentSizes(long sizes[]) {
680 List<Segment> segments = new ArrayList<>();
681 Segment cur = new Segment();
682 for (int i = 0; i < sizes.length; i++) {
683 int l = log(sizes[i]);
684 if (l != cur.log && cur.bytes > 0) {
685 segments.add(cur);
686 cur = new Segment();
687 cur.start = i;
688 cur.log = l;
689 }
690
691 cur.log = l;
692 cur.end = i + 1;
693 cur.bytes += sizes[i];
694 }
695 segments.add(cur);
696 return segments;
697 }
698
699 private static Optional<Segment> autoCompactCandidate(long sizes[]) {
700 if (sizes.length == 0) {
701 return Optional.empty();
702 }
703
704
705
706
707
708
709
710 List<Segment> segments = segmentSizes(sizes);
711 segments = segments.stream().filter(s -> s.size() > 1)
712 .collect(Collectors.toList());
713 if (segments.isEmpty()) {
714 return Optional.empty();
715 }
716
717 Optional<Segment> optMinSeg = segments.stream()
718 .min(Comparator.comparing(s -> Integer.valueOf(s.log)));
719
720 Segment smallCollected = optMinSeg.get();
721 while (smallCollected.start > 0) {
722 int prev = smallCollected.start - 1;
723 long prevSize = sizes[prev];
724 if (log(smallCollected.bytes) < log(prevSize)) {
725 break;
726 }
727 smallCollected.start = prev;
728 smallCollected.bytes += prevSize;
729 }
730
731 return Optional.of(smallCollected);
732 }
733
734
735
736
737
738
739
740 private void autoCompact() throws IOException {
741 Optional<Segment> cand = autoCompactCandidate(tableSizes());
742 if (cand.isPresent()) {
743 if (!compactRange(cand.get().start, cand.get().end - 1)) {
744 stats.failed++;
745 }
746 }
747 }
748
749
750 private static long OVERHEAD = 91;
751
752 private long[] tableSizes() throws IOException {
753 long[] sizes = new long[stack.size()];
754 for (int i = 0; i < stack.size(); i++) {
755
756
757
758 sizes[i] = stack.get(i).reftableReader.size() - OVERHEAD;
759 }
760 return sizes;
761 }
762
763 void compactFully() throws IOException {
764 if (!compactRange(0, stack.size() - 1)) {
765 stats.failed++;
766 }
767 }
768 }