View Javadoc
1   /*
2    * Copyright (C) 2011, 2013 Google Inc., and others. and others
3    *
4    * This program and the accompanying materials are made available under the
5    * terms of the Eclipse Distribution License v. 1.0 which is available at
6    * https://www.eclipse.org/org/documents/edl-v10.php.
7    *
8    * SPDX-License-Identifier: BSD-3-Clause
9    */
10  
11  package org.eclipse.jgit.internal.storage.dfs;
12  
13  import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
14  import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
15  
16  import java.util.Arrays;
17  import java.util.Comparator;
18  
19  import org.eclipse.jgit.annotations.NonNull;
20  import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
21  import org.eclipse.jgit.internal.storage.pack.PackExt;
22  import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
23  import org.eclipse.jgit.storage.pack.PackStatistics;
24  
25  /**
26   * Description of a DFS stored pack/index file.
27   * <p>
28   * Implementors may extend this class and add additional data members.
29   * <p>
30   * Instances of this class are cached with the DfsPackFile, and should not be
31   * modified once initialized and presented to the JGit DFS library.
32   */
33  public class DfsPackDescription {
34  	/**
35  	 * Comparator for packs when looking up objects in indexes.
36  	 * <p>
37  	 * This comparator tries to position packs in the order readers should examine
38  	 * them when looking for objects by SHA-1. The default tries to sort packs
39  	 * with more recent modification dates before older packs, and packs with
40  	 * fewer objects before packs with more objects.
41  	 * <p>
42  	 * Uses {@link PackSource#DEFAULT_COMPARATOR} for the portion of comparison
43  	 * where packs are sorted by source.
44  	 *
45  	 * @return comparator.
46  	 */
47  	public static Comparator<DfsPackDescription> objectLookupComparator() {
48  		return objectLookupComparator(PackSource.DEFAULT_COMPARATOR);
49  	}
50  
51  	/**
52  	 * Comparator for packs when looking up objects in indexes.
53  	 * <p>
54  	 * This comparator tries to position packs in the order readers should examine
55  	 * them when looking for objects by SHA-1. The default tries to sort packs
56  	 * with more recent modification dates before older packs, and packs with
57  	 * fewer objects before packs with more objects.
58  	 *
59  	 * @param packSourceComparator
60  	 *            comparator for the {@link PackSource}, used as the first step in
61  	 *            comparison.
62  	 * @return comparator.
63  	 */
64  	public static Comparator<DfsPackDescription> objectLookupComparator(
65  			Comparator<PackSource> packSourceComparator) {
66  		return Comparator.comparing(
67  					DfsPackDescription::getPackSource, packSourceComparator)
68  			.thenComparing((a, b) -> {
69  				PackSource as = a.getPackSource();
70  				PackSource bs = b.getPackSource();
71  
72  				// Tie break GC type packs by smallest first. There should be at most
73  				// one of each source, but when multiple exist concurrent GCs may have
74  				// run. Preferring the smaller file selects higher quality delta
75  				// compression, placing less demand on the DfsBlockCache.
76  				if (as == bs && isGC(as)) {
77  					int cmp = Long.signum(a.getFileSize(PACK) - b.getFileSize(PACK));
78  					if (cmp != 0) {
79  						return cmp;
80  					}
81  				}
82  
83  				// Newer packs should sort first.
84  				int cmp = Long.signum(b.getLastModified() - a.getLastModified());
85  				if (cmp != 0) {
86  					return cmp;
87  				}
88  
89  				// Break ties on smaller index. Readers may get lucky and find
90  				// the object they care about in the smaller index. This also pushes
91  				// big historical packs to the end of the list, due to more objects.
92  				return Long.signum(a.getObjectCount() - b.getObjectCount());
93  			});
94  	}
95  
96  	static Comparator<DfsPackDescription> reftableComparator() {
97  		return (a, b) -> {
98  				// GC, COMPACT reftables first by reversing default order.
99  				int c = PackSource.DEFAULT_COMPARATOR.reversed()
100 						.compare(a.getPackSource(), b.getPackSource());
101 				if (c != 0) {
102 					return c;
103 				}
104 
105 				// Lower maxUpdateIndex first.
106 				c = Long.signum(a.getMaxUpdateIndex() - b.getMaxUpdateIndex());
107 				if (c != 0) {
108 					return c;
109 				}
110 
111 				// Older reftable first.
112 				return Long.signum(a.getLastModified() - b.getLastModified());
113 			};
114 	}
115 
116 	static Comparator<DfsPackDescription> reuseComparator() {
117 		return (a, b) -> {
118 			PackSource as = a.getPackSource();
119 			PackSource bs = b.getPackSource();
120 
121 			if (as == bs && DfsPackDescription.isGC(as)) {
122 				// Push smaller GC files last; these likely have higher quality
123 				// delta compression and the contained representation should be
124 				// favored over other files.
125 				return Long.signum(b.getFileSize(PACK) - a.getFileSize(PACK));
126 			}
127 
128 			// DfsPackDescription.compareTo already did a reasonable sort.
129 			// Rely on Arrays.sort being stable, leaving equal elements.
130 			return 0;
131 		};
132 	}
133 
134 	private final DfsRepositoryDescription repoDesc;
135 	private final String packName;
136 	private PackSource packSource;
137 	private long lastModified;
138 	private long[] sizeMap;
139 	private int[] blockSizeMap;
140 	private long objectCount;
141 	private long deltaCount;
142 	private long minUpdateIndex;
143 	private long maxUpdateIndex;
144 
145 	private PackStatistics packStats;
146 	private ReftableWriter.Stats refStats;
147 	private int extensions;
148 	private int indexVersion;
149 	private long estimatedPackSize;
150 
151 	/**
152 	 * Initialize a description by pack name and repository.
153 	 * <p>
154 	 * The corresponding index file is assumed to exist. If this is not true
155 	 * implementors must extend the class and override
156 	 * {@link #getFileName(PackExt)}.
157 	 * <p>
158 	 * Callers should also try to fill in other fields if they are reasonably
159 	 * free to access at the time this instance is being initialized.
160 	 *
161 	 * @param name
162 	 *            name of the pack file. Must end with ".pack".
163 	 * @param repoDesc
164 	 *            description of the repo containing the pack file.
165 	 * @param packSource
166 	 *            the source of the pack.
167 	 */
168 	public DfsPackDescription(DfsRepositoryDescription repoDesc, String name,
169 			@NonNull PackSource packSource) {
170 		this.repoDesc = repoDesc;
171 		int dot = name.lastIndexOf('.');
172 		this.packName = (dot < 0) ? name : name.substring(0, dot);
173 		this.packSource = packSource;
174 
175 		int extCnt = PackExt.values().length;
176 		sizeMap = new long[extCnt];
177 		blockSizeMap = new int[extCnt];
178 	}
179 
180 	/**
181 	 * Get description of the repository.
182 	 *
183 	 * @return description of the repository.
184 	 */
185 	public DfsRepositoryDescription getRepositoryDescription() {
186 		return repoDesc;
187 	}
188 
189 	/**
190 	 * Adds the pack file extension to the known list.
191 	 *
192 	 * @param ext
193 	 *            the file extension
194 	 */
195 	public void addFileExt(PackExt ext) {
196 		extensions |= ext.getBit();
197 	}
198 
199 	/**
200 	 * Whether the pack file extension is known to exist.
201 	 *
202 	 * @param ext
203 	 *            the file extension
204 	 * @return whether the pack file extension is known to exist.
205 	 */
206 	public boolean hasFileExt(PackExt ext) {
207 		return (extensions & ext.getBit()) != 0;
208 	}
209 
210 	/**
211 	 * Get file name
212 	 *
213 	 * @param ext
214 	 *            the file extension
215 	 * @return name of the file.
216 	 */
217 	public String getFileName(PackExt ext) {
218 		return packName + '.' + ext.getExtension();
219 	}
220 
221 	/**
222 	 * Get cache key for use by the block cache.
223 	 *
224 	 * @param ext
225 	 *            the file extension.
226 	 * @return cache key for use by the block cache.
227 	 */
228 	public DfsStreamKey getStreamKey(PackExt ext) {
229 		return DfsStreamKey.of(getRepositoryDescription(), getFileName(ext),
230 				ext);
231 	}
232 
233 	/**
234 	 * Get the source of the pack.
235 	 *
236 	 * @return the source of the pack.
237 	 */
238 	@NonNull
239 	public PackSource getPackSource() {
240 		return packSource;
241 	}
242 
243 	/**
244 	 * Set the source of the pack.
245 	 *
246 	 * @param source
247 	 *            the source of the pack.
248 	 * @return {@code this}
249 	 */
250 	public DfsPackDescription setPackSource(@NonNull PackSource source) {
251 		packSource = source;
252 		return this;
253 	}
254 
255 	/**
256 	 * Get time the pack was created, in milliseconds.
257 	 *
258 	 * @return time the pack was created, in milliseconds.
259 	 */
260 	public long getLastModified() {
261 		return lastModified;
262 	}
263 
264 	/**
265 	 * Set time the pack was created, in milliseconds.
266 	 *
267 	 * @param timeMillis
268 	 *            time the pack was created, in milliseconds. 0 if not known.
269 	 * @return {@code this}
270 	 */
271 	public DfsPackDescription setLastModified(long timeMillis) {
272 		lastModified = timeMillis;
273 		return this;
274 	}
275 
276 	/**
277 	 * Get minUpdateIndex for the reftable, if present.
278 	 *
279 	 * @return minUpdateIndex for the reftable, if present.
280 	 */
281 	public long getMinUpdateIndex() {
282 		return minUpdateIndex;
283 	}
284 
285 	/**
286 	 * Set minUpdateIndex for the reftable.
287 	 *
288 	 * @param min
289 	 *            minUpdateIndex for the reftable.
290 	 * @return {@code this}
291 	 */
292 	public DfsPackDescription setMinUpdateIndex(long min) {
293 		minUpdateIndex = min;
294 		return this;
295 	}
296 
297 	/**
298 	 * Get maxUpdateIndex for the reftable, if present.
299 	 *
300 	 * @return maxUpdateIndex for the reftable, if present.
301 	 */
302 	public long getMaxUpdateIndex() {
303 		return maxUpdateIndex;
304 	}
305 
306 	/**
307 	 * Set maxUpdateIndex for the reftable.
308 	 *
309 	 * @param max
310 	 *            maxUpdateIndex for the reftable.
311 	 * @return {@code this}
312 	 */
313 	public DfsPackDescription setMaxUpdateIndex(long max) {
314 		maxUpdateIndex = max;
315 		return this;
316 	}
317 
318 	/**
319 	 * Set size of the file in bytes.
320 	 *
321 	 * @param ext
322 	 *            the file extension.
323 	 * @param bytes
324 	 *            size of the file in bytes. If 0 the file is not known and will
325 	 *            be determined on first read.
326 	 * @return {@code this}
327 	 */
328 	public DfsPackDescription setFileSize(PackExt ext, long bytes) {
329 		int i = ext.getPosition();
330 		if (i >= sizeMap.length) {
331 			sizeMap = Arrays.copyOf(sizeMap, i + 1);
332 		}
333 		sizeMap[i] = Math.max(0, bytes);
334 		return this;
335 	}
336 
337 	/**
338 	 * Get size of the file, in bytes.
339 	 *
340 	 * @param ext
341 	 *            the file extension.
342 	 * @return size of the file, in bytes. If 0 the file size is not yet known.
343 	 */
344 	public long getFileSize(PackExt ext) {
345 		int i = ext.getPosition();
346 		return i < sizeMap.length ? sizeMap[i] : 0;
347 	}
348 
349 	/**
350 	 * Get blockSize of the file, in bytes.
351 	 *
352 	 * @param ext
353 	 *            the file extension.
354 	 * @return blockSize of the file, in bytes. If 0 the blockSize size is not
355 	 *         yet known and may be discovered when opening the file.
356 	 */
357 	public int getBlockSize(PackExt ext) {
358 		int i = ext.getPosition();
359 		return i < blockSizeMap.length ? blockSizeMap[i] : 0;
360 	}
361 
362 	/**
363 	 * Set blockSize of the file, in bytes.
364 	 *
365 	 * @param ext
366 	 *            the file extension.
367 	 * @param blockSize
368 	 *            blockSize of the file, in bytes. If 0 the blockSize is not
369 	 *            known and will be determined on first read.
370 	 * @return {@code this}
371 	 */
372 	public DfsPackDescription setBlockSize(PackExt ext, int blockSize) {
373 		int i = ext.getPosition();
374 		if (i >= blockSizeMap.length) {
375 			blockSizeMap = Arrays.copyOf(blockSizeMap, i + 1);
376 		}
377 		blockSizeMap[i] = Math.max(0, blockSize);
378 		return this;
379 	}
380 
381 	/**
382 	 * Set estimated size of the .pack file in bytes.
383 	 *
384 	 * @param estimatedPackSize
385 	 *            estimated size of the .pack file in bytes. If 0 the pack file
386 	 *            size is unknown.
387 	 * @return {@code this}
388 	 */
389 	public DfsPackDescription setEstimatedPackSize(long estimatedPackSize) {
390 		this.estimatedPackSize = Math.max(0, estimatedPackSize);
391 		return this;
392 	}
393 
394 	/**
395 	 * Get estimated size of the .pack file in bytes.
396 	 *
397 	 * @return estimated size of the .pack file in bytes. If 0 the pack file
398 	 *         size is unknown.
399 	 */
400 	public long getEstimatedPackSize() {
401 		return estimatedPackSize;
402 	}
403 
404 	/**
405 	 * Get number of objects in the pack.
406 	 *
407 	 * @return number of objects in the pack.
408 	 */
409 	public long getObjectCount() {
410 		return objectCount;
411 	}
412 
413 	/**
414 	 * Set number of objects in the pack.
415 	 *
416 	 * @param cnt
417 	 *            number of objects in the pack.
418 	 * @return {@code this}
419 	 */
420 	public DfsPackDescription setObjectCount(long cnt) {
421 		objectCount = Math.max(0, cnt);
422 		return this;
423 	}
424 
425 	/**
426 	 * Get number of delta compressed objects in the pack.
427 	 *
428 	 * @return number of delta compressed objects in the pack.
429 	 */
430 	public long getDeltaCount() {
431 		return deltaCount;
432 	}
433 
434 	/**
435 	 * Set number of delta compressed objects in the pack.
436 	 *
437 	 * @param cnt
438 	 *            number of delta compressed objects in the pack.
439 	 * @return {@code this}
440 	 */
441 	public DfsPackDescription setDeltaCount(long cnt) {
442 		deltaCount = Math.max(0, cnt);
443 		return this;
444 	}
445 
446 	/**
447 	 * Get statistics from PackWriter, if the pack was built with it.
448 	 *
449 	 * @return statistics from PackWriter, if the pack was built with it.
450 	 *         Generally this is only available for packs created by
451 	 *         DfsGarbageCollector or DfsPackCompactor, and only when the pack
452 	 *         is being committed to the repository.
453 	 */
454 	public PackStatistics getPackStats() {
455 		return packStats;
456 	}
457 
458 	DfsPackDescription setPackStats(PackStatistics stats) {
459 		this.packStats = stats;
460 		setFileSize(PACK, stats.getTotalBytes());
461 		setObjectCount(stats.getTotalObjects());
462 		setDeltaCount(stats.getTotalDeltas());
463 		return this;
464 	}
465 
466 	/**
467 	 * Get stats from the sibling reftable, if created.
468 	 *
469 	 * @return stats from the sibling reftable, if created.
470 	 */
471 	public ReftableWriter.Stats getReftableStats() {
472 		return refStats;
473 	}
474 
475 	void setReftableStats(ReftableWriter.Stats stats) {
476 		this.refStats = stats;
477 		setMinUpdateIndex(stats.minUpdateIndex());
478 		setMaxUpdateIndex(stats.maxUpdateIndex());
479 		setFileSize(REFTABLE, stats.totalBytes());
480 		setBlockSize(REFTABLE, stats.refBlockSize());
481 	}
482 
483 	/**
484 	 * Discard the pack statistics, if it was populated.
485 	 *
486 	 * @return {@code this}
487 	 */
488 	public DfsPackDescription clearPackStats() {
489 		packStats = null;
490 		refStats = null;
491 		return this;
492 	}
493 
494 	/**
495 	 * Get the version of the index file written.
496 	 *
497 	 * @return the version of the index file written.
498 	 */
499 	public int getIndexVersion() {
500 		return indexVersion;
501 	}
502 
503 	/**
504 	 * Set the version of the index file written.
505 	 *
506 	 * @param version
507 	 *            the version of the index file written.
508 	 * @return {@code this}
509 	 */
510 	public DfsPackDescription setIndexVersion(int version) {
511 		indexVersion = version;
512 		return this;
513 	}
514 
515 	/** {@inheritDoc} */
516 	@Override
517 	public int hashCode() {
518 		return packName.hashCode();
519 	}
520 
521 	/** {@inheritDoc} */
522 	@Override
523 	public boolean equals(Object b) {
524 		if (b instanceof DfsPackDescription) {
525 			DfsPackDescription desc = (DfsPackDescription) b;
526 			return packName.equals(desc.packName) &&
527 					getRepositoryDescription().equals(desc.getRepositoryDescription());
528 		}
529 		return false;
530 	}
531 
532 	static boolean isGC(PackSource s) {
533 		switch (s) {
534 		case GC:
535 		case GC_REST:
536 		case GC_TXN:
537 			return true;
538 		default:
539 			return false;
540 		}
541 	}
542 
543 	/** {@inheritDoc} */
544 	@Override
545 	public String toString() {
546 		return getFileName(PackExt.PACK);
547 	}
548 }