View Javadoc
1   /*
2    * Copyright (C) 2015, Google Inc.
3    *
4    * This program and the accompanying materials are made available
5    * under the terms of the Eclipse Distribution License v1.0 which
6    * accompanies this distribution, is reproduced below, and is
7    * available at http://www.eclipse.org/org/documents/edl-v10.php
8    *
9    * All rights reserved.
10   *
11   * Redistribution and use in source and binary forms, with or
12   * without modification, are permitted provided that the following
13   * conditions are met:
14   *
15   * - Redistributions of source code must retain the above copyright
16   *   notice, this list of conditions and the following disclaimer.
17   *
18   * - Redistributions in binary form must reproduce the above
19   *   copyright notice, this list of conditions and the following
20   *   disclaimer in the documentation and/or other materials provided
21   *   with the distribution.
22   *
23   * - Neither the name of the Eclipse Foundation, Inc. nor the
24   *   names of its contributors may be used to endorse or promote
25   *   products derived from this software without specific prior
26   *   written permission.
27   *
28   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
29   * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
30   * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
31   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
33   * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
35   * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
36   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
37   * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
38   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
39   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
40   * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
41   */
42  
43  package org.eclipse.jgit.storage.pack;
44  
45  import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
46  import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT;
47  import static org.eclipse.jgit.lib.Constants.OBJ_TAG;
48  import static org.eclipse.jgit.lib.Constants.OBJ_TREE;
49  
50  import java.text.MessageFormat;
51  import java.util.HashMap;
52  import java.util.List;
53  import java.util.Map;
54  import java.util.Set;
55  
56  import org.eclipse.jgit.internal.JGitText;
57  import org.eclipse.jgit.internal.storage.pack.CachedPack;
58  import org.eclipse.jgit.lib.ObjectId;
59  
60  /**
61   * Statistics about {@link org.eclipse.jgit.internal.storage.pack.PackWriter}
62   * pack creation.
63   *
64   * @since 4.1
65   */
66  public class PackStatistics {
67  	/**
68  	 * Statistics about a single type of object (commits, tags, trees and
69  	 * blobs).
70  	 */
71  	public static class ObjectType {
72  		/**
73  		 * POJO for accumulating the ObjectType statistics.
74  		 */
75  		public static class Accumulator {
76  			/** Count of objects of this type. */
77  			public long cntObjects;
78  
79  			/** Count of deltas of this type. */
80  			public long cntDeltas;
81  
82  			/** Count of reused objects of this type. */
83  			public long reusedObjects;
84  
85  			/** Count of reused deltas of this type. */
86  			public long reusedDeltas;
87  
88  			/** Count of bytes for all objects of this type. */
89  			public long bytes;
90  
91  			/** Count of delta bytes for objects of this type. */
92  			public long deltaBytes;
93  		}
94  
95  		private ObjectType.Accumulator objectType;
96  
97  		/**
98  		 * Creates a new {@link ObjectType} object from the accumulator.
99  		 *
100 		 * @param accumulator
101 		 *            the accumulator of the statistics
102 		 */
103 		public ObjectType(ObjectType.Accumulator accumulator) {
104 			/*
105 			 * For efficiency this wraps and serves up the Accumulator object
106 			 * rather than making a deep clone. Normal usage of PackWriter is to
107 			 * create a single pack/index/bitmap and only call getStatistics()
108 			 * after all work is complete.
109 			 */
110 			objectType = accumulator;
111 		}
112 
113 		/**
114 		 * @return total number of objects output. This total includes the value
115 		 *         of {@link #getDeltas()}.
116 		 */
117 		public long getObjects() {
118 			return objectType.cntObjects;
119 		}
120 
121 		/**
122 		 * @return total number of deltas output. This may be lower than the
123 		 *         actual number of deltas if a cached pack was reused.
124 		 */
125 		public long getDeltas() {
126 			return objectType.cntDeltas;
127 		}
128 
129 		/**
130 		 * @return number of objects whose existing representation was reused in
131 		 *         the output. This count includes {@link #getReusedDeltas()}.
132 		 */
133 		public long getReusedObjects() {
134 			return objectType.reusedObjects;
135 		}
136 
137 		/**
138 		 * @return number of deltas whose existing representation was reused in
139 		 *         the output, as their base object was also output or was
140 		 *         assumed present for a thin pack. This may be lower than the
141 		 *         actual number of reused deltas if a cached pack was reused.
142 		 */
143 		public long getReusedDeltas() {
144 			return objectType.reusedDeltas;
145 		}
146 
147 		/**
148 		 * @return total number of bytes written. This size includes the object
149 		 *         headers as well as the compressed data. This size also
150 		 *         includes all of {@link #getDeltaBytes()}.
151 		 */
152 		public long getBytes() {
153 			return objectType.bytes;
154 		}
155 
156 		/**
157 		 * @return number of delta bytes written. This size includes the object
158 		 *         headers for the delta objects.
159 		 */
160 		public long getDeltaBytes() {
161 			return objectType.deltaBytes;
162 		}
163 	}
164 
165 	/**
166 	 * POJO for accumulating the statistics.
167 	 */
168 	public static class Accumulator {
169 		/**
170 		 * The count of references in the ref advertisement.
171 		 *
172 		 * @since 4.11
173 		 */
174 		public long advertised;
175 
176 		/**
177 		 * The count of client wants.
178 		 *
179 		 * @since 4.11
180 		 */
181 		public long wants;
182 
183 		/**
184 		 * The count of client haves.
185 		 *
186 		 * @since 4.11
187 		 */
188 		public long haves;
189 
190 		/**
191 		 * Time in ms spent in the negotiation phase. For non-bidirectional
192 		 * transports (e.g., HTTP), this is only for the final request that
193 		 * sends back the pack file.
194 		 *
195 		 * @since 4.11
196 		 */
197 		public long timeNegotiating;
198 
199 		/** The set of objects to be included in the pack. */
200 		public Set<ObjectId> interestingObjects;
201 
202 		/** The set of objects to be excluded from the pack. */
203 		public Set<ObjectId> uninterestingObjects;
204 
205 		/** The set of shallow commits on the client. */
206 		public Set<ObjectId> clientShallowCommits;
207 
208 		/** The collection of reused packs in the upload. */
209 		public List<CachedPack> reusedPacks;
210 
211 		/** Commits with no parents. */
212 		public Set<ObjectId> rootCommits;
213 
214 		/** If a shallow pack, the depth in commits. */
215 		public int depth;
216 
217 		/**
218 		 * The count of objects in the pack that went through the delta search
219 		 * process in order to find a potential delta base.
220 		 */
221 		public int deltaSearchNonEdgeObjects;
222 
223 		/**
224 		 * The count of objects in the pack that went through delta base search
225 		 * and found a suitable base. This is a subset of
226 		 * deltaSearchNonEdgeObjects.
227 		 */
228 		public int deltasFound;
229 
230 		/** The total count of objects in the pack. */
231 		public long totalObjects;
232 
233 		/**
234 		 * The count of objects that needed to be discovered through an object
235 		 * walk because they were not found in bitmap indices.
236 		 */
237 		public long bitmapIndexMisses;
238 
239 		/** The total count of deltas output. */
240 		public long totalDeltas;
241 
242 		/** The count of reused objects in the pack. */
243 		public long reusedObjects;
244 
245 		/** The count of reused deltas in the pack. */
246 		public long reusedDeltas;
247 
248 		/** The count of total bytes in the pack. */
249 		public long totalBytes;
250 
251 		/** The size of the thin pack in bytes, if a thin pack was generated. */
252 		public long thinPackBytes;
253 
254 		/** Time in ms spent counting the objects that will go into the pack. */
255 		public long timeCounting;
256 
257 		/** Time in ms spent searching for objects to reuse. */
258 		public long timeSearchingForReuse;
259 
260 		/** Time in ms spent searching for sizes of objects. */
261 		public long timeSearchingForSizes;
262 
263 		/** Time in ms spent compressing the pack. */
264 		public long timeCompressing;
265 
266 		/** Time in ms spent writing the pack. */
267 		public long timeWriting;
268 
269 		/** Number of trees traversed in the walk when writing the pack.
270 		 * @since 5.4*/
271 		public long treesTraversed;
272 
273 		/**
274 		 * Amount of packfile uris sent to the client to download via HTTP.
275 		 *
276 		 * @since 5.6
277 		 */
278 		public long offloadedPackfiles;
279 
280 		/**
281 		 * Total size (in bytes) offloaded to HTTP downloads.
282 		 *
283 		 * @since 5.6
284 		 */
285 		public long offloadedPackfileSize;
286 
287 		/**
288 		 * Statistics about each object type in the pack (commits, tags, trees
289 		 * and blobs.)
290 		 */
291 		public ObjectType.Accumulator[] objectTypes;
292 
293 		{
294 			objectTypes = new ObjectType.Accumulator[5];
295 			objectTypes[OBJ_COMMIT] = new ObjectType.Accumulator();
296 			objectTypes[OBJ_TREE] = new ObjectType.Accumulator();
297 			objectTypes[OBJ_BLOB] = new ObjectType.Accumulator();
298 			objectTypes[OBJ_TAG] = new ObjectType.Accumulator();
299 		}
300 	}
301 
302 	private Accumulator statistics;
303 
304 	/**
305 	 * Creates a new {@link org.eclipse.jgit.storage.pack.PackStatistics} object
306 	 * from the accumulator.
307 	 *
308 	 * @param accumulator
309 	 *            the accumulator of the statistics
310 	 */
311 	public PackStatistics(Accumulator accumulator) {
312 		/*
313 		 * For efficiency this wraps and serves up the Accumulator object rather
314 		 * than making a deep clone. Normal usage of PackWriter is to create a
315 		 * single pack/index/bitmap and only call getStatistics() after all work
316 		 * is complete.
317 		 */
318 		statistics = accumulator;
319 	}
320 
321 	/**
322 	 * Get the count of references in the ref advertisement.
323 	 *
324 	 * @return count of refs in the ref advertisement.
325 	 * @since 4.11
326 	 */
327 	public long getAdvertised() {
328 		return statistics.advertised;
329 	}
330 
331 	/**
332 	 * Get the count of client wants.
333 	 *
334 	 * @return count of client wants.
335 	 * @since 4.11
336 	 */
337 	public long getWants() {
338 		return statistics.wants;
339 	}
340 
341 	/**
342 	 * Get the count of client haves.
343 	 *
344 	 * @return count of client haves.
345 	 * @since 4.11
346 	 */
347 	public long getHaves() {
348 		return statistics.haves;
349 	}
350 
351 	/**
352 	 * Time in ms spent in the negotiation phase. For non-bidirectional
353 	 * transports (e.g., HTTP), this is only for the final request that sends
354 	 * back the pack file.
355 	 *
356 	 * @return time for ref advertisement in ms.
357 	 * @since 4.11
358 	 */
359 	public long getTimeNegotiating() {
360 		return statistics.timeNegotiating;
361 	}
362 
363 	/**
364 	 * Get unmodifiable collection of objects to be included in the pack.
365 	 *
366 	 * @return unmodifiable collection of objects to be included in the pack.
367 	 *         May be {@code null} if the pack was hand-crafted in a unit test.
368 	 */
369 	public Set<ObjectId> getInterestingObjects() {
370 		return statistics.interestingObjects;
371 	}
372 
373 	/**
374 	 * Get unmodifiable collection of objects that should be excluded from the
375 	 * pack
376 	 *
377 	 * @return unmodifiable collection of objects that should be excluded from
378 	 *         the pack, as the peer that will receive the pack already has
379 	 *         these objects.
380 	 */
381 	public Set<ObjectId> getUninterestingObjects() {
382 		return statistics.uninterestingObjects;
383 	}
384 
385 	/**
386 	 * Get unmodifiable collection of objects that were shallow commits on the
387 	 * client.
388 	 *
389 	 * @return unmodifiable collection of objects that were shallow commits on
390 	 *         the client.
391 	 */
392 	public Set<ObjectId> getClientShallowCommits() {
393 		return statistics.clientShallowCommits;
394 	}
395 
396 	/**
397 	 * Get unmodifiable list of the cached packs that were reused in the output
398 	 *
399 	 * @return unmodifiable list of the cached packs that were reused in the
400 	 *         output, if any were selected for reuse.
401 	 */
402 	public List<CachedPack> getReusedPacks() {
403 		return statistics.reusedPacks;
404 	}
405 
406 	/**
407 	 * Get unmodifiable collection of the root commits of the history.
408 	 *
409 	 * @return unmodifiable collection of the root commits of the history.
410 	 */
411 	public Set<ObjectId> getRootCommits() {
412 		return statistics.rootCommits;
413 	}
414 
415 	/**
416 	 * Get number of objects in the output pack that went through the delta
417 	 * search process in order to find a potential delta base.
418 	 *
419 	 * @return number of objects in the output pack that went through the delta
420 	 *         search process in order to find a potential delta base.
421 	 */
422 	public int getDeltaSearchNonEdgeObjects() {
423 		return statistics.deltaSearchNonEdgeObjects;
424 	}
425 
426 	/**
427 	 * Get number of objects in the output pack that went through delta base
428 	 * search and found a suitable base.
429 	 *
430 	 * @return number of objects in the output pack that went through delta base
431 	 *         search and found a suitable base. This is a subset of
432 	 *         {@link #getDeltaSearchNonEdgeObjects()}.
433 	 */
434 	public int getDeltasFound() {
435 		return statistics.deltasFound;
436 	}
437 
438 	/**
439 	 * Get total number of objects output.
440 	 *
441 	 * @return total number of objects output. This total includes the value of
442 	 *         {@link #getTotalDeltas()}.
443 	 */
444 	public long getTotalObjects() {
445 		return statistics.totalObjects;
446 	}
447 
448 	/**
449 	 * Get the count of objects that needed to be discovered through an object
450 	 * walk because they were not found in bitmap indices.
451 	 *
452 	 * @return the count of objects that needed to be discovered through an
453 	 *         object walk because they were not found in bitmap indices.
454 	 *         Returns -1 if no bitmap indices were found.
455 	 */
456 	public long getBitmapIndexMisses() {
457 		return statistics.bitmapIndexMisses;
458 	}
459 
460 	/**
461 	 * Get total number of deltas output.
462 	 *
463 	 * @return total number of deltas output. This may be lower than the actual
464 	 *         number of deltas if a cached pack was reused.
465 	 */
466 	public long getTotalDeltas() {
467 		return statistics.totalDeltas;
468 	}
469 
470 	/**
471 	 * Get number of objects whose existing representation was reused in the
472 	 * output.
473 	 *
474 	 * @return number of objects whose existing representation was reused in the
475 	 *         output. This count includes {@link #getReusedDeltas()}.
476 	 */
477 	public long getReusedObjects() {
478 		return statistics.reusedObjects;
479 	}
480 
481 	/**
482 	 * Get number of deltas whose existing representation was reused in the
483 	 * output.
484 	 *
485 	 * @return number of deltas whose existing representation was reused in the
486 	 *         output, as their base object was also output or was assumed
487 	 *         present for a thin pack. This may be lower than the actual number
488 	 *         of reused deltas if a cached pack was reused.
489 	 */
490 	public long getReusedDeltas() {
491 		return statistics.reusedDeltas;
492 	}
493 
494 	/**
495 	 * Get total number of bytes written.
496 	 *
497 	 * @return total number of bytes written. This size includes the pack
498 	 *         header, trailer, thin pack, and reused cached pack(s).
499 	 */
500 	public long getTotalBytes() {
501 		return statistics.totalBytes;
502 	}
503 
504 	/**
505 	 * Get size of the thin pack in bytes.
506 	 *
507 	 * @return size of the thin pack in bytes, if a thin pack was generated. A
508 	 *         thin pack is created when the client already has objects and some
509 	 *         deltas are created against those objects, or if a cached pack is
510 	 *         being used and some deltas will reference objects in the cached
511 	 *         pack. This size does not include the pack header or trailer.
512 	 */
513 	public long getThinPackBytes() {
514 		return statistics.thinPackBytes;
515 	}
516 
517 	/**
518 	 * Get information about this type of object in the pack.
519 	 *
520 	 * @param typeCode
521 	 *            object type code, e.g. OBJ_COMMIT or OBJ_TREE.
522 	 * @return information about this type of object in the pack.
523 	 */
524 	public ObjectType byObjectType(int typeCode) {
525 		return new ObjectType(statistics.objectTypes[typeCode]);
526 	}
527 
528 	/**
529 	 * Whether the resulting pack file was a shallow pack.
530 	 *
531 	 * @return {@code true} if the resulting pack file was a shallow pack.
532 	 */
533 	public boolean isShallow() {
534 		return statistics.depth > 0;
535 	}
536 
537 	/**
538 	 * Get depth (in commits) the pack includes if shallow.
539 	 *
540 	 * @return depth (in commits) the pack includes if shallow.
541 	 */
542 	public int getDepth() {
543 		return statistics.depth;
544 	}
545 
546 	/**
547 	 * Get time in milliseconds spent enumerating the objects that need to be
548 	 * included in the output.
549 	 *
550 	 * @return time in milliseconds spent enumerating the objects that need to
551 	 *         be included in the output. This time includes any restarts that
552 	 *         occur when a cached pack is selected for reuse.
553 	 */
554 	public long getTimeCounting() {
555 		return statistics.timeCounting;
556 	}
557 
558 	/**
559 	 * Get time in milliseconds spent matching existing representations against
560 	 * objects that will be transmitted.
561 	 *
562 	 * @return time in milliseconds spent matching existing representations
563 	 *         against objects that will be transmitted, or that the client can
564 	 *         be assumed to already have.
565 	 */
566 	public long getTimeSearchingForReuse() {
567 		return statistics.timeSearchingForReuse;
568 	}
569 
570 	/**
571 	 * Get time in milliseconds spent finding the sizes of all objects that will
572 	 * enter the delta compression search window.
573 	 *
574 	 * @return time in milliseconds spent finding the sizes of all objects that
575 	 *         will enter the delta compression search window. The sizes need to
576 	 *         be known to better match similar objects together and improve
577 	 *         delta compression ratios.
578 	 */
579 	public long getTimeSearchingForSizes() {
580 		return statistics.timeSearchingForSizes;
581 	}
582 
583 	/**
584 	 * Get time in milliseconds spent on delta compression.
585 	 *
586 	 * @return time in milliseconds spent on delta compression. This is observed
587 	 *         wall-clock time and does not accurately track CPU time used when
588 	 *         multiple threads were used to perform the delta compression.
589 	 */
590 	public long getTimeCompressing() {
591 		return statistics.timeCompressing;
592 	}
593 
594 	/**
595 	 * Get time in milliseconds spent writing the pack output, from start of
596 	 * header until end of trailer.
597 	 *
598 	 * @return time in milliseconds spent writing the pack output, from start of
599 	 *         header until end of trailer. The transfer speed can be
600 	 *         approximated by dividing {@link #getTotalBytes()} by this value.
601 	 */
602 	public long getTimeWriting() {
603 		return statistics.timeWriting;
604 	}
605 
606 	/**
607 	 * @return number of trees traversed in the walk when writing the pack.
608 	 * @since 5.4
609 	 */
610 	public long getTreesTraversed() {
611 		return statistics.treesTraversed;
612 	}
613 
614 	/**
615 	 * @return amount of packfiles offloaded (sent as "packfile-uri")/
616 	 * @since 5.6
617 	 */
618 	public long getOffloadedPackfiles() {
619 		return statistics.offloadedPackfiles;
620 	}
621 
622 	/**
623 	 * @return total size (in bytes) offloaded to HTTP downloads.
624 	 * @since 5.6
625 	 */
626 	public long getOffloadedPackfilesSize() {
627 		return statistics.offloadedPackfileSize;
628 	}
629 
630 	/**
631 	 * Get total time spent processing this pack.
632 	 *
633 	 * @return total time spent processing this pack.
634 	 */
635 	public long getTimeTotal() {
636 		return statistics.timeCounting + statistics.timeSearchingForReuse
637 				+ statistics.timeSearchingForSizes + statistics.timeCompressing
638 				+ statistics.timeWriting;
639 	}
640 
641 	/**
642 	 * Get the average output speed in terms of bytes-per-second.
643 	 *
644 	 * @return the average output speed in terms of bytes-per-second.
645 	 *         {@code getTotalBytes() / (getTimeWriting() / 1000.0)}.
646 	 */
647 	public double getTransferRate() {
648 		return getTotalBytes() / (getTimeWriting() / 1000.0);
649 	}
650 
651 	/**
652 	 * Get formatted message string for display to clients.
653 	 *
654 	 * @return formatted message string for display to clients.
655 	 */
656 	public String getMessage() {
657 		return MessageFormat.format(JGitText.get().packWriterStatistics,
658 				Long.valueOf(statistics.totalObjects),
659 				Long.valueOf(statistics.totalDeltas),
660 				Long.valueOf(statistics.reusedObjects),
661 				Long.valueOf(statistics.reusedDeltas));
662 	}
663 
664 	/**
665 	 * Get a map containing ObjectType statistics.
666 	 *
667 	 * @return a map containing ObjectType statistics.
668 	 */
669 	public Map<Integer, ObjectType> getObjectTypes() {
670 		HashMap<Integer, ObjectType> map = new HashMap<>();
671 		map.put(Integer.valueOf(OBJ_BLOB), new ObjectType(
672 				statistics.objectTypes[OBJ_BLOB]));
673 		map.put(Integer.valueOf(OBJ_COMMIT), new ObjectType(
674 				statistics.objectTypes[OBJ_COMMIT]));
675 		map.put(Integer.valueOf(OBJ_TAG), new ObjectType(
676 				statistics.objectTypes[OBJ_TAG]));
677 		map.put(Integer.valueOf(OBJ_TREE), new ObjectType(
678 				statistics.objectTypes[OBJ_TREE]));
679 		return map;
680 	}
681 }