View Javadoc
1   /*
2    * Copyright (C) 2012, Google Inc.
3    * and other copyright owners as documented in the project's IP log.
4    *
5    * This program and the accompanying materials are made available
6    * under the terms of the Eclipse Distribution License v1.0 which
7    * accompanies this distribution, is reproduced below, and is
8    * available at http://www.eclipse.org/org/documents/edl-v10.php
9    *
10   * All rights reserved.
11   *
12   * Redistribution and use in source and binary forms, with or
13   * without modification, are permitted provided that the following
14   * conditions are met:
15   *
16   * - Redistributions of source code must retain the above copyright
17   *   notice, this list of conditions and the following disclaimer.
18   *
19   * - Redistributions in binary form must reproduce the above
20   *   copyright notice, this list of conditions and the following
21   *   disclaimer in the documentation and/or other materials provided
22   *   with the distribution.
23   *
24   * - Neither the name of the Eclipse Foundation, Inc. nor the
25   *   names of its contributors may be used to endorse or promote
26   *   products derived from this software without specific prior
27   *   written permission.
28   *
29   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
30   * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
31   * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
32   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
34   * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
35   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
36   * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
37   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
38   * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
39   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
40   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
41   * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42   */
43  
44  package org.eclipse.jgit.internal.storage.file;
45  
46  import java.text.MessageFormat;
47  import java.util.Iterator;
48  import java.util.NoSuchElementException;
49  
50  import com.googlecode.javaewah.EWAHCompressedBitmap;
51  import com.googlecode.javaewah.IntIterator;
52  
53  import org.eclipse.jgit.internal.JGitText;
54  import org.eclipse.jgit.lib.AnyObjectId;
55  import org.eclipse.jgit.lib.BitmapIndex;
56  import org.eclipse.jgit.lib.BitmapObject;
57  import org.eclipse.jgit.lib.Constants;
58  import org.eclipse.jgit.lib.ObjectId;
59  import org.eclipse.jgit.lib.ObjectIdOwnerMap;
60  import org.eclipse.jgit.util.BlockList;
61  
62  /** A compressed bitmap representation of the entire object graph. */
63  public class BitmapIndexImpl implements BitmapIndex {
64  	private static final int EXTRA_BITS = 10 * 1024;
65  
66  	private final PackBitmapIndex packIndex;
67  
68  	private final MutableBitmapIndex mutableIndex;
69  
70  	private final int indexObjectCount;
71  
72  	/**
73  	 * Creates a BitmapIndex that is back by Compressed bitmaps.
74  	 *
75  	 * @param packIndex
76  	 *            the bitmap index for the pack.
77  	 */
78  	public BitmapIndexImpl(PackBitmapIndex packIndex) {
79  		this.packIndex = packIndex;
80  		mutableIndex = new MutableBitmapIndex();
81  		indexObjectCount = packIndex.getObjectCount();
82  	}
83  
84  	PackBitmapIndex getPackBitmapIndex() {
85  		return packIndex;
86  	}
87  
88  	public CompressedBitmap getBitmap(AnyObjectId objectId) {
89  		EWAHCompressedBitmap compressed = packIndex.getBitmap(objectId);
90  		if (compressed == null)
91  			return null;
92  		return new CompressedBitmap(compressed);
93  	}
94  
95  	public CompressedBitmapBuilder newBitmapBuilder() {
96  		return new CompressedBitmapBuilder();
97  	}
98  
99  	private int findPosition(AnyObjectId objectId) {
100 		int position = packIndex.findPosition(objectId);
101 		if (position < 0) {
102 			position = mutableIndex.findPosition(objectId);
103 			if (position >= 0)
104 				position += indexObjectCount;
105 		}
106 		return position;
107 	}
108 
109 	private int addObject(AnyObjectId objectId, int type) {
110 		int position = findPosition(objectId);
111 		if (position < 0) {
112 			position = mutableIndex.addObject(objectId, type);
113 			position += indexObjectCount;
114 		}
115 		return position;
116 	}
117 
118 	private static final class ComboBitset {
119 		private InflatingBitSet inflatingBitmap;
120 
121 		private BitSet toAdd;
122 
123 		private BitSet toRemove;
124 
125 		private ComboBitset() {
126 			this(new EWAHCompressedBitmap());
127 		}
128 
129 		private ComboBitset(EWAHCompressedBitmap bitmap) {
130 			this.inflatingBitmap = new InflatingBitSet(bitmap);
131 		}
132 
133 		EWAHCompressedBitmap combine() {
134 			EWAHCompressedBitmap toAddCompressed = null;
135 			if (toAdd != null) {
136 				toAddCompressed = toAdd.toEWAHCompressedBitmap();
137 				toAdd = null;
138 			}
139 
140 			EWAHCompressedBitmap toRemoveCompressed = null;
141 			if (toRemove != null) {
142 				toRemoveCompressed = toRemove.toEWAHCompressedBitmap();
143 				toRemove = null;
144 			}
145 
146 			if (toAddCompressed != null)
147 				or(toAddCompressed);
148 			if (toRemoveCompressed != null)
149 				andNot(toRemoveCompressed);
150 			return inflatingBitmap.getBitmap();
151 		}
152 
153 		void or(EWAHCompressedBitmap inbits) {
154 			if (toRemove != null)
155 				combine();
156 			inflatingBitmap = inflatingBitmap.or(inbits);
157 		}
158 
159 		void andNot(EWAHCompressedBitmap inbits) {
160 			if (toAdd != null || toRemove != null)
161 				combine();
162 			inflatingBitmap = inflatingBitmap.andNot(inbits);
163 		}
164 
165 		void xor(EWAHCompressedBitmap inbits) {
166 			if (toAdd != null || toRemove != null)
167 				combine();
168 			inflatingBitmap = inflatingBitmap.xor(inbits);
169 		}
170 
171 		boolean contains(int position) {
172 			if (toRemove != null && toRemove.get(position))
173 				return false;
174 			if (toAdd != null && toAdd.get(position))
175 				return true;
176 			return inflatingBitmap.contains(position);
177 		}
178 
179 		void remove(int position) {
180 			if (toAdd != null)
181 				toAdd.clear(position);
182 
183 			if (inflatingBitmap.maybeContains(position)) {
184 				if (toRemove == null)
185 					toRemove = new BitSet(position + EXTRA_BITS);
186 				toRemove.set(position);
187 			}
188 		}
189 
190 		void set(int position) {
191 			if (toRemove != null)
192 				toRemove.clear(position);
193 
194 			if (toAdd == null)
195 				toAdd = new BitSet(position + EXTRA_BITS);
196 			toAdd.set(position);
197 		}
198 	}
199 
200 	private final class CompressedBitmapBuilder implements BitmapBuilder {
201 		private ComboBitset bitset = new ComboBitset();
202 
203 		public boolean add(AnyObjectId objectId, int type) {
204 			int position = addObject(objectId, type);
205 			if (bitset.contains(position))
206 				return false;
207 
208 			Bitmap entry = getBitmap(objectId);
209 			if (entry != null) {
210 				or(entry);
211 				return false;
212 			}
213 
214 			bitset.set(position);
215 			return true;
216 		}
217 
218 		public boolean contains(AnyObjectId objectId) {
219 			int position = findPosition(objectId);
220 			return 0 <= position && bitset.contains(position);
221 		}
222 
223 		public void remove(AnyObjectId objectId) {
224 			int position = findPosition(objectId);
225 			if (0 <= position)
226 				bitset.remove(position);
227 		}
228 
229 		public CompressedBitmapBuilder or(Bitmap other) {
230 			if (isSameCompressedBitmap(other)) {
231 				bitset.or(((CompressedBitmap) other).bitmap);
232 			} else if (isSameCompressedBitmapBuilder(other)) {
233 				CompressedBitmapBuilder b = (CompressedBitmapBuilder) other;
234 				bitset.or(b.bitset.combine());
235 			} else {
236 				throw new IllegalArgumentException();
237 			}
238 			return this;
239 		}
240 
241 		public CompressedBitmapBuilder andNot(Bitmap other) {
242 			if (isSameCompressedBitmap(other)) {
243 				bitset.andNot(((CompressedBitmap) other).bitmap);
244 			} else if (isSameCompressedBitmapBuilder(other)) {
245 				CompressedBitmapBuilder b = (CompressedBitmapBuilder) other;
246 				bitset.andNot(b.bitset.combine());
247 			} else {
248 				throw new IllegalArgumentException();
249 			}
250 			return this;
251 		}
252 
253 		public CompressedBitmapBuilder xor(Bitmap other) {
254 			if (isSameCompressedBitmap(other)) {
255 				bitset.xor(((CompressedBitmap) other).bitmap);
256 			} else if (isSameCompressedBitmapBuilder(other)) {
257 				CompressedBitmapBuilder b = (CompressedBitmapBuilder) other;
258 				bitset.xor(b.bitset.combine());
259 			} else {
260 				throw new IllegalArgumentException();
261 			}
262 			return this;
263 		}
264 
265 		/** @return the fully built immutable bitmap */
266 		public CompressedBitmap build() {
267 			return new CompressedBitmap(bitset.combine());
268 		}
269 
270 		public Iterator<BitmapObject> iterator() {
271 			return build().iterator();
272 		}
273 
274 		public int cardinality() {
275 			return bitset.combine().cardinality();
276 		}
277 
278 		public boolean removeAllOrNone(PackBitmapIndex index) {
279 			if (!packIndex.equals(index))
280 				return false;
281 
282 			EWAHCompressedBitmap curr = bitset.combine()
283 					.xor(ones(indexObjectCount));
284 
285 			IntIterator ii = curr.intIterator();
286 			if (ii.hasNext() && ii.next() < indexObjectCount)
287 				return false;
288 			bitset = new ComboBitset(curr);
289 			return true;
290 		}
291 
292 		private BitmapIndexImpl getBitmapIndex() {
293 			return BitmapIndexImpl.this;
294 		}
295 	}
296 
297 	final class CompressedBitmap implements Bitmap {
298 		private final EWAHCompressedBitmap bitmap;
299 
300 		private CompressedBitmap(EWAHCompressedBitmap bitmap) {
301 			this.bitmap = bitmap;
302 		}
303 
304 		public CompressedBitmap or(Bitmap other) {
305 			return new CompressedBitmap(bitmap.or(bitmapOf(other)));
306 		}
307 
308 		public CompressedBitmap andNot(Bitmap other) {
309 			return new CompressedBitmap(bitmap.andNot(bitmapOf(other)));
310 		}
311 
312 		public CompressedBitmap xor(Bitmap other) {
313 			return new CompressedBitmap(bitmap.xor(bitmapOf(other)));
314 		}
315 
316 		private EWAHCompressedBitmap bitmapOf(Bitmap other) {
317 			if (isSameCompressedBitmap(other))
318 				return ((CompressedBitmap) other).bitmap;
319 			if (isSameCompressedBitmapBuilder(other))
320 				return ((CompressedBitmapBuilder) other).build().bitmap;
321 			CompressedBitmapBuilder builder = newBitmapBuilder();
322 			builder.or(other);
323 			return builder.build().bitmap;
324 		}
325 
326 		private final IntIterator ofObjectType(int type) {
327 			return packIndex.ofObjectType(bitmap, type).intIterator();
328 		}
329 
330 		public Iterator<BitmapObject> iterator() {
331 			final IntIterator dynamic = bitmap.andNot(ones(indexObjectCount))
332 					.intIterator();
333 			final IntIterator commits = ofObjectType(Constants.OBJ_COMMIT);
334 			final IntIterator trees = ofObjectType(Constants.OBJ_TREE);
335 			final IntIterator blobs = ofObjectType(Constants.OBJ_BLOB);
336 			final IntIterator tags = ofObjectType(Constants.OBJ_TAG);
337 			return new Iterator<BitmapObject>() {
338 				private final BitmapObjectImpl out = new BitmapObjectImpl();
339 				private int type;
340 				private IntIterator cached = dynamic;
341 
342 				public boolean hasNext() {
343 					if (!cached.hasNext()) {
344 						if (commits.hasNext()) {
345 							type = Constants.OBJ_COMMIT;
346 							cached = commits;
347 						} else if (trees.hasNext()) {
348 							type = Constants.OBJ_TREE;
349 							cached = trees;
350 						} else if (blobs.hasNext()) {
351 							type = Constants.OBJ_BLOB;
352 							cached = blobs;
353 						} else if (tags.hasNext()) {
354 							type = Constants.OBJ_TAG;
355 							cached = tags;
356 						} else {
357 							return false;
358 						}
359 					}
360 					return true;
361 				}
362 
363 				public BitmapObject next() {
364 					if (!hasNext())
365 						throw new NoSuchElementException();
366 
367 					int position = cached.next();
368 					if (position < indexObjectCount) {
369 						out.type = type;
370 						out.objectId = packIndex.getObject(position);
371 					} else {
372 						position -= indexObjectCount;
373 						MutableEntry entry = mutableIndex.getObject(position);
374 						out.type = entry.type;
375 						out.objectId = entry;
376 					}
377 					return out;
378 				}
379 
380 				public void remove() {
381 					throw new UnsupportedOperationException();
382 				}
383 			};
384 		}
385 
386 		EWAHCompressedBitmap getEwahCompressedBitmap() {
387 			return bitmap;
388 		}
389 
390 		private BitmapIndexImpl getPackBitmapIndex() {
391 			return BitmapIndexImpl.this;
392 		}
393 	}
394 
395 	private static final class MutableBitmapIndex {
396 		private final ObjectIdOwnerMap<MutableEntry>
397 				revMap = new ObjectIdOwnerMap<MutableEntry>();
398 
399 		private final BlockList<MutableEntry>
400 				revList = new BlockList<MutableEntry>();
401 
402 		int findPosition(AnyObjectId objectId) {
403 			MutableEntry entry = revMap.get(objectId);
404 			if (entry == null)
405 				return -1;
406 			return entry.position;
407 		}
408 
409 		MutableEntry getObject(int position) {
410 			try {
411 				MutableEntry entry = revList.get(position);
412 				if (entry == null)
413 					throw new IllegalArgumentException(MessageFormat.format(
414 							JGitText.get().objectNotFound,
415 							String.valueOf(position)));
416 				return entry;
417 			} catch (IndexOutOfBoundsException ex) {
418 				throw new IllegalArgumentException(ex);
419 			}
420 		}
421 
422 		int addObject(AnyObjectId objectId, int type) {
423 			MutableEntry entry = new MutableEntry(
424 					objectId, type, revList.size());
425 			revList.add(entry);
426 			revMap.add(entry);
427 			return entry.position;
428 		}
429 	}
430 
431 	private static final class MutableEntry extends ObjectIdOwnerMap.Entry {
432 		private final int type;
433 
434 		private final int position;
435 
436 		MutableEntry(AnyObjectId objectId, int type, int position) {
437 			super(objectId);
438 			this.type = type;
439 			this.position = position;
440 		}
441 	}
442 
443 	private static final class BitmapObjectImpl extends BitmapObject {
444 		private ObjectId objectId;
445 
446 		private int type;
447 
448 		@Override
449 		public ObjectId getObjectId() {
450 			return objectId;
451 		}
452 
453 		@Override
454 		public int getType() {
455 			return type;
456 		}
457 	}
458 
459 	private boolean isSameCompressedBitmap(Bitmap other) {
460 		if (other instanceof CompressedBitmap) {
461 			CompressedBitmap b = (CompressedBitmap) other;
462 			return this == b.getPackBitmapIndex();
463 		}
464 		return false;
465 	}
466 
467 	private boolean isSameCompressedBitmapBuilder(Bitmap other) {
468 		if (other instanceof CompressedBitmapBuilder) {
469 			CompressedBitmapBuilder b = (CompressedBitmapBuilder) other;
470 			return this == b.getBitmapIndex();
471 		}
472 		return false;
473 	}
474 
475 	private static final EWAHCompressedBitmap ones(int sizeInBits) {
476 		EWAHCompressedBitmap mask = new EWAHCompressedBitmap();
477 		mask.addStreamOfEmptyWords(
478 				true, sizeInBits / EWAHCompressedBitmap.wordinbits);
479 		int remaining = sizeInBits % EWAHCompressedBitmap.wordinbits;
480 		if (remaining > 0)
481 			mask.add((1L << remaining) - 1, remaining);
482 		return mask;
483 	}
484 }