View Javadoc
1   /*
2    * Copyright (C) 2012, Google Inc.
3    * and other copyright owners as documented in the project's IP log.
4    *
5    * This program and the accompanying materials are made available
6    * under the terms of the Eclipse Distribution License v1.0 which
7    * accompanies this distribution, is reproduced below, and is
8    * available at http://www.eclipse.org/org/documents/edl-v10.php
9    *
10   * All rights reserved.
11   *
12   * Redistribution and use in source and binary forms, with or
13   * without modification, are permitted provided that the following
14   * conditions are met:
15   *
16   * - Redistributions of source code must retain the above copyright
17   *   notice, this list of conditions and the following disclaimer.
18   *
19   * - Redistributions in binary form must reproduce the above
20   *   copyright notice, this list of conditions and the following
21   *   disclaimer in the documentation and/or other materials provided
22   *   with the distribution.
23   *
24   * - Neither the name of the Eclipse Foundation, Inc. nor the
25   *   names of its contributors may be used to endorse or promote
26   *   products derived from this software without specific prior
27   *   written permission.
28   *
29   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
30   * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
31   * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
32   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
34   * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
35   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
36   * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
37   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
38   * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
39   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
40   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
41   * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42   */
43  
44  package org.eclipse.jgit.internal.storage.file;
45  
46  import java.text.MessageFormat;
47  import java.util.Collections;
48  import java.util.Comparator;
49  import java.util.Iterator;
50  import java.util.List;
51  import java.util.NoSuchElementException;
52  
53  import com.googlecode.javaewah.EWAHCompressedBitmap;
54  
55  import org.eclipse.jgit.internal.JGitText;
56  import org.eclipse.jgit.internal.storage.file.BitmapIndexImpl.CompressedBitmap;
57  import org.eclipse.jgit.internal.storage.pack.ObjectToPack;
58  import org.eclipse.jgit.lib.AnyObjectId;
59  import org.eclipse.jgit.lib.BitmapIndex.Bitmap;
60  import org.eclipse.jgit.lib.BitmapIndex.BitmapBuilder;
61  import org.eclipse.jgit.lib.Constants;
62  import org.eclipse.jgit.lib.ObjectId;
63  import org.eclipse.jgit.lib.ObjectIdOwnerMap;
64  import org.eclipse.jgit.util.BlockList;
65  
66  /**
67   * Helper for constructing {@link PackBitmapIndex}es.
68   */
69  public class PackBitmapIndexBuilder extends BasePackBitmapIndex {
70  	private static final int MAX_XOR_OFFSET_SEARCH = 10;
71  
72  	private final EWAHCompressedBitmap commits;
73  	private final EWAHCompressedBitmap trees;
74  	private final EWAHCompressedBitmap blobs;
75  	private final EWAHCompressedBitmap tags;
76  	private final BlockList<PositionEntry> byOffset;
77  	private final BlockList<StoredBitmap>
78  			byAddOrder = new BlockList<StoredBitmap>();
79  	private final ObjectIdOwnerMap<PositionEntry>
80  			positionEntries = new ObjectIdOwnerMap<PositionEntry>();
81  
82  	/**
83  	 * Creates a PackBitmapIndex used for building the contents of an index
84  	 * file.
85  	 *
86  	 * @param objects
87  	 *            objects sorted by name. The list must be initially sorted by
88  	 *            ObjectId (name); it will be resorted in place.
89  	 */
90  	public PackBitmapIndexBuilder(List<ObjectToPack> objects) {
91  		super(new ObjectIdOwnerMap<StoredBitmap>());
92  		byOffset = new BlockList<>(objects.size());
93  		sortByOffsetAndIndex(byOffset, positionEntries, objects);
94  
95  		// 64 objects fit in a single long word (64 bits).
96  		// On average a repository is 30% commits, 30% trees, 30% blobs.
97  		// Initialize bitmap capacity for worst case to minimize growing.
98  		int sizeInWords = Math.max(4, byOffset.size() / 64 / 3);
99  		commits = new EWAHCompressedBitmap(sizeInWords);
100 		trees = new EWAHCompressedBitmap(sizeInWords);
101 		blobs = new EWAHCompressedBitmap(sizeInWords);
102 		tags = new EWAHCompressedBitmap(sizeInWords);
103 		for (int i = 0; i < objects.size(); i++) {
104 			int type = objects.get(i).getType();
105 			switch (type) {
106 			case Constants.OBJ_COMMIT:
107 				commits.set(i);
108 				break;
109 			case Constants.OBJ_TREE:
110 				trees.set(i);
111 				break;
112 			case Constants.OBJ_BLOB:
113 				blobs.set(i);
114 				break;
115 			case Constants.OBJ_TAG:
116 				tags.set(i);
117 				break;
118 			default:
119 				throw new IllegalArgumentException(MessageFormat.format(
120 						JGitText.get().badObjectType, String.valueOf(type)));
121 			}
122 		}
123 		commits.trim();
124 		trees.trim();
125 		blobs.trim();
126 		tags.trim();
127 	}
128 
129 	private static void sortByOffsetAndIndex(BlockList<PositionEntry> byOffset,
130 			ObjectIdOwnerMap<PositionEntry> positionEntries,
131 			List<ObjectToPack> entries) {
132 		for (int i = 0; i < entries.size(); i++) {
133 			positionEntries.add(new PositionEntry(entries.get(i), i));
134 		}
135 		Collections.sort(entries, new Comparator<ObjectToPack>() {
136 			public int compare(ObjectToPack a, ObjectToPack b) {
137 				return Long.signum(a.getOffset() - b.getOffset());
138 			}
139 		});
140 		for (int i = 0; i < entries.size(); i++) {
141 			PositionEntry e = positionEntries.get(entries.get(i));
142 			e.offsetPosition = i;
143 			byOffset.add(e);
144 		}
145 	}
146 
147 	/** @return set of objects included in the pack. */
148 	public ObjectIdOwnerMap<ObjectIdOwnerMap.Entry> getObjectSet() {
149 		ObjectIdOwnerMap<ObjectIdOwnerMap.Entry> r = new ObjectIdOwnerMap<>();
150 		for (PositionEntry e : byOffset) {
151 			r.add(new ObjectIdOwnerMap.Entry(e) {
152 				// A new entry that copies the ObjectId
153 			});
154 		}
155 		return r;
156 	}
157 
158 	/**
159 	 * Stores the bitmap for the objectId.
160 	 *
161 	 * @param objectId
162 	 *            the object id key for the bitmap.
163 	 * @param bitmap
164 	 *            the bitmap
165 	 * @param flags
166 	 *            the flags to be stored with the bitmap
167 	 */
168 	public void addBitmap(AnyObjectId objectId, Bitmap bitmap, int flags) {
169 		if (bitmap instanceof BitmapBuilder)
170 			bitmap = ((BitmapBuilder) bitmap).build();
171 
172 		EWAHCompressedBitmap compressed;
173 		if (bitmap instanceof CompressedBitmap)
174 			compressed = ((CompressedBitmap) bitmap).getEwahCompressedBitmap();
175 		else
176 			throw new IllegalArgumentException(bitmap.getClass().toString());
177 
178 		addBitmap(objectId, compressed, flags);
179 	}
180 
181 	/**
182 	 * Stores the bitmap for the objectId.
183 	 *
184 	 * @param objectId
185 	 *            the object id key for the bitmap.
186 	 * @param bitmap
187 	 *            the bitmap
188 	 * @param flags
189 	 *            the flags to be stored with the bitmap
190 	 */
191 	public void addBitmap(
192 			AnyObjectId objectId, EWAHCompressedBitmap bitmap, int flags) {
193 		bitmap.trim();
194 		StoredBitmap result = new StoredBitmap(objectId, bitmap, null, flags);
195 		getBitmaps().add(result);
196 		byAddOrder.add(result);
197 	}
198 
199 	@Override
200 	public EWAHCompressedBitmap ofObjectType(
201 			EWAHCompressedBitmap bitmap, int type) {
202 		switch (type) {
203 		case Constants.OBJ_BLOB:
204 			return getBlobs().and(bitmap);
205 		case Constants.OBJ_TREE:
206 			return getTrees().and(bitmap);
207 		case Constants.OBJ_COMMIT:
208 			return getCommits().and(bitmap);
209 		case Constants.OBJ_TAG:
210 			return getTags().and(bitmap);
211 		}
212 		throw new IllegalArgumentException();
213 	}
214 
215 	@Override
216 	public int findPosition(AnyObjectId objectId) {
217 		PositionEntry entry = positionEntries.get(objectId);
218 		if (entry == null)
219 			return -1;
220 		return entry.offsetPosition;
221 	}
222 
223 	@Override
224 	public ObjectId getObject(int position) throws IllegalArgumentException {
225 		ObjectId objectId = byOffset.get(position);
226 		if (objectId == null)
227 			throw new IllegalArgumentException();
228 		return objectId;
229 	}
230 
231 	/** @return the commit object bitmap. */
232 	public EWAHCompressedBitmap getCommits() {
233 		return commits;
234 	}
235 
236 	/** @return the tree object bitmap. */
237 	public EWAHCompressedBitmap getTrees() {
238 		return trees;
239 	}
240 
241 	/** @return the blob object bitmap. */
242 	public EWAHCompressedBitmap getBlobs() {
243 		return blobs;
244 	}
245 
246 	/** @return the tag object bitmap. */
247 	public EWAHCompressedBitmap getTags() {
248 		return tags;
249 	}
250 
251 	/** @return the index storage options. */
252 	public int getOptions() {
253 		return PackBitmapIndexV1.OPT_FULL;
254 	}
255 
256 	/** @return the number of bitmaps. */
257 	public int getBitmapCount() {
258 		return getBitmaps().size();
259 	}
260 
261 	/** Removes all the bitmaps entries added. */
262 	public void clearBitmaps() {
263 		byAddOrder.clear();
264 		getBitmaps().clear();
265 	}
266 
267 	@Override
268 	public int getObjectCount() {
269 		return byOffset.size();
270 	}
271 
272 	/** @return an iterator over the xor compressed entries. */
273 	public Iterable<StoredEntry> getCompressedBitmaps() {
274 		// Add order is from oldest to newest. The reverse add order is the
275 		// output order.
276 		return new Iterable<StoredEntry>() {
277 			public Iterator<StoredEntry> iterator() {
278 				return new Iterator<StoredEntry>() {
279 					private int index = byAddOrder.size() - 1;
280 
281 					public boolean hasNext() {
282 						return index >= 0;
283 					}
284 
285 					public StoredEntry next() {
286 						if (!hasNext())
287 							throw new NoSuchElementException();
288 						StoredBitmap item = byAddOrder.get(index);
289 						int bestXorOffset = 0;
290 						EWAHCompressedBitmap bestBitmap = item.getBitmap();
291 
292 						// Attempt to compress the bitmap with an XOR of the
293 						// previously written entries.
294 						for (int i = 1; i <= MAX_XOR_OFFSET_SEARCH; i++) {
295 							int curr = i + index;
296 							if (curr >= byAddOrder.size())
297 								break;
298 
299 							StoredBitmap other = byAddOrder.get(curr);
300 							EWAHCompressedBitmap bitmap = other.getBitmap()
301 									.xor(item.getBitmap());
302 
303 							if (bitmap.sizeInBytes()
304 									< bestBitmap.sizeInBytes()) {
305 								bestBitmap = bitmap;
306 								bestXorOffset = i;
307 							}
308 						}
309 						index--;
310 
311 						PositionEntry entry = positionEntries.get(item);
312 						if (entry == null)
313 							throw new IllegalStateException();
314 						return new StoredEntry(entry.namePosition, bestBitmap,
315 								bestXorOffset, item.getFlags());
316 					}
317 
318 					public void remove() {
319 						throw new UnsupportedOperationException();
320 					}
321 				};
322 			}
323 		};
324 	}
325 
326 	/** Data object for the on disk representation of a bitmap entry. */
327 	public static final class StoredEntry {
328 		private final long objectId;
329 		private final EWAHCompressedBitmap bitmap;
330 		private final int xorOffset;
331 		private final int flags;
332 
333 		private StoredEntry(long objectId, EWAHCompressedBitmap bitmap,
334 				int xorOffset, int flags) {
335 			this.objectId = objectId;
336 			this.bitmap = bitmap;
337 			this.xorOffset = xorOffset;
338 			this.flags = flags;
339 		}
340 
341 		/** @return the bitmap */
342 		public EWAHCompressedBitmap getBitmap() {
343 			return bitmap;
344 		}
345 
346 		/** @return the xorOffset */
347 		public int getXorOffset() {
348 			return xorOffset;
349 		}
350 
351 		/** @return the flags */
352 		public int getFlags() {
353 			return flags;
354 		}
355 
356 		/** @return the ObjectId */
357 		public long getObjectId() {
358 			return objectId;
359 		}
360 	}
361 
362 	private static final class PositionEntry extends ObjectIdOwnerMap.Entry {
363 		private final int namePosition;
364 
365 		private int offsetPosition;
366 
367 		private PositionEntry(AnyObjectId objectId, int namePosition) {
368 			super(objectId);
369 			this.namePosition = namePosition;
370 		}
371 	}
372 }