View Javadoc
1   /*
2    * Copyright (C) 2017, Google Inc.
3    * and other copyright owners as documented in the project's IP log.
4    *
5    * This program and the accompanying materials are made available
6    * under the terms of the Eclipse Distribution License v1.0 which
7    * accompanies this distribution, is reproduced below, and is
8    * available at http://www.eclipse.org/org/documents/edl-v10.php
9    *
10   * All rights reserved.
11   *
12   * Redistribution and use in source and binary forms, with or
13   * without modification, are permitted provided that the following
14   * conditions are met:
15   *
16   * - Redistributions of source code must retain the above copyright
17   *   notice, this list of conditions and the following disclaimer.
18   *
19   * - Redistributions in binary form must reproduce the above
20   *   copyright notice, this list of conditions and the following
21   *   disclaimer in the documentation and/or other materials provided
22   *   with the distribution.
23   *
24   * - Neither the name of the Eclipse Foundation, Inc. nor the
25   *   names of its contributors may be used to endorse or promote
26   *   products derived from this software without specific prior
27   *   written permission.
28   *
29   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
30   * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
31   * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
32   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
34   * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
35   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
36   * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
37   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
38   * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
39   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
40   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
41   * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42   */
43  
44  package org.eclipse.jgit.internal.storage.dfs;
45  
46  import java.io.EOFException;
47  import java.io.IOException;
48  import java.nio.ByteBuffer;
49  import java.text.MessageFormat;
50  
51  import org.eclipse.jgit.errors.PackInvalidException;
52  import org.eclipse.jgit.internal.storage.pack.PackExt;
53  
54  /** Block based file stored in {@link DfsBlockCache}. */
55  abstract class BlockBasedFile {
56  	/** Cache that owns this file and its data. */
57  	final DfsBlockCache cache;
58  
59  	/** Unique identity of this file while in-memory. */
60  	final DfsStreamKey key;
61  
62  	/** Description of the associated pack file's storage. */
63  	final DfsPackDescription desc;
64  	final PackExt ext;
65  
66  	/**
67  	 * Preferred alignment for loading blocks from the backing file.
68  	 * <p>
69  	 * It is initialized to 0 and filled in on the first read made from the
70  	 * file. Block sizes may be odd, e.g. 4091, caused by the underling DFS
71  	 * storing 4091 user bytes and 5 bytes block metadata into a lower level
72  	 * 4096 byte block on disk.
73  	 */
74  	volatile int blockSize;
75  
76  	/**
77  	 * Total number of bytes in this pack file.
78  	 * <p>
79  	 * This field initializes to -1 and gets populated when a block is loaded.
80  	 */
81  	volatile long length;
82  
83  	/** True once corruption has been detected that cannot be worked around. */
84  	volatile boolean invalid;
85  
86  	/** Exception that caused the packfile to be flagged as invalid */
87  	protected volatile Exception invalidatingCause;
88  
89  	BlockBasedFile(DfsBlockCache cache, DfsPackDescription desc, PackExt ext) {
90  		this.cache = cache;
91  		this.key = desc.getStreamKey(ext);
92  		this.desc = desc;
93  		this.ext = ext;
94  	}
95  
96  	String getFileName() {
97  		return desc.getFileName(ext);
98  	}
99  
100 	boolean invalid() {
101 		return invalid;
102 	}
103 
104 	void setInvalid() {
105 		invalid = true;
106 	}
107 
108 	void setBlockSize(int newSize) {
109 		blockSize = newSize;
110 	}
111 
112 	long alignToBlock(long pos) {
113 		int size = blockSize;
114 		if (size == 0)
115 			size = cache.getBlockSize();
116 		return (pos / size) * size;
117 	}
118 
119 	int blockSize(ReadableChannel rc) {
120 		// If the block alignment is not yet known, discover it. Prefer the
121 		// larger size from either the cache or the file itself.
122 		int size = blockSize;
123 		if (size == 0) {
124 			size = rc.blockSize();
125 			if (size <= 0)
126 				size = cache.getBlockSize();
127 			else if (size < cache.getBlockSize())
128 				size = (cache.getBlockSize() / size) * size;
129 			blockSize = size;
130 		}
131 		return size;
132 	}
133 
134 	DfsBlock getOrLoadBlock(long pos, DfsReader ctx) throws IOException {
135 		try (LazyChannel c = new LazyChannel(ctx, desc, ext)) {
136 			return cache.getOrLoad(this, pos, ctx, c);
137 		}
138 	}
139 
140 	DfsBlock readOneBlock(long pos, DfsReader ctx, ReadableChannel rc)
141 			throws IOException {
142 		if (invalid) {
143 			throw new PackInvalidException(getFileName(), invalidatingCause);
144 		}
145 
146 		ctx.stats.readBlock++;
147 		long start = System.nanoTime();
148 		try {
149 			int size = blockSize(rc);
150 			pos = (pos / size) * size;
151 
152 			// If the size of the file is not yet known, try to discover it.
153 			// Channels may choose to return -1 to indicate they don't
154 			// know the length yet, in this case read up to the size unit
155 			// given by the caller, then recheck the length.
156 			long len = length;
157 			if (len < 0) {
158 				len = rc.size();
159 				if (0 <= len)
160 					length = len;
161 			}
162 
163 			if (0 <= len && len < pos + size)
164 				size = (int) (len - pos);
165 			if (size <= 0)
166 				throw new EOFException(MessageFormat.format(
167 						DfsText.get().shortReadOfBlock, Long.valueOf(pos),
168 						getFileName(), Long.valueOf(0), Long.valueOf(0)));
169 
170 			byte[] buf = new byte[size];
171 			rc.position(pos);
172 			int cnt = read(rc, ByteBuffer.wrap(buf, 0, size));
173 			ctx.stats.readBlockBytes += cnt;
174 			if (cnt != size) {
175 				if (0 <= len) {
176 					throw new EOFException(MessageFormat.format(
177 							DfsText.get().shortReadOfBlock, Long.valueOf(pos),
178 							getFileName(), Integer.valueOf(size),
179 							Integer.valueOf(cnt)));
180 				}
181 
182 				// Assume the entire thing was read in a single shot, compact
183 				// the buffer to only the space required.
184 				byte[] n = new byte[cnt];
185 				System.arraycopy(buf, 0, n, 0, n.length);
186 				buf = n;
187 			} else if (len < 0) {
188 				// With no length at the start of the read, the channel should
189 				// have the length available at the end.
190 				length = len = rc.size();
191 			}
192 
193 			return new DfsBlock(key, pos, buf);
194 		} finally {
195 			ctx.stats.readBlockMicros += elapsedMicros(start);
196 		}
197 	}
198 
199 	static int read(ReadableChannel rc, ByteBuffer buf) throws IOException {
200 		int n;
201 		do {
202 			n = rc.read(buf);
203 		} while (0 < n && buf.hasRemaining());
204 		return buf.position();
205 	}
206 
207 	static long elapsedMicros(long start) {
208 		return (System.nanoTime() - start) / 1000L;
209 	}
210 
211 	/**
212 	 * A supplier of readable channel that opens the channel lazily.
213 	 */
214 	private static class LazyChannel
215 			implements AutoCloseable, DfsBlockCache.ReadableChannelSupplier {
216 		private final DfsReader ctx;
217 		private final DfsPackDescription desc;
218 		private final PackExt ext;
219 
220 		private ReadableChannel rc;
221 
222 		LazyChannel(DfsReader ctx, DfsPackDescription desc, PackExt ext) {
223 			this.ctx = ctx;
224 			this.desc = desc;
225 			this.ext = ext;
226 		}
227 
228 		@Override
229 		public ReadableChannel get() throws IOException {
230 			if (rc == null) {
231 				rc = ctx.db.openFile(desc, ext);
232 			}
233 			return rc;
234 		}
235 
236 		@Override
237 		public void close() throws IOException {
238 			if (rc != null) {
239 				rc.close();
240 			}
241 		}
242 	}
243 }