View Javadoc
1   /*
2    * Copyright (C) 2017, Google Inc. and others
3    *
4    * This program and the accompanying materials are made available under the
5    * terms of the Eclipse Distribution License v. 1.0 which is available at
6    * https://www.eclipse.org/org/documents/edl-v10.php.
7    *
8    * SPDX-License-Identifier: BSD-3-Clause
9    */
10  
11  package org.eclipse.jgit.internal.storage.reftable;
12  
13  import static java.nio.charset.StandardCharsets.UTF_8;
14  import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.FILE_HEADER_LEN;
15  import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.INDEX_BLOCK_TYPE;
16  import static org.eclipse.jgit.internal.storage.reftable.ReftableConstants.LOG_BLOCK_TYPE;
17  import static org.eclipse.jgit.lib.Constants.OBJECT_ID_LENGTH;
18  
19  import java.io.IOException;
20  import java.io.OutputStream;
21  import java.util.Arrays;
22  import java.util.zip.Deflater;
23  import java.util.zip.DeflaterOutputStream;
24  
25  import org.eclipse.jgit.internal.JGitText;
26  import org.eclipse.jgit.lib.ObjectId;
27  import org.eclipse.jgit.util.NB;
28  import org.eclipse.jgit.util.io.CountingOutputStream;
29  
30  /**
31   * Wrapper to assist formatting a reftable to an {@link OutputStream}.
32   * <p>
33   * Internally buffers at block size boundaries, flushing only complete blocks to
34   * the {@code OutputStream}.
35   */
36  class ReftableOutputStream extends OutputStream {
37  	private final byte[] tmp = new byte[10];
38  	private final CountingOutputStream out;
39  	private final boolean alignBlocks;
40  
41  	private Deflater deflater;
42  	private DeflaterOutputStream compressor;
43  
44  	private int blockType;
45  	private int blockSize;
46  	private int blockStart;
47  	private byte[] blockBuf;
48  	private int cur;
49  	private long paddingUsed;
50  
51  	ReftableOutputStream(OutputStream os, int bs, boolean align) {
52  		blockSize = bs;
53  		blockBuf = new byte[bs];
54  		alignBlocks = align;
55  		out = new CountingOutputStream(os);
56  	}
57  
58  	void setBlockSize(int bs) {
59  		blockSize = bs;
60  	}
61  
62  	/** {@inheritDoc} */
63  	@Override
64  	public void write(int b) {
65  		ensureBytesAvailableInBlockBuf(1);
66  		blockBuf[cur++] = (byte) b;
67  	}
68  
69  	/** {@inheritDoc} */
70  	@Override
71  	public void write(byte[] b, int off, int cnt) {
72  		ensureBytesAvailableInBlockBuf(cnt);
73  		System.arraycopy(b, off, blockBuf, cur, cnt);
74  		cur += cnt;
75  	}
76  
77  	int bytesWrittenInBlock() {
78  		return cur;
79  	}
80  
81  	int bytesAvailableInBlock() {
82  		return blockSize - cur;
83  	}
84  
85  	long paddingUsed() {
86  		return paddingUsed;
87  	}
88  
89  	/** @return bytes flushed; excludes {@link #bytesWrittenInBlock()}. */
90  	long size() {
91  		return out.getCount();
92  	}
93  
94  	static int computeVarintSize(long val) {
95  		int n = 1;
96  		for (; (val >>>= 7) != 0; n++) {
97  			val--;
98  		}
99  		return n;
100 	}
101 
102 	void writeVarint(long val) {
103 		int n = tmp.length;
104 		tmp[--n] = (byte) (val & 0x7f);
105 		while ((val >>>= 7) != 0) {
106 			tmp[--n] = (byte) (0x80 | (--val & 0x7F));
107 		}
108 		write(tmp, n, tmp.length - n);
109 	}
110 
111 	void writeInt16(int val) {
112 		ensureBytesAvailableInBlockBuf(2);
113 		NB.encodeInt16(blockBuf, cur, val);
114 		cur += 2;
115 	}
116 
117 	void writeInt24(int val) {
118 		ensureBytesAvailableInBlockBuf(3);
119 		NB.encodeInt24(blockBuf, cur, val);
120 		cur += 3;
121 	}
122 
123 	void writeId(ObjectId id) {
124 		ensureBytesAvailableInBlockBuf(OBJECT_ID_LENGTH);
125 		id.copyRawTo(blockBuf, cur);
126 		cur += OBJECT_ID_LENGTH;
127 	}
128 
129 	void writeVarintString(String s) {
130 		writeVarintString(s.getBytes(UTF_8));
131 	}
132 
133 	void writeVarintString(byte[] msg) {
134 		writeVarint(msg.length);
135 		write(msg, 0, msg.length);
136 	}
137 
138 	private void ensureBytesAvailableInBlockBuf(int cnt) {
139 		if (cur + cnt > blockBuf.length) {
140 			int n = Math.max(cur + cnt, blockBuf.length * 2);
141 			blockBuf = Arrays.copyOf(blockBuf, n);
142 		}
143 	}
144 
145 	void flushFileHeader() throws IOException {
146 		if (cur == FILE_HEADER_LEN && out.getCount() == 0) {
147 			out.write(blockBuf, 0, cur);
148 			cur = 0;
149 		}
150 	}
151 
152 	void beginBlock(byte type) {
153 		blockType = type;
154 		blockStart = cur;
155 		cur += 4; // reserve space for 4-byte block header.
156 	}
157 
158 	void flushBlock() throws IOException {
159 		if (cur > blockSize && blockType != INDEX_BLOCK_TYPE) {
160 			throw new IOException(JGitText.get().overflowedReftableBlock);
161 		}
162 		NB.encodeInt32(blockBuf, blockStart, (blockType << 24) | cur);
163 
164 		if (blockType == LOG_BLOCK_TYPE) {
165 			// Log blocks are deflated after the block header.
166 			out.write(blockBuf, 0, 4);
167 			if (deflater != null) {
168 				deflater.reset();
169 			} else {
170 				deflater = new Deflater(Deflater.BEST_COMPRESSION);
171 				compressor = new DeflaterOutputStream(out, deflater);
172 			}
173 			compressor.write(blockBuf, 4, cur - 4);
174 			compressor.finish();
175 		} else {
176 			// Other blocks are uncompressed.
177 			out.write(blockBuf, 0, cur);
178 		}
179 
180 		cur = 0;
181 		blockType = 0;
182 		blockStart = 0;
183 	}
184 
185 	void padBetweenBlocksToNextBlock() throws IOException {
186 		if (alignBlocks) {
187 			long m = size() % blockSize;
188 			if (m > 0) {
189 				int pad = blockSize - (int) m;
190 				ensureBytesAvailableInBlockBuf(pad);
191 				Arrays.fill(blockBuf, 0, pad, (byte) 0);
192 				out.write(blockBuf, 0, pad);
193 				paddingUsed += pad;
194 			}
195 		}
196 	}
197 
198 	int estimatePadBetweenBlocks(int currentBlockSize) {
199 		if (alignBlocks) {
200 			long m = (size() + currentBlockSize) % blockSize;
201 			return m > 0 ? blockSize - (int) m : 0;
202 		}
203 		return 0;
204 	}
205 
206 	void finishFile() throws IOException {
207 		// File footer doesn't need patching for the block start.
208 		// Just flush what has been buffered.
209 		out.write(blockBuf, 0, cur);
210 		cur = 0;
211 
212 		if (deflater != null) {
213 			deflater.end();
214 		}
215 	}
216 }