1
2
3
4
5
6
7
8
9
10
11
12
13 package org.eclipse.jgit.internal.storage.file;
14
15 import java.io.File;
16 import java.io.FileOutputStream;
17 import java.io.IOException;
18 import java.io.InputStream;
19 import java.io.RandomAccessFile;
20 import java.nio.file.StandardCopyOption;
21 import java.security.MessageDigest;
22 import java.text.MessageFormat;
23 import java.util.Arrays;
24 import java.util.List;
25 import java.util.zip.CRC32;
26 import java.util.zip.Deflater;
27
28 import org.eclipse.jgit.errors.LockFailedException;
29 import org.eclipse.jgit.internal.JGitText;
30 import org.eclipse.jgit.lib.AnyObjectId;
31 import org.eclipse.jgit.lib.Constants;
32 import org.eclipse.jgit.lib.CoreConfig;
33 import org.eclipse.jgit.lib.ObjectId;
34 import org.eclipse.jgit.lib.ProgressMonitor;
35 import org.eclipse.jgit.storage.pack.PackConfig;
36 import org.eclipse.jgit.transport.PackParser;
37 import org.eclipse.jgit.transport.PackedObjectInfo;
38 import org.eclipse.jgit.util.FileUtils;
39 import org.eclipse.jgit.util.NB;
40
41
42
43
44
45
46
47
48 public class ObjectDirectoryPackParser extends PackParser {
49 private final FileObjectDatabase db;
50
51
52 private final CRC32 crc;
53
54
55 private final MessageDigest tailDigest;
56
57
58 private int indexVersion;
59
60
61 private boolean keepEmpty;
62
63
64 private File tmpPack;
65
66
67
68
69
70 private File tmpIdx;
71
72
73 private RandomAccessFile out;
74
75
76 private long origEnd;
77
78
79 private byte[] origHash;
80
81
82 private long packEnd;
83
84
85 private byte[] packHash;
86
87
88 private Deflater def;
89
90
91 private PackFile newPack;
92
93 private PackConfig pconfig;
94
95 ObjectDirectoryPackParser(FileObjectDatabase odb, InputStream src) {
96 super(odb, src);
97 this.db = odb;
98 this.pconfig = new PackConfig(odb.getConfig());
99 this.crc = new CRC32();
100 this.tailDigest = Constants.newMessageDigest();
101
102 indexVersion = db.getConfig().get(CoreConfig.KEY).getPackIndexVersion();
103 }
104
105
106
107
108
109
110
111
112
113 public void setIndexVersion(int version) {
114 indexVersion = version;
115 }
116
117
118
119
120
121
122
123
124
125
126
127 public void setKeepEmpty(boolean empty) {
128 keepEmpty = empty;
129 }
130
131
132
133
134
135
136
137
138
139 public PackFile getPackFile() {
140 return newPack;
141 }
142
143
144 @Override
145 public long getPackSize() {
146 if (newPack == null)
147 return super.getPackSize();
148
149 File pack = newPack.getPackFile();
150 long size = pack.length();
151 String p = pack.getAbsolutePath();
152 String i = p.substring(0, p.length() - ".pack".length()) + ".idx";
153 File idx = new File(i);
154 if (idx.exists() && idx.isFile())
155 size += idx.length();
156 return size;
157 }
158
159
160 @Override
161 public PackLock parse(ProgressMonitor./../../org/eclipse/jgit/lib/ProgressMonitor.html#ProgressMonitor">ProgressMonitor receiving, ProgressMonitor resolving)
162 throws IOException {
163 tmpPack = File.createTempFile("incoming_", ".pack", db.getDirectory());
164 tmpIdx = new File(db.getDirectory(), baseName(tmpPack) + ".idx");
165 try {
166 out = new RandomAccessFile(tmpPack, "rw");
167
168 super.parse(receiving, resolving);
169
170 out.seek(packEnd);
171 out.write(packHash);
172 out.getChannel().force(true);
173 out.close();
174
175 writeIdx();
176
177 tmpPack.setReadOnly();
178 tmpIdx.setReadOnly();
179
180 return renameAndOpenPack(getLockMessage());
181 } finally {
182 if (def != null)
183 def.end();
184 try {
185 if (out != null && out.getChannel().isOpen())
186 out.close();
187 } catch (IOException closeError) {
188
189 }
190 cleanupTemporaryFiles();
191 }
192 }
193
194
195 @Override
196 protected void onPackHeader(long objectCount) throws IOException {
197
198 }
199
200
201 @Override
202 protected void onBeginWholeObject(long streamPosition, int type,
203 long inflatedSize) throws IOException {
204 crc.reset();
205 }
206
207
208 @Override
209 protected void onEndWholeObject(PackedObjectInfo info) throws IOException {
210 info.setCRC((int) crc.getValue());
211 }
212
213
214 @Override
215 protected void onBeginOfsDelta(long streamPosition,
216 long baseStreamPosition, long inflatedSize) throws IOException {
217 crc.reset();
218 }
219
220
221 @Override
222 protected void onBeginRefDelta(long streamPosition, AnyObjectId baseId,
223 long inflatedSize) throws IOException {
224 crc.reset();
225 }
226
227
228 @Override
229 protected UnresolvedDelta onEndDelta() throws IOException {
230 UnresolvedDelta delta = new UnresolvedDelta();
231 delta.setCRC((int) crc.getValue());
232 return delta;
233 }
234
235
236 @Override
237 protected void onInflatedObjectData(PackedObjectInfo obj, int typeCode,
238 byte[] data) throws IOException {
239
240 }
241
242
243 @Override
244 protected void onObjectHeader(Source src, byte[] raw, int pos, int len)
245 throws IOException {
246 crc.update(raw, pos, len);
247 }
248
249
250 @Override
251 protected void onObjectData(Source src, byte[] raw, int pos, int len)
252 throws IOException {
253 crc.update(raw, pos, len);
254 }
255
256
257 @Override
258 protected void onStoreStream(byte[] raw, int pos, int len)
259 throws IOException {
260 out.write(raw, pos, len);
261 }
262
263
264 @Override
265 protected void onPackFooter(byte[] hash) throws IOException {
266 packEnd = out.getFilePointer();
267 origEnd = packEnd;
268 origHash = hash;
269 packHash = hash;
270 }
271
272
273 @Override
274 protected ObjectTypeAndSize seekDatabase(UnresolvedDelta delta,
275 ObjectTypeAndSize info) throws IOException {
276 out.seek(delta.getOffset());
277 crc.reset();
278 return readObjectHeader(info);
279 }
280
281
282 @Override
283 protected ObjectTypeAndSize seekDatabase(PackedObjectInfo obj,
284 ObjectTypeAndSize info) throws IOException {
285 out.seek(obj.getOffset());
286 crc.reset();
287 return readObjectHeader(info);
288 }
289
290
291 @Override
292 protected int readDatabase(byte[] dst, int pos, int cnt) throws IOException {
293 return out.read(dst, pos, cnt);
294 }
295
296
297 @Override
298 protected boolean checkCRC(int oldCRC) {
299 return oldCRC == (int) crc.getValue();
300 }
301
302 private static String baseName(File tmpPack) {
303 String name = tmpPack.getName();
304 return name.substring(0, name.lastIndexOf('.'));
305 }
306
307 private void cleanupTemporaryFiles() {
308 if (tmpIdx != null && !tmpIdx.delete() && tmpIdx.exists())
309 tmpIdx.deleteOnExit();
310 if (tmpPack != null && !tmpPack.delete() && tmpPack.exists())
311 tmpPack.deleteOnExit();
312 }
313
314
315 @Override
316 protected boolean onAppendBase(final int typeCode, final byte[] data,
317 final PackedObjectInfo info) throws IOException {
318 info.setOffset(packEnd);
319
320 final byte[] buf = buffer();
321 int sz = data.length;
322 int len = 0;
323 buf[len++] = (byte) ((typeCode << 4) | (sz & 15));
324 sz >>>= 4;
325 while (sz > 0) {
326 buf[len - 1] |= (byte) 0x80;
327 buf[len++] = (byte) (sz & 0x7f);
328 sz >>>= 7;
329 }
330
331 tailDigest.update(buf, 0, len);
332 crc.reset();
333 crc.update(buf, 0, len);
334 out.seek(packEnd);
335 out.write(buf, 0, len);
336 packEnd += len;
337
338 if (def == null)
339 def = new Deflater(Deflater.DEFAULT_COMPRESSION, false);
340 else
341 def.reset();
342 def.setInput(data);
343 def.finish();
344
345 while (!def.finished()) {
346 len = def.deflate(buf);
347 tailDigest.update(buf, 0, len);
348 crc.update(buf, 0, len);
349 out.write(buf, 0, len);
350 packEnd += len;
351 }
352
353 info.setCRC((int) crc.getValue());
354 return true;
355 }
356
357
358 @Override
359 protected void onEndThinPack() throws IOException {
360 final byte[] buf = buffer();
361
362 final MessageDigest origDigest = Constants.newMessageDigest();
363 final MessageDigest tailDigest2 = Constants.newMessageDigest();
364 final MessageDigest packDigest = Constants.newMessageDigest();
365
366 long origRemaining = origEnd;
367 out.seek(0);
368 out.readFully(buf, 0, 12);
369 origDigest.update(buf, 0, 12);
370 origRemaining -= 12;
371
372 NB.encodeInt32(buf, 8, getObjectCount());
373 out.seek(0);
374 out.write(buf, 0, 12);
375 packDigest.update(buf, 0, 12);
376
377 for (;;) {
378 final int n = out.read(buf);
379 if (n < 0)
380 break;
381 if (origRemaining != 0) {
382 final int origCnt = (int) Math.min(n, origRemaining);
383 origDigest.update(buf, 0, origCnt);
384 origRemaining -= origCnt;
385 if (origRemaining == 0)
386 tailDigest2.update(buf, origCnt, n - origCnt);
387 } else
388 tailDigest2.update(buf, 0, n);
389
390 packDigest.update(buf, 0, n);
391 }
392
393 if (!Arrays.equals(origDigest.digest(), origHash) || !Arrays
394 .equals(tailDigest2.digest(), this.tailDigest.digest()))
395 throw new IOException(
396 JGitText.get().packCorruptedWhileWritingToFilesystem);
397
398 packHash = packDigest.digest();
399 }
400
401 private void writeIdx() throws IOException {
402 List<PackedObjectInfo> list = getSortedObjectList(null );
403 try (FileOutputStream os = new FileOutputStream(tmpIdx)) {
404 final PackIndexWriter iw;
405 if (indexVersion <= 0)
406 iw = PackIndexWriter.createOldestPossible(os, list);
407 else
408 iw = PackIndexWriter.createVersion(os, indexVersion);
409 iw.write(list, packHash);
410 os.getChannel().force(true);
411 }
412 }
413
414 private PackLock renameAndOpenPack(String lockMessage)
415 throws IOException {
416 if (!keepEmpty && getObjectCount() == 0) {
417 cleanupTemporaryFiles();
418 return null;
419 }
420
421 final MessageDigest d = Constants.newMessageDigest();
422 final byte[] oeBytes = new byte[Constants.OBJECT_ID_LENGTH];
423 for (int i = 0; i < getObjectCount(); i++) {
424 final PackedObjectInfo oe = getObject(i);
425 oe.copyRawTo(oeBytes, 0);
426 d.update(oeBytes);
427 }
428
429 final String name = ObjectId.fromRaw(d.digest()).name();
430 final File packDir = new File(db.getDirectory(), "pack");
431 final File finalPack = new File(packDir, "pack-" + name + ".pack");
432 final File finalIdx = new File(packDir, "pack-" + name + ".idx");
433 final PackLocknal/storage/file/PackLock.html#PackLock">PackLock keep = new PackLock(finalPack, db.getFS());
434
435 if (!packDir.exists() && !packDir.mkdir() && !packDir.exists()) {
436
437
438
439 cleanupTemporaryFiles();
440 throw new IOException(MessageFormat.format(
441 JGitText.get().cannotCreateDirectory, packDir
442 .getAbsolutePath()));
443 }
444
445 if (finalPack.exists()) {
446
447
448 cleanupTemporaryFiles();
449 return null;
450 }
451
452 if (lockMessage != null) {
453
454
455
456 try {
457 if (!keep.lock(lockMessage))
458 throw new LockFailedException(finalPack,
459 MessageFormat.format(
460 JGitText.get().cannotLockPackIn, finalPack));
461 } catch (IOException e) {
462 cleanupTemporaryFiles();
463 throw e;
464 }
465 }
466
467 try {
468 FileUtils.rename(tmpPack, finalPack,
469 StandardCopyOption.ATOMIC_MOVE);
470 } catch (IOException e) {
471 cleanupTemporaryFiles();
472 keep.unlock();
473 throw new IOException(MessageFormat.format(
474 JGitText.get().cannotMovePackTo, finalPack), e);
475 }
476
477 try {
478 FileUtils.rename(tmpIdx, finalIdx, StandardCopyOption.ATOMIC_MOVE);
479 } catch (IOException e) {
480 cleanupTemporaryFiles();
481 keep.unlock();
482 if (!finalPack.delete())
483 finalPack.deleteOnExit();
484 throw new IOException(MessageFormat.format(
485 JGitText.get().cannotMoveIndexTo, finalIdx), e);
486 }
487
488 boolean interrupted = false;
489 try {
490 FileSnapshot snapshot = FileSnapshot.save(finalPack);
491 if (pconfig.doWaitPreventRacyPack(snapshot.size())) {
492 snapshot.waitUntilNotRacy();
493 }
494 } catch (InterruptedException e) {
495 interrupted = true;
496 }
497 try {
498 newPack = db.openPack(finalPack);
499 } catch (IOException err) {
500 keep.unlock();
501 if (finalPack.exists())
502 FileUtils.delete(finalPack);
503 if (finalIdx.exists())
504 FileUtils.delete(finalIdx);
505 throw err;
506 } finally {
507 if (interrupted) {
508
509 Thread.currentThread().interrupt();
510 }
511 }
512
513 return lockMessage != null ? keep : null;
514 }
515 }