1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46 package org.eclipse.jgit.internal.storage.file;
47
48 import java.io.File;
49 import java.io.FileOutputStream;
50 import java.io.IOException;
51 import java.io.InputStream;
52 import java.io.RandomAccessFile;
53 import java.nio.file.StandardCopyOption;
54 import java.security.MessageDigest;
55 import java.text.MessageFormat;
56 import java.util.Arrays;
57 import java.util.List;
58 import java.util.zip.CRC32;
59 import java.util.zip.Deflater;
60
61 import org.eclipse.jgit.errors.LockFailedException;
62 import org.eclipse.jgit.internal.JGitText;
63 import org.eclipse.jgit.lib.AnyObjectId;
64 import org.eclipse.jgit.lib.Constants;
65 import org.eclipse.jgit.lib.CoreConfig;
66 import org.eclipse.jgit.lib.ObjectId;
67 import org.eclipse.jgit.lib.ProgressMonitor;
68 import org.eclipse.jgit.storage.pack.PackConfig;
69 import org.eclipse.jgit.transport.PackParser;
70 import org.eclipse.jgit.transport.PackedObjectInfo;
71 import org.eclipse.jgit.util.FileUtils;
72 import org.eclipse.jgit.util.NB;
73
74
75
76
77
78
79
80
81 public class ObjectDirectoryPackParser extends PackParser {
82 private final FileObjectDatabase db;
83
84
85 private final CRC32 crc;
86
87
88 private final MessageDigest tailDigest;
89
90
91 private int indexVersion;
92
93
94 private boolean keepEmpty;
95
96
97 private File tmpPack;
98
99
100
101
102
103 private File tmpIdx;
104
105
106 private RandomAccessFile out;
107
108
109 private long origEnd;
110
111
112 private byte[] origHash;
113
114
115 private long packEnd;
116
117
118 private byte[] packHash;
119
120
121 private Deflater def;
122
123
124 private PackFile newPack;
125
126 private PackConfig pconfig;
127
128 ObjectDirectoryPackParser(FileObjectDatabase odb, InputStream src) {
129 super(odb, src);
130 this.db = odb;
131 this.pconfig = new PackConfig(odb.getConfig());
132 this.crc = new CRC32();
133 this.tailDigest = Constants.newMessageDigest();
134
135 indexVersion = db.getConfig().get(CoreConfig.KEY).getPackIndexVersion();
136 }
137
138
139
140
141
142
143
144
145
146 public void setIndexVersion(int version) {
147 indexVersion = version;
148 }
149
150
151
152
153
154
155
156
157
158
159
160 public void setKeepEmpty(boolean empty) {
161 keepEmpty = empty;
162 }
163
164
165
166
167
168
169
170
171
172 public PackFile getPackFile() {
173 return newPack;
174 }
175
176
177 @Override
178 public long getPackSize() {
179 if (newPack == null)
180 return super.getPackSize();
181
182 File pack = newPack.getPackFile();
183 long size = pack.length();
184 String p = pack.getAbsolutePath();
185 String i = p.substring(0, p.length() - ".pack".length()) + ".idx";
186 File idx = new File(i);
187 if (idx.exists() && idx.isFile())
188 size += idx.length();
189 return size;
190 }
191
192
193 @Override
194 public PackLock parse(ProgressMonitor./../../org/eclipse/jgit/lib/ProgressMonitor.html#ProgressMonitor">ProgressMonitor receiving, ProgressMonitor resolving)
195 throws IOException {
196 tmpPack = File.createTempFile("incoming_", ".pack", db.getDirectory());
197 tmpIdx = new File(db.getDirectory(), baseName(tmpPack) + ".idx");
198 try {
199 out = new RandomAccessFile(tmpPack, "rw");
200
201 super.parse(receiving, resolving);
202
203 out.seek(packEnd);
204 out.write(packHash);
205 out.getChannel().force(true);
206 out.close();
207
208 writeIdx();
209
210 tmpPack.setReadOnly();
211 tmpIdx.setReadOnly();
212
213 return renameAndOpenPack(getLockMessage());
214 } finally {
215 if (def != null)
216 def.end();
217 try {
218 if (out != null && out.getChannel().isOpen())
219 out.close();
220 } catch (IOException closeError) {
221
222 }
223 cleanupTemporaryFiles();
224 }
225 }
226
227
228 @Override
229 protected void onPackHeader(long objectCount) throws IOException {
230
231 }
232
233
234 @Override
235 protected void onBeginWholeObject(long streamPosition, int type,
236 long inflatedSize) throws IOException {
237 crc.reset();
238 }
239
240
241 @Override
242 protected void onEndWholeObject(PackedObjectInfo info) throws IOException {
243 info.setCRC((int) crc.getValue());
244 }
245
246
247 @Override
248 protected void onBeginOfsDelta(long streamPosition,
249 long baseStreamPosition, long inflatedSize) throws IOException {
250 crc.reset();
251 }
252
253
254 @Override
255 protected void onBeginRefDelta(long streamPosition, AnyObjectId baseId,
256 long inflatedSize) throws IOException {
257 crc.reset();
258 }
259
260
261 @Override
262 protected UnresolvedDelta onEndDelta() throws IOException {
263 UnresolvedDelta delta = new UnresolvedDelta();
264 delta.setCRC((int) crc.getValue());
265 return delta;
266 }
267
268
269 @Override
270 protected void onInflatedObjectData(PackedObjectInfo obj, int typeCode,
271 byte[] data) throws IOException {
272
273 }
274
275
276 @Override
277 protected void onObjectHeader(Source src, byte[] raw, int pos, int len)
278 throws IOException {
279 crc.update(raw, pos, len);
280 }
281
282
283 @Override
284 protected void onObjectData(Source src, byte[] raw, int pos, int len)
285 throws IOException {
286 crc.update(raw, pos, len);
287 }
288
289
290 @Override
291 protected void onStoreStream(byte[] raw, int pos, int len)
292 throws IOException {
293 out.write(raw, pos, len);
294 }
295
296
297 @Override
298 protected void onPackFooter(byte[] hash) throws IOException {
299 packEnd = out.getFilePointer();
300 origEnd = packEnd;
301 origHash = hash;
302 packHash = hash;
303 }
304
305
306 @Override
307 protected ObjectTypeAndSize seekDatabase(UnresolvedDelta delta,
308 ObjectTypeAndSize info) throws IOException {
309 out.seek(delta.getOffset());
310 crc.reset();
311 return readObjectHeader(info);
312 }
313
314
315 @Override
316 protected ObjectTypeAndSize seekDatabase(PackedObjectInfo obj,
317 ObjectTypeAndSize info) throws IOException {
318 out.seek(obj.getOffset());
319 crc.reset();
320 return readObjectHeader(info);
321 }
322
323
324 @Override
325 protected int readDatabase(byte[] dst, int pos, int cnt) throws IOException {
326 return out.read(dst, pos, cnt);
327 }
328
329
330 @Override
331 protected boolean checkCRC(int oldCRC) {
332 return oldCRC == (int) crc.getValue();
333 }
334
335 private static String baseName(File tmpPack) {
336 String name = tmpPack.getName();
337 return name.substring(0, name.lastIndexOf('.'));
338 }
339
340 private void cleanupTemporaryFiles() {
341 if (tmpIdx != null && !tmpIdx.delete() && tmpIdx.exists())
342 tmpIdx.deleteOnExit();
343 if (tmpPack != null && !tmpPack.delete() && tmpPack.exists())
344 tmpPack.deleteOnExit();
345 }
346
347
348 @Override
349 protected boolean onAppendBase(final int typeCode, final byte[] data,
350 final PackedObjectInfo info) throws IOException {
351 info.setOffset(packEnd);
352
353 final byte[] buf = buffer();
354 int sz = data.length;
355 int len = 0;
356 buf[len++] = (byte) ((typeCode << 4) | (sz & 15));
357 sz >>>= 4;
358 while (sz > 0) {
359 buf[len - 1] |= (byte) 0x80;
360 buf[len++] = (byte) (sz & 0x7f);
361 sz >>>= 7;
362 }
363
364 tailDigest.update(buf, 0, len);
365 crc.reset();
366 crc.update(buf, 0, len);
367 out.seek(packEnd);
368 out.write(buf, 0, len);
369 packEnd += len;
370
371 if (def == null)
372 def = new Deflater(Deflater.DEFAULT_COMPRESSION, false);
373 else
374 def.reset();
375 def.setInput(data);
376 def.finish();
377
378 while (!def.finished()) {
379 len = def.deflate(buf);
380 tailDigest.update(buf, 0, len);
381 crc.update(buf, 0, len);
382 out.write(buf, 0, len);
383 packEnd += len;
384 }
385
386 info.setCRC((int) crc.getValue());
387 return true;
388 }
389
390
391 @Override
392 protected void onEndThinPack() throws IOException {
393 final byte[] buf = buffer();
394
395 final MessageDigest origDigest = Constants.newMessageDigest();
396 final MessageDigest tailDigest2 = Constants.newMessageDigest();
397 final MessageDigest packDigest = Constants.newMessageDigest();
398
399 long origRemaining = origEnd;
400 out.seek(0);
401 out.readFully(buf, 0, 12);
402 origDigest.update(buf, 0, 12);
403 origRemaining -= 12;
404
405 NB.encodeInt32(buf, 8, getObjectCount());
406 out.seek(0);
407 out.write(buf, 0, 12);
408 packDigest.update(buf, 0, 12);
409
410 for (;;) {
411 final int n = out.read(buf);
412 if (n < 0)
413 break;
414 if (origRemaining != 0) {
415 final int origCnt = (int) Math.min(n, origRemaining);
416 origDigest.update(buf, 0, origCnt);
417 origRemaining -= origCnt;
418 if (origRemaining == 0)
419 tailDigest2.update(buf, origCnt, n - origCnt);
420 } else
421 tailDigest2.update(buf, 0, n);
422
423 packDigest.update(buf, 0, n);
424 }
425
426 if (!Arrays.equals(origDigest.digest(), origHash) || !Arrays
427 .equals(tailDigest2.digest(), this.tailDigest.digest()))
428 throw new IOException(
429 JGitText.get().packCorruptedWhileWritingToFilesystem);
430
431 packHash = packDigest.digest();
432 }
433
434 private void writeIdx() throws IOException {
435 List<PackedObjectInfo> list = getSortedObjectList(null );
436 try (FileOutputStream os = new FileOutputStream(tmpIdx)) {
437 final PackIndexWriter iw;
438 if (indexVersion <= 0)
439 iw = PackIndexWriter.createOldestPossible(os, list);
440 else
441 iw = PackIndexWriter.createVersion(os, indexVersion);
442 iw.write(list, packHash);
443 os.getChannel().force(true);
444 }
445 }
446
447 private PackLock renameAndOpenPack(String lockMessage)
448 throws IOException {
449 if (!keepEmpty && getObjectCount() == 0) {
450 cleanupTemporaryFiles();
451 return null;
452 }
453
454 final MessageDigest d = Constants.newMessageDigest();
455 final byte[] oeBytes = new byte[Constants.OBJECT_ID_LENGTH];
456 for (int i = 0; i < getObjectCount(); i++) {
457 final PackedObjectInfo oe = getObject(i);
458 oe.copyRawTo(oeBytes, 0);
459 d.update(oeBytes);
460 }
461
462 final String name = ObjectId.fromRaw(d.digest()).name();
463 final File packDir = new File(db.getDirectory(), "pack");
464 final File finalPack = new File(packDir, "pack-" + name + ".pack");
465 final File finalIdx = new File(packDir, "pack-" + name + ".idx");
466 final PackLocknal/storage/file/PackLock.html#PackLock">PackLock keep = new PackLock(finalPack, db.getFS());
467
468 if (!packDir.exists() && !packDir.mkdir() && !packDir.exists()) {
469
470
471
472 cleanupTemporaryFiles();
473 throw new IOException(MessageFormat.format(
474 JGitText.get().cannotCreateDirectory, packDir
475 .getAbsolutePath()));
476 }
477
478 if (finalPack.exists()) {
479
480
481 cleanupTemporaryFiles();
482 return null;
483 }
484
485 if (lockMessage != null) {
486
487
488
489 try {
490 if (!keep.lock(lockMessage))
491 throw new LockFailedException(finalPack,
492 MessageFormat.format(
493 JGitText.get().cannotLockPackIn, finalPack));
494 } catch (IOException e) {
495 cleanupTemporaryFiles();
496 throw e;
497 }
498 }
499
500 try {
501 FileUtils.rename(tmpPack, finalPack,
502 StandardCopyOption.ATOMIC_MOVE);
503 } catch (IOException e) {
504 cleanupTemporaryFiles();
505 keep.unlock();
506 throw new IOException(MessageFormat.format(
507 JGitText.get().cannotMovePackTo, finalPack), e);
508 }
509
510 try {
511 FileUtils.rename(tmpIdx, finalIdx, StandardCopyOption.ATOMIC_MOVE);
512 } catch (IOException e) {
513 cleanupTemporaryFiles();
514 keep.unlock();
515 if (!finalPack.delete())
516 finalPack.deleteOnExit();
517 throw new IOException(MessageFormat.format(
518 JGitText.get().cannotMoveIndexTo, finalIdx), e);
519 }
520
521 boolean interrupted = false;
522 try {
523 FileSnapshot snapshot = FileSnapshot.save(finalPack);
524 if (pconfig.doWaitPreventRacyPack(snapshot.size())) {
525 snapshot.waitUntilNotRacy();
526 }
527 } catch (InterruptedException e) {
528 interrupted = true;
529 }
530 try {
531 newPack = db.openPack(finalPack);
532 } catch (IOException err) {
533 keep.unlock();
534 if (finalPack.exists())
535 FileUtils.delete(finalPack);
536 if (finalIdx.exists())
537 FileUtils.delete(finalIdx);
538 throw err;
539 } finally {
540 if (interrupted) {
541
542 Thread.currentThread().interrupt();
543 }
544 }
545
546 return lockMessage != null ? keep : null;
547 }
548 }