1
2
3
4
5
6
7
8
9
10
11
12
13 package org.eclipse.jgit.internal.storage.file;
14
15 import java.io.File;
16 import java.io.FileOutputStream;
17 import java.io.IOException;
18 import java.io.InputStream;
19 import java.io.RandomAccessFile;
20 import java.nio.file.StandardCopyOption;
21 import java.security.MessageDigest;
22 import java.text.MessageFormat;
23 import java.util.Arrays;
24 import java.util.List;
25 import java.util.zip.CRC32;
26 import java.util.zip.Deflater;
27
28 import org.eclipse.jgit.errors.LockFailedException;
29 import org.eclipse.jgit.internal.JGitText;
30 import org.eclipse.jgit.internal.storage.pack.PackExt;
31 import org.eclipse.jgit.lib.AnyObjectId;
32 import org.eclipse.jgit.lib.Constants;
33 import org.eclipse.jgit.lib.CoreConfig;
34 import org.eclipse.jgit.lib.ObjectId;
35 import org.eclipse.jgit.lib.ProgressMonitor;
36 import org.eclipse.jgit.storage.pack.PackConfig;
37 import org.eclipse.jgit.transport.PackLock;
38 import org.eclipse.jgit.transport.PackParser;
39 import org.eclipse.jgit.transport.PackedObjectInfo;
40 import org.eclipse.jgit.util.FileUtils;
41 import org.eclipse.jgit.util.NB;
42
43
44
45
46
47
48
49
50 public class ObjectDirectoryPackParser extends PackParser {
51 private final FileObjectDatabase db;
52
53
54 private final CRC32 crc;
55
56
57 private final MessageDigest tailDigest;
58
59
60 private int indexVersion;
61
62
63 private boolean keepEmpty;
64
65
66 private File tmpPack;
67
68
69
70
71
72 private File tmpIdx;
73
74
75 private RandomAccessFile out;
76
77
78 private long origEnd;
79
80
81 private byte[] origHash;
82
83
84 private long packEnd;
85
86
87 private byte[] packHash;
88
89
90 private Deflater def;
91
92
93 private Pack newPack;
94
95 private PackConfig pconfig;
96
97 ObjectDirectoryPackParser(FileObjectDatabase odb, InputStream src) {
98 super(odb, src);
99 this.db = odb;
100 this.pconfig = new PackConfig(odb.getConfig());
101 this.crc = new CRC32();
102 this.tailDigest = Constants.newMessageDigest();
103
104 indexVersion = db.getConfig().get(CoreConfig.KEY).getPackIndexVersion();
105 }
106
107
108
109
110
111
112
113
114
115 public void setIndexVersion(int version) {
116 indexVersion = version;
117 }
118
119
120
121
122
123
124
125
126
127
128
129 public void setKeepEmpty(boolean empty) {
130 keepEmpty = empty;
131 }
132
133
134
135
136
137
138
139
140
141 public Pack getPack() {
142 return newPack;
143 }
144
145
146 @Override
147 public long getPackSize() {
148 if (newPack == null)
149 return super.getPackSize();
150
151 File pack = newPack.getPackFile();
152 long size = pack.length();
153 String p = pack.getAbsolutePath();
154 String i = p.substring(0, p.length() - ".pack".length()) + ".idx";
155 File idx = new File(i);
156 if (idx.isFile())
157 size += idx.length();
158 return size;
159 }
160
161
162 @Override
163 public PackLock parse(ProgressMonitor receiving, ProgressMonitor resolving)
164 throws IOException {
165 tmpPack = File.createTempFile("incoming_", ".pack", db.getDirectory());
166 tmpIdx = new File(db.getDirectory(), baseName(tmpPack) + ".idx");
167 try {
168 out = new RandomAccessFile(tmpPack, "rw");
169
170 super.parse(receiving, resolving);
171
172 out.seek(packEnd);
173 out.write(packHash);
174 out.getChannel().force(true);
175 out.close();
176
177 writeIdx();
178
179 tmpPack.setReadOnly();
180 tmpIdx.setReadOnly();
181
182 return renameAndOpenPack(getLockMessage());
183 } finally {
184 if (def != null)
185 def.end();
186 try {
187 if (out != null && out.getChannel().isOpen())
188 out.close();
189 } catch (IOException closeError) {
190
191 }
192 cleanupTemporaryFiles();
193 }
194 }
195
196
197 @Override
198 protected void onPackHeader(long objectCount) throws IOException {
199
200 }
201
202
203 @Override
204 protected void onBeginWholeObject(long streamPosition, int type,
205 long inflatedSize) throws IOException {
206 crc.reset();
207 }
208
209
210 @Override
211 protected void onEndWholeObject(PackedObjectInfo info) throws IOException {
212 info.setCRC((int) crc.getValue());
213 }
214
215
216 @Override
217 protected void onBeginOfsDelta(long streamPosition,
218 long baseStreamPosition, long inflatedSize) throws IOException {
219 crc.reset();
220 }
221
222
223 @Override
224 protected void onBeginRefDelta(long streamPosition, AnyObjectId baseId,
225 long inflatedSize) throws IOException {
226 crc.reset();
227 }
228
229
230 @Override
231 protected UnresolvedDelta onEndDelta() throws IOException {
232 UnresolvedDelta delta = new UnresolvedDelta();
233 delta.setCRC((int) crc.getValue());
234 return delta;
235 }
236
237
238 @Override
239 protected void onInflatedObjectData(PackedObjectInfo obj, int typeCode,
240 byte[] data) throws IOException {
241
242 }
243
244
245 @Override
246 protected void onObjectHeader(Source src, byte[] raw, int pos, int len)
247 throws IOException {
248 crc.update(raw, pos, len);
249 }
250
251
252 @Override
253 protected void onObjectData(Source src, byte[] raw, int pos, int len)
254 throws IOException {
255 crc.update(raw, pos, len);
256 }
257
258
259 @Override
260 protected void onStoreStream(byte[] raw, int pos, int len)
261 throws IOException {
262 out.write(raw, pos, len);
263 }
264
265
266 @Override
267 protected void onPackFooter(byte[] hash) throws IOException {
268 packEnd = out.getFilePointer();
269 origEnd = packEnd;
270 origHash = hash;
271 packHash = hash;
272 }
273
274
275 @Override
276 protected ObjectTypeAndSize seekDatabase(UnresolvedDelta delta,
277 ObjectTypeAndSize info) throws IOException {
278 out.seek(delta.getOffset());
279 crc.reset();
280 return readObjectHeader(info);
281 }
282
283
284 @Override
285 protected ObjectTypeAndSize seekDatabase(PackedObjectInfo obj,
286 ObjectTypeAndSize info) throws IOException {
287 out.seek(obj.getOffset());
288 crc.reset();
289 return readObjectHeader(info);
290 }
291
292
293 @Override
294 protected int readDatabase(byte[] dst, int pos, int cnt) throws IOException {
295 return out.read(dst, pos, cnt);
296 }
297
298
299 @Override
300 protected boolean checkCRC(int oldCRC) {
301 return oldCRC == (int) crc.getValue();
302 }
303
304 private static String baseName(File tmpPack) {
305 String name = tmpPack.getName();
306 return name.substring(0, name.lastIndexOf('.'));
307 }
308
309 private void cleanupTemporaryFiles() {
310 if (tmpIdx != null && !tmpIdx.delete() && tmpIdx.exists())
311 tmpIdx.deleteOnExit();
312 if (tmpPack != null && !tmpPack.delete() && tmpPack.exists())
313 tmpPack.deleteOnExit();
314 }
315
316
317 @Override
318 protected boolean onAppendBase(final int typeCode, final byte[] data,
319 final PackedObjectInfo info) throws IOException {
320 info.setOffset(packEnd);
321
322 final byte[] buf = buffer();
323 int sz = data.length;
324 int len = 0;
325 buf[len++] = (byte) ((typeCode << 4) | (sz & 15));
326 sz >>>= 4;
327 while (sz > 0) {
328 buf[len - 1] |= (byte) 0x80;
329 buf[len++] = (byte) (sz & 0x7f);
330 sz >>>= 7;
331 }
332
333 tailDigest.update(buf, 0, len);
334 crc.reset();
335 crc.update(buf, 0, len);
336 out.seek(packEnd);
337 out.write(buf, 0, len);
338 packEnd += len;
339
340 if (def == null)
341 def = new Deflater(Deflater.DEFAULT_COMPRESSION, false);
342 else
343 def.reset();
344 def.setInput(data);
345 def.finish();
346
347 while (!def.finished()) {
348 len = def.deflate(buf);
349 tailDigest.update(buf, 0, len);
350 crc.update(buf, 0, len);
351 out.write(buf, 0, len);
352 packEnd += len;
353 }
354
355 info.setCRC((int) crc.getValue());
356 return true;
357 }
358
359
360 @Override
361 protected void onEndThinPack() throws IOException {
362 final byte[] buf = buffer();
363
364 final MessageDigest origDigest = Constants.newMessageDigest();
365 final MessageDigest tailDigest2 = Constants.newMessageDigest();
366 final MessageDigest packDigest = Constants.newMessageDigest();
367
368 long origRemaining = origEnd;
369 out.seek(0);
370 out.readFully(buf, 0, 12);
371 origDigest.update(buf, 0, 12);
372 origRemaining -= 12;
373
374 NB.encodeInt32(buf, 8, getObjectCount());
375 out.seek(0);
376 out.write(buf, 0, 12);
377 packDigest.update(buf, 0, 12);
378
379 for (;;) {
380 final int n = out.read(buf);
381 if (n < 0)
382 break;
383 if (origRemaining != 0) {
384 final int origCnt = (int) Math.min(n, origRemaining);
385 origDigest.update(buf, 0, origCnt);
386 origRemaining -= origCnt;
387 if (origRemaining == 0)
388 tailDigest2.update(buf, origCnt, n - origCnt);
389 } else
390 tailDigest2.update(buf, 0, n);
391
392 packDigest.update(buf, 0, n);
393 }
394
395 if (!Arrays.equals(origDigest.digest(), origHash) || !Arrays
396 .equals(tailDigest2.digest(), this.tailDigest.digest()))
397 throw new IOException(
398 JGitText.get().packCorruptedWhileWritingToFilesystem);
399
400 packHash = packDigest.digest();
401 }
402
403 private void writeIdx() throws IOException {
404 List<PackedObjectInfo> list = getSortedObjectList(null );
405 try (FileOutputStream os = new FileOutputStream(tmpIdx)) {
406 final PackIndexWriter iw;
407 if (indexVersion <= 0)
408 iw = PackIndexWriter.createOldestPossible(os, list);
409 else
410 iw = PackIndexWriter.createVersion(os, indexVersion);
411 iw.write(list, packHash);
412 os.getChannel().force(true);
413 }
414 }
415
416 private PackLock renameAndOpenPack(String lockMessage)
417 throws IOException {
418 if (!keepEmpty && getObjectCount() == 0) {
419 cleanupTemporaryFiles();
420 return null;
421 }
422
423 final MessageDigest d = Constants.newMessageDigest();
424 final byte[] oeBytes = new byte[Constants.OBJECT_ID_LENGTH];
425 for (int i = 0; i < getObjectCount(); i++) {
426 final PackedObjectInfo oe = getObject(i);
427 oe.copyRawTo(oeBytes, 0);
428 d.update(oeBytes);
429 }
430
431 ObjectId id = ObjectId.fromRaw(d.digest());
432 File packDir = new File(db.getDirectory(), "pack");
433 PackFile finalPack = new PackFile(packDir, id, PackExt.PACK);
434 PackFile finalIdx = finalPack.create(PackExt.INDEX);
435 final PackLockImpl keep = new PackLockImpl(finalPack, db.getFS());
436
437 if (!packDir.exists() && !packDir.mkdir() && !packDir.exists()) {
438
439
440
441 cleanupTemporaryFiles();
442 throw new IOException(MessageFormat.format(
443 JGitText.get().cannotCreateDirectory, packDir
444 .getAbsolutePath()));
445 }
446
447 if (finalPack.exists()) {
448
449
450 cleanupTemporaryFiles();
451 return null;
452 }
453
454 if (lockMessage != null) {
455
456
457
458 try {
459 if (!keep.lock(lockMessage))
460 throw new LockFailedException(finalPack,
461 MessageFormat.format(
462 JGitText.get().cannotLockPackIn, finalPack));
463 } catch (IOException e) {
464 cleanupTemporaryFiles();
465 throw e;
466 }
467 }
468
469 try {
470 FileUtils.rename(tmpPack, finalPack,
471 StandardCopyOption.ATOMIC_MOVE);
472 } catch (IOException e) {
473 cleanupTemporaryFiles();
474 keep.unlock();
475 throw new IOException(MessageFormat.format(
476 JGitText.get().cannotMovePackTo, finalPack), e);
477 }
478
479 try {
480 FileUtils.rename(tmpIdx, finalIdx, StandardCopyOption.ATOMIC_MOVE);
481 } catch (IOException e) {
482 cleanupTemporaryFiles();
483 keep.unlock();
484 if (!finalPack.delete())
485 finalPack.deleteOnExit();
486 throw new IOException(MessageFormat.format(
487 JGitText.get().cannotMoveIndexTo, finalIdx), e);
488 }
489
490 boolean interrupted = false;
491 try {
492 FileSnapshot snapshot = FileSnapshot.save(finalPack);
493 if (pconfig.doWaitPreventRacyPack(snapshot.size())) {
494 snapshot.waitUntilNotRacy();
495 }
496 } catch (InterruptedException e) {
497 interrupted = true;
498 }
499 try {
500 newPack = db.openPack(finalPack);
501 } catch (IOException err) {
502 keep.unlock();
503 if (finalPack.exists())
504 FileUtils.delete(finalPack);
505 if (finalIdx.exists())
506 FileUtils.delete(finalIdx);
507 throw err;
508 } finally {
509 if (interrupted) {
510
511 Thread.currentThread().interrupt();
512 }
513 }
514
515 return lockMessage != null ? keep : null;
516 }
517 }