1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45 package org.eclipse.jgit.transport;
46
47 import static org.eclipse.jgit.lib.RefDatabase.ALL;
48
49 import java.io.File;
50 import java.io.FileNotFoundException;
51 import java.io.FileOutputStream;
52 import java.io.IOException;
53 import java.text.MessageFormat;
54 import java.util.ArrayList;
55 import java.util.Collection;
56 import java.util.HashMap;
57 import java.util.HashSet;
58 import java.util.Iterator;
59 import java.util.LinkedList;
60 import java.util.List;
61 import java.util.Map;
62 import java.util.Set;
63
64 import org.eclipse.jgit.errors.CompoundException;
65 import org.eclipse.jgit.errors.CorruptObjectException;
66 import org.eclipse.jgit.errors.MissingObjectException;
67 import org.eclipse.jgit.errors.TransportException;
68 import org.eclipse.jgit.internal.JGitText;
69 import org.eclipse.jgit.internal.storage.file.ObjectDirectory;
70 import org.eclipse.jgit.internal.storage.file.PackIndex;
71 import org.eclipse.jgit.internal.storage.file.PackLock;
72 import org.eclipse.jgit.internal.storage.file.UnpackedObject;
73 import org.eclipse.jgit.lib.AnyObjectId;
74 import org.eclipse.jgit.lib.Constants;
75 import org.eclipse.jgit.lib.FileMode;
76 import org.eclipse.jgit.lib.MutableObjectId;
77 import org.eclipse.jgit.lib.ObjectChecker;
78 import org.eclipse.jgit.lib.ObjectId;
79 import org.eclipse.jgit.lib.ObjectInserter;
80 import org.eclipse.jgit.lib.ObjectLoader;
81 import org.eclipse.jgit.lib.ObjectReader;
82 import org.eclipse.jgit.lib.ProgressMonitor;
83 import org.eclipse.jgit.lib.Ref;
84 import org.eclipse.jgit.lib.Repository;
85 import org.eclipse.jgit.revwalk.DateRevQueue;
86 import org.eclipse.jgit.revwalk.RevCommit;
87 import org.eclipse.jgit.revwalk.RevFlag;
88 import org.eclipse.jgit.revwalk.RevObject;
89 import org.eclipse.jgit.revwalk.RevTag;
90 import org.eclipse.jgit.revwalk.RevTree;
91 import org.eclipse.jgit.revwalk.RevWalk;
92 import org.eclipse.jgit.treewalk.TreeWalk;
93 import org.eclipse.jgit.util.FileUtils;
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116 class WalkFetchConnection extends BaseFetchConnection {
117
118 private final Repository local;
119
120
121 private final ObjectChecker objCheck;
122
123
124
125
126
127
128
129
130 private final List<WalkRemoteObjectDatabase> remotes;
131
132
133 private int lastRemoteIdx;
134
135 private final RevWalk revWalk;
136
137 private final TreeWalk treeWalk;
138
139
140 private final RevFlag COMPLETE;
141
142
143 private final RevFlag IN_WORK_QUEUE;
144
145
146 private final RevFlag LOCALLY_SEEN;
147
148
149 private final DateRevQueue localCommitQueue;
150
151
152 private LinkedList<ObjectId> workQueue;
153
154
155 private final LinkedList<WalkRemoteObjectDatabase> noPacksYet;
156
157
158 private final LinkedList<WalkRemoteObjectDatabase> noAlternatesYet;
159
160
161 private final LinkedList<RemotePack> unfetchedPacks;
162
163
164
165
166
167
168
169
170 private final Set<String> packsConsidered;
171
172 private final MutableObjectId idBuffer = new MutableObjectId();
173
174
175
176
177
178
179
180
181 private final HashMap<ObjectId, List<Throwable>> fetchErrors;
182
183 private String lockMessage;
184
185 private final List<PackLock> packLocks;
186
187
188 private final ObjectInserter inserter;
189
190
191 private final ObjectReader reader;
192
193 WalkFetchConnection(final WalkTransport t, final WalkRemoteObjectDatabase w) {
194 Transport wt = (Transport)t;
195 local = wt.local;
196 objCheck = wt.getObjectChecker();
197 inserter = local.newObjectInserter();
198 reader = local.newObjectReader();
199
200 remotes = new ArrayList<WalkRemoteObjectDatabase>();
201 remotes.add(w);
202
203 unfetchedPacks = new LinkedList<RemotePack>();
204 packsConsidered = new HashSet<String>();
205
206 noPacksYet = new LinkedList<WalkRemoteObjectDatabase>();
207 noPacksYet.add(w);
208
209 noAlternatesYet = new LinkedList<WalkRemoteObjectDatabase>();
210 noAlternatesYet.add(w);
211
212 fetchErrors = new HashMap<ObjectId, List<Throwable>>();
213 packLocks = new ArrayList<PackLock>(4);
214
215 revWalk = new RevWalk(reader);
216 revWalk.setRetainBody(false);
217 treeWalk = new TreeWalk(reader);
218 COMPLETE = revWalk.newFlag("COMPLETE");
219 IN_WORK_QUEUE = revWalk.newFlag("IN_WORK_QUEUE");
220 LOCALLY_SEEN = revWalk.newFlag("LOCALLY_SEEN");
221
222 localCommitQueue = new DateRevQueue();
223 workQueue = new LinkedList<ObjectId>();
224 }
225
226 public boolean didFetchTestConnectivity() {
227 return true;
228 }
229
230 @Override
231 protected void doFetch(final ProgressMonitor monitor,
232 final Collection<Ref> want, final Set<ObjectId> have)
233 throws TransportException {
234 markLocalRefsComplete(have);
235 queueWants(want);
236
237 while (!monitor.isCancelled() && !workQueue.isEmpty()) {
238 final ObjectId id = workQueue.removeFirst();
239 if (!(id instanceof RevObject) || !((RevObject) id).has(COMPLETE))
240 downloadObject(monitor, id);
241 process(id);
242 }
243 }
244
245 public Collection<PackLock> getPackLocks() {
246 return packLocks;
247 }
248
249 public void setPackLockMessage(final String message) {
250 lockMessage = message;
251 }
252
253 @Override
254 public void close() {
255 inserter.close();
256 reader.close();
257 for (final RemotePack p : unfetchedPacks) {
258 if (p.tmpIdx != null)
259 p.tmpIdx.delete();
260 }
261 for (final WalkRemoteObjectDatabase r : remotes)
262 r.close();
263 }
264
265 private void queueWants(final Collection<Ref> want)
266 throws TransportException {
267 final HashSet<ObjectId> inWorkQueue = new HashSet<ObjectId>();
268 for (final Ref r : want) {
269 final ObjectId id = r.getObjectId();
270 try {
271 final RevObject obj = revWalk.parseAny(id);
272 if (obj.has(COMPLETE))
273 continue;
274 if (inWorkQueue.add(id)) {
275 obj.add(IN_WORK_QUEUE);
276 workQueue.add(obj);
277 }
278 } catch (MissingObjectException e) {
279 if (inWorkQueue.add(id))
280 workQueue.add(id);
281 } catch (IOException e) {
282 throw new TransportException(MessageFormat.format(JGitText.get().cannotRead, id.name()), e);
283 }
284 }
285 }
286
287 private void process(final ObjectId id) throws TransportException {
288 final RevObject obj;
289 try {
290 if (id instanceof RevObject) {
291 obj = (RevObject) id;
292 if (obj.has(COMPLETE))
293 return;
294 revWalk.parseHeaders(obj);
295 } else {
296 obj = revWalk.parseAny(id);
297 if (obj.has(COMPLETE))
298 return;
299 }
300 } catch (IOException e) {
301 throw new TransportException(MessageFormat.format(JGitText.get().cannotRead, id.name()), e);
302 }
303
304 switch (obj.getType()) {
305 case Constants.OBJ_BLOB:
306 processBlob(obj);
307 break;
308 case Constants.OBJ_TREE:
309 processTree(obj);
310 break;
311 case Constants.OBJ_COMMIT:
312 processCommit(obj);
313 break;
314 case Constants.OBJ_TAG:
315 processTag(obj);
316 break;
317 default:
318 throw new TransportException(MessageFormat.format(JGitText.get().unknownObjectType, id.name()));
319 }
320
321
322
323
324 fetchErrors.remove(id);
325 }
326
327 private void processBlob(final RevObject obj) throws TransportException {
328 try {
329 if (reader.has(obj, Constants.OBJ_BLOB))
330 obj.add(COMPLETE);
331 else
332 throw new TransportException(MessageFormat.format(JGitText
333 .get().cannotReadBlob, obj.name()),
334 new MissingObjectException(obj, Constants.TYPE_BLOB));
335 } catch (IOException error) {
336 throw new TransportException(MessageFormat.format(
337 JGitText.get().cannotReadBlob, obj.name()), error);
338 }
339 }
340
341 private void processTree(final RevObject obj) throws TransportException {
342 try {
343 treeWalk.reset(obj);
344 while (treeWalk.next()) {
345 final FileMode mode = treeWalk.getFileMode(0);
346 final int sType = mode.getObjectType();
347
348 switch (sType) {
349 case Constants.OBJ_BLOB:
350 case Constants.OBJ_TREE:
351 treeWalk.getObjectId(idBuffer, 0);
352 needs(revWalk.lookupAny(idBuffer, sType));
353 continue;
354
355 default:
356 if (FileMode.GITLINK.equals(mode))
357 continue;
358 treeWalk.getObjectId(idBuffer, 0);
359 throw new CorruptObjectException(MessageFormat.format(JGitText.get().invalidModeFor
360 , mode, idBuffer.name(), treeWalk.getPathString(), obj.getId().name()));
361 }
362 }
363 } catch (IOException ioe) {
364 throw new TransportException(MessageFormat.format(JGitText.get().cannotReadTree, obj.name()), ioe);
365 }
366 obj.add(COMPLETE);
367 }
368
369 private void processCommit(final RevObject obj) throws TransportException {
370 final RevCommit commit = (RevCommit) obj;
371 markLocalCommitsComplete(commit.getCommitTime());
372 needs(commit.getTree());
373 for (final RevCommit p : commit.getParents())
374 needs(p);
375 obj.add(COMPLETE);
376 }
377
378 private void processTag(final RevObject obj) {
379 final RevTag tag = (RevTag) obj;
380 needs(tag.getObject());
381 obj.add(COMPLETE);
382 }
383
384 private void needs(final RevObject obj) {
385 if (obj.has(COMPLETE))
386 return;
387 if (!obj.has(IN_WORK_QUEUE)) {
388 obj.add(IN_WORK_QUEUE);
389 workQueue.add(obj);
390 }
391 }
392
393 private void downloadObject(final ProgressMonitor pm, final AnyObjectId id)
394 throws TransportException {
395 if (alreadyHave(id))
396 return;
397
398 for (;;) {
399
400
401
402
403 if (downloadPackedObject(pm, id))
404 return;
405
406
407
408
409 final String idStr = id.name();
410 final String subdir = idStr.substring(0, 2);
411 final String file = idStr.substring(2);
412 final String looseName = subdir + "/" + file;
413
414 for (int i = lastRemoteIdx; i < remotes.size(); i++) {
415 if (downloadLooseObject(id, looseName, remotes.get(i))) {
416 lastRemoteIdx = i;
417 return;
418 }
419 }
420 for (int i = 0; i < lastRemoteIdx; i++) {
421 if (downloadLooseObject(id, looseName, remotes.get(i))) {
422 lastRemoteIdx = i;
423 return;
424 }
425 }
426
427
428
429 while (!noPacksYet.isEmpty()) {
430 final WalkRemoteObjectDatabase wrr = noPacksYet.removeFirst();
431 final Collection<String> packNameList;
432 try {
433 pm.beginTask(JGitText.get().listingPacks,
434 ProgressMonitor.UNKNOWN);
435 packNameList = wrr.getPackNames();
436 } catch (IOException e) {
437
438
439 recordError(id, e);
440 continue;
441 } finally {
442 pm.endTask();
443 }
444
445 if (packNameList == null || packNameList.isEmpty())
446 continue;
447 for (final String packName : packNameList) {
448 if (packsConsidered.add(packName))
449 unfetchedPacks.add(new RemotePack(wrr, packName));
450 }
451 if (downloadPackedObject(pm, id))
452 return;
453 }
454
455
456
457 Collection<WalkRemoteObjectDatabase> al = expandOneAlternate(id, pm);
458 if (al != null && !al.isEmpty()) {
459 for (final WalkRemoteObjectDatabase alt : al) {
460 remotes.add(alt);
461 noPacksYet.add(alt);
462 noAlternatesYet.add(alt);
463 }
464 continue;
465 }
466
467
468
469 List<Throwable> failures = fetchErrors.get(id);
470 final TransportException te;
471
472 te = new TransportException(MessageFormat.format(JGitText.get().cannotGet, id.name()));
473 if (failures != null && !failures.isEmpty()) {
474 if (failures.size() == 1)
475 te.initCause(failures.get(0));
476 else
477 te.initCause(new CompoundException(failures));
478 }
479 throw te;
480 }
481 }
482
483 private boolean alreadyHave(final AnyObjectId id) throws TransportException {
484 try {
485 return reader.has(id);
486 } catch (IOException error) {
487 throw new TransportException(MessageFormat.format(
488 JGitText.get().cannotReadObject, id.name()), error);
489 }
490 }
491
492 private boolean downloadPackedObject(final ProgressMonitor monitor,
493 final AnyObjectId id) throws TransportException {
494
495
496
497 final Iterator<RemotePack> packItr = unfetchedPacks.iterator();
498 while (packItr.hasNext() && !monitor.isCancelled()) {
499 final RemotePack pack = packItr.next();
500 try {
501 pack.openIndex(monitor);
502 } catch (IOException err) {
503
504
505
506
507
508 recordError(id, err);
509 packItr.remove();
510 continue;
511 }
512
513 if (monitor.isCancelled()) {
514
515
516
517
518 return false;
519 }
520
521 if (!pack.index.hasObject(id)) {
522
523
524 continue;
525 }
526
527
528
529
530
531 try {
532 pack.downloadPack(monitor);
533 } catch (IOException err) {
534
535
536
537
538
539 recordError(id, err);
540 continue;
541 } finally {
542
543
544
545
546
547
548 try {
549 if (pack.tmpIdx != null)
550 FileUtils.delete(pack.tmpIdx);
551 } catch (IOException e) {
552 throw new TransportException(e.getMessage(), e);
553 }
554 packItr.remove();
555 }
556
557 if (!alreadyHave(id)) {
558
559
560
561
562 recordError(id, new FileNotFoundException(MessageFormat.format(
563 JGitText.get().objectNotFoundIn, id.name(), pack.packName)));
564 continue;
565 }
566
567
568
569 final Iterator<ObjectId> pending = swapFetchQueue();
570 while (pending.hasNext()) {
571 final ObjectId p = pending.next();
572 if (pack.index.hasObject(p)) {
573 pending.remove();
574 process(p);
575 } else {
576 workQueue.add(p);
577 }
578 }
579 return true;
580
581 }
582 return false;
583 }
584
585 private Iterator<ObjectId> swapFetchQueue() {
586 final Iterator<ObjectId> r = workQueue.iterator();
587 workQueue = new LinkedList<ObjectId>();
588 return r;
589 }
590
591 private boolean downloadLooseObject(final AnyObjectId id,
592 final String looseName, final WalkRemoteObjectDatabase remote)
593 throws TransportException {
594 try {
595 final byte[] compressed = remote.open(looseName).toArray();
596 verifyAndInsertLooseObject(id, compressed);
597 return true;
598 } catch (FileNotFoundException e) {
599
600
601
602 recordError(id, e);
603 return false;
604 } catch (IOException e) {
605 throw new TransportException(MessageFormat.format(JGitText.get().cannotDownload, id.name()), e);
606 }
607 }
608
609 private void verifyAndInsertLooseObject(final AnyObjectId id,
610 final byte[] compressed) throws IOException {
611 final ObjectLoader uol;
612 try {
613 uol = UnpackedObject.parse(compressed, id);
614 } catch (CorruptObjectException parsingError) {
615
616
617
618
619
620
621
622
623
624
625
626 final FileNotFoundException e;
627 e = new FileNotFoundException(id.name());
628 e.initCause(parsingError);
629 throw e;
630 }
631
632 final int type = uol.getType();
633 final byte[] raw = uol.getCachedBytes();
634 if (objCheck != null) {
635 try {
636 objCheck.check(type, raw);
637 } catch (CorruptObjectException e) {
638 throw new TransportException(MessageFormat.format(JGitText.get().transportExceptionInvalid
639 , Constants.typeString(type), id.name(), e.getMessage()));
640 }
641 }
642
643 ObjectId act = inserter.insert(type, raw);
644 if (!AnyObjectId.equals(id, act)) {
645 throw new TransportException(MessageFormat.format(
646 JGitText.get().incorrectHashFor, id.name(), act.name(),
647 Constants.typeString(type),
648 Integer.valueOf(compressed.length)));
649 }
650 inserter.flush();
651 }
652
653 private Collection<WalkRemoteObjectDatabase> expandOneAlternate(
654 final AnyObjectId id, final ProgressMonitor pm) {
655 while (!noAlternatesYet.isEmpty()) {
656 final WalkRemoteObjectDatabase wrr = noAlternatesYet.removeFirst();
657 try {
658 pm.beginTask(JGitText.get().listingAlternates, ProgressMonitor.UNKNOWN);
659 Collection<WalkRemoteObjectDatabase> altList = wrr
660 .getAlternates();
661 if (altList != null && !altList.isEmpty())
662 return altList;
663 } catch (IOException e) {
664
665
666 recordError(id, e);
667 } finally {
668 pm.endTask();
669 }
670 }
671 return null;
672 }
673
674 private void markLocalRefsComplete(final Set<ObjectId> have) throws TransportException {
675 Map<String, Ref> refs;
676 try {
677 refs = local.getRefDatabase().getRefs(ALL);
678 } catch (IOException e) {
679 throw new TransportException(e.getMessage(), e);
680 }
681 for (final Ref r : refs.values()) {
682 try {
683 markLocalObjComplete(revWalk.parseAny(r.getObjectId()));
684 } catch (IOException readError) {
685 throw new TransportException(MessageFormat.format(JGitText.get().localRefIsMissingObjects, r.getName()), readError);
686 }
687 }
688 for (final ObjectId id : have) {
689 try {
690 markLocalObjComplete(revWalk.parseAny(id));
691 } catch (IOException readError) {
692 throw new TransportException(MessageFormat.format(JGitText.get().transportExceptionMissingAssumed, id.name()), readError);
693 }
694 }
695 }
696
697 private void markLocalObjComplete(RevObject obj) throws IOException {
698 while (obj.getType() == Constants.OBJ_TAG) {
699 obj.add(COMPLETE);
700 obj = ((RevTag) obj).getObject();
701 revWalk.parseHeaders(obj);
702 }
703
704 switch (obj.getType()) {
705 case Constants.OBJ_BLOB:
706 obj.add(COMPLETE);
707 break;
708 case Constants.OBJ_COMMIT:
709 pushLocalCommit((RevCommit) obj);
710 break;
711 case Constants.OBJ_TREE:
712 markTreeComplete((RevTree) obj);
713 break;
714 }
715 }
716
717 private void markLocalCommitsComplete(final int until)
718 throws TransportException {
719 try {
720 for (;;) {
721 final RevCommit c = localCommitQueue.peek();
722 if (c == null || c.getCommitTime() < until)
723 return;
724 localCommitQueue.next();
725
726 markTreeComplete(c.getTree());
727 for (final RevCommit p : c.getParents())
728 pushLocalCommit(p);
729 }
730 } catch (IOException err) {
731 throw new TransportException(JGitText.get().localObjectsIncomplete, err);
732 }
733 }
734
735 private void pushLocalCommit(final RevCommit p)
736 throws MissingObjectException, IOException {
737 if (p.has(LOCALLY_SEEN))
738 return;
739 revWalk.parseHeaders(p);
740 p.add(LOCALLY_SEEN);
741 p.add(COMPLETE);
742 p.carry(COMPLETE);
743 localCommitQueue.add(p);
744 }
745
746 private void markTreeComplete(final RevTree tree) throws IOException {
747 if (tree.has(COMPLETE))
748 return;
749 tree.add(COMPLETE);
750 treeWalk.reset(tree);
751 while (treeWalk.next()) {
752 final FileMode mode = treeWalk.getFileMode(0);
753 final int sType = mode.getObjectType();
754
755 switch (sType) {
756 case Constants.OBJ_BLOB:
757 treeWalk.getObjectId(idBuffer, 0);
758 revWalk.lookupAny(idBuffer, sType).add(COMPLETE);
759 continue;
760
761 case Constants.OBJ_TREE: {
762 treeWalk.getObjectId(idBuffer, 0);
763 final RevObject o = revWalk.lookupAny(idBuffer, sType);
764 if (!o.has(COMPLETE)) {
765 o.add(COMPLETE);
766 treeWalk.enterSubtree();
767 }
768 continue;
769 }
770 default:
771 if (FileMode.GITLINK.equals(mode))
772 continue;
773 treeWalk.getObjectId(idBuffer, 0);
774 throw new CorruptObjectException(MessageFormat.format(JGitText.get().corruptObjectInvalidMode3
775 , mode, idBuffer.name(), treeWalk.getPathString(), tree.name()));
776 }
777 }
778 }
779
780 private void recordError(final AnyObjectId id, final Throwable what) {
781 final ObjectId objId = id.copy();
782 List<Throwable> errors = fetchErrors.get(objId);
783 if (errors == null) {
784 errors = new ArrayList<Throwable>(2);
785 fetchErrors.put(objId, errors);
786 }
787 errors.add(what);
788 }
789
790 private class RemotePack {
791 final WalkRemoteObjectDatabase connection;
792
793 final String packName;
794
795 final String idxName;
796
797 File tmpIdx;
798
799 PackIndex index;
800
801 RemotePack(final WalkRemoteObjectDatabase c, final String pn) {
802 connection = c;
803 packName = pn;
804 idxName = packName.substring(0, packName.length() - 5) + ".idx";
805
806 String tn = idxName;
807 if (tn.startsWith("pack-"))
808 tn = tn.substring(5);
809 if (tn.endsWith(".idx"))
810 tn = tn.substring(0, tn.length() - 4);
811
812 if (local.getObjectDatabase() instanceof ObjectDirectory) {
813 tmpIdx = new File(((ObjectDirectory) local.getObjectDatabase())
814 .getDirectory(),
815 "walk-" + tn + ".walkidx");
816 }
817 }
818
819 void openIndex(final ProgressMonitor pm) throws IOException {
820 if (index != null)
821 return;
822 if (tmpIdx == null)
823 tmpIdx = File.createTempFile("jgit-walk-", ".idx");
824 else if (tmpIdx.isFile()) {
825 try {
826 index = PackIndex.open(tmpIdx);
827 return;
828 } catch (FileNotFoundException err) {
829
830 }
831 }
832
833 final WalkRemoteObjectDatabase.FileStream s;
834 s = connection.open("pack/" + idxName);
835 pm.beginTask("Get " + idxName.substring(0, 12) + "..idx",
836 s.length < 0 ? ProgressMonitor.UNKNOWN
837 : (int) (s.length / 1024));
838 try {
839 final FileOutputStream fos = new FileOutputStream(tmpIdx);
840 try {
841 final byte[] buf = new byte[2048];
842 int cnt;
843 while (!pm.isCancelled() && (cnt = s.in.read(buf)) >= 0) {
844 fos.write(buf, 0, cnt);
845 pm.update(cnt / 1024);
846 }
847 } finally {
848 fos.close();
849 }
850 } catch (IOException err) {
851 FileUtils.delete(tmpIdx);
852 throw err;
853 } finally {
854 s.in.close();
855 }
856 pm.endTask();
857
858 if (pm.isCancelled()) {
859 FileUtils.delete(tmpIdx);
860 return;
861 }
862
863 try {
864 index = PackIndex.open(tmpIdx);
865 } catch (IOException e) {
866 FileUtils.delete(tmpIdx);
867 throw e;
868 }
869 }
870
871 void downloadPack(final ProgressMonitor monitor) throws IOException {
872 String name = "pack/" + packName;
873 WalkRemoteObjectDatabase.FileStream s = connection.open(name);
874 PackParser parser = inserter.newPackParser(s.in);
875 parser.setAllowThin(false);
876 parser.setObjectChecker(objCheck);
877 parser.setLockMessage(lockMessage);
878 PackLock lock = parser.parse(monitor);
879 if (lock != null)
880 packLocks.add(lock);
881 inserter.flush();
882 }
883 }
884 }