1
2
3
4
5
6
7
8
9
10
11
12 package org.eclipse.jgit.transport;
13
14 import java.io.File;
15 import java.io.FileNotFoundException;
16 import java.io.FileOutputStream;
17 import java.io.IOException;
18 import java.text.MessageFormat;
19 import java.util.ArrayList;
20 import java.util.Collection;
21 import java.util.HashMap;
22 import java.util.HashSet;
23 import java.util.Iterator;
24 import java.util.LinkedList;
25 import java.util.List;
26 import java.util.Set;
27
28 import org.eclipse.jgit.errors.CompoundException;
29 import org.eclipse.jgit.errors.CorruptObjectException;
30 import org.eclipse.jgit.errors.MissingObjectException;
31 import org.eclipse.jgit.errors.TransportException;
32 import org.eclipse.jgit.internal.JGitText;
33 import org.eclipse.jgit.internal.storage.file.ObjectDirectory;
34 import org.eclipse.jgit.internal.storage.file.PackIndex;
35 import org.eclipse.jgit.internal.storage.file.UnpackedObject;
36 import org.eclipse.jgit.lib.AnyObjectId;
37 import org.eclipse.jgit.lib.Constants;
38 import org.eclipse.jgit.lib.FileMode;
39 import org.eclipse.jgit.lib.MutableObjectId;
40 import org.eclipse.jgit.lib.ObjectChecker;
41 import org.eclipse.jgit.lib.ObjectId;
42 import org.eclipse.jgit.lib.ObjectInserter;
43 import org.eclipse.jgit.lib.ObjectLoader;
44 import org.eclipse.jgit.lib.ObjectReader;
45 import org.eclipse.jgit.lib.ProgressMonitor;
46 import org.eclipse.jgit.lib.Ref;
47 import org.eclipse.jgit.lib.Repository;
48 import org.eclipse.jgit.revwalk.DateRevQueue;
49 import org.eclipse.jgit.revwalk.RevCommit;
50 import org.eclipse.jgit.revwalk.RevFlag;
51 import org.eclipse.jgit.revwalk.RevObject;
52 import org.eclipse.jgit.revwalk.RevTag;
53 import org.eclipse.jgit.revwalk.RevTree;
54 import org.eclipse.jgit.revwalk.RevWalk;
55 import org.eclipse.jgit.treewalk.TreeWalk;
56 import org.eclipse.jgit.util.FileUtils;
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79 class WalkFetchConnection extends BaseFetchConnection {
80
81 final Repository local;
82
83
84 final ObjectChecker objCheck;
85
86
87
88
89
90
91
92
93 private final List<WalkRemoteObjectDatabase> remotes;
94
95
96 private int lastRemoteIdx;
97
98 private final RevWalk revWalk;
99
100 private final TreeWalk treeWalk;
101
102
103 private final RevFlag COMPLETE;
104
105
106 private final RevFlag IN_WORK_QUEUE;
107
108
109 private final RevFlag LOCALLY_SEEN;
110
111
112 private final DateRevQueue localCommitQueue;
113
114
115 private LinkedList<ObjectId> workQueue;
116
117
118 private final LinkedList<WalkRemoteObjectDatabase> noPacksYet;
119
120
121 private final LinkedList<WalkRemoteObjectDatabase> noAlternatesYet;
122
123
124 private final LinkedList<RemotePack> unfetchedPacks;
125
126
127
128
129
130
131
132
133 private final Set<String> packsConsidered;
134
135 private final MutableObjectId idBuffer = new MutableObjectId();
136
137
138
139
140
141
142
143
144 private final HashMap<ObjectId, List<Throwable>> fetchErrors;
145
146 String lockMessage;
147
148 final List<PackLock> packLocks;
149
150
151 final ObjectInserter inserter;
152
153
154 private final ObjectReader reader;
155
156 WalkFetchConnection(WalkTransport t, WalkRemoteObjectDatabase w) {
157 Transport wt = (Transport)t;
158 local = wt.local;
159 objCheck = wt.getObjectChecker();
160 inserter = local.newObjectInserter();
161 reader = inserter.newReader();
162
163 remotes = new ArrayList<>();
164 remotes.add(w);
165
166 unfetchedPacks = new LinkedList<>();
167 packsConsidered = new HashSet<>();
168
169 noPacksYet = new LinkedList<>();
170 noPacksYet.add(w);
171
172 noAlternatesYet = new LinkedList<>();
173 noAlternatesYet.add(w);
174
175 fetchErrors = new HashMap<>();
176 packLocks = new ArrayList<>(4);
177
178 revWalk = new RevWalk(reader);
179 revWalk.setRetainBody(false);
180 treeWalk = new TreeWalk(reader);
181 COMPLETE = revWalk.newFlag("COMPLETE");
182 IN_WORK_QUEUE = revWalk.newFlag("IN_WORK_QUEUE");
183 LOCALLY_SEEN = revWalk.newFlag("LOCALLY_SEEN");
184
185 localCommitQueue = new DateRevQueue();
186 workQueue = new LinkedList<>();
187 }
188
189
190 @Override
191 public boolean didFetchTestConnectivity() {
192 return true;
193 }
194
195
196 @Override
197 protected void doFetch(final ProgressMonitor monitor,
198 final Collection<Ref> want, final Set<ObjectId> have)
199 throws TransportException {
200 markLocalRefsComplete(have);
201 queueWants(want);
202
203 while (!monitor.isCancelled() && !workQueue.isEmpty()) {
204 final ObjectId id = workQueue.removeFirst();
205 if (!(id instanceof RevObject) || !((RevObject) id).has(COMPLETE))
206 downloadObject(monitor, id);
207 process(id);
208 }
209
210 try {
211 inserter.flush();
212 } catch (IOException e) {
213 throw new TransportException(e.getMessage(), e);
214 }
215 }
216
217
218 @Override
219 public Collection<PackLock> getPackLocks() {
220 return packLocks;
221 }
222
223
224 @Override
225 public void setPackLockMessage(String message) {
226 lockMessage = message;
227 }
228
229
230 @Override
231 public void close() {
232 inserter.close();
233 reader.close();
234 for (RemotePack p : unfetchedPacks) {
235 if (p.tmpIdx != null)
236 p.tmpIdx.delete();
237 }
238 for (WalkRemoteObjectDatabase r : remotes)
239 r.close();
240 }
241
242 private void queueWants(Collection<Ref> want)
243 throws TransportException {
244 final HashSet<ObjectId> inWorkQueue = new HashSet<>();
245 for (Ref r : want) {
246 final ObjectId id = r.getObjectId();
247 if (id == null) {
248 throw new NullPointerException(MessageFormat.format(
249 JGitText.get().transportProvidedRefWithNoObjectId, r.getName()));
250 }
251 try {
252 final RevObject obj = revWalk.parseAny(id);
253 if (obj.has(COMPLETE))
254 continue;
255 if (inWorkQueue.add(id)) {
256 obj.add(IN_WORK_QUEUE);
257 workQueue.add(obj);
258 }
259 } catch (MissingObjectException e) {
260 if (inWorkQueue.add(id))
261 workQueue.add(id);
262 } catch (IOException e) {
263 throw new TransportException(MessageFormat.format(JGitText.get().cannotRead, id.name()), e);
264 }
265 }
266 }
267
268 private void process(ObjectId id) throws TransportException {
269 final RevObject obj;
270 try {
271 if (id instanceof RevObject) {
272 obj = (RevObject) id;
273 if (obj.has(COMPLETE))
274 return;
275 revWalk.parseHeaders(obj);
276 } else {
277 obj = revWalk.parseAny(id);
278 if (obj.has(COMPLETE))
279 return;
280 }
281 } catch (IOException e) {
282 throw new TransportException(MessageFormat.format(JGitText.get().cannotRead, id.name()), e);
283 }
284
285 switch (obj.getType()) {
286 case Constants.OBJ_BLOB:
287 processBlob(obj);
288 break;
289 case Constants.OBJ_TREE:
290 processTree(obj);
291 break;
292 case Constants.OBJ_COMMIT:
293 processCommit(obj);
294 break;
295 case Constants.OBJ_TAG:
296 processTag(obj);
297 break;
298 default:
299 throw new TransportException(MessageFormat.format(JGitText.get().unknownObjectType, id.name()));
300 }
301
302
303
304
305 fetchErrors.remove(id);
306 }
307
308 private void processBlob(RevObject obj) throws TransportException {
309 try {
310 if (reader.has(obj, Constants.OBJ_BLOB))
311 obj.add(COMPLETE);
312 else
313 throw new TransportException(MessageFormat.format(JGitText
314 .get().cannotReadBlob, obj.name()),
315 new MissingObjectException(obj, Constants.TYPE_BLOB));
316 } catch (IOException error) {
317 throw new TransportException(MessageFormat.format(
318 JGitText.get().cannotReadBlob, obj.name()), error);
319 }
320 }
321
322 private void processTree(RevObject obj) throws TransportException {
323 try {
324 treeWalk.reset(obj);
325 while (treeWalk.next()) {
326 final FileMode mode = treeWalk.getFileMode(0);
327 final int sType = mode.getObjectType();
328
329 switch (sType) {
330 case Constants.OBJ_BLOB:
331 case Constants.OBJ_TREE:
332 treeWalk.getObjectId(idBuffer, 0);
333 needs(revWalk.lookupAny(idBuffer, sType));
334 continue;
335
336 default:
337 if (FileMode.GITLINK.equals(mode))
338 continue;
339 treeWalk.getObjectId(idBuffer, 0);
340 throw new CorruptObjectException(MessageFormat.format(JGitText.get().invalidModeFor
341 , mode, idBuffer.name(), treeWalk.getPathString(), obj.getId().name()));
342 }
343 }
344 } catch (IOException ioe) {
345 throw new TransportException(MessageFormat.format(JGitText.get().cannotReadTree, obj.name()), ioe);
346 }
347 obj.add(COMPLETE);
348 }
349
350 private void processCommit(RevObject obj) throws TransportException {
351 final RevCommit commit = (RevCommit) obj;
352 markLocalCommitsComplete(commit.getCommitTime());
353 needs(commit.getTree());
354 for (RevCommit p : commit.getParents())
355 needs(p);
356 obj.add(COMPLETE);
357 }
358
359 private void processTag(RevObject obj) {
360 final RevTag tag = (RevTag) obj;
361 needs(tag.getObject());
362 obj.add(COMPLETE);
363 }
364
365 private void needs(RevObject obj) {
366 if (obj.has(COMPLETE))
367 return;
368 if (!obj.has(IN_WORK_QUEUE)) {
369 obj.add(IN_WORK_QUEUE);
370 workQueue.add(obj);
371 }
372 }
373
374 private void downloadObject(ProgressMonitor pm, AnyObjectId id)
375 throws TransportException {
376 if (alreadyHave(id))
377 return;
378
379 for (;;) {
380
381
382
383
384 if (downloadPackedObject(pm, id))
385 return;
386
387
388
389
390 final String idStr = id.name();
391 final String subdir = idStr.substring(0, 2);
392 final String file = idStr.substring(2);
393 final String looseName = subdir + "/" + file;
394
395 for (int i = lastRemoteIdx; i < remotes.size(); i++) {
396 if (downloadLooseObject(id, looseName, remotes.get(i))) {
397 lastRemoteIdx = i;
398 return;
399 }
400 }
401 for (int i = 0; i < lastRemoteIdx; i++) {
402 if (downloadLooseObject(id, looseName, remotes.get(i))) {
403 lastRemoteIdx = i;
404 return;
405 }
406 }
407
408
409
410 while (!noPacksYet.isEmpty()) {
411 final WalkRemoteObjectDatabase wrr = noPacksYet.removeFirst();
412 final Collection<String> packNameList;
413 try {
414 pm.beginTask(JGitText.get().listingPacks,
415 ProgressMonitor.UNKNOWN);
416 packNameList = wrr.getPackNames();
417 } catch (IOException e) {
418
419
420 recordError(id, e);
421 continue;
422 } finally {
423 pm.endTask();
424 }
425
426 if (packNameList == null || packNameList.isEmpty())
427 continue;
428 for (String packName : packNameList) {
429 if (packsConsidered.add(packName))
430 unfetchedPacks.add(new RemotePack(wrr, packName));
431 }
432 if (downloadPackedObject(pm, id))
433 return;
434 }
435
436
437
438 Collection<WalkRemoteObjectDatabase> al = expandOneAlternate(id, pm);
439 if (al != null && !al.isEmpty()) {
440 for (WalkRemoteObjectDatabase alt : al) {
441 remotes.add(alt);
442 noPacksYet.add(alt);
443 noAlternatesYet.add(alt);
444 }
445 continue;
446 }
447
448
449
450 List<Throwable> failures = fetchErrors.get(id);
451 final TransportException te;
452
453 te = new TransportException(MessageFormat.format(JGitText.get().cannotGet, id.name()));
454 if (failures != null && !failures.isEmpty()) {
455 if (failures.size() == 1)
456 te.initCause(failures.get(0));
457 else
458 te.initCause(new CompoundException(failures));
459 }
460 throw te;
461 }
462 }
463
464 private boolean alreadyHave(AnyObjectId id) throws TransportException {
465 try {
466 return reader.has(id);
467 } catch (IOException error) {
468 throw new TransportException(MessageFormat.format(
469 JGitText.get().cannotReadObject, id.name()), error);
470 }
471 }
472
473 private boolean downloadPackedObject(final ProgressMonitor monitor,
474 final AnyObjectId id) throws TransportException {
475
476
477
478 final Iterator<RemotePack> packItr = unfetchedPacks.iterator();
479 while (packItr.hasNext() && !monitor.isCancelled()) {
480 final RemotePack pack = packItr.next();
481 try {
482 pack.openIndex(monitor);
483 } catch (IOException err) {
484
485
486
487
488
489 recordError(id, err);
490 packItr.remove();
491 continue;
492 }
493
494 if (monitor.isCancelled()) {
495
496
497
498
499 return false;
500 }
501
502 if (!pack.index.hasObject(id)) {
503
504
505 continue;
506 }
507
508
509
510
511
512 Throwable e1 = null;
513 try {
514 pack.downloadPack(monitor);
515 } catch (IOException err) {
516
517
518
519
520
521 recordError(id, err);
522 e1 = err;
523 continue;
524 } finally {
525
526
527
528
529
530
531 try {
532 if (pack.tmpIdx != null)
533 FileUtils.delete(pack.tmpIdx);
534 } catch (IOException e) {
535 if (e1 != null) {
536 e.addSuppressed(e1);
537 }
538 throw new TransportException(e.getMessage(), e);
539 }
540 packItr.remove();
541 }
542
543 if (!alreadyHave(id)) {
544
545
546
547
548 recordError(id, new FileNotFoundException(MessageFormat.format(
549 JGitText.get().objectNotFoundIn, id.name(), pack.packName)));
550 continue;
551 }
552
553
554
555 final Iterator<ObjectId> pending = swapFetchQueue();
556 while (pending.hasNext()) {
557 final ObjectId p = pending.next();
558 if (pack.index.hasObject(p)) {
559 pending.remove();
560 process(p);
561 } else {
562 workQueue.add(p);
563 }
564 }
565 return true;
566
567 }
568 return false;
569 }
570
571 private Iterator<ObjectId> swapFetchQueue() {
572 final Iterator<ObjectId> r = workQueue.iterator();
573 workQueue = new LinkedList<>();
574 return r;
575 }
576
577 private boolean downloadLooseObject(final AnyObjectId id,
578 final String looseName, final WalkRemoteObjectDatabase remote)
579 throws TransportException {
580 try {
581 final byte[] compressed = remote.open(looseName).toArray();
582 verifyAndInsertLooseObject(id, compressed);
583 return true;
584 } catch (FileNotFoundException e) {
585
586
587
588 recordError(id, e);
589 return false;
590 } catch (IOException e) {
591 throw new TransportException(MessageFormat.format(JGitText.get().cannotDownload, id.name()), e);
592 }
593 }
594
595 private void verifyAndInsertLooseObject(final AnyObjectId id,
596 final byte[] compressed) throws IOException {
597 final ObjectLoader uol;
598 try {
599 uol = UnpackedObject.parse(compressed, id);
600 } catch (CorruptObjectException parsingError) {
601
602
603
604
605
606
607
608
609
610
611
612 final FileNotFoundException e;
613 e = new FileNotFoundException(id.name());
614 e.initCause(parsingError);
615 throw e;
616 }
617
618 final int type = uol.getType();
619 final byte[] raw = uol.getCachedBytes();
620 if (objCheck != null) {
621 try {
622 objCheck.check(id, type, raw);
623 } catch (CorruptObjectException e) {
624 throw new TransportException(MessageFormat.format(
625 JGitText.get().transportExceptionInvalid,
626 Constants.typeString(type), id.name(), e.getMessage()));
627 }
628 }
629
630 ObjectId act = inserter.insert(type, raw);
631 if (!AnyObjectId.isEqual(id, act)) {
632 throw new TransportException(MessageFormat.format(
633 JGitText.get().incorrectHashFor, id.name(), act.name(),
634 Constants.typeString(type),
635 Integer.valueOf(compressed.length)));
636 }
637 }
638
639 private Collection<WalkRemoteObjectDatabase> expandOneAlternate(
640 final AnyObjectId id, final ProgressMonitor pm) {
641 while (!noAlternatesYet.isEmpty()) {
642 final WalkRemoteObjectDatabase wrr = noAlternatesYet.removeFirst();
643 try {
644 pm.beginTask(JGitText.get().listingAlternates, ProgressMonitor.UNKNOWN);
645 Collection<WalkRemoteObjectDatabase> altList = wrr
646 .getAlternates();
647 if (altList != null && !altList.isEmpty())
648 return altList;
649 } catch (IOException e) {
650
651
652 recordError(id, e);
653 } finally {
654 pm.endTask();
655 }
656 }
657 return null;
658 }
659
660 private void markLocalRefsComplete(Set<ObjectId> have) throws TransportException {
661 List<Ref> refs;
662 try {
663 refs = local.getRefDatabase().getRefs();
664 } catch (IOException e) {
665 throw new TransportException(e.getMessage(), e);
666 }
667 for (Ref r : refs) {
668 try {
669 markLocalObjComplete(revWalk.parseAny(r.getObjectId()));
670 } catch (IOException readError) {
671 throw new TransportException(MessageFormat.format(JGitText.get().localRefIsMissingObjects, r.getName()), readError);
672 }
673 }
674 for (ObjectId id : have) {
675 try {
676 markLocalObjComplete(revWalk.parseAny(id));
677 } catch (IOException readError) {
678 throw new TransportException(MessageFormat.format(JGitText.get().transportExceptionMissingAssumed, id.name()), readError);
679 }
680 }
681 }
682
683 private void markLocalObjComplete(RevObject obj) throws IOException {
684 while (obj.getType() == Constants.OBJ_TAG) {
685 obj.add(COMPLETE);
686 obj = ((RevTag) obj).getObject();
687 revWalk.parseHeaders(obj);
688 }
689
690 switch (obj.getType()) {
691 case Constants.OBJ_BLOB:
692 obj.add(COMPLETE);
693 break;
694 case Constants.OBJ_COMMIT:
695 pushLocalCommit((RevCommit) obj);
696 break;
697 case Constants.OBJ_TREE:
698 markTreeComplete((RevTree) obj);
699 break;
700 }
701 }
702
703 private void markLocalCommitsComplete(int until)
704 throws TransportException {
705 try {
706 for (;;) {
707 final RevCommit c = localCommitQueue.peek();
708 if (c == null || c.getCommitTime() < until)
709 return;
710 localCommitQueue.next();
711
712 markTreeComplete(c.getTree());
713 for (RevCommit p : c.getParents())
714 pushLocalCommit(p);
715 }
716 } catch (IOException err) {
717 throw new TransportException(JGitText.get().localObjectsIncomplete, err);
718 }
719 }
720
721 private void pushLocalCommit(RevCommit p)
722 throws MissingObjectException, IOException {
723 if (p.has(LOCALLY_SEEN))
724 return;
725 revWalk.parseHeaders(p);
726 p.add(LOCALLY_SEEN);
727 p.add(COMPLETE);
728 p.carry(COMPLETE);
729 localCommitQueue.add(p);
730 }
731
732 private void markTreeComplete(RevTree tree) throws IOException {
733 if (tree.has(COMPLETE))
734 return;
735 tree.add(COMPLETE);
736 treeWalk.reset(tree);
737 while (treeWalk.next()) {
738 final FileMode mode = treeWalk.getFileMode(0);
739 final int sType = mode.getObjectType();
740
741 switch (sType) {
742 case Constants.OBJ_BLOB:
743 treeWalk.getObjectId(idBuffer, 0);
744 revWalk.lookupAny(idBuffer, sType).add(COMPLETE);
745 continue;
746
747 case Constants.OBJ_TREE: {
748 treeWalk.getObjectId(idBuffer, 0);
749 final RevObject o = revWalk.lookupAny(idBuffer, sType);
750 if (!o.has(COMPLETE)) {
751 o.add(COMPLETE);
752 treeWalk.enterSubtree();
753 }
754 continue;
755 }
756 default:
757 if (FileMode.GITLINK.equals(mode))
758 continue;
759 treeWalk.getObjectId(idBuffer, 0);
760 throw new CorruptObjectException(MessageFormat.format(JGitText.get().corruptObjectInvalidMode3
761 , mode, idBuffer.name(), treeWalk.getPathString(), tree.name()));
762 }
763 }
764 }
765
766 private void recordError(AnyObjectId id, Throwable what) {
767 final ObjectId objId = id.copy();
768 List<Throwable> errors = fetchErrors.get(objId);
769 if (errors == null) {
770 errors = new ArrayList<>(2);
771 fetchErrors.put(objId, errors);
772 }
773 errors.add(what);
774 }
775
776 private class RemotePack {
777 final WalkRemoteObjectDatabase connection;
778
779 final String packName;
780
781 final String idxName;
782
783 File tmpIdx;
784
785 PackIndex index;
786
787 RemotePack(WalkRemoteObjectDatabase c, String pn) {
788 connection = c;
789 packName = pn;
790 idxName = packName.substring(0, packName.length() - 5) + ".idx";
791
792 String tn = idxName;
793 if (tn.startsWith("pack-"))
794 tn = tn.substring(5);
795 if (tn.endsWith(".idx"))
796 tn = tn.substring(0, tn.length() - 4);
797
798 if (local.getObjectDatabase() instanceof ObjectDirectory) {
799 tmpIdx = new File(((ObjectDirectory) local.getObjectDatabase())
800 .getDirectory(),
801 "walk-" + tn + ".walkidx");
802 }
803 }
804
805 void openIndex(ProgressMonitor pm) throws IOException {
806 if (index != null)
807 return;
808 if (tmpIdx == null)
809 tmpIdx = File.createTempFile("jgit-walk-", ".idx");
810 else if (tmpIdx.isFile()) {
811 try {
812 index = PackIndex.open(tmpIdx);
813 return;
814 } catch (FileNotFoundException err) {
815
816 }
817 }
818
819 final WalkRemoteObjectDatabase.FileStream s;
820 s = connection.open("pack/" + idxName);
821 pm.beginTask("Get " + idxName.substring(0, 12) + "..idx",
822 s.length < 0 ? ProgressMonitor.UNKNOWN
823 : (int) (s.length / 1024));
824 try (FileOutputStream fos = new FileOutputStream(tmpIdx)) {
825 final byte[] buf = new byte[2048];
826 int cnt;
827 while (!pm.isCancelled() && (cnt = s.in.read(buf)) >= 0) {
828 fos.write(buf, 0, cnt);
829 pm.update(cnt / 1024);
830 }
831 } catch (IOException err) {
832 FileUtils.delete(tmpIdx);
833 throw err;
834 } finally {
835 s.in.close();
836 }
837 pm.endTask();
838
839 if (pm.isCancelled()) {
840 FileUtils.delete(tmpIdx);
841 return;
842 }
843
844 try {
845 index = PackIndex.open(tmpIdx);
846 } catch (IOException e) {
847 FileUtils.delete(tmpIdx);
848 throw e;
849 }
850 }
851
852 void downloadPack(ProgressMonitor monitor) throws IOException {
853 String name = "pack/" + packName;
854 WalkRemoteObjectDatabase.FileStream s = connection.open(name);
855 try {
856 PackParser parser = inserter.newPackParser(s.in);
857 parser.setAllowThin(false);
858 parser.setObjectChecker(objCheck);
859 parser.setLockMessage(lockMessage);
860 PackLock lock = parser.parse(monitor);
861 if (lock != null)
862 packLocks.add(lock);
863 } finally {
864 s.in.close();
865 }
866 }
867 }
868 }