View Javadoc
1   /*
2    * Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com>
3    * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others
4    *
5    * This program and the accompanying materials are made available under the
6    * terms of the Eclipse Distribution License v. 1.0 which is available at
7    * https://www.eclipse.org/org/documents/edl-v10.php.
8    *
9    * SPDX-License-Identifier: BSD-3-Clause
10   */
11  
12  package org.eclipse.jgit.transport;
13  
14  import java.io.File;
15  import java.io.FileNotFoundException;
16  import java.io.FileOutputStream;
17  import java.io.IOException;
18  import java.text.MessageFormat;
19  import java.util.ArrayList;
20  import java.util.Collection;
21  import java.util.HashMap;
22  import java.util.HashSet;
23  import java.util.Iterator;
24  import java.util.LinkedList;
25  import java.util.List;
26  import java.util.Set;
27  
28  import org.eclipse.jgit.errors.CompoundException;
29  import org.eclipse.jgit.errors.CorruptObjectException;
30  import org.eclipse.jgit.errors.MissingObjectException;
31  import org.eclipse.jgit.errors.TransportException;
32  import org.eclipse.jgit.internal.JGitText;
33  import org.eclipse.jgit.internal.storage.file.ObjectDirectory;
34  import org.eclipse.jgit.internal.storage.file.PackIndex;
35  import org.eclipse.jgit.internal.storage.file.PackLock;
36  import org.eclipse.jgit.internal.storage.file.UnpackedObject;
37  import org.eclipse.jgit.lib.AnyObjectId;
38  import org.eclipse.jgit.lib.Constants;
39  import org.eclipse.jgit.lib.FileMode;
40  import org.eclipse.jgit.lib.MutableObjectId;
41  import org.eclipse.jgit.lib.ObjectChecker;
42  import org.eclipse.jgit.lib.ObjectId;
43  import org.eclipse.jgit.lib.ObjectInserter;
44  import org.eclipse.jgit.lib.ObjectLoader;
45  import org.eclipse.jgit.lib.ObjectReader;
46  import org.eclipse.jgit.lib.ProgressMonitor;
47  import org.eclipse.jgit.lib.Ref;
48  import org.eclipse.jgit.lib.Repository;
49  import org.eclipse.jgit.revwalk.DateRevQueue;
50  import org.eclipse.jgit.revwalk.RevCommit;
51  import org.eclipse.jgit.revwalk.RevFlag;
52  import org.eclipse.jgit.revwalk.RevObject;
53  import org.eclipse.jgit.revwalk.RevTag;
54  import org.eclipse.jgit.revwalk.RevTree;
55  import org.eclipse.jgit.revwalk.RevWalk;
56  import org.eclipse.jgit.treewalk.TreeWalk;
57  import org.eclipse.jgit.util.FileUtils;
58  
59  /**
60   * Generic fetch support for dumb transport protocols.
61   * <p>
62   * Since there are no Git-specific smarts on the remote side of the connection
63   * the client side must determine which objects it needs to copy in order to
64   * completely fetch the requested refs and their history. The generic walk
65   * support in this class parses each individual object (once it has been copied
66   * to the local repository) and examines the list of objects that must also be
67   * copied to create a complete history. Objects which are already available
68   * locally are retained (and not copied), saving bandwidth for incremental
69   * fetches. Pack files are copied from the remote repository only as a last
70   * resort, as the entire pack must be copied locally in order to access any
71   * single object.
72   * <p>
73   * This fetch connection does not actually perform the object data transfer.
74   * Instead it delegates the transfer to a {@link WalkRemoteObjectDatabase},
75   * which knows how to read individual files from the remote repository and
76   * supply the data as a standard Java InputStream.
77   *
78   * @see WalkRemoteObjectDatabase
79   */
80  class WalkFetchConnection extends BaseFetchConnection {
81  	/** The repository this transport fetches into, or pushes out of. */
82  	final Repository local;
83  
84  	/** If not null the validator for received objects. */
85  	final ObjectChecker objCheck;
86  
87  	/**
88  	 * List of all remote repositories we may need to get objects out of.
89  	 * <p>
90  	 * The first repository in the list is the one we were asked to fetch from;
91  	 * the remaining repositories point to the alternate locations we can fetch
92  	 * objects through.
93  	 */
94  	private final List<WalkRemoteObjectDatabase> remotes;
95  
96  	/** Most recently used item in {@link #remotes}. */
97  	private int lastRemoteIdx;
98  
99  	private final RevWalk revWalk;
100 
101 	private final TreeWalk treeWalk;
102 
103 	/** Objects whose direct dependents we know we have (or will have). */
104 	private final RevFlag COMPLETE;
105 
106 	/** Objects that have already entered {@link #workQueue}. */
107 	private final RevFlag IN_WORK_QUEUE;
108 
109 	/** Commits that have already entered {@link #localCommitQueue}. */
110 	private final RevFlag LOCALLY_SEEN;
111 
112 	/** Commits already reachable from all local refs. */
113 	private final DateRevQueue localCommitQueue;
114 
115 	/** Objects we need to copy from the remote repository. */
116 	private LinkedList<ObjectId> workQueue;
117 
118 	/** Databases we have not yet obtained the list of packs from. */
119 	private final LinkedList<WalkRemoteObjectDatabase> noPacksYet;
120 
121 	/** Databases we have not yet obtained the alternates from. */
122 	private final LinkedList<WalkRemoteObjectDatabase> noAlternatesYet;
123 
124 	/** Packs we have discovered, but have not yet fetched locally. */
125 	private final LinkedList<RemotePack> unfetchedPacks;
126 
127 	/**
128 	 * Packs whose indexes we have looked at in {@link #unfetchedPacks}.
129 	 * <p>
130 	 * We try to avoid getting duplicate copies of the same pack through
131 	 * multiple alternates by only looking at packs whose names are not yet in
132 	 * this collection.
133 	 */
134 	private final Set<String> packsConsidered;
135 
136 	private final MutableObjectIdml#MutableObjectId">MutableObjectId idBuffer = new MutableObjectId();
137 
138 	/**
139 	 * Errors received while trying to obtain an object.
140 	 * <p>
141 	 * If the fetch winds up failing because we cannot locate a specific object
142 	 * then we need to report all errors related to that object back to the
143 	 * caller as there may be cascading failures.
144 	 */
145 	private final HashMap<ObjectId, List<Throwable>> fetchErrors;
146 
147 	String lockMessage;
148 
149 	final List<PackLock> packLocks;
150 
151 	/** Inserter to write objects onto {@link #local}. */
152 	final ObjectInserter inserter;
153 
154 	/** Inserter to read objects from {@link #local}. */
155 	private final ObjectReader reader;
156 
157 	WalkFetchConnection(WalkTransport t, WalkRemoteObjectDatabase w) {
158 		Transport wt = (Transport)t;
159 		local = wt.local;
160 		objCheck = wt.getObjectChecker();
161 		inserter = local.newObjectInserter();
162 		reader = inserter.newReader();
163 
164 		remotes = new ArrayList<>();
165 		remotes.add(w);
166 
167 		unfetchedPacks = new LinkedList<>();
168 		packsConsidered = new HashSet<>();
169 
170 		noPacksYet = new LinkedList<>();
171 		noPacksYet.add(w);
172 
173 		noAlternatesYet = new LinkedList<>();
174 		noAlternatesYet.add(w);
175 
176 		fetchErrors = new HashMap<>();
177 		packLocks = new ArrayList<>(4);
178 
179 		revWalk = new RevWalk(reader);
180 		revWalk.setRetainBody(false);
181 		treeWalk = new TreeWalk(reader);
182 		COMPLETE = revWalk.newFlag("COMPLETE"); //$NON-NLS-1$
183 		IN_WORK_QUEUE = revWalk.newFlag("IN_WORK_QUEUE"); //$NON-NLS-1$
184 		LOCALLY_SEEN = revWalk.newFlag("LOCALLY_SEEN"); //$NON-NLS-1$
185 
186 		localCommitQueue = new DateRevQueue();
187 		workQueue = new LinkedList<>();
188 	}
189 
190 	/** {@inheritDoc} */
191 	@Override
192 	public boolean didFetchTestConnectivity() {
193 		return true;
194 	}
195 
196 	/** {@inheritDoc} */
197 	@Override
198 	protected void doFetch(final ProgressMonitor monitor,
199 			final Collection<Ref> want, final Set<ObjectId> have)
200 			throws TransportException {
201 		markLocalRefsComplete(have);
202 		queueWants(want);
203 
204 		while (!monitor.isCancelled() && !workQueue.isEmpty()) {
205 			final ObjectId id = workQueue.removeFirst();
206 			if (!(id instanceof RevObject../../../../org/eclipse/jgit/revwalk/RevObject.html#RevObject">RevObject) || !((RevObject) id).has(COMPLETE))
207 				downloadObject(monitor, id);
208 			process(id);
209 		}
210 
211 		try {
212 			inserter.flush();
213 		} catch (IOException e) {
214 			throw new TransportException(e.getMessage(), e);
215 		}
216 	}
217 
218 	/** {@inheritDoc} */
219 	@Override
220 	public Collection<PackLock> getPackLocks() {
221 		return packLocks;
222 	}
223 
224 	/** {@inheritDoc} */
225 	@Override
226 	public void setPackLockMessage(String message) {
227 		lockMessage = message;
228 	}
229 
230 	/** {@inheritDoc} */
231 	@Override
232 	public void close() {
233 		inserter.close();
234 		reader.close();
235 		for (RemotePack p : unfetchedPacks) {
236 			if (p.tmpIdx != null)
237 				p.tmpIdx.delete();
238 		}
239 		for (WalkRemoteObjectDatabase r : remotes)
240 			r.close();
241 	}
242 
243 	private void queueWants(Collection<Ref> want)
244 			throws TransportException {
245 		final HashSet<ObjectId> inWorkQueue = new HashSet<>();
246 		for (Ref r : want) {
247 			final ObjectId id = r.getObjectId();
248 			if (id == null) {
249 				throw new NullPointerException(MessageFormat.format(
250 						JGitText.get().transportProvidedRefWithNoObjectId, r.getName()));
251 			}
252 			try {
253 				final RevObject obj = revWalk.parseAny(id);
254 				if (obj.has(COMPLETE))
255 					continue;
256 				if (inWorkQueue.add(id)) {
257 					obj.add(IN_WORK_QUEUE);
258 					workQueue.add(obj);
259 				}
260 			} catch (MissingObjectException e) {
261 				if (inWorkQueue.add(id))
262 					workQueue.add(id);
263 			} catch (IOException e) {
264 				throw new TransportException(MessageFormat.format(JGitText.get().cannotRead, id.name()), e);
265 			}
266 		}
267 	}
268 
269 	private void process(ObjectId id) throws TransportException {
270 		final RevObject obj;
271 		try {
272 			if (id instanceof RevObject) {
273 				obj = (RevObject) id;
274 				if (obj.has(COMPLETE))
275 					return;
276 				revWalk.parseHeaders(obj);
277 			} else {
278 				obj = revWalk.parseAny(id);
279 				if (obj.has(COMPLETE))
280 					return;
281 			}
282 		} catch (IOException e) {
283 			throw new TransportException(MessageFormat.format(JGitText.get().cannotRead, id.name()), e);
284 		}
285 
286 		switch (obj.getType()) {
287 		case Constants.OBJ_BLOB:
288 			processBlob(obj);
289 			break;
290 		case Constants.OBJ_TREE:
291 			processTree(obj);
292 			break;
293 		case Constants.OBJ_COMMIT:
294 			processCommit(obj);
295 			break;
296 		case Constants.OBJ_TAG:
297 			processTag(obj);
298 			break;
299 		default:
300 			throw new TransportException(MessageFormat.format(JGitText.get().unknownObjectType, id.name()));
301 		}
302 
303 		// If we had any prior errors fetching this object they are
304 		// now resolved, as the object was parsed successfully.
305 		//
306 		fetchErrors.remove(id);
307 	}
308 
309 	private void processBlob(RevObject obj) throws TransportException {
310 		try {
311 			if (reader.has(obj, Constants.OBJ_BLOB))
312 				obj.add(COMPLETE);
313 			else
314 				throw new TransportException(MessageFormat.format(JGitText
315 						.get().cannotReadBlob, obj.name()),
316 						new MissingObjectException(obj, Constants.TYPE_BLOB));
317 		} catch (IOException error) {
318 			throw new TransportException(MessageFormat.format(
319 					JGitText.get().cannotReadBlob, obj.name()), error);
320 		}
321 	}
322 
323 	private void processTree(RevObject obj) throws TransportException {
324 		try {
325 			treeWalk.reset(obj);
326 			while (treeWalk.next()) {
327 				final FileMode mode = treeWalk.getFileMode(0);
328 				final int sType = mode.getObjectType();
329 
330 				switch (sType) {
331 				case Constants.OBJ_BLOB:
332 				case Constants.OBJ_TREE:
333 					treeWalk.getObjectId(idBuffer, 0);
334 					needs(revWalk.lookupAny(idBuffer, sType));
335 					continue;
336 
337 				default:
338 					if (FileMode.GITLINK.equals(mode))
339 						continue;
340 					treeWalk.getObjectId(idBuffer, 0);
341 					throw new CorruptObjectException(MessageFormat.format(JGitText.get().invalidModeFor
342 							, mode, idBuffer.name(), treeWalk.getPathString(), obj.getId().name()));
343 				}
344 			}
345 		} catch (IOException ioe) {
346 			throw new TransportException(MessageFormat.format(JGitText.get().cannotReadTree, obj.name()), ioe);
347 		}
348 		obj.add(COMPLETE);
349 	}
350 
351 	private void processCommit(RevObject obj) throws TransportException {
352 		final RevCommit../../../org/eclipse/jgit/revwalk/RevCommit.html#RevCommit">RevCommit commit = (RevCommit) obj;
353 		markLocalCommitsComplete(commit.getCommitTime());
354 		needs(commit.getTree());
355 		for (RevCommit p : commit.getParents())
356 			needs(p);
357 		obj.add(COMPLETE);
358 	}
359 
360 	private void processTag(RevObject obj) {
361 		final RevTagf="../../../../org/eclipse/jgit/revwalk/RevTag.html#RevTag">RevTag tag = (RevTag) obj;
362 		needs(tag.getObject());
363 		obj.add(COMPLETE);
364 	}
365 
366 	private void needs(RevObject obj) {
367 		if (obj.has(COMPLETE))
368 			return;
369 		if (!obj.has(IN_WORK_QUEUE)) {
370 			obj.add(IN_WORK_QUEUE);
371 			workQueue.add(obj);
372 		}
373 	}
374 
375 	private void downloadObject(ProgressMonitor pm, AnyObjectId id)
376 			throws TransportException {
377 		if (alreadyHave(id))
378 			return;
379 
380 		for (;;) {
381 			// Try a pack file we know about, but don't have yet. Odds are
382 			// that if it has this object, it has others related to it so
383 			// getting the pack is a good bet.
384 			//
385 			if (downloadPackedObject(pm, id))
386 				return;
387 
388 			// Search for a loose object over all alternates, starting
389 			// from the one we last successfully located an object through.
390 			//
391 			final String idStr = id.name();
392 			final String subdir = idStr.substring(0, 2);
393 			final String file = idStr.substring(2);
394 			final String looseName = subdir + "/" + file; //$NON-NLS-1$
395 
396 			for (int i = lastRemoteIdx; i < remotes.size(); i++) {
397 				if (downloadLooseObject(id, looseName, remotes.get(i))) {
398 					lastRemoteIdx = i;
399 					return;
400 				}
401 			}
402 			for (int i = 0; i < lastRemoteIdx; i++) {
403 				if (downloadLooseObject(id, looseName, remotes.get(i))) {
404 					lastRemoteIdx = i;
405 					return;
406 				}
407 			}
408 
409 			// Try to obtain more pack information and search those.
410 			//
411 			while (!noPacksYet.isEmpty()) {
412 				final WalkRemoteObjectDatabase wrr = noPacksYet.removeFirst();
413 				final Collection<String> packNameList;
414 				try {
415 					pm.beginTask(JGitText.get().listingPacks,
416 							ProgressMonitor.UNKNOWN);
417 					packNameList = wrr.getPackNames();
418 				} catch (IOException e) {
419 					// Try another repository.
420 					//
421 					recordError(id, e);
422 					continue;
423 				} finally {
424 					pm.endTask();
425 				}
426 
427 				if (packNameList == null || packNameList.isEmpty())
428 					continue;
429 				for (String packName : packNameList) {
430 					if (packsConsidered.add(packName))
431 						unfetchedPacks.add(new RemotePack(wrr, packName));
432 				}
433 				if (downloadPackedObject(pm, id))
434 					return;
435 			}
436 
437 			// Try to expand the first alternate we haven't expanded yet.
438 			//
439 			Collection<WalkRemoteObjectDatabase> al = expandOneAlternate(id, pm);
440 			if (al != null && !al.isEmpty()) {
441 				for (WalkRemoteObjectDatabase alt : al) {
442 					remotes.add(alt);
443 					noPacksYet.add(alt);
444 					noAlternatesYet.add(alt);
445 				}
446 				continue;
447 			}
448 
449 			// We could not obtain the object. There may be reasons why.
450 			//
451 			List<Throwable> failures = fetchErrors.get(id);
452 			final TransportException te;
453 
454 			te = new TransportException(MessageFormat.format(JGitText.get().cannotGet, id.name()));
455 			if (failures != null && !failures.isEmpty()) {
456 				if (failures.size() == 1)
457 					te.initCause(failures.get(0));
458 				else
459 					te.initCause(new CompoundException(failures));
460 			}
461 			throw te;
462 		}
463 	}
464 
465 	private boolean alreadyHave(AnyObjectId id) throws TransportException {
466 		try {
467 			return reader.has(id);
468 		} catch (IOException error) {
469 			throw new TransportException(MessageFormat.format(
470 					JGitText.get().cannotReadObject, id.name()), error);
471 		}
472 	}
473 
474 	private boolean downloadPackedObject(final ProgressMonitor monitor,
475 			final AnyObjectId id) throws TransportException {
476 		// Search for the object in a remote pack whose index we have,
477 		// but whose pack we do not yet have.
478 		//
479 		final Iterator<RemotePack> packItr = unfetchedPacks.iterator();
480 		while (packItr.hasNext() && !monitor.isCancelled()) {
481 			final RemotePack pack = packItr.next();
482 			try {
483 				pack.openIndex(monitor);
484 			} catch (IOException err) {
485 				// If the index won't open its either not found or
486 				// its a format we don't recognize. In either case
487 				// we may still be able to obtain the object from
488 				// another source, so don't consider it a failure.
489 				//
490 				recordError(id, err);
491 				packItr.remove();
492 				continue;
493 			}
494 
495 			if (monitor.isCancelled()) {
496 				// If we were cancelled while the index was opening
497 				// the open may have aborted. We can't search an
498 				// unopen index.
499 				//
500 				return false;
501 			}
502 
503 			if (!pack.index.hasObject(id)) {
504 				// Not in this pack? Try another.
505 				//
506 				continue;
507 			}
508 
509 			// It should be in the associated pack. Download that
510 			// and attach it to the local repository so we can use
511 			// all of the contained objects.
512 			//
513 			try {
514 				pack.downloadPack(monitor);
515 			} catch (IOException err) {
516 				// If the pack failed to download, index correctly,
517 				// or open in the local repository we may still be
518 				// able to obtain this object from another pack or
519 				// an alternate.
520 				//
521 				recordError(id, err);
522 				continue;
523 			} finally {
524 				// If the pack was good its in the local repository
525 				// and Repository.getObjectDatabase().has(id) will
526 				// succeed in the future, so we do not need this
527 				// data any more. If it failed the index and pack
528 				// are unusable and we shouldn't consult them again.
529 				//
530 				try {
531 					if (pack.tmpIdx != null)
532 						FileUtils.delete(pack.tmpIdx);
533 				} catch (IOException e) {
534 					throw new TransportException(e.getMessage(), e);
535 				}
536 				packItr.remove();
537 			}
538 
539 			if (!alreadyHave(id)) {
540 				// What the hell? This pack claimed to have
541 				// the object, but after indexing we didn't
542 				// actually find it in the pack.
543 				//
544 				recordError(id, new FileNotFoundException(MessageFormat.format(
545 						JGitText.get().objectNotFoundIn, id.name(), pack.packName)));
546 				continue;
547 			}
548 
549 			// Complete any other objects that we can.
550 			//
551 			final Iterator<ObjectId> pending = swapFetchQueue();
552 			while (pending.hasNext()) {
553 				final ObjectId p = pending.next();
554 				if (pack.index.hasObject(p)) {
555 					pending.remove();
556 					process(p);
557 				} else {
558 					workQueue.add(p);
559 				}
560 			}
561 			return true;
562 
563 		}
564 		return false;
565 	}
566 
567 	private Iterator<ObjectId> swapFetchQueue() {
568 		final Iterator<ObjectId> r = workQueue.iterator();
569 		workQueue = new LinkedList<>();
570 		return r;
571 	}
572 
573 	private boolean downloadLooseObject(final AnyObjectId id,
574 			final String looseName, final WalkRemoteObjectDatabase remote)
575 			throws TransportException {
576 		try {
577 			final byte[] compressed = remote.open(looseName).toArray();
578 			verifyAndInsertLooseObject(id, compressed);
579 			return true;
580 		} catch (FileNotFoundException e) {
581 			// Not available in a loose format from this alternate?
582 			// Try another strategy to get the object.
583 			//
584 			recordError(id, e);
585 			return false;
586 		} catch (IOException e) {
587 			throw new TransportException(MessageFormat.format(JGitText.get().cannotDownload, id.name()), e);
588 		}
589 	}
590 
591 	private void verifyAndInsertLooseObject(final AnyObjectId id,
592 			final byte[] compressed) throws IOException {
593 		final ObjectLoader uol;
594 		try {
595 			uol = UnpackedObject.parse(compressed, id);
596 		} catch (CorruptObjectException parsingError) {
597 			// Some HTTP servers send back a "200 OK" status with an HTML
598 			// page that explains the requested file could not be found.
599 			// These servers are most certainly misconfigured, but many
600 			// of them exist in the world, and many of those are hosting
601 			// Git repositories.
602 			//
603 			// Since an HTML page is unlikely to hash to one of our loose
604 			// objects we treat this condition as a FileNotFoundException
605 			// and attempt to recover by getting the object from another
606 			// source.
607 			//
608 			final FileNotFoundException e;
609 			e = new FileNotFoundException(id.name());
610 			e.initCause(parsingError);
611 			throw e;
612 		}
613 
614 		final int type = uol.getType();
615 		final byte[] raw = uol.getCachedBytes();
616 		if (objCheck != null) {
617 			try {
618 				objCheck.check(id, type, raw);
619 			} catch (CorruptObjectException e) {
620 				throw new TransportException(MessageFormat.format(
621 						JGitText.get().transportExceptionInvalid,
622 						Constants.typeString(type), id.name(), e.getMessage()));
623 			}
624 		}
625 
626 		ObjectId act = inserter.insert(type, raw);
627 		if (!AnyObjectId.isEqual(id, act)) {
628 			throw new TransportException(MessageFormat.format(
629 					JGitText.get().incorrectHashFor, id.name(), act.name(),
630 					Constants.typeString(type),
631 					Integer.valueOf(compressed.length)));
632 		}
633 	}
634 
635 	private Collection<WalkRemoteObjectDatabase> expandOneAlternate(
636 			final AnyObjectId id, final ProgressMonitor pm) {
637 		while (!noAlternatesYet.isEmpty()) {
638 			final WalkRemoteObjectDatabase wrr = noAlternatesYet.removeFirst();
639 			try {
640 				pm.beginTask(JGitText.get().listingAlternates, ProgressMonitor.UNKNOWN);
641 				Collection<WalkRemoteObjectDatabase> altList = wrr
642 						.getAlternates();
643 				if (altList != null && !altList.isEmpty())
644 					return altList;
645 			} catch (IOException e) {
646 				// Try another repository.
647 				//
648 				recordError(id, e);
649 			} finally {
650 				pm.endTask();
651 			}
652 		}
653 		return null;
654 	}
655 
656 	private void markLocalRefsComplete(Set<ObjectId> have) throws TransportException {
657 		List<Ref> refs;
658 		try {
659 			refs = local.getRefDatabase().getRefs();
660 		} catch (IOException e) {
661 			throw new TransportException(e.getMessage(), e);
662 		}
663 		for (Ref r : refs) {
664 			try {
665 				markLocalObjComplete(revWalk.parseAny(r.getObjectId()));
666 			} catch (IOException readError) {
667 				throw new TransportException(MessageFormat.format(JGitText.get().localRefIsMissingObjects, r.getName()), readError);
668 			}
669 		}
670 		for (ObjectId id : have) {
671 			try {
672 				markLocalObjComplete(revWalk.parseAny(id));
673 			} catch (IOException readError) {
674 				throw new TransportException(MessageFormat.format(JGitText.get().transportExceptionMissingAssumed, id.name()), readError);
675 			}
676 		}
677 	}
678 
679 	private void markLocalObjComplete(RevObject obj) throws IOException {
680 		while (obj.getType() == Constants.OBJ_TAG) {
681 			obj.add(COMPLETE);
682 			obj = ((RevTag) obj).getObject();
683 			revWalk.parseHeaders(obj);
684 		}
685 
686 		switch (obj.getType()) {
687 		case Constants.OBJ_BLOB:
688 			obj.add(COMPLETE);
689 			break;
690 		case Constants.OBJ_COMMIT:
691 			pushLocalCommit((RevCommit) obj);
692 			break;
693 		case Constants.OBJ_TREE:
694 			markTreeComplete((RevTree) obj);
695 			break;
696 		}
697 	}
698 
699 	private void markLocalCommitsComplete(int until)
700 			throws TransportException {
701 		try {
702 			for (;;) {
703 				final RevCommit c = localCommitQueue.peek();
704 				if (c == null || c.getCommitTime() < until)
705 					return;
706 				localCommitQueue.next();
707 
708 				markTreeComplete(c.getTree());
709 				for (RevCommit p : c.getParents())
710 					pushLocalCommit(p);
711 			}
712 		} catch (IOException err) {
713 			throw new TransportException(JGitText.get().localObjectsIncomplete, err);
714 		}
715 	}
716 
717 	private void pushLocalCommit(RevCommit p)
718 			throws MissingObjectException, IOException {
719 		if (p.has(LOCALLY_SEEN))
720 			return;
721 		revWalk.parseHeaders(p);
722 		p.add(LOCALLY_SEEN);
723 		p.add(COMPLETE);
724 		p.carry(COMPLETE);
725 		localCommitQueue.add(p);
726 	}
727 
728 	private void markTreeComplete(RevTree tree) throws IOException {
729 		if (tree.has(COMPLETE))
730 			return;
731 		tree.add(COMPLETE);
732 		treeWalk.reset(tree);
733 		while (treeWalk.next()) {
734 			final FileMode mode = treeWalk.getFileMode(0);
735 			final int sType = mode.getObjectType();
736 
737 			switch (sType) {
738 			case Constants.OBJ_BLOB:
739 				treeWalk.getObjectId(idBuffer, 0);
740 				revWalk.lookupAny(idBuffer, sType).add(COMPLETE);
741 				continue;
742 
743 			case Constants.OBJ_TREE: {
744 				treeWalk.getObjectId(idBuffer, 0);
745 				final RevObject o = revWalk.lookupAny(idBuffer, sType);
746 				if (!o.has(COMPLETE)) {
747 					o.add(COMPLETE);
748 					treeWalk.enterSubtree();
749 				}
750 				continue;
751 			}
752 			default:
753 				if (FileMode.GITLINK.equals(mode))
754 					continue;
755 				treeWalk.getObjectId(idBuffer, 0);
756 				throw new CorruptObjectException(MessageFormat.format(JGitText.get().corruptObjectInvalidMode3
757 						, mode, idBuffer.name(), treeWalk.getPathString(), tree.name()));
758 			}
759 		}
760 	}
761 
762 	private void recordError(AnyObjectId id, Throwable what) {
763 		final ObjectId objId = id.copy();
764 		List<Throwable> errors = fetchErrors.get(objId);
765 		if (errors == null) {
766 			errors = new ArrayList<>(2);
767 			fetchErrors.put(objId, errors);
768 		}
769 		errors.add(what);
770 	}
771 
772 	private class RemotePack {
773 		final WalkRemoteObjectDatabase connection;
774 
775 		final String packName;
776 
777 		final String idxName;
778 
779 		File tmpIdx;
780 
781 		PackIndex index;
782 
783 		RemotePack(WalkRemoteObjectDatabase c, String pn) {
784 			connection = c;
785 			packName = pn;
786 			idxName = packName.substring(0, packName.length() - 5) + ".idx"; //$NON-NLS-1$
787 
788 			String tn = idxName;
789 			if (tn.startsWith("pack-")) //$NON-NLS-1$
790 				tn = tn.substring(5);
791 			if (tn.endsWith(".idx")) //$NON-NLS-1$
792 				tn = tn.substring(0, tn.length() - 4);
793 
794 			if (local.getObjectDatabase() instanceof ObjectDirectory) {
795 				tmpIdx = new File(((ObjectDirectory) local.getObjectDatabase())
796 								.getDirectory(),
797 						"walk-" + tn + ".walkidx"); //$NON-NLS-1$ //$NON-NLS-2$
798 			}
799 		}
800 
801 		void openIndex(ProgressMonitor pm) throws IOException {
802 			if (index != null)
803 				return;
804 			if (tmpIdx == null)
805 				tmpIdx = File.createTempFile("jgit-walk-", ".idx"); //$NON-NLS-1$ //$NON-NLS-2$
806 			else if (tmpIdx.isFile()) {
807 				try {
808 					index = PackIndex.open(tmpIdx);
809 					return;
810 				} catch (FileNotFoundException err) {
811 					// Fall through and get the file.
812 				}
813 			}
814 
815 			final WalkRemoteObjectDatabase.FileStream s;
816 			s = connection.open("pack/" + idxName); //$NON-NLS-1$
817 			pm.beginTask("Get " + idxName.substring(0, 12) + "..idx", //$NON-NLS-1$ //$NON-NLS-2$
818 					s.length < 0 ? ProgressMonitor.UNKNOWN
819 							: (int) (s.length / 1024));
820 			try (FileOutputStream fos = new FileOutputStream(tmpIdx)) {
821 				final byte[] buf = new byte[2048];
822 				int cnt;
823 				while (!pm.isCancelled() && (cnt = s.in.read(buf)) >= 0) {
824 					fos.write(buf, 0, cnt);
825 					pm.update(cnt / 1024);
826 				}
827 			} catch (IOException err) {
828 				FileUtils.delete(tmpIdx);
829 				throw err;
830 			} finally {
831 				s.in.close();
832 			}
833 			pm.endTask();
834 
835 			if (pm.isCancelled()) {
836 				FileUtils.delete(tmpIdx);
837 				return;
838 			}
839 
840 			try {
841 				index = PackIndex.open(tmpIdx);
842 			} catch (IOException e) {
843 				FileUtils.delete(tmpIdx);
844 				throw e;
845 			}
846 		}
847 
848 		void downloadPack(ProgressMonitor monitor) throws IOException {
849 			String name = "pack/" + packName; //$NON-NLS-1$
850 			WalkRemoteObjectDatabase.FileStream s = connection.open(name);
851 			try {
852 				PackParser parser = inserter.newPackParser(s.in);
853 				parser.setAllowThin(false);
854 				parser.setObjectChecker(objCheck);
855 				parser.setLockMessage(lockMessage);
856 				PackLock lock = parser.parse(monitor);
857 				if (lock != null)
858 					packLocks.add(lock);
859 			} finally {
860 				s.in.close();
861 			}
862 		}
863 	}
864 }