1
2
3
4
5
6
7
8
9
10
11
12 package org.eclipse.jgit.transport;
13
14 import static java.nio.charset.StandardCharsets.UTF_8;
15 import static org.eclipse.jgit.transport.ReceiveCommand.Result.NOT_ATTEMPTED;
16 import static org.eclipse.jgit.transport.ReceiveCommand.Result.OK;
17 import static org.eclipse.jgit.transport.ReceiveCommand.Result.REJECTED_NONFASTFORWARD;
18 import static org.eclipse.jgit.transport.ReceiveCommand.Type.UPDATE_NONFASTFORWARD;
19
20 import java.io.File;
21 import java.io.IOException;
22 import java.io.OutputStreamWriter;
23 import java.io.Writer;
24 import java.text.MessageFormat;
25 import java.util.ArrayList;
26 import java.util.Collection;
27 import java.util.Collections;
28 import java.util.HashMap;
29 import java.util.HashSet;
30 import java.util.Iterator;
31 import java.util.Map;
32 import java.util.Set;
33 import java.util.concurrent.TimeUnit;
34
35 import org.eclipse.jgit.errors.MissingObjectException;
36 import org.eclipse.jgit.errors.NotSupportedException;
37 import org.eclipse.jgit.errors.TransportException;
38 import org.eclipse.jgit.internal.JGitText;
39 import org.eclipse.jgit.internal.storage.file.LockFile;
40 import org.eclipse.jgit.lib.BatchRefUpdate;
41 import org.eclipse.jgit.lib.BatchingProgressMonitor;
42 import org.eclipse.jgit.lib.Constants;
43 import org.eclipse.jgit.lib.ObjectId;
44 import org.eclipse.jgit.lib.ObjectIdRef;
45 import org.eclipse.jgit.lib.ProgressMonitor;
46 import org.eclipse.jgit.lib.Ref;
47 import org.eclipse.jgit.lib.RefDatabase;
48 import org.eclipse.jgit.revwalk.ObjectWalk;
49 import org.eclipse.jgit.revwalk.RevWalk;
50 import org.eclipse.jgit.util.StringUtils;
51
52 class FetchProcess {
53
54 private final Transport transport;
55
56
57 private final Collection<RefSpec> toFetch;
58
59
60 private final HashMap<ObjectId, Ref> askFor = new HashMap<>();
61
62
63 private final HashSet<ObjectId> have = new HashSet<>();
64
65
66 private final ArrayList<TrackingRefUpdate> localUpdates = new ArrayList<>();
67
68
69 private final ArrayList<FetchHeadRecord> fetchHeadUpdates = new ArrayList<>();
70
71 private final ArrayList<PackLock> packLocks = new ArrayList<>();
72
73 private FetchConnection conn;
74
75 private Map<String, Ref> localRefs;
76
77 FetchProcess(Transport t, Collection<RefSpec> f) {
78 transport = t;
79 toFetch = f;
80 }
81
82 void execute(ProgressMonitor monitor, FetchResult result,
83 String initialBranch)
84 throws NotSupportedException, TransportException {
85 askFor.clear();
86 localUpdates.clear();
87 fetchHeadUpdates.clear();
88 packLocks.clear();
89 localRefs = null;
90
91 Throwable e1 = null;
92 try {
93 executeImp(monitor, result, initialBranch);
94 } catch (NotSupportedException | TransportException err) {
95 e1 = err;
96 throw err;
97 } finally {
98 try {
99 for (PackLock lock : packLocks) {
100 lock.unlock();
101 }
102 } catch (IOException e) {
103 if (e1 != null) {
104 e.addSuppressed(e1);
105 }
106 throw new TransportException(e.getMessage(), e);
107 }
108 }
109 }
110
111 private boolean isInitialBranchMissing(Map<String, Ref> refsMap,
112 String initialBranch) {
113 if (StringUtils.isEmptyOrNull(initialBranch) || refsMap.isEmpty()) {
114 return false;
115 }
116 if (refsMap.containsKey(initialBranch)
117 || refsMap.containsKey(Constants.R_HEADS + initialBranch)
118 || refsMap.containsKey(Constants.R_TAGS + initialBranch)) {
119 return false;
120 }
121 return true;
122 }
123
124 private void executeImp(final ProgressMonitor monitor,
125 final FetchResult result, String initialBranch)
126 throws NotSupportedException, TransportException {
127 final TagOpt tagopt = transport.getTagOpt();
128 String getTags = (tagopt == TagOpt.NO_TAGS) ? null : Constants.R_TAGS;
129 String getHead = null;
130 try {
131
132
133 Ref head = transport.local.exactRef(Constants.HEAD);
134 ObjectId id = head != null ? head.getObjectId() : null;
135 if (id == null || id.equals(ObjectId.zeroId())) {
136 getHead = Constants.HEAD;
137 }
138 } catch (IOException e) {
139
140 }
141 conn = transport.openFetch(toFetch, getTags, getHead);
142 try {
143 Map<String, Ref> refsMap = conn.getRefsMap();
144 if (isInitialBranchMissing(refsMap, initialBranch)) {
145 throw new TransportException(MessageFormat.format(
146 JGitText.get().remoteBranchNotFound, initialBranch));
147 }
148 result.setAdvertisedRefs(transport.getURI(), refsMap);
149 result.peerUserAgent = conn.getPeerUserAgent();
150 final Set<Ref> matched = new HashSet<>();
151 for (RefSpec spec : toFetch) {
152 if (spec.getSource() == null)
153 throw new TransportException(MessageFormat.format(
154 JGitText.get().sourceRefNotSpecifiedForRefspec, spec));
155
156 if (spec.isWildcard())
157 expandWildcard(spec, matched);
158 else
159 expandSingle(spec, matched);
160 }
161
162 Collection<Ref> additionalTags = Collections.<Ref> emptyList();
163 if (tagopt == TagOpt.AUTO_FOLLOW)
164 additionalTags = expandAutoFollowTags();
165 else if (tagopt == TagOpt.FETCH_TAGS)
166 expandFetchTags();
167
168 final boolean includedTags;
169 if (!askFor.isEmpty() && !askForIsComplete()) {
170 fetchObjects(monitor);
171 includedTags = conn.didFetchIncludeTags();
172
173
174
175
176 closeConnection(result);
177 } else {
178 includedTags = false;
179 }
180
181 if (tagopt == TagOpt.AUTO_FOLLOW && !additionalTags.isEmpty()) {
182
183
184
185 have.addAll(askFor.keySet());
186 askFor.clear();
187 for (Ref r : additionalTags) {
188 ObjectId id = r.getPeeledObjectId();
189 if (id == null)
190 id = r.getObjectId();
191 if (localHasObject(id))
192 wantTag(r);
193 }
194
195 if (!askFor.isEmpty() && (!includedTags || !askForIsComplete())) {
196 reopenConnection();
197 if (!askFor.isEmpty())
198 fetchObjects(monitor);
199 }
200 }
201 } finally {
202 closeConnection(result);
203 }
204
205 BatchRefUpdate batch = transport.local.getRefDatabase()
206 .newBatchUpdate()
207 .setAllowNonFastForwards(true)
208 .setRefLogMessage("fetch", true);
209 try (RevWalk walk = new RevWalk(transport.local)) {
210 walk.setRetainBody(false);
211 if (monitor instanceof BatchingProgressMonitor) {
212 ((BatchingProgressMonitor) monitor).setDelayStart(
213 250, TimeUnit.MILLISECONDS);
214 }
215 if (transport.isRemoveDeletedRefs()) {
216 deleteStaleTrackingRefs(result, batch);
217 }
218 addUpdateBatchCommands(result, batch);
219 for (ReceiveCommand cmd : batch.getCommands()) {
220 cmd.updateType(walk);
221 if (cmd.getType() == UPDATE_NONFASTFORWARD
222 && cmd instanceof TrackingRefUpdate.Command
223 && !((TrackingRefUpdate.Command) cmd).canForceUpdate())
224 cmd.setResult(REJECTED_NONFASTFORWARD);
225 }
226 if (transport.isDryRun()) {
227 for (ReceiveCommand cmd : batch.getCommands()) {
228 if (cmd.getResult() == NOT_ATTEMPTED)
229 cmd.setResult(OK);
230 }
231 } else {
232 batch.execute(walk, monitor);
233 }
234 } catch (TransportException e) {
235 throw e;
236 } catch (IOException err) {
237 throw new TransportException(MessageFormat.format(
238 JGitText.get().failureUpdatingTrackingRef,
239 getFirstFailedRefName(batch), err.getMessage()), err);
240 }
241
242 if (!fetchHeadUpdates.isEmpty()) {
243 try {
244 updateFETCH_HEAD(result);
245 } catch (IOException err) {
246 throw new TransportException(MessageFormat.format(
247 JGitText.get().failureUpdatingFETCH_HEAD, err.getMessage()), err);
248 }
249 }
250 }
251
252 private void addUpdateBatchCommands(FetchResult result,
253 BatchRefUpdate batch) throws TransportException {
254 Map<String, ObjectId> refs = new HashMap<>();
255 for (TrackingRefUpdate u : localUpdates) {
256
257 ObjectId existing = refs.get(u.getLocalName());
258 if (existing == null) {
259 refs.put(u.getLocalName(), u.getNewObjectId());
260 result.add(u);
261 batch.addCommand(u.asReceiveCommand());
262 } else if (!existing.equals(u.getNewObjectId())) {
263 throw new TransportException(MessageFormat
264 .format(JGitText.get().duplicateRef, u.getLocalName()));
265 }
266 }
267 }
268
269 private void fetchObjects(ProgressMonitor monitor)
270 throws TransportException {
271 try {
272 conn.setPackLockMessage("jgit fetch " + transport.uri);
273 conn.fetch(monitor, askFor.values(), have);
274 } finally {
275 packLocks.addAll(conn.getPackLocks());
276 }
277 if (transport.isCheckFetchedObjects()
278 && !conn.didFetchTestConnectivity() && !askForIsComplete())
279 throw new TransportException(transport.getURI(),
280 JGitText.get().peerDidNotSupplyACompleteObjectGraph);
281 }
282
283 private void closeConnection(FetchResult result) {
284 if (conn != null) {
285 conn.close();
286 result.addMessages(conn.getMessages());
287 conn = null;
288 }
289 }
290
291 private void reopenConnection() throws NotSupportedException,
292 TransportException {
293 if (conn != null)
294 return;
295
296
297 Set<String> prefixes = new HashSet<>();
298 for (Ref toGet : askFor.values()) {
299 String src = toGet.getName();
300 prefixes.add(src);
301 prefixes.add(Constants.R_REFS + src);
302 prefixes.add(Constants.R_HEADS + src);
303 prefixes.add(Constants.R_TAGS + src);
304 }
305 conn = transport.openFetch(Collections.emptyList(),
306 prefixes.toArray(new String[0]));
307
308
309
310
311
312
313
314
315
316 final HashMap<ObjectId, Ref> avail = new HashMap<>();
317 for (Ref r : conn.getRefs())
318 avail.put(r.getObjectId(), r);
319
320 final Collection<Ref> wants = new ArrayList<>(askFor.values());
321 askFor.clear();
322 for (Ref want : wants) {
323 final Ref newRef = avail.get(want.getObjectId());
324 if (newRef != null) {
325 askFor.put(newRef.getObjectId(), newRef);
326 } else {
327 removeFetchHeadRecord(want.getObjectId());
328 removeTrackingRefUpdate(want.getObjectId());
329 }
330 }
331 }
332
333 private void removeTrackingRefUpdate(ObjectId want) {
334 final Iterator<TrackingRefUpdate> i = localUpdates.iterator();
335 while (i.hasNext()) {
336 final TrackingRefUpdate u = i.next();
337 if (u.getNewObjectId().equals(want))
338 i.remove();
339 }
340 }
341
342 private void removeFetchHeadRecord(ObjectId want) {
343 final Iterator<FetchHeadRecord> i = fetchHeadUpdates.iterator();
344 while (i.hasNext()) {
345 final FetchHeadRecord fh = i.next();
346 if (fh.newValue.equals(want))
347 i.remove();
348 }
349 }
350
351 private void updateFETCH_HEAD(FetchResult result) throws IOException {
352 File meta = transport.local.getDirectory();
353 if (meta == null)
354 return;
355 final LockFile lock = new LockFile(new File(meta, "FETCH_HEAD"));
356 try {
357 if (lock.lock()) {
358 try (Writer w = new OutputStreamWriter(
359 lock.getOutputStream(), UTF_8)) {
360 for (FetchHeadRecord h : fetchHeadUpdates) {
361 h.write(w);
362 result.add(h);
363 }
364 }
365 lock.commit();
366 }
367 } finally {
368 lock.unlock();
369 }
370 }
371
372 private boolean askForIsComplete() throws TransportException {
373 try {
374 try (ObjectWalk ow = new ObjectWalk(transport.local)) {
375 for (ObjectId want : askFor.keySet())
376 ow.markStart(ow.parseAny(want));
377 for (Ref ref : localRefs().values())
378 ow.markUninteresting(ow.parseAny(ref.getObjectId()));
379 ow.checkConnectivity();
380 }
381 return true;
382 } catch (MissingObjectException e) {
383 return false;
384 } catch (IOException e) {
385 throw new TransportException(JGitText.get().unableToCheckConnectivity, e);
386 }
387 }
388
389 private void expandWildcard(RefSpec spec, Set<Ref> matched)
390 throws TransportException {
391 for (Ref src : conn.getRefs()) {
392 if (spec.matchSource(src) && matched.add(src))
393 want(src, spec.expandFromSource(src));
394 }
395 }
396
397 private void expandSingle(RefSpec spec, Set<Ref> matched)
398 throws TransportException {
399 String want = spec.getSource();
400 if (ObjectId.isId(want)) {
401 want(ObjectId.fromString(want));
402 return;
403 }
404
405 Ref src = conn.getRef(want);
406 if (src == null) {
407 throw new TransportException(MessageFormat.format(JGitText.get().remoteDoesNotHaveSpec, want));
408 }
409 if (matched.add(src)) {
410 want(src, spec);
411 }
412 }
413
414 private boolean localHasObject(ObjectId id) throws TransportException {
415 try {
416 return transport.local.getObjectDatabase().has(id);
417 } catch (IOException err) {
418 throw new TransportException(
419 MessageFormat.format(
420 JGitText.get().readingObjectsFromLocalRepositoryFailed,
421 err.getMessage()),
422 err);
423 }
424 }
425
426 private Collection<Ref> expandAutoFollowTags() throws TransportException {
427 final Collection<Ref> additionalTags = new ArrayList<>();
428 final Map<String, Ref> haveRefs = localRefs();
429 for (Ref r : conn.getRefs()) {
430 if (!isTag(r))
431 continue;
432
433 Ref local = haveRefs.get(r.getName());
434 if (local != null)
435
436
437 continue;
438
439 ObjectId obj = r.getPeeledObjectId();
440 if (obj == null)
441 obj = r.getObjectId();
442
443 if (askFor.containsKey(obj) || localHasObject(obj))
444 wantTag(r);
445 else
446 additionalTags.add(r);
447 }
448 return additionalTags;
449 }
450
451 private void expandFetchTags() throws TransportException {
452 final Map<String, Ref> haveRefs = localRefs();
453 for (Ref r : conn.getRefs()) {
454 if (!isTag(r)) {
455 continue;
456 }
457 ObjectId id = r.getObjectId();
458 if (id == null) {
459 continue;
460 }
461 final Ref local = haveRefs.get(r.getName());
462 if (local == null || !id.equals(local.getObjectId())) {
463 wantTag(r);
464 }
465 }
466 }
467
468 private void wantTag(Ref r) throws TransportException {
469 want(r, new RefSpec().setSource(r.getName())
470 .setDestination(r.getName()).setForceUpdate(true));
471 }
472
473 private void want(Ref src, RefSpec spec)
474 throws TransportException {
475 final ObjectId newId = src.getObjectId();
476 if (newId == null) {
477 throw new NullPointerException(MessageFormat.format(
478 JGitText.get().transportProvidedRefWithNoObjectId,
479 src.getName()));
480 }
481 if (spec.getDestination() != null) {
482 final TrackingRefUpdate tru = createUpdate(spec, newId);
483 if (newId.equals(tru.getOldObjectId()))
484 return;
485 localUpdates.add(tru);
486 }
487
488 askFor.put(newId, src);
489
490 final FetchHeadRecord fhr = new FetchHeadRecord();
491 fhr.newValue = newId;
492 fhr.notForMerge = spec.getDestination() != null;
493 fhr.sourceName = src.getName();
494 fhr.sourceURI = transport.getURI();
495 fetchHeadUpdates.add(fhr);
496 }
497
498 private void want(ObjectId id) {
499 askFor.put(id,
500 new ObjectIdRef.Unpeeled(Ref.Storage.NETWORK, id.name(), id));
501 }
502
503 private TrackingRefUpdate createUpdate(RefSpec spec, ObjectId newId)
504 throws TransportException {
505 Ref ref = localRefs().get(spec.getDestination());
506 ObjectId oldId = ref != null && ref.getObjectId() != null
507 ? ref.getObjectId()
508 : ObjectId.zeroId();
509 return new TrackingRefUpdate(
510 spec.isForceUpdate(),
511 spec.getSource(),
512 spec.getDestination(),
513 oldId,
514 newId);
515 }
516
517 private Map<String, Ref> localRefs() throws TransportException {
518 if (localRefs == null) {
519 try {
520 localRefs = transport.local.getRefDatabase()
521 .getRefs(RefDatabase.ALL);
522 } catch (IOException err) {
523 throw new TransportException(JGitText.get().cannotListRefs, err);
524 }
525 }
526 return localRefs;
527 }
528
529 private void deleteStaleTrackingRefs(FetchResult result,
530 BatchRefUpdate batch) throws IOException {
531 Set<Ref> processed = new HashSet<>();
532 for (Ref ref : localRefs().values()) {
533 if (ref.isSymbolic()) {
534 continue;
535 }
536 String refname = ref.getName();
537 for (RefSpec spec : toFetch) {
538 if (spec.matchDestination(refname)) {
539 RefSpec s = spec.expandFromDestination(refname);
540 if (result.getAdvertisedRef(s.getSource()) == null
541 && processed.add(ref)) {
542 deleteTrackingRef(result, batch, s, ref);
543 }
544 }
545 }
546 }
547 }
548
549 private void deleteTrackingRef(final FetchResult result,
550 final BatchRefUpdate batch, final RefSpec spec, final Ref localRef) {
551 if (localRef.getObjectId() == null)
552 return;
553 TrackingRefUpdate update = new TrackingRefUpdate(
554 true,
555 spec.getSource(),
556 localRef.getName(),
557 localRef.getObjectId(),
558 ObjectId.zeroId());
559 result.add(update);
560 batch.addCommand(update.asReceiveCommand());
561 }
562
563 private static boolean isTag(Ref r) {
564 return isTag(r.getName());
565 }
566
567 private static boolean isTag(String name) {
568 return name.startsWith(Constants.R_TAGS);
569 }
570
571 private static String getFirstFailedRefName(BatchRefUpdate batch) {
572 for (ReceiveCommand cmd : batch.getCommands()) {
573 if (cmd.getResult() != ReceiveCommand.Result.OK)
574 return cmd.getRefName();
575 }
576 return "";
577 }
578 }