View Javadoc
1   /*
2    * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com>
3    * Copyright (C) 2008, 2022 Shawn O. Pearce <spearce@spearce.org> and others
4    *
5    * This program and the accompanying materials are made available under the
6    * terms of the Eclipse Distribution License v. 1.0 which is available at
7    * https://www.eclipse.org/org/documents/edl-v10.php.
8    *
9    * SPDX-License-Identifier: BSD-3-Clause
10   */
11  
12  package org.eclipse.jgit.transport;
13  
14  import static org.eclipse.jgit.transport.GitProtocolConstants.CAPABILITY_ATOMIC;
15  
16  import java.io.IOException;
17  import java.io.InputStream;
18  import java.io.OutputStream;
19  import java.text.MessageFormat;
20  import java.util.Collection;
21  import java.util.HashSet;
22  import java.util.List;
23  import java.util.Map;
24  import java.util.Set;
25  
26  import org.eclipse.jgit.errors.NoRemoteRepositoryException;
27  import org.eclipse.jgit.errors.NotSupportedException;
28  import org.eclipse.jgit.errors.PackProtocolException;
29  import org.eclipse.jgit.errors.TooLargeObjectInPackException;
30  import org.eclipse.jgit.errors.TooLargePackException;
31  import org.eclipse.jgit.errors.TransportException;
32  import org.eclipse.jgit.internal.JGitText;
33  import org.eclipse.jgit.internal.storage.pack.PackWriter;
34  import org.eclipse.jgit.lib.ObjectId;
35  import org.eclipse.jgit.lib.ProgressMonitor;
36  import org.eclipse.jgit.lib.Ref;
37  import org.eclipse.jgit.transport.RemoteRefUpdate.Status;
38  
39  /**
40   * Push implementation using the native Git pack transfer service.
41   * <p>
42   * This is the canonical implementation for transferring objects to the remote
43   * repository from the local repository by talking to the 'git-receive-pack'
44   * service. Objects are packed on the local side into a pack file and then sent
45   * to the remote repository.
46   * <p>
47   * This connection requires only a bi-directional pipe or socket, and thus is
48   * easily wrapped up into a local process pipe, anonymous TCP socket, or a
49   * command executed through an SSH tunnel.
50   * <p>
51   * This implementation honors
52   * {@link org.eclipse.jgit.transport.Transport#isPushThin()} option.
53   * <p>
54   * Concrete implementations should just call
55   * {@link #init(java.io.InputStream, java.io.OutputStream)} and
56   * {@link #readAdvertisedRefs()} methods in constructor or before any use. They
57   * should also handle resources releasing in {@link #close()} method if needed.
58   */
59  public abstract class BasePackPushConnection extends BasePackConnection implements
60  		PushConnection {
61  	/**
62  	 * The client expects a status report after the server processes the pack.
63  	 * @since 2.0
64  	 */
65  	public static final String CAPABILITY_REPORT_STATUS = GitProtocolConstants.CAPABILITY_REPORT_STATUS;
66  
67  	/**
68  	 * The server supports deleting refs.
69  	 * @since 2.0
70  	 */
71  	public static final String CAPABILITY_DELETE_REFS = GitProtocolConstants.CAPABILITY_DELETE_REFS;
72  
73  	/**
74  	 * The server supports packs with OFS deltas.
75  	 * @since 2.0
76  	 */
77  	public static final String CAPABILITY_OFS_DELTA = GitProtocolConstants.CAPABILITY_OFS_DELTA;
78  
79  	/**
80  	 * The client supports using the 64K side-band for progress messages.
81  	 * @since 2.0
82  	 */
83  	public static final String CAPABILITY_SIDE_BAND_64K = GitProtocolConstants.CAPABILITY_SIDE_BAND_64K;
84  
85  	/**
86  	 * The server supports the receiving of push options.
87  	 * @since 4.5
88  	 */
89  	public static final String CAPABILITY_PUSH_OPTIONS = GitProtocolConstants.CAPABILITY_PUSH_OPTIONS;
90  
91  	private final boolean thinPack;
92  	private final boolean atomic;
93  
94  	/** A list of option strings associated with this push. */
95  	private List<String> pushOptions;
96  
97  	private boolean capableAtomic;
98  	private boolean capableDeleteRefs;
99  	private boolean capableReport;
100 	private boolean capableSideBand;
101 	private boolean capableOfsDelta;
102 	private boolean capablePushOptions;
103 
104 	private boolean sentCommand;
105 	private boolean writePack;
106 
107 	/** Time in milliseconds spent transferring the pack data. */
108 	private long packTransferTime;
109 
110 	/**
111 	 * Create a new connection to push using the native git transport.
112 	 *
113 	 * @param packTransport
114 	 *            the transport.
115 	 */
116 	public BasePackPushConnection(PackTransport packTransport) {
117 		super(packTransport);
118 		thinPack = transport.isPushThin();
119 		atomic = transport.isPushAtomic();
120 		pushOptions = transport.getPushOptions();
121 	}
122 
123 	/** {@inheritDoc} */
124 	@Override
125 	public void push(final ProgressMonitor monitor,
126 			final Map<String, RemoteRefUpdate> refUpdates)
127 			throws TransportException {
128 		push(monitor, refUpdates, null);
129 	}
130 
131 	/** {@inheritDoc} */
132 	@Override
133 	public void push(final ProgressMonitor monitor,
134 			final Map<String, RemoteRefUpdate> refUpdates, OutputStream outputStream)
135 			throws TransportException {
136 		markStartedOperation();
137 		doPush(monitor, refUpdates, outputStream);
138 	}
139 
140 	/** {@inheritDoc} */
141 	@Override
142 	protected TransportException noRepository(Throwable cause) {
143 		// Sadly we cannot tell the "invalid URI" case from "push not allowed".
144 		// Opening a fetch connection can help us tell the difference, as any
145 		// useful repository is going to support fetch if it also would allow
146 		// push. So if fetch throws NoRemoteRepositoryException we know the
147 		// URI is wrong. Otherwise we can correctly state push isn't allowed
148 		// as the fetch connection opened successfully.
149 		//
150 		TransportException te;
151 		try {
152 			transport.openFetch().close();
153 			te = new TransportException(uri, JGitText.get().pushNotPermitted);
154 		} catch (NoRemoteRepositoryException e) {
155 			// Fetch concluded the repository doesn't exist.
156 			te = e;
157 		} catch (NotSupportedException | TransportException e) {
158 			te = new TransportException(uri, JGitText.get().pushNotPermitted, e);
159 		}
160 		te.addSuppressed(cause);
161 		return te;
162 	}
163 
164 	/**
165 	 * Push one or more objects and update the remote repository.
166 	 *
167 	 * @param monitor
168 	 *            progress monitor to receive status updates.
169 	 * @param refUpdates
170 	 *            update commands to be applied to the remote repository.
171 	 * @param outputStream
172 	 *            output stream to write sideband messages to
173 	 * @throws org.eclipse.jgit.errors.TransportException
174 	 *             if any exception occurs.
175 	 * @since 3.0
176 	 */
177 	protected void doPush(final ProgressMonitor monitor,
178 			final Map<String, RemoteRefUpdate> refUpdates,
179 			OutputStream outputStream) throws TransportException {
180 		try {
181 			writeCommands(refUpdates.values(), monitor, outputStream);
182 
183 			if (pushOptions != null && capablePushOptions)
184 				transmitOptions();
185 			if (writePack)
186 				writePack(refUpdates, monitor);
187 			if (sentCommand) {
188 				if (capableReport)
189 					readStatusReport(refUpdates);
190 				if (capableSideBand) {
191 					// Ensure the data channel is at EOF, so we know we have
192 					// read all side-band data from all channels and have a
193 					// complete copy of the messages (if any) buffered from
194 					// the other data channels.
195 					//
196 					int b = in.read();
197 					if (0 <= b) {
198 						throw new TransportException(uri, MessageFormat.format(
199 								JGitText.get().expectedEOFReceived,
200 								Character.valueOf((char) b)));
201 					}
202 				}
203 			}
204 		} catch (TransportException e) {
205 			throw e;
206 		} catch (Exception e) {
207 			throw new TransportException(uri, e.getMessage(), e);
208 		} finally {
209 			if (in instanceof SideBandInputStream) {
210 				((SideBandInputStream) in).drainMessages();
211 			}
212 			close();
213 		}
214 	}
215 
216 	private void writeCommands(final Collection<RemoteRefUpdate> refUpdates,
217 			final ProgressMonitor monitor, OutputStream outputStream) throws IOException {
218 		final String capabilities = enableCapabilities(monitor, outputStream);
219 		if (atomic && !capableAtomic) {
220 			throw new TransportException(uri,
221 					JGitText.get().atomicPushNotSupported);
222 		}
223 
224 		if (pushOptions != null && !capablePushOptions) {
225 			throw new TransportException(uri,
226 					MessageFormat.format(JGitText.get().pushOptionsNotSupported,
227 							pushOptions.toString()));
228 		}
229 
230 		for (RemoteRefUpdate rru : refUpdates) {
231 			if (!capableDeleteRefs && rru.isDelete()) {
232 				rru.setStatus(Status.REJECTED_NODELETE);
233 				continue;
234 			}
235 
236 			final StringBuilder sb = new StringBuilder();
237 			ObjectId oldId = rru.getExpectedOldObjectId();
238 			if (oldId == null) {
239 				final Ref advertised = getRef(rru.getRemoteName());
240 				oldId = advertised != null ? advertised.getObjectId() : null;
241 				if (oldId == null) {
242 					oldId = ObjectId.zeroId();
243 				}
244 			}
245 			sb.append(oldId.name());
246 			sb.append(' ');
247 			sb.append(rru.getNewObjectId().name());
248 			sb.append(' ');
249 			sb.append(rru.getRemoteName());
250 			if (!sentCommand) {
251 				sentCommand = true;
252 				sb.append(capabilities);
253 			}
254 
255 			pckOut.writeString(sb.toString());
256 			rru.setStatus(Status.AWAITING_REPORT);
257 			if (!rru.isDelete())
258 				writePack = true;
259 		}
260 
261 		if (monitor.isCancelled())
262 			throw new TransportException(uri, JGitText.get().pushCancelled);
263 		pckOut.end();
264 		outNeedsEnd = false;
265 	}
266 
267 	private void transmitOptions() throws IOException {
268 		for (String pushOption : pushOptions) {
269 			pckOut.writeString(pushOption);
270 		}
271 
272 		pckOut.end();
273 	}
274 
275 	private String enableCapabilities(final ProgressMonitor monitor,
276 			OutputStream outputStream) {
277 		final StringBuilder line = new StringBuilder();
278 		if (atomic)
279 			capableAtomic = wantCapability(line, CAPABILITY_ATOMIC);
280 		capableReport = wantCapability(line, CAPABILITY_REPORT_STATUS);
281 		capableDeleteRefs = wantCapability(line, CAPABILITY_DELETE_REFS);
282 		capableOfsDelta = wantCapability(line, CAPABILITY_OFS_DELTA);
283 
284 		if (pushOptions != null) {
285 			capablePushOptions = wantCapability(line, CAPABILITY_PUSH_OPTIONS);
286 		}
287 
288 		capableSideBand = wantCapability(line, CAPABILITY_SIDE_BAND_64K);
289 		if (capableSideBand) {
290 			in = new SideBandInputStream(in, monitor, getMessageWriter(),
291 					outputStream);
292 			pckIn = new PacketLineIn(in);
293 		}
294 		addUserAgentCapability(line);
295 
296 		if (line.length() > 0)
297 			line.setCharAt(0, '\0');
298 		return line.toString();
299 	}
300 
301 	private void writePack(final Map<String, RemoteRefUpdate> refUpdates,
302 			final ProgressMonitor monitor) throws IOException {
303 		Set<ObjectId> remoteObjects = new HashSet<>();
304 		Set<ObjectId> newObjects = new HashSet<>();
305 
306 		try (PackWriter writer = new PackWriter(transport.getPackConfig(),
307 				local.newObjectReader())) {
308 
309 			for (Ref r : getRefs()) {
310 				// only add objects that we actually have
311 				ObjectId oid = r.getObjectId();
312 				if (local.getObjectDatabase().has(oid))
313 					remoteObjects.add(oid);
314 			}
315 			remoteObjects.addAll(additionalHaves);
316 			for (RemoteRefUpdate r : refUpdates.values()) {
317 				if (!ObjectId.zeroId().equals(r.getNewObjectId()))
318 					newObjects.add(r.getNewObjectId());
319 			}
320 
321 			writer.setIndexDisabled(true);
322 			writer.setUseCachedPacks(true);
323 			writer.setUseBitmaps(true);
324 			writer.setThin(thinPack);
325 			writer.setReuseValidatingObjects(false);
326 			writer.setDeltaBaseAsOffset(capableOfsDelta);
327 			writer.preparePack(monitor, newObjects, remoteObjects);
328 
329 			OutputStream packOut = out;
330 			if (capableSideBand) {
331 				packOut = new CheckingSideBandOutputStream(in, out);
332 			}
333 			writer.writePack(monitor, monitor, packOut);
334 
335 			packTransferTime = writer.getStatistics().getTimeWriting();
336 		}
337 	}
338 
339 	private void readStatusReport(Map<String, RemoteRefUpdate> refUpdates)
340 			throws IOException {
341 		final String unpackLine = readStringLongTimeout();
342 		if (!unpackLine.startsWith("unpack ")) //$NON-NLS-1$
343 			throw new PackProtocolException(uri, MessageFormat
344 					.format(JGitText.get().unexpectedReportLine, unpackLine));
345 		final String unpackStatus = unpackLine.substring("unpack ".length()); //$NON-NLS-1$
346 		if (unpackStatus.startsWith("error Pack exceeds the limit of")) {//$NON-NLS-1$
347 			throw new TooLargePackException(uri,
348 					unpackStatus.substring("error ".length())); //$NON-NLS-1$
349 		} else if (unpackStatus.startsWith("error Object too large")) {//$NON-NLS-1$
350 			throw new TooLargeObjectInPackException(uri,
351 					unpackStatus.substring("error ".length())); //$NON-NLS-1$
352 		} else if (!unpackStatus.equals("ok")) { //$NON-NLS-1$
353 			throw new TransportException(uri, MessageFormat.format(
354 					JGitText.get().errorOccurredDuringUnpackingOnTheRemoteEnd, unpackStatus));
355 		}
356 
357 		for (String refLine : pckIn.readStrings()) {
358 			boolean ok = false;
359 			int refNameEnd = -1;
360 			if (refLine.startsWith("ok ")) { //$NON-NLS-1$
361 				ok = true;
362 				refNameEnd = refLine.length();
363 			} else if (refLine.startsWith("ng ")) { //$NON-NLS-1$
364 				ok = false;
365 				refNameEnd = refLine.indexOf(' ', 3);
366 			}
367 			if (refNameEnd == -1)
368 				throw new PackProtocolException(MessageFormat.format(JGitText.get().unexpectedReportLine2
369 						, uri, refLine));
370 			final String refName = refLine.substring(3, refNameEnd);
371 			final String message = (ok ? null : refLine
372 					.substring(refNameEnd + 1));
373 
374 			final RemoteRefUpdate rru = refUpdates.get(refName);
375 			if (rru == null)
376 				throw new PackProtocolException(MessageFormat.format(JGitText.get().unexpectedRefReport, uri, refName));
377 			if (ok) {
378 				rru.setStatus(Status.OK);
379 			} else {
380 				rru.setStatus(Status.REJECTED_OTHER_REASON);
381 				rru.setMessage(message);
382 			}
383 		}
384 		for (RemoteRefUpdate rru : refUpdates.values()) {
385 			if (rru.getStatus() == Status.AWAITING_REPORT)
386 				throw new PackProtocolException(MessageFormat.format(
387 						JGitText.get().expectedReportForRefNotReceived , uri, rru.getRemoteName()));
388 		}
389 	}
390 
391 	private String readStringLongTimeout() throws IOException {
392 		if (timeoutIn == null)
393 			return pckIn.readString();
394 
395 		// The remote side may need a lot of time to choke down the pack
396 		// we just sent them. There may be many deltas that need to be
397 		// resolved by the remote. Its hard to say how long the other
398 		// end is going to be silent. Taking 10x the configured timeout
399 		// or the time spent transferring the pack, whichever is larger,
400 		// gives the other side some reasonable window to process the data,
401 		// but this is just a wild guess.
402 		//
403 		final int oldTimeout = timeoutIn.getTimeout();
404 		final int sendTime = (int) Math.min(packTransferTime, 28800000L);
405 		try {
406 			int timeout = 10 * Math.max(sendTime, oldTimeout);
407 			timeoutIn.setTimeout((timeout < 0) ? Integer.MAX_VALUE : timeout);
408 			return pckIn.readString();
409 		} finally {
410 			timeoutIn.setTimeout(oldTimeout);
411 		}
412 	}
413 
414 	/**
415 	 * Gets the list of option strings associated with this push.
416 	 *
417 	 * @return pushOptions
418 	 * @since 4.5
419 	 */
420 	public List<String> getPushOptions() {
421 		return pushOptions;
422 	}
423 
424 	private static class CheckingSideBandOutputStream extends OutputStream {
425 		private final InputStream in;
426 		private final OutputStream out;
427 
428 		CheckingSideBandOutputStream(InputStream in, OutputStream out) {
429 			this.in = in;
430 			this.out = out;
431 		}
432 
433 		@Override
434 		public void write(int b) throws IOException {
435 			write(new byte[] { (byte) b });
436 		}
437 
438 		@Override
439 		public void write(byte[] buf, int ptr, int cnt) throws IOException {
440 			try {
441 				out.write(buf, ptr, cnt);
442 			} catch (IOException e) {
443 				throw checkError(e);
444 			}
445 		}
446 
447 		@Override
448 		public void flush() throws IOException {
449 			try {
450 				out.flush();
451 			} catch (IOException e) {
452 				throw checkError(e);
453 			}
454 		}
455 
456 		private IOException checkError(IOException e1) {
457 			try {
458 				in.read();
459 			} catch (TransportException e2) {
460 				return e2;
461 			} catch (IOException e2) {
462 				return e1;
463 			}
464 			return e1;
465 		}
466 	}
467 }