View Javadoc
1   /*
2    * Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com>
3    * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others
4    *
5    * This program and the accompanying materials are made available under the
6    * terms of the Eclipse Distribution License v. 1.0 which is available at
7    * https://www.eclipse.org/org/documents/edl-v10.php.
8    *
9    * SPDX-License-Identifier: BSD-3-Clause
10   */
11  
12  package org.eclipse.jgit.transport;
13  
14  import static org.eclipse.jgit.transport.GitProtocolConstants.CAPABILITY_ATOMIC;
15  
16  import java.io.IOException;
17  import java.io.InputStream;
18  import java.io.OutputStream;
19  import java.text.MessageFormat;
20  import java.util.Collection;
21  import java.util.HashSet;
22  import java.util.List;
23  import java.util.Map;
24  import java.util.Set;
25  
26  import org.eclipse.jgit.errors.NoRemoteRepositoryException;
27  import org.eclipse.jgit.errors.NotSupportedException;
28  import org.eclipse.jgit.errors.PackProtocolException;
29  import org.eclipse.jgit.errors.TooLargeObjectInPackException;
30  import org.eclipse.jgit.errors.TooLargePackException;
31  import org.eclipse.jgit.errors.TransportException;
32  import org.eclipse.jgit.internal.JGitText;
33  import org.eclipse.jgit.internal.storage.pack.PackWriter;
34  import org.eclipse.jgit.lib.ObjectId;
35  import org.eclipse.jgit.lib.ProgressMonitor;
36  import org.eclipse.jgit.lib.Ref;
37  import org.eclipse.jgit.transport.RemoteRefUpdate.Status;
38  
39  /**
40   * Push implementation using the native Git pack transfer service.
41   * <p>
42   * This is the canonical implementation for transferring objects to the remote
43   * repository from the local repository by talking to the 'git-receive-pack'
44   * service. Objects are packed on the local side into a pack file and then sent
45   * to the remote repository.
46   * <p>
47   * This connection requires only a bi-directional pipe or socket, and thus is
48   * easily wrapped up into a local process pipe, anonymous TCP socket, or a
49   * command executed through an SSH tunnel.
50   * <p>
51   * This implementation honors
52   * {@link org.eclipse.jgit.transport.Transport#isPushThin()} option.
53   * <p>
54   * Concrete implementations should just call
55   * {@link #init(java.io.InputStream, java.io.OutputStream)} and
56   * {@link #readAdvertisedRefs()} methods in constructor or before any use. They
57   * should also handle resources releasing in {@link #close()} method if needed.
58   */
59  public abstract class BasePackPushConnection extends BasePackConnection implements
60  		PushConnection {
61  	/**
62  	 * The client expects a status report after the server processes the pack.
63  	 * @since 2.0
64  	 */
65  	public static final String CAPABILITY_REPORT_STATUS = GitProtocolConstants.CAPABILITY_REPORT_STATUS;
66  
67  	/**
68  	 * The server supports deleting refs.
69  	 * @since 2.0
70  	 */
71  	public static final String CAPABILITY_DELETE_REFS = GitProtocolConstants.CAPABILITY_DELETE_REFS;
72  
73  	/**
74  	 * The server supports packs with OFS deltas.
75  	 * @since 2.0
76  	 */
77  	public static final String CAPABILITY_OFS_DELTA = GitProtocolConstants.CAPABILITY_OFS_DELTA;
78  
79  	/**
80  	 * The client supports using the 64K side-band for progress messages.
81  	 * @since 2.0
82  	 */
83  	public static final String CAPABILITY_SIDE_BAND_64K = GitProtocolConstants.CAPABILITY_SIDE_BAND_64K;
84  
85  	/**
86  	 * The server supports the receiving of push options.
87  	 * @since 4.5
88  	 */
89  	public static final String CAPABILITY_PUSH_OPTIONS = GitProtocolConstants.CAPABILITY_PUSH_OPTIONS;
90  
91  	private final boolean thinPack;
92  	private final boolean atomic;
93  
94  	/** A list of option strings associated with this push. */
95  	private List<String> pushOptions;
96  
97  	private boolean capableAtomic;
98  	private boolean capableDeleteRefs;
99  	private boolean capableReport;
100 	private boolean capableSideBand;
101 	private boolean capableOfsDelta;
102 	private boolean capablePushOptions;
103 
104 	private boolean sentCommand;
105 	private boolean writePack;
106 
107 	/** Time in milliseconds spent transferring the pack data. */
108 	private long packTransferTime;
109 
110 	/**
111 	 * Create a new connection to push using the native git transport.
112 	 *
113 	 * @param packTransport
114 	 *            the transport.
115 	 */
116 	public BasePackPushConnection(PackTransport packTransport) {
117 		super(packTransport);
118 		thinPack = transport.isPushThin();
119 		atomic = transport.isPushAtomic();
120 		pushOptions = transport.getPushOptions();
121 	}
122 
123 	/** {@inheritDoc} */
124 	@Override
125 	public void push(final ProgressMonitor monitor,
126 			final Map<String, RemoteRefUpdate> refUpdates)
127 			throws TransportException {
128 		push(monitor, refUpdates, null);
129 	}
130 
131 	/** {@inheritDoc} */
132 	@Override
133 	public void push(final ProgressMonitor monitor,
134 			final Map<String, RemoteRefUpdate> refUpdates, OutputStream outputStream)
135 			throws TransportException {
136 		markStartedOperation();
137 		doPush(monitor, refUpdates, outputStream);
138 	}
139 
140 	/** {@inheritDoc} */
141 	@Override
142 	protected TransportException noRepository() {
143 		// Sadly we cannot tell the "invalid URI" case from "push not allowed".
144 		// Opening a fetch connection can help us tell the difference, as any
145 		// useful repository is going to support fetch if it also would allow
146 		// push. So if fetch throws NoRemoteRepositoryException we know the
147 		// URI is wrong. Otherwise we can correctly state push isn't allowed
148 		// as the fetch connection opened successfully.
149 		//
150 		try {
151 			transport.openFetch().close();
152 		} catch (NotSupportedException e) {
153 			// Fall through.
154 		} catch (NoRemoteRepositoryException e) {
155 			// Fetch concluded the repository doesn't exist.
156 			//
157 			return e;
158 		} catch (TransportException e) {
159 			// Fall through.
160 		}
161 		return new TransportException(uri, JGitText.get().pushNotPermitted);
162 	}
163 
164 	/**
165 	 * Push one or more objects and update the remote repository.
166 	 *
167 	 * @param monitor
168 	 *            progress monitor to receive status updates.
169 	 * @param refUpdates
170 	 *            update commands to be applied to the remote repository.
171 	 * @param outputStream
172 	 *            output stream to write sideband messages to
173 	 * @throws org.eclipse.jgit.errors.TransportException
174 	 *             if any exception occurs.
175 	 * @since 3.0
176 	 */
177 	protected void doPush(final ProgressMonitor monitor,
178 			final Map<String, RemoteRefUpdate> refUpdates,
179 			OutputStream outputStream) throws TransportException {
180 		try {
181 			writeCommands(refUpdates.values(), monitor, outputStream);
182 
183 			if (pushOptions != null && capablePushOptions)
184 				transmitOptions();
185 			if (writePack)
186 				writePack(refUpdates, monitor);
187 			if (sentCommand) {
188 				if (capableReport)
189 					readStatusReport(refUpdates);
190 				if (capableSideBand) {
191 					// Ensure the data channel is at EOF, so we know we have
192 					// read all side-band data from all channels and have a
193 					// complete copy of the messages (if any) buffered from
194 					// the other data channels.
195 					//
196 					int b = in.read();
197 					if (0 <= b)
198 						throw new TransportException(uri, MessageFormat.format(
199 								JGitText.get().expectedEOFReceived,
200 								Character.valueOf((char) b)));
201 				}
202 			}
203 		} catch (TransportException e) {
204 			throw e;
205 		} catch (Exception e) {
206 			throw new TransportException(uri, e.getMessage(), e);
207 		} finally {
208 			close();
209 		}
210 	}
211 
212 	private void writeCommands(final Collection<RemoteRefUpdate> refUpdates,
213 			final ProgressMonitor monitor, OutputStream outputStream) throws IOException {
214 		final String capabilities = enableCapabilities(monitor, outputStream);
215 		if (atomic && !capableAtomic) {
216 			throw new TransportException(uri,
217 					JGitText.get().atomicPushNotSupported);
218 		}
219 
220 		if (pushOptions != null && !capablePushOptions) {
221 			throw new TransportException(uri,
222 					MessageFormat.format(JGitText.get().pushOptionsNotSupported,
223 							pushOptions.toString()));
224 		}
225 
226 		for (RemoteRefUpdate rru : refUpdates) {
227 			if (!capableDeleteRefs && rru.isDelete()) {
228 				rru.setStatus(Status.REJECTED_NODELETE);
229 				continue;
230 			}
231 
232 			final StringBuilder sb = new StringBuilder();
233 			ObjectId oldId = rru.getExpectedOldObjectId();
234 			if (oldId == null) {
235 				final Ref advertised = getRef(rru.getRemoteName());
236 				oldId = advertised != null ? advertised.getObjectId() : null;
237 				if (oldId == null) {
238 					oldId = ObjectId.zeroId();
239 				}
240 			}
241 			sb.append(oldId.name());
242 			sb.append(' ');
243 			sb.append(rru.getNewObjectId().name());
244 			sb.append(' ');
245 			sb.append(rru.getRemoteName());
246 			if (!sentCommand) {
247 				sentCommand = true;
248 				sb.append(capabilities);
249 			}
250 
251 			pckOut.writeString(sb.toString());
252 			rru.setStatus(Status.AWAITING_REPORT);
253 			if (!rru.isDelete())
254 				writePack = true;
255 		}
256 
257 		if (monitor.isCancelled())
258 			throw new TransportException(uri, JGitText.get().pushCancelled);
259 		pckOut.end();
260 		outNeedsEnd = false;
261 	}
262 
263 	private void transmitOptions() throws IOException {
264 		for (String pushOption : pushOptions) {
265 			pckOut.writeString(pushOption);
266 		}
267 
268 		pckOut.end();
269 	}
270 
271 	private String enableCapabilities(final ProgressMonitor monitor,
272 			OutputStream outputStream) {
273 		final StringBuilder line = new StringBuilder();
274 		if (atomic)
275 			capableAtomic = wantCapability(line, CAPABILITY_ATOMIC);
276 		capableReport = wantCapability(line, CAPABILITY_REPORT_STATUS);
277 		capableDeleteRefs = wantCapability(line, CAPABILITY_DELETE_REFS);
278 		capableOfsDelta = wantCapability(line, CAPABILITY_OFS_DELTA);
279 
280 		if (pushOptions != null) {
281 			capablePushOptions = wantCapability(line, CAPABILITY_PUSH_OPTIONS);
282 		}
283 
284 		capableSideBand = wantCapability(line, CAPABILITY_SIDE_BAND_64K);
285 		if (capableSideBand) {
286 			in = new SideBandInputStream(in, monitor, getMessageWriter(),
287 					outputStream);
288 			pckIn = new PacketLineIn(in);
289 		}
290 		addUserAgentCapability(line);
291 
292 		if (line.length() > 0)
293 			line.setCharAt(0, '\0');
294 		return line.toString();
295 	}
296 
297 	private void writePack(final Map<String, RemoteRefUpdate> refUpdates,
298 			final ProgressMonitor monitor) throws IOException {
299 		Set<ObjectId> remoteObjects = new HashSet<>();
300 		Set<ObjectId> newObjects = new HashSet<>();
301 
302 		try (PackWriter writer = new PackWriter(transport.getPackConfig(),
303 				local.newObjectReader())) {
304 
305 			for (Ref r : getRefs()) {
306 				// only add objects that we actually have
307 				ObjectId oid = r.getObjectId();
308 				if (local.getObjectDatabase().has(oid))
309 					remoteObjects.add(oid);
310 			}
311 			remoteObjects.addAll(additionalHaves);
312 			for (RemoteRefUpdate r : refUpdates.values()) {
313 				if (!ObjectId.zeroId().equals(r.getNewObjectId()))
314 					newObjects.add(r.getNewObjectId());
315 			}
316 
317 			writer.setIndexDisabled(true);
318 			writer.setUseCachedPacks(true);
319 			writer.setUseBitmaps(true);
320 			writer.setThin(thinPack);
321 			writer.setReuseValidatingObjects(false);
322 			writer.setDeltaBaseAsOffset(capableOfsDelta);
323 			writer.preparePack(monitor, newObjects, remoteObjects);
324 
325 			OutputStream packOut = out;
326 			if (capableSideBand) {
327 				packOut = new CheckingSideBandOutputStream(in, out);
328 			}
329 			writer.writePack(monitor, monitor, packOut);
330 
331 			packTransferTime = writer.getStatistics().getTimeWriting();
332 		}
333 	}
334 
335 	private void readStatusReport(Map<String, RemoteRefUpdate> refUpdates)
336 			throws IOException {
337 		final String unpackLine = readStringLongTimeout();
338 		if (!unpackLine.startsWith("unpack ")) //$NON-NLS-1$
339 			throw new PackProtocolException(uri, MessageFormat
340 					.format(JGitText.get().unexpectedReportLine, unpackLine));
341 		final String unpackStatus = unpackLine.substring("unpack ".length()); //$NON-NLS-1$
342 		if (unpackStatus.startsWith("error Pack exceeds the limit of")) {//$NON-NLS-1$
343 			throw new TooLargePackException(uri,
344 					unpackStatus.substring("error ".length())); //$NON-NLS-1$
345 		} else if (unpackStatus.startsWith("error Object too large")) {//$NON-NLS-1$
346 			throw new TooLargeObjectInPackException(uri,
347 					unpackStatus.substring("error ".length())); //$NON-NLS-1$
348 		} else if (!unpackStatus.equals("ok")) { //$NON-NLS-1$
349 			throw new TransportException(uri, MessageFormat.format(
350 					JGitText.get().errorOccurredDuringUnpackingOnTheRemoteEnd, unpackStatus));
351 		}
352 
353 		for (String refLine : pckIn.readStrings()) {
354 			boolean ok = false;
355 			int refNameEnd = -1;
356 			if (refLine.startsWith("ok ")) { //$NON-NLS-1$
357 				ok = true;
358 				refNameEnd = refLine.length();
359 			} else if (refLine.startsWith("ng ")) { //$NON-NLS-1$
360 				ok = false;
361 				refNameEnd = refLine.indexOf(' ', 3);
362 			}
363 			if (refNameEnd == -1)
364 				throw new PackProtocolException(MessageFormat.format(JGitText.get().unexpectedReportLine2
365 						, uri, refLine));
366 			final String refName = refLine.substring(3, refNameEnd);
367 			final String message = (ok ? null : refLine
368 					.substring(refNameEnd + 1));
369 
370 			final RemoteRefUpdate rru = refUpdates.get(refName);
371 			if (rru == null)
372 				throw new PackProtocolException(MessageFormat.format(JGitText.get().unexpectedRefReport, uri, refName));
373 			if (ok) {
374 				rru.setStatus(Status.OK);
375 			} else {
376 				rru.setStatus(Status.REJECTED_OTHER_REASON);
377 				rru.setMessage(message);
378 			}
379 		}
380 		for (RemoteRefUpdate rru : refUpdates.values()) {
381 			if (rru.getStatus() == Status.AWAITING_REPORT)
382 				throw new PackProtocolException(MessageFormat.format(
383 						JGitText.get().expectedReportForRefNotReceived , uri, rru.getRemoteName()));
384 		}
385 	}
386 
387 	private String readStringLongTimeout() throws IOException {
388 		if (timeoutIn == null)
389 			return pckIn.readString();
390 
391 		// The remote side may need a lot of time to choke down the pack
392 		// we just sent them. There may be many deltas that need to be
393 		// resolved by the remote. Its hard to say how long the other
394 		// end is going to be silent. Taking 10x the configured timeout
395 		// or the time spent transferring the pack, whichever is larger,
396 		// gives the other side some reasonable window to process the data,
397 		// but this is just a wild guess.
398 		//
399 		final int oldTimeout = timeoutIn.getTimeout();
400 		final int sendTime = (int) Math.min(packTransferTime, 28800000L);
401 		try {
402 			int timeout = 10 * Math.max(sendTime, oldTimeout);
403 			timeoutIn.setTimeout((timeout < 0) ? Integer.MAX_VALUE : timeout);
404 			return pckIn.readString();
405 		} finally {
406 			timeoutIn.setTimeout(oldTimeout);
407 		}
408 	}
409 
410 	/**
411 	 * Gets the list of option strings associated with this push.
412 	 *
413 	 * @return pushOptions
414 	 * @since 4.5
415 	 */
416 	public List<String> getPushOptions() {
417 		return pushOptions;
418 	}
419 
420 	private static class CheckingSideBandOutputStream extends OutputStream {
421 		private final InputStream in;
422 		private final OutputStream out;
423 
424 		CheckingSideBandOutputStream(InputStream in, OutputStream out) {
425 			this.in = in;
426 			this.out = out;
427 		}
428 
429 		@Override
430 		public void write(int b) throws IOException {
431 			write(new byte[] { (byte) b });
432 		}
433 
434 		@Override
435 		public void write(byte[] buf, int ptr, int cnt) throws IOException {
436 			try {
437 				out.write(buf, ptr, cnt);
438 			} catch (IOException e) {
439 				throw checkError(e);
440 			}
441 		}
442 
443 		@Override
444 		public void flush() throws IOException {
445 			try {
446 				out.flush();
447 			} catch (IOException e) {
448 				throw checkError(e);
449 			}
450 		}
451 
452 		private IOException checkError(IOException e1) {
453 			try {
454 				in.read();
455 			} catch (TransportException e2) {
456 				return e2;
457 			} catch (IOException e2) {
458 				return e1;
459 			}
460 			return e1;
461 		}
462 	}
463 }