1 /* 2 * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> 3 * and other copyright owners as documented in the project's IP log. 4 * 5 * This program and the accompanying materials are made available 6 * under the terms of the Eclipse Distribution License v1.0 which 7 * accompanies this distribution, is reproduced below, and is 8 * available at http://www.eclipse.org/org/documents/edl-v10.php 9 * 10 * All rights reserved. 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials provided 22 * with the distribution. 23 * 24 * - Neither the name of the Eclipse Foundation, Inc. nor the 25 * names of its contributors may be used to endorse or promote 26 * products derived from this software without specific prior 27 * written permission. 28 * 29 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND 30 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, 31 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 32 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 33 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 34 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 35 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 36 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 37 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 38 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 39 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 40 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 41 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 42 */ 43 44 package org.eclipse.jgit.transport; 45 46 import java.io.BufferedReader; 47 import java.io.ByteArrayOutputStream; 48 import java.io.FileNotFoundException; 49 import java.io.IOException; 50 import java.io.InputStream; 51 import java.io.InputStreamReader; 52 import java.io.OutputStream; 53 import java.text.MessageFormat; 54 import java.util.ArrayList; 55 import java.util.Collection; 56 import java.util.Map; 57 58 import org.eclipse.jgit.errors.TransportException; 59 import org.eclipse.jgit.internal.JGitText; 60 import org.eclipse.jgit.internal.storage.file.RefDirectory; 61 import org.eclipse.jgit.lib.Constants; 62 import org.eclipse.jgit.lib.ObjectId; 63 import org.eclipse.jgit.lib.ObjectIdRef; 64 import org.eclipse.jgit.lib.ProgressMonitor; 65 import org.eclipse.jgit.lib.Ref; 66 import org.eclipse.jgit.util.IO; 67 68 /** 69 * Transfers object data through a dumb transport. 70 * <p> 71 * Implementations are responsible for resolving path names relative to the 72 * <code>objects/</code> subdirectory of a single remote Git repository or 73 * naked object database and make the content available as a Java input stream 74 * for reading during fetch. The actual object traversal logic to determine the 75 * names of files to retrieve is handled through the generic, protocol 76 * independent {@link WalkFetchConnection}. 77 */ 78 abstract class WalkRemoteObjectDatabase { 79 static final String ROOT_DIR = "../"; //$NON-NLS-1$ 80 81 static final String INFO_PACKS = "info/packs"; //$NON-NLS-1$ 82 83 static final String INFO_ALTERNATES = "info/alternates"; //$NON-NLS-1$ 84 85 static final String INFO_HTTP_ALTERNATES = "info/http-alternates"; //$NON-NLS-1$ 86 87 static final String INFO_REFS = ROOT_DIR + Constants.INFO_REFS; 88 89 abstract URIish getURI(); 90 91 /** 92 * Obtain the list of available packs (if any). 93 * <p> 94 * Pack names should be the file name in the packs directory, that is 95 * <code>pack-035760ab452d6eebd123add421f253ce7682355a.pack</code>. Index 96 * names should not be included in the returned collection. 97 * 98 * @return list of pack names; null or empty list if none are available. 99 * @throws IOException 100 * The connection is unable to read the remote repository's list 101 * of available pack files. 102 */ 103 abstract Collection<String> getPackNames() throws IOException; 104 105 /** 106 * Obtain alternate connections to alternate object databases (if any). 107 * <p> 108 * Alternates are typically read from the file {@link #INFO_ALTERNATES} or 109 * {@link #INFO_HTTP_ALTERNATES}. The content of each line must be resolved 110 * by the implementation and a new database reference should be returned to 111 * represent the additional location. 112 * <p> 113 * Alternates may reuse the same network connection handle, however the 114 * fetch connection will {@link #close()} each created alternate. 115 * 116 * @return list of additional object databases the caller could fetch from; 117 * null or empty list if none are configured. 118 * @throws IOException 119 * The connection is unable to read the remote repository's list 120 * of configured alternates. 121 */ 122 abstract Collection<WalkRemoteObjectDatabase> getAlternates() 123 throws IOException; 124 125 /** 126 * Open a single file for reading. 127 * <p> 128 * Implementors should make every attempt possible to ensure 129 * {@link FileNotFoundException} is used when the remote object does not 130 * exist. However when fetching over HTTP some misconfigured servers may 131 * generate a 200 OK status message (rather than a 404 Not Found) with an 132 * HTML formatted message explaining the requested resource does not exist. 133 * Callers such as {@link WalkFetchConnection} are prepared to handle this 134 * by validating the content received, and assuming content that fails to 135 * match its hash is an incorrectly phrased FileNotFoundException. 136 * <p> 137 * This method is recommended for already compressed files like loose objects 138 * and pack files. For text files, see {@link #openReader(String)}. 139 * 140 * @param path 141 * location of the file to read, relative to this objects 142 * directory (e.g. 143 * <code>cb/95df6ab7ae9e57571511ef451cf33767c26dd2</code> or 144 * <code>pack/pack-035760ab452d6eebd123add421f253ce7682355a.pack</code>). 145 * @return a stream to read from the file. Never null. 146 * @throws FileNotFoundException 147 * the requested file does not exist at the given location. 148 * @throws IOException 149 * The connection is unable to read the remote's file, and the 150 * failure occurred prior to being able to determine if the file 151 * exists, or after it was determined to exist but before the 152 * stream could be created. 153 */ 154 abstract FileStream open(String path) throws FileNotFoundException, 155 IOException; 156 157 /** 158 * Create a new connection for a discovered alternate object database 159 * <p> 160 * This method is typically called by {@link #readAlternates(String)} when 161 * subclasses us the generic alternate parsing logic for their 162 * implementation of {@link #getAlternates()}. 163 * 164 * @param location 165 * the location of the new alternate, relative to the current 166 * object database. 167 * @return a new database connection that can read from the specified 168 * alternate. 169 * @throws IOException 170 * The database connection cannot be established with the 171 * alternate, such as if the alternate location does not 172 * actually exist and the connection's constructor attempts to 173 * verify that. 174 */ 175 abstract WalkRemoteObjectDatabase openAlternate(String location) 176 throws IOException; 177 178 /** 179 * Close any resources used by this connection. 180 * <p> 181 * If the remote repository is contacted by a network socket this method 182 * must close that network socket, disconnecting the two peers. If the 183 * remote repository is actually local (same system) this method must close 184 * any open file handles used to read the "remote" repository. 185 */ 186 abstract void close(); 187 188 /** 189 * Delete a file from the object database. 190 * <p> 191 * Path may start with <code>../</code> to request deletion of a file that 192 * resides in the repository itself. 193 * <p> 194 * When possible empty directories must be removed, up to but not including 195 * the current object database directory itself. 196 * <p> 197 * This method does not support deletion of directories. 198 * 199 * @param path 200 * name of the item to be removed, relative to the current object 201 * database. 202 * @throws IOException 203 * deletion is not supported, or deletion failed. 204 */ 205 void deleteFile(String path) throws IOException { 206 throw new IOException(MessageFormat.format(JGitText.get().deletingNotSupported, path)); 207 } 208 209 /** 210 * Open a remote file for writing. 211 * <p> 212 * Path may start with <code>../</code> to request writing of a file that 213 * resides in the repository itself. 214 * <p> 215 * The requested path may or may not exist. If the path already exists as a 216 * file the file should be truncated and completely replaced. 217 * <p> 218 * This method creates any missing parent directories, if necessary. 219 * 220 * @param path 221 * name of the file to write, relative to the current object 222 * database. 223 * @return stream to write into this file. Caller must close the stream to 224 * complete the write request. The stream is not buffered and each 225 * write may cause a network request/response so callers should 226 * buffer to smooth out small writes. 227 * @param monitor 228 * (optional) progress monitor to post write completion to during 229 * the stream's close method. 230 * @param monitorTask 231 * (optional) task name to display during the close method. 232 * @throws IOException 233 * writing is not supported, or attempting to write the file 234 * failed, possibly due to permissions or remote disk full, etc. 235 */ 236 OutputStream writeFile(final String path, final ProgressMonitor monitor, 237 final String monitorTask) throws IOException { 238 throw new IOException(MessageFormat.format(JGitText.get().writingNotSupported, path)); 239 } 240 241 /** 242 * Atomically write a remote file. 243 * <p> 244 * This method attempts to perform as atomic of an update as it can, 245 * reducing (or eliminating) the time that clients might be able to see 246 * partial file content. This method is not suitable for very large 247 * transfers as the complete content must be passed as an argument. 248 * <p> 249 * Path may start with <code>../</code> to request writing of a file that 250 * resides in the repository itself. 251 * <p> 252 * The requested path may or may not exist. If the path already exists as a 253 * file the file should be truncated and completely replaced. 254 * <p> 255 * This method creates any missing parent directories, if necessary. 256 * 257 * @param path 258 * name of the file to write, relative to the current object 259 * database. 260 * @param data 261 * complete new content of the file. 262 * @throws IOException 263 * writing is not supported, or attempting to write the file 264 * failed, possibly due to permissions or remote disk full, etc. 265 */ 266 void writeFile(String path, byte[] data) throws IOException { 267 try (OutputStream os = writeFile(path, null, null)) { 268 os.write(data); 269 } 270 } 271 272 /** 273 * Delete a loose ref from the remote repository. 274 * 275 * @param name 276 * name of the ref within the ref space, for example 277 * <code>refs/heads/pu</code>. 278 * @throws IOException 279 * deletion is not supported, or deletion failed. 280 */ 281 void deleteRef(String name) throws IOException { 282 deleteFile(ROOT_DIR + name); 283 } 284 285 /** 286 * Delete a reflog from the remote repository. 287 * 288 * @param name 289 * name of the ref within the ref space, for example 290 * <code>refs/heads/pu</code>. 291 * @throws IOException 292 * deletion is not supported, or deletion failed. 293 */ 294 void deleteRefLog(String name) throws IOException { 295 deleteFile(ROOT_DIR + Constants.LOGS + "/" + name); //$NON-NLS-1$ 296 } 297 298 /** 299 * Overwrite (or create) a loose ref in the remote repository. 300 * <p> 301 * This method creates any missing parent directories, if necessary. 302 * 303 * @param name 304 * name of the ref within the ref space, for example 305 * <code>refs/heads/pu</code>. 306 * @param value 307 * new value to store in this ref. Must not be null. 308 * @throws IOException 309 * writing is not supported, or attempting to write the file 310 * failed, possibly due to permissions or remote disk full, etc. 311 */ 312 void writeRef(String name, ObjectId value) throws IOException { 313 final ByteArrayOutputStream b; 314 315 b = new ByteArrayOutputStream(Constants.OBJECT_ID_STRING_LENGTH + 1); 316 value.copyTo(b); 317 b.write('\n'); 318 319 writeFile(ROOT_DIR + name, b.toByteArray()); 320 } 321 322 /** 323 * Rebuild the {@link #INFO_PACKS} for dumb transport clients. 324 * <p> 325 * This method rebuilds the contents of the {@link #INFO_PACKS} file to 326 * match the passed list of pack names. 327 * 328 * @param packNames 329 * names of available pack files, in the order they should appear 330 * in the file. Valid pack name strings are of the form 331 * <code>pack-035760ab452d6eebd123add421f253ce7682355a.pack</code>. 332 * @throws IOException 333 * writing is not supported, or attempting to write the file 334 * failed, possibly due to permissions or remote disk full, etc. 335 */ 336 void writeInfoPacks(Collection<String> packNames) throws IOException { 337 final StringBuilder w = new StringBuilder(); 338 for (String n : packNames) { 339 w.append("P "); //$NON-NLS-1$ 340 w.append(n); 341 w.append('\n'); 342 } 343 writeFile(INFO_PACKS, Constants.encodeASCII(w.toString())); 344 } 345 346 /** 347 * Open a buffered reader around a file. 348 * <p> 349 * This method is suitable for for reading line-oriented resources like 350 * <code>info/packs</code>, <code>info/refs</code>, and the alternates list. 351 * 352 * @return a stream to read from the file. Never null. 353 * @param path 354 * location of the file to read, relative to this objects 355 * directory (e.g. <code>info/packs</code>). 356 * @throws FileNotFoundException 357 * the requested file does not exist at the given location. 358 * @throws IOException 359 * The connection is unable to read the remote's file, and the 360 * failure occurred prior to being able to determine if the file 361 * exists, or after it was determined to exist but before the 362 * stream could be created. 363 */ 364 BufferedReader openReader(String path) throws IOException { 365 final InputStream is = open(path).in; 366 return new BufferedReader(new InputStreamReader(is, Constants.CHARSET)); 367 } 368 369 /** 370 * Read a standard Git alternates file to discover other object databases. 371 * <p> 372 * This method is suitable for reading the standard formats of the 373 * alternates file, such as found in <code>objects/info/alternates</code> 374 * or <code>objects/info/http-alternates</code> within a Git repository. 375 * <p> 376 * Alternates appear one per line, with paths expressed relative to this 377 * object database. 378 * 379 * @param listPath 380 * location of the alternate file to read, relative to this 381 * object database (e.g. <code>info/alternates</code>). 382 * @return the list of discovered alternates. Empty list if the file exists, 383 * but no entries were discovered. 384 * @throws FileNotFoundException 385 * the requested file does not exist at the given location. 386 * @throws IOException 387 * The connection is unable to read the remote's file, and the 388 * failure occurred prior to being able to determine if the file 389 * exists, or after it was determined to exist but before the 390 * stream could be created. 391 */ 392 Collection<WalkRemoteObjectDatabase> readAlternates(final String listPath) 393 throws IOException { 394 try (BufferedReader br = openReader(listPath)) { 395 final Collection<WalkRemoteObjectDatabase> alts = new ArrayList<>(); 396 for (;;) { 397 String line = br.readLine(); 398 if (line == null) 399 break; 400 if (!line.endsWith("/")) //$NON-NLS-1$ 401 line += "/"; //$NON-NLS-1$ 402 alts.add(openAlternate(line)); 403 } 404 return alts; 405 } 406 } 407 408 /** 409 * Read a standard Git packed-refs file to discover known references. 410 * 411 * @param avail 412 * return collection of references. Any existing entries will be 413 * replaced if they are found in the packed-refs file. 414 * @throws org.eclipse.jgit.errors.TransportException 415 * an error occurred reading from the packed refs file. 416 */ 417 protected void readPackedRefs(Map<String, Ref> avail) 418 throws TransportException { 419 try (BufferedReader br = openReader(ROOT_DIR + Constants.PACKED_REFS)) { 420 readPackedRefsImpl(avail, br); 421 } catch (FileNotFoundException notPacked) { 422 // Perhaps it wasn't worthwhile, or is just an older repository. 423 } catch (IOException e) { 424 throw new TransportException(getURI(), JGitText.get().errorInPackedRefs, e); 425 } 426 } 427 428 private void readPackedRefsImpl(final Map<String, Ref> avail, 429 final BufferedReader br) throws IOException { 430 Ref last = null; 431 boolean peeled = false; 432 for (;;) { 433 String line = br.readLine(); 434 if (line == null) 435 break; 436 if (line.charAt(0) == '#') { 437 if (line.startsWith(RefDirectory.PACKED_REFS_HEADER)) { 438 line = line.substring(RefDirectory.PACKED_REFS_HEADER.length()); 439 peeled = line.contains(RefDirectory.PACKED_REFS_PEELED); 440 } 441 continue; 442 } 443 if (line.charAt(0) == '^') { 444 if (last == null) 445 throw new TransportException(JGitText.get().peeledLineBeforeRef); 446 final ObjectId id = ObjectId.fromString(line.substring(1)); 447 last = new ObjectIdRef.PeeledTag(Ref.Storage.PACKED, last 448 .getName(), last.getObjectId(), id); 449 avail.put(last.getName(), last); 450 continue; 451 } 452 453 final int sp = line.indexOf(' '); 454 if (sp < 0) 455 throw new TransportException(MessageFormat.format(JGitText.get().unrecognizedRef, line)); 456 final ObjectId id = ObjectId.fromString(line.substring(0, sp)); 457 final String name = line.substring(sp + 1); 458 if (peeled) 459 last = new ObjectIdRef.PeeledNonTag(Ref.Storage.PACKED, name, id); 460 else 461 last = new ObjectIdRef.Unpeeled(Ref.Storage.PACKED, name, id); 462 avail.put(last.getName(), last); 463 } 464 } 465 466 static final class FileStream { 467 final InputStream in; 468 469 final long length; 470 471 /** 472 * Create a new stream of unknown length. 473 * 474 * @param i 475 * stream containing the file data. This stream will be 476 * closed by the caller when reading is complete. 477 */ 478 FileStream(InputStream i) { 479 in = i; 480 length = -1; 481 } 482 483 /** 484 * Create a new stream of known length. 485 * 486 * @param i 487 * stream containing the file data. This stream will be 488 * closed by the caller when reading is complete. 489 * @param n 490 * total number of bytes available for reading through 491 * <code>i</code>. 492 */ 493 FileStream(InputStream i, long n) { 494 in = i; 495 length = n; 496 } 497 498 byte[] toArray() throws IOException { 499 try { 500 if (length >= 0) { 501 final byte[] r = new byte[(int) length]; 502 IO.readFully(in, r, 0, r.length); 503 return r; 504 } 505 506 final ByteArrayOutputStream r = new ByteArrayOutputStream(); 507 final byte[] buf = new byte[2048]; 508 int n; 509 while ((n = in.read(buf)) >= 0) 510 r.write(buf, 0, n); 511 return r.toByteArray(); 512 } finally { 513 in.close(); 514 } 515 } 516 } 517 }