/*
    * Copyright (C) 2011, Google Inc. and others
    *
    * This program and the accompanying materials are made available under the
    * terms of the Eclipse Distribution License v. 1.0 which is available at
    * https://www.eclipse.org/org/documents/edl-v10.php.
    *
    * SPDX-License-Identifier: BSD-3-Clause
    */
  
  package org.eclipse.jgit.internal.storage.dfs;
  
  import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.COMPACT;
  import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC;
  import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_REST;
  import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC_TXN;
  import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.INSERT;
  import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.RECEIVE;
  import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.UNREACHABLE_GARBAGE;
  import static org.eclipse.jgit.internal.storage.dfs.DfsPackCompactor.configureReftable;
  import static org.eclipse.jgit.internal.storage.pack.PackExt.BITMAP_INDEX;
  import static org.eclipse.jgit.internal.storage.pack.PackExt.INDEX;
  import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK;
  import static org.eclipse.jgit.internal.storage.pack.PackExt.REFTABLE;
  import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE;
  
  import java.io.IOException;
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.Calendar;
  import java.util.Collection;
  import java.util.EnumSet;
  import java.util.GregorianCalendar;
  import java.util.HashSet;
  import java.util.List;
  import java.util.Set;
  import java.util.concurrent.TimeUnit;
  
  import org.eclipse.jgit.internal.JGitText;
  import org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource;
  import org.eclipse.jgit.internal.storage.file.PackIndex;
  import org.eclipse.jgit.internal.storage.file.PackReverseIndex;
  import org.eclipse.jgit.internal.storage.pack.PackExt;
  import org.eclipse.jgit.internal.storage.pack.PackWriter;
  import org.eclipse.jgit.internal.storage.reftable.ReftableCompactor;
  import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
  import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
  import org.eclipse.jgit.internal.storage.reftree.RefTreeNames;
  import org.eclipse.jgit.lib.AnyObjectId;
  import org.eclipse.jgit.lib.Constants;
  import org.eclipse.jgit.lib.NullProgressMonitor;
  import org.eclipse.jgit.lib.ObjectId;
  import org.eclipse.jgit.lib.ObjectIdSet;
  import org.eclipse.jgit.lib.ProgressMonitor;
  import org.eclipse.jgit.lib.Ref;
  import org.eclipse.jgit.lib.RefDatabase;
  import org.eclipse.jgit.revwalk.RevWalk;
  import org.eclipse.jgit.storage.pack.PackConfig;
  import org.eclipse.jgit.storage.pack.PackStatistics;
  import org.eclipse.jgit.util.SystemReader;
  import org.eclipse.jgit.util.io.CountingOutputStream;
  
  /**
   * Repack and garbage collect a repository.
   */
  public class DfsGarbageCollector {
  	private final DfsRepository repo;
  	private final RefDatabase refdb;
  	private final DfsObjDatabase objdb;
  
  	private final List<DfsPackDescription> newPackDesc;
  	private final List<PackStatistics> newPackStats;
  	private final List<ObjectIdSet> newPackObj;
  
  	private DfsReader ctx;
  
  	private PackConfig packConfig;
  	private ReftableConfig reftableConfig;
  	private boolean convertToReftable = true;
  	private boolean includeDeletes;
  	private long reftableInitialMinUpdateIndex = 1;
  	private long reftableInitialMaxUpdateIndex = 1;
  
  	// See packIsCoalesceableGarbage(), below, for how these two variables
  	// interact.
  	private long coalesceGarbageLimit = 50 << 20;
  	private long garbageTtlMillis = TimeUnit.DAYS.toMillis(1);
  
  	private long startTimeMillis;
  	private List<DfsPackFile> packsBefore;
  	private List<DfsReftable> reftablesBefore;
  	private List<DfsPackFile> expiredGarbagePacks;
  
  	private Collection<Ref> refsBefore;
  	private Set<ObjectId> allHeadsAndTags;
  	private Set<ObjectId> allTags;
  	private Set<ObjectId> nonHeads;
  	private Set<ObjectId> txnHeads;
  	private Set<ObjectId> tagTargets;
 
 	/**
 	 * Initialize a garbage collector.
 	 *
 	 * @param repository
 	 *            repository objects to be packed will be read from.
 	 */
 	public DfsGarbageCollector(DfsRepository repository) {
 		repo = repository;
 		refdb = repo.getRefDatabase();
 		objdb = repo.getObjectDatabase();
 		newPackDesc = new ArrayList<>(4);
 		newPackStats = new ArrayList<>(4);
 		newPackObj = new ArrayList<>(4);
 
 		packConfig = new PackConfig(repo);
 		packConfig.setIndexVersion(2);
 	}
 
 	/**
 	 * Get configuration used to generate the new pack file.
 	 *
 	 * @return configuration used to generate the new pack file.
 	 */
 	public PackConfig getPackConfig() {
 		return packConfig;
 	}
 
 	/**
 	 * Set the new configuration to use when creating the pack file.
 	 *
 	 * @param newConfig
 	 *            the new configuration to use when creating the pack file.
 	 * @return {@code this}
 	 */
 	public DfsGarbageCollector setPackConfig(PackConfig newConfig) {
 		packConfig = newConfig;
 		return this;
 	}
 
 	/**
 	 * Set configuration to write a reftable.
 	 *
 	 * @param cfg
 	 *            configuration to write a reftable. Reftable writing is
 	 *            disabled (default) when {@code cfg} is {@code null}.
 	 * @return {@code this}
 	 */
 	public DfsGarbageCollector setReftableConfig(ReftableConfig cfg) {
 		reftableConfig = cfg;
 		return this;
 	}
 
 	/**
 	 * Whether the garbage collector should convert references to reftable.
 	 *
 	 * @param convert
 	 *            if {@code true}, {@link #setReftableConfig(ReftableConfig)}
 	 *            has been set non-null, and a GC reftable doesn't yet exist,
 	 *            the garbage collector will make one by scanning the existing
 	 *            references, and writing a new reftable. Default is
 	 *            {@code true}.
 	 * @return {@code this}
 	 */
 	public DfsGarbageCollector setConvertToReftable(boolean convert) {
 		convertToReftable = convert;
 		return this;
 	}
 
 	/**
 	 * Whether the garbage collector will include tombstones for deleted
 	 * references in the reftable.
 	 *
 	 * @param include
 	 *            if {@code true}, the garbage collector will include tombstones
 	 *            for deleted references in the reftable. Default is
 	 *            {@code false}.
 	 * @return {@code this}
 	 */
 	public DfsGarbageCollector setIncludeDeletes(boolean include) {
 		includeDeletes = include;
 		return this;
 	}
 
 	/**
 	 * Set minUpdateIndex for the initial reftable created during conversion.
 	 *
 	 * @param u
 	 *            minUpdateIndex for the initial reftable created by scanning
 	 *            {@link org.eclipse.jgit.internal.storage.dfs.DfsRefDatabase#getRefs(String)}.
 	 *            Ignored unless caller has also set
 	 *            {@link #setReftableConfig(ReftableConfig)}. Defaults to
 	 *            {@code 1}. Must be {@code u >= 0}.
 	 * @return {@code this}
 	 */
 	public DfsGarbageCollector setReftableInitialMinUpdateIndex(long u) {
 		reftableInitialMinUpdateIndex = Math.max(u, 0);
 		return this;
 	}
 
 	/**
 	 * Set maxUpdateIndex for the initial reftable created during conversion.
 	 *
 	 * @param u
 	 *            maxUpdateIndex for the initial reftable created by scanning
 	 *            {@link org.eclipse.jgit.internal.storage.dfs.DfsRefDatabase#getRefs(String)}.
 	 *            Ignored unless caller has also set
 	 *            {@link #setReftableConfig(ReftableConfig)}. Defaults to
 	 *            {@code 1}. Must be {@code u >= 0}.
 	 * @return {@code this}
 	 */
 	public DfsGarbageCollector setReftableInitialMaxUpdateIndex(long u) {
 		reftableInitialMaxUpdateIndex = Math.max(0, u);
 		return this;
 	}
 
 	/**
 	 * Get coalesce garbage limit
 	 *
 	 * @return coalesce garbage limit, packs smaller than this size will be
 	 *         repacked.
 	 */
 	public long getCoalesceGarbageLimit() {
 		return coalesceGarbageLimit;
 	}
 
 	/**
 	 * Set the byte size limit for garbage packs to be repacked.
 	 * <p>
 	 * Any UNREACHABLE_GARBAGE pack smaller than this limit will be repacked at
 	 * the end of the run. This allows the garbage collector to coalesce
 	 * unreachable objects into a single file.
 	 * <p>
 	 * If an UNREACHABLE_GARBAGE pack is already larger than this limit it will
 	 * be left alone by the garbage collector. This avoids unnecessary disk IO
 	 * reading and copying the objects.
 	 * <p>
 	 * If limit is set to 0 the UNREACHABLE_GARBAGE coalesce is disabled.<br>
 	 * If limit is set to {@link java.lang.Long#MAX_VALUE}, everything is
 	 * coalesced.
 	 * <p>
 	 * Keeping unreachable garbage prevents race conditions with repository
 	 * changes that may suddenly need an object whose only copy was stored in
 	 * the UNREACHABLE_GARBAGE pack.
 	 *
 	 * @param limit
 	 *            size in bytes.
 	 * @return {@code this}
 	 */
 	public DfsGarbageCollector setCoalesceGarbageLimit(long limit) {
 		coalesceGarbageLimit = limit;
 		return this;
 	}
 
 	/**
 	 * Get time to live for garbage packs.
 	 *
 	 * @return garbage packs older than this limit (in milliseconds) will be
 	 *         pruned as part of the garbage collection process if the value is
 	 *         &gt; 0, otherwise garbage packs are retained.
 	 */
 	public long getGarbageTtlMillis() {
 		return garbageTtlMillis;
 	}
 
 	/**
 	 * Set the time to live for garbage objects.
 	 * <p>
 	 * Any UNREACHABLE_GARBAGE older than this limit will be pruned at the end
 	 * of the run.
 	 * <p>
 	 * If timeToLiveMillis is set to 0, UNREACHABLE_GARBAGE purging is disabled.
 	 *
 	 * @param ttl
 	 *            Time to live whatever unit is specified.
 	 * @param unit
 	 *            The specified time unit.
 	 * @return {@code this}
 	 */
 	public DfsGarbageCollector setGarbageTtl(long ttl, TimeUnit unit) {
 		garbageTtlMillis = unit.toMillis(ttl);
 		return this;
 	}
 
 	/**
 	 * Create a single new pack file containing all of the live objects.
 	 * <p>
 	 * This method safely decides which packs can be expired after the new pack
 	 * is created by validating the references have not been modified in an
 	 * incompatible way.
 	 *
 	 * @param pm
 	 *            progress monitor to receive updates on as packing may take a
 	 *            while, depending on the size of the repository.
 	 * @return true if the repack was successful without race conditions. False
 	 *         if a race condition was detected and the repack should be run
 	 *         again later.
 	 * @throws java.io.IOException
 	 *             a new pack cannot be created.
 	 */
 	public boolean pack(ProgressMonitor pm) throws IOException {
 		if (pm == null)
 			pm = NullProgressMonitor.INSTANCE;
 		if (packConfig.getIndexVersion() != 2)
 			throw new IllegalStateException(
 					JGitText.get().supportOnlyPackIndexVersion2);
 
 		startTimeMillis = SystemReader.getInstance().getCurrentTime();
 		ctx = objdb.newReader();
 		try {
 			refdb.refresh();
 			objdb.clearCache();
 
 			refsBefore = getAllRefs();
 			readPacksBefore();
 			readReftablesBefore();
 
 			Set<ObjectId> allHeads = new HashSet<>();
 			allHeadsAndTags = new HashSet<>();
 			allTags = new HashSet<>();
 			nonHeads = new HashSet<>();
 			txnHeads = new HashSet<>();
 			tagTargets = new HashSet<>();
 			for (Ref ref : refsBefore) {
 				if (ref.isSymbolic() || ref.getObjectId() == null) {
 					continue;
 				}
 				if (isHead(ref)) {
 					allHeads.add(ref.getObjectId());
 				} else if (isTag(ref)) {
 					allTags.add(ref.getObjectId());
 				} else if (RefTreeNames.isRefTree(refdb, ref.getName())) {
 					txnHeads.add(ref.getObjectId());
 				} else {
 					nonHeads.add(ref.getObjectId());
 				}
 				if (ref.getPeeledObjectId() != null) {
 					tagTargets.add(ref.getPeeledObjectId());
 				}
 			}
 			// Don't exclude tags that are also branch tips.
 			allTags.removeAll(allHeads);
 			allHeadsAndTags.addAll(allHeads);
 			allHeadsAndTags.addAll(allTags);
 
 			// Hoist all branch tips and tags earlier in the pack file
 			tagTargets.addAll(allHeadsAndTags);
 
 			// Combine the GC_REST objects into the GC pack if requested
 			if (packConfig.getSinglePack()) {
 				allHeadsAndTags.addAll(nonHeads);
 				nonHeads.clear();
 			}
 
 			boolean rollback = true;
 			try {
 				packHeads(pm);
 				packRest(pm);
 				packRefTreeGraph(pm);
 				packGarbage(pm);
 				objdb.commitPack(newPackDesc, toPrune());
 				rollback = false;
 				return true;
 			} finally {
 				if (rollback)
 					objdb.rollbackPack(newPackDesc);
 			}
 		} finally {
 			ctx.close();
 		}
 	}
 
 	private Collection<Ref> getAllRefs() throws IOException {
 		Collection<Ref> refs = refdb.getRefs();
 		List<Ref> addl = refdb.getAdditionalRefs();
 		if (!addl.isEmpty()) {
 			List<Ref> all = new ArrayList<>(refs.size() + addl.size());
 			all.addAll(refs);
 			// add additional refs which start with refs/
 			for (Ref r : addl) {
 				if (r.getName().startsWith(Constants.R_REFS)) {
 					all.add(r);
 				}
 			}
 			return all;
 		}
 		return refs;
 	}
 
 	private void readPacksBefore() throws IOException {
 		DfsPackFile[] packs = objdb.getPacks();
 		packsBefore = new ArrayList<>(packs.length);
 		expiredGarbagePacks = new ArrayList<>(packs.length);
 
 		long now = SystemReader.getInstance().getCurrentTime();
 		for (DfsPackFile p : packs) {
 			DfsPackDescription d = p.getPackDescription();
 			if (d.getPackSource() != UNREACHABLE_GARBAGE) {
 				packsBefore.add(p);
 			} else if (packIsExpiredGarbage(d, now)) {
 				expiredGarbagePacks.add(p);
 			} else if (packIsCoalesceableGarbage(d, now)) {
 				packsBefore.add(p);
 			}
 		}
 	}
 
 	private void readReftablesBefore() throws IOException {
 		DfsReftable[] tables = objdb.getReftables();
 		reftablesBefore = new ArrayList<>(Arrays.asList(tables));
 	}
 
 	private boolean packIsExpiredGarbage(DfsPackDescription d, long now) {
 		// Consider the garbage pack as expired when it's older than
 		// garbagePackTtl. This check gives concurrent inserter threads
 		// sufficient time to identify an object is not in the graph and should
 		// have a new copy written, rather than relying on something from an
 		// UNREACHABLE_GARBAGE pack.
 		return d.getPackSource() == UNREACHABLE_GARBAGE
 				&& garbageTtlMillis > 0
 				&& now - d.getLastModified() >= garbageTtlMillis;
 	}
 
 	private boolean packIsCoalesceableGarbage(DfsPackDescription d, long now) {
 		// An UNREACHABLE_GARBAGE pack can be coalesced if its size is less than
 		// the coalesceGarbageLimit and either garbageTtl is zero or if the pack
 		// is created in a close time interval (on a single calendar day when
 		// the garbageTtl is more than one day or one third of the garbageTtl).
 		//
 		// When the garbageTtl is more than 24 hours, garbage packs that are
 		// created within a single calendar day are coalesced together. This
 		// would make the effective ttl of the garbage pack as garbageTtl+23:59
 		// and limit the number of garbage to a maximum number of
 		// garbageTtl_in_days + 1 (assuming all of them are less than the size
 		// of coalesceGarbageLimit).
 		//
 		// When the garbageTtl is less than or equal to 24 hours, garbage packs
 		// that are created within a one third of garbageTtl are coalesced
 		// together. This would make the effective ttl of the garbage packs as
 		// garbageTtl + (garbageTtl / 3) and would limit the number of garbage
 		// packs to a maximum number of 4 (assuming all of them are less than
 		// the size of coalesceGarbageLimit).
 
 		if (d.getPackSource() != UNREACHABLE_GARBAGE
 				|| d.getFileSize(PackExt.PACK) >= coalesceGarbageLimit) {
 			return false;
 		}
 
 		if (garbageTtlMillis == 0) {
 			return true;
 		}
 
 		long lastModified = d.getLastModified();
 		long dayStartLastModified = dayStartInMillis(lastModified);
 		long dayStartToday = dayStartInMillis(now);
 
 		if (dayStartLastModified != dayStartToday) {
 			return false; // this pack is not created today.
 		}
 
 		if (garbageTtlMillis > TimeUnit.DAYS.toMillis(1)) {
 			return true; // ttl is more than one day and pack is created today.
 		}
 
 		long timeInterval = garbageTtlMillis / 3;
 		if (timeInterval == 0) {
 			return false; // ttl is too small, don't try to coalesce.
 		}
 
 		long modifiedTimeSlot = (lastModified - dayStartLastModified) / timeInterval;
 		long presentTimeSlot = (now - dayStartToday) / timeInterval;
 		return modifiedTimeSlot == presentTimeSlot;
 	}
 
 	private static long dayStartInMillis(long timeInMillis) {
 		Calendar cal = new GregorianCalendar(
 				SystemReader.getInstance().getTimeZone());
 		cal.setTimeInMillis(timeInMillis);
 		cal.set(Calendar.HOUR_OF_DAY, 0);
 		cal.set(Calendar.MINUTE, 0);
 		cal.set(Calendar.SECOND, 0);
 		cal.set(Calendar.MILLISECOND, 0);
 		return cal.getTimeInMillis();
 	}
 
 	/**
 	 * Get all of the source packs that fed into this compaction.
 	 *
 	 * @return all of the source packs that fed into this compaction.
 	 */
 	public Set<DfsPackDescription> getSourcePacks() {
 		return toPrune();
 	}
 
 	/**
 	 * Get new packs created by this compaction.
 	 *
 	 * @return new packs created by this compaction.
 	 */
 	public List<DfsPackDescription> getNewPacks() {
 		return newPackDesc;
 	}
 
 	/**
 	 * Get statistics corresponding to the {@link #getNewPacks()}.
 	 * <p>
 	 * The elements can be null if the stat is not available for the pack file.
 	 *
 	 * @return statistics corresponding to the {@link #getNewPacks()}.
 	 */
 	public List<PackStatistics> getNewPackStatistics() {
 		return newPackStats;
 	}
 
 	private Set<DfsPackDescription> toPrune() {
 		Set<DfsPackDescription> toPrune = new HashSet<>();
 		for (DfsPackFile pack : packsBefore) {
 			toPrune.add(pack.getPackDescription());
 		}
 		if (reftableConfig != null) {
 			for (DfsReftable table : reftablesBefore) {
 				toPrune.add(table.getPackDescription());
 			}
 		}
 		for (DfsPackFile pack : expiredGarbagePacks) {
 			toPrune.add(pack.getPackDescription());
 		}
 		return toPrune;
 	}
 
 	private void packHeads(ProgressMonitor pm) throws IOException {
 		if (allHeadsAndTags.isEmpty()) {
 			writeReftable();
 			return;
 		}
 
 		try (PackWriter pw = newPackWriter()) {
 			pw.setTagTargets(tagTargets);
 			pw.preparePack(pm, allHeadsAndTags, NONE, NONE, allTags);
 			if (0 < pw.getObjectCount()) {
 				long estSize = estimateGcPackSize(INSERT, RECEIVE, COMPACT, GC);
 				writePack(GC, pw, pm, estSize);
 			} else {
 				writeReftable();
 			}
 		}
 	}
 
 	private void packRest(ProgressMonitor pm) throws IOException {
 		if (nonHeads.isEmpty())
 			return;
 
 		try (PackWriter pw = newPackWriter()) {
 			for (ObjectIdSet packedObjs : newPackObj)
 				pw.excludeObjects(packedObjs);
 			pw.preparePack(pm, nonHeads, allHeadsAndTags);
 			if (0 < pw.getObjectCount())
 				writePack(GC_REST, pw, pm,
 						estimateGcPackSize(INSERT, RECEIVE, COMPACT, GC_REST));
 		}
 	}
 
 	private void packRefTreeGraph(ProgressMonitor pm) throws IOException {
 		if (txnHeads.isEmpty())
 			return;
 
 		try (PackWriter pw = newPackWriter()) {
 			for (ObjectIdSet packedObjs : newPackObj)
 				pw.excludeObjects(packedObjs);
 			pw.preparePack(pm, txnHeads, NONE);
 			if (0 < pw.getObjectCount())
 				writePack(GC_TXN, pw, pm, 0 /* unknown pack size */);
 		}
 	}
 
 	private void packGarbage(ProgressMonitor pm) throws IOException {
 		PackConfig cfg = new PackConfig(packConfig);
 		cfg.setReuseDeltas(true);
 		cfg.setReuseObjects(true);
 		cfg.setDeltaCompress(false);
 		cfg.setBuildBitmaps(false);
 
 		try (PackWriternal/storage/pack/PackWriter.html#PackWriter">PackWriter pw = new PackWriter(cfg, ctx);
 				RevWalk pool = new RevWalk(ctx)) {
 			pw.setDeltaBaseAsOffset(true);
 			pw.setReuseDeltaCommits(true);
 			pm.beginTask(JGitText.get().findingGarbage, objectsBefore());
 			long estimatedPackSize = 12 + 20; // header and trailer sizes.
 			for (DfsPackFile oldPack : packsBefore) {
 				PackIndex oldIdx = oldPack.getPackIndex(ctx);
 				PackReverseIndex oldRevIdx = oldPack.getReverseIdx(ctx);
 				long maxOffset = oldPack.getPackDescription().getFileSize(PACK)
 						- 20; // pack size - trailer size.
 				for (PackIndex.MutableEntry ent : oldIdx) {
 					pm.update(1);
 					ObjectId id = ent.toObjectId();
 					if (pool.lookupOrNull(id) != null || anyPackHas(id))
 						continue;
 
 					long offset = ent.getOffset();
 					int type = oldPack.getObjectType(ctx, offset);
 					pw.addObject(pool.lookupAny(id, type));
 					long objSize = oldRevIdx.findNextOffset(offset, maxOffset)
 							- offset;
 					estimatedPackSize += objSize;
 				}
 			}
 			pm.endTask();
 			if (0 < pw.getObjectCount())
 				writePack(UNREACHABLE_GARBAGE, pw, pm, estimatedPackSize);
 		}
 	}
 
 	private boolean anyPackHas(AnyObjectId id) {
 		for (ObjectIdSet packedObjs : newPackObj)
 			if (packedObjs.contains(id))
 				return true;
 		return false;
 	}
 
 	private static boolean isHead(Ref ref) {
 		return ref.getName().startsWith(Constants.R_HEADS);
 	}
 
 	private static boolean isTag(Ref ref) {
 		return ref.getName().startsWith(Constants.R_TAGS);
 	}
 
 	private int objectsBefore() {
 		int cnt = 0;
 		for (DfsPackFile p : packsBefore)
 			cnt += (int) p.getPackDescription().getObjectCount();
 		return cnt;
 	}
 
 	private PackWriter newPackWriter() {
 		PackWriter pw = new PackWriter(packConfig, ctx);
 		pw.setDeltaBaseAsOffset(true);
 		pw.setReuseDeltaCommits(false);
 		return pw;
 	}
 
 	private long estimateGcPackSize(PackSource first, PackSource... rest) {
 		EnumSet<PackSource> sourceSet = EnumSet.of(first, rest);
 		// Every pack file contains 12 bytes of header and 20 bytes of trailer.
 		// Include the final pack file header and trailer size here and ignore
 		// the same from individual pack files.
 		long size = 32;
 		for (DfsPackDescription pack : getSourcePacks()) {
 			if (sourceSet.contains(pack.getPackSource())) {
 				size += pack.getFileSize(PACK) - 32;
 			}
 		}
 		return size;
 	}
 
 	private DfsPackDescription writePack(PackSource source, PackWriter pw,
 			ProgressMonitor pm, long estimatedPackSize) throws IOException {
 		DfsPackDescription pack = repo.getObjectDatabase().newPack(source,
 				estimatedPackSize);
 
 		if (source == GC && reftableConfig != null) {
 			writeReftable(pack);
 		}
 
 		try (DfsOutputStream out = objdb.writeFile(pack, PACK)) {
 			pw.writePack(pm, pm, out);
 			pack.addFileExt(PACK);
 			pack.setBlockSize(PACK, out.blockSize());
 		}
 
 		try (DfsOutputStream out = objdb.writeFile(pack, INDEX)) {
 			CountingOutputStream cnt = new CountingOutputStream(out);
 			pw.writeIndex(cnt);
 			pack.addFileExt(INDEX);
 			pack.setFileSize(INDEX, cnt.getCount());
 			pack.setBlockSize(INDEX, out.blockSize());
 			pack.setIndexVersion(pw.getIndexVersion());
 		}
 
 		if (pw.prepareBitmapIndex(pm)) {
 			try (DfsOutputStream out = objdb.writeFile(pack, BITMAP_INDEX)) {
 				CountingOutputStream cnt = new CountingOutputStream(out);
 				pw.writeBitmapIndex(cnt);
 				pack.addFileExt(BITMAP_INDEX);
 				pack.setFileSize(BITMAP_INDEX, cnt.getCount());
 				pack.setBlockSize(BITMAP_INDEX, out.blockSize());
 			}
 		}
 
 		PackStatistics stats = pw.getStatistics();
 		pack.setPackStats(stats);
 		pack.setLastModified(startTimeMillis);
 		newPackDesc.add(pack);
 		newPackStats.add(stats);
 		newPackObj.add(pw.getObjectSet());
 		return pack;
 	}
 
 	private void writeReftable() throws IOException {
 		if (reftableConfig != null) {
 			DfsPackDescription pack = objdb.newPack(GC);
 			newPackDesc.add(pack);
 			newPackStats.add(null);
 			writeReftable(pack);
 		}
 	}
 
 	private void writeReftable(DfsPackDescription pack) throws IOException {
 		if (convertToReftable && !hasGcReftable()) {
 			writeReftable(pack, refsBefore);
 			return;
 		}
 
 		try (DfsReftableStack stack = DfsReftableStack.open(ctx, reftablesBefore);
 		     DfsOutputStream out = objdb.writeFile(pack, REFTABLE)) {
 			ReftableCompactor compact = new ReftableCompactor(out);
 			compact.addAll(stack.readers());
 			compact.setIncludeDeletes(includeDeletes);
 			compact.setConfig(configureReftable(reftableConfig, out));
 			compact.compact();
 			pack.addFileExt(REFTABLE);
 			pack.setReftableStats(compact.getStats());
 		}
 	}
 
 	private boolean hasGcReftable() {
 		for (DfsReftable table : reftablesBefore) {
 			if (table.getPackDescription().getPackSource() == GC) {
 				return true;
 			}
 		}
 		return false;
 	}
 
 	private void writeReftable(DfsPackDescription pack, Collection<Ref> refs)
 			throws IOException {
 		try (DfsOutputStream out = objdb.writeFile(pack, REFTABLE)) {
 			ReftableConfig cfg = configureReftable(reftableConfig, out);
 			ReftableWriter writer = new ReftableWriter(cfg, out)
 					.setMinUpdateIndex(reftableInitialMinUpdateIndex)
 					.setMaxUpdateIndex(reftableInitialMaxUpdateIndex).begin()
 					.sortAndWriteRefs(refs).finish();
 			pack.addFileExt(REFTABLE);
 			pack.setReftableStats(writer.getStats());
 		}
 	}
 }