1
2
3
4
5
6
7
8
9
10 package org.eclipse.jgit.gitrepo;
11
12 import static java.nio.charset.StandardCharsets.UTF_8;
13 import static org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME;
14 import static org.eclipse.jgit.lib.Constants.R_REMOTES;
15 import static org.eclipse.jgit.lib.Constants.R_TAGS;
16
17 import java.io.File;
18 import java.io.FileInputStream;
19 import java.io.IOException;
20 import java.io.InputStream;
21 import java.net.URI;
22 import java.text.MessageFormat;
23 import java.util.ArrayList;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.Objects;
27 import java.util.StringJoiner;
28 import java.util.TreeMap;
29
30 import org.eclipse.jgit.annotations.NonNull;
31 import org.eclipse.jgit.annotations.Nullable;
32 import org.eclipse.jgit.api.Git;
33 import org.eclipse.jgit.api.GitCommand;
34 import org.eclipse.jgit.api.SubmoduleAddCommand;
35 import org.eclipse.jgit.api.errors.ConcurrentRefUpdateException;
36 import org.eclipse.jgit.api.errors.GitAPIException;
37 import org.eclipse.jgit.api.errors.InvalidRefNameException;
38 import org.eclipse.jgit.api.errors.JGitInternalException;
39 import org.eclipse.jgit.dircache.DirCache;
40 import org.eclipse.jgit.dircache.DirCacheBuilder;
41 import org.eclipse.jgit.dircache.DirCacheEntry;
42 import org.eclipse.jgit.gitrepo.ManifestParser.IncludedFileReader;
43 import org.eclipse.jgit.gitrepo.RepoProject.CopyFile;
44 import org.eclipse.jgit.gitrepo.RepoProject.LinkFile;
45 import org.eclipse.jgit.gitrepo.internal.RepoText;
46 import org.eclipse.jgit.internal.JGitText;
47 import org.eclipse.jgit.lib.CommitBuilder;
48 import org.eclipse.jgit.lib.Config;
49 import org.eclipse.jgit.lib.Constants;
50 import org.eclipse.jgit.lib.FileMode;
51 import org.eclipse.jgit.lib.ObjectId;
52 import org.eclipse.jgit.lib.ObjectInserter;
53 import org.eclipse.jgit.lib.PersonIdent;
54 import org.eclipse.jgit.lib.ProgressMonitor;
55 import org.eclipse.jgit.lib.Ref;
56 import org.eclipse.jgit.lib.RefDatabase;
57 import org.eclipse.jgit.lib.RefUpdate;
58 import org.eclipse.jgit.lib.RefUpdate.Result;
59 import org.eclipse.jgit.lib.Repository;
60 import org.eclipse.jgit.revwalk.RevCommit;
61 import org.eclipse.jgit.revwalk.RevWalk;
62 import org.eclipse.jgit.treewalk.TreeWalk;
63 import org.eclipse.jgit.util.FileUtils;
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82 public class RepoCommand extends GitCommand<RevCommit> {
83 private static final int LOCK_FAILURE_MAX_RETRIES = 5;
84
85
86 private static final int LOCK_FAILURE_MIN_RETRY_DELAY_MILLIS = 50;
87
88 private static final int LOCK_FAILURE_MAX_RETRY_DELAY_MILLIS = 5000;
89
90 private String manifestPath;
91 private String baseUri;
92 private URI targetUri;
93 private String groupsParam;
94 private String branch;
95 private String targetBranch = Constants.HEAD;
96 private boolean recordRemoteBranch = true;
97 private boolean recordSubmoduleLabels = true;
98 private boolean recordShallowSubmodules = true;
99 private PersonIdent author;
100 private RemoteReader callback;
101 private InputStream inputStream;
102 private IncludedFileReader includedReader;
103 private boolean ignoreRemoteFailures = false;
104
105 private ProgressMonitor monitor;
106
107
108
109
110
111
112
113
114
115
116
117 public interface RemoteReader {
118
119
120
121
122
123
124
125
126
127
128
129
130
131 @Nullable
132 public ObjectId sha1(String uri, String ref) throws GitAPIException;
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151 @Deprecated
152 public default byte[] readFile(String uri, String ref, String path)
153 throws GitAPIException, IOException {
154 return readFileWithMode(uri, ref, path).getContents();
155 }
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178 @NonNull
179 public RemoteFile readFileWithMode(String uri, String ref, String path)
180 throws GitAPIException, IOException;
181 }
182
183
184
185
186
187
188
189 public static final class RemoteFile {
190 @NonNull
191 private final byte[] contents;
192
193 @NonNull
194 private final FileMode fileMode;
195
196
197
198
199
200
201
202 public RemoteFile(@NonNull byte[] contents,
203 @NonNull FileMode fileMode) {
204 this.contents = Objects.requireNonNull(contents);
205 this.fileMode = Objects.requireNonNull(fileMode);
206 }
207
208
209
210
211
212
213
214
215
216 @NonNull
217 public byte[] getContents() {
218 return contents;
219 }
220
221
222
223
224 @NonNull
225 public FileMode getFileMode() {
226 return fileMode;
227 }
228
229 }
230
231
232 public static class DefaultRemoteReader implements RemoteReader {
233
234 @Override
235 public ObjectId sha1(String uri, String ref) throws GitAPIException {
236 Map<String, Ref> map = Git
237 .lsRemoteRepository()
238 .setRemote(uri)
239 .callAsMap();
240 Ref r = RefDatabase.findRef(map, ref);
241 return r != null ? r.getObjectId() : null;
242 }
243
244 @Override
245 public RemoteFile readFileWithMode(String uri, String ref, String path)
246 throws GitAPIException, IOException {
247 File dir = FileUtils.createTempDir("jgit_", ".git", null);
248 try (Git git = Git.cloneRepository().setBare(true).setDirectory(dir)
249 .setURI(uri).call()) {
250 Repository repo = git.getRepository();
251 ObjectId refCommitId = sha1(uri, ref);
252 if (refCommitId == null) {
253 throw new InvalidRefNameException(MessageFormat
254 .format(JGitText.get().refNotResolved, ref));
255 }
256 RevCommit commit = repo.parseCommit(refCommitId);
257 TreeWalk tw = TreeWalk.forPath(repo, path, commit.getTree());
258
259
260
261 return new RemoteFile(
262 tw.getObjectReader().open(tw.getObjectId(0))
263 .getCachedBytes(Integer.MAX_VALUE),
264 tw.getFileMode(0));
265 } finally {
266 FileUtils.delete(dir, FileUtils.RECURSIVE);
267 }
268 }
269 }
270
271 @SuppressWarnings("serial")
272 private static class ManifestErrorException extends GitAPIException {
273 ManifestErrorException(Throwable cause) {
274 super(RepoText.get().invalidManifest, cause);
275 }
276 }
277
278 @SuppressWarnings("serial")
279 private static class RemoteUnavailableException extends GitAPIException {
280 RemoteUnavailableException(String uri) {
281 super(MessageFormat.format(RepoText.get().errorRemoteUnavailable, uri));
282 }
283 }
284
285
286
287
288
289
290
291 public RepoCommand(Repository repo) {
292 super(repo);
293 }
294
295
296
297
298
299
300
301
302
303
304 public RepoCommand setPath(String path) {
305 this.manifestPath = path;
306 return this;
307 }
308
309
310
311
312
313
314
315
316
317
318
319 public RepoCommand setInputStream(InputStream inputStream) {
320 this.inputStream = inputStream;
321 return this;
322 }
323
324
325
326
327
328
329
330
331
332
333
334
335 public RepoCommand setURI(String uri) {
336 this.baseUri = uri;
337 return this;
338 }
339
340
341
342
343
344
345
346
347
348
349 public RepoCommand setTargetURI(String uri) {
350
351
352
353
354 this.targetUri = URI.create(uri + "/");
355 return this;
356 }
357
358
359
360
361
362
363
364 public RepoCommand setGroups(String groups) {
365 this.groupsParam = groups;
366 return this;
367 }
368
369
370
371
372
373
374
375
376
377
378
379
380 public RepoCommand setBranch(String branch) {
381 this.branch = branch;
382 return this;
383 }
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399 public RepoCommand setTargetBranch(String branch) {
400 this.targetBranch = Constants.R_HEADS + branch;
401 return this;
402 }
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423 public RepoCommand setRecordRemoteBranch(boolean enable) {
424 this.recordRemoteBranch = enable;
425 return this;
426 }
427
428
429
430
431
432
433
434
435
436
437
438 public RepoCommand setRecordSubmoduleLabels(boolean enable) {
439 this.recordSubmoduleLabels = enable;
440 return this;
441 }
442
443
444
445
446
447
448
449
450
451
452
453 public RepoCommand setRecommendShallow(boolean enable) {
454 this.recordShallowSubmodules = enable;
455 return this;
456 }
457
458
459
460
461
462
463
464
465
466
467 public RepoCommand setProgressMonitor(ProgressMonitor monitor) {
468 this.monitor = monitor;
469 return this;
470 }
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487 public RepoCommand setIgnoreRemoteFailures(boolean ignore) {
488 this.ignoreRemoteFailures = ignore;
489 return this;
490 }
491
492
493
494
495
496
497
498
499
500
501
502 public RepoCommand setAuthor(PersonIdent author) {
503 this.author = author;
504 return this;
505 }
506
507
508
509
510
511
512
513
514
515
516
517 public RepoCommand setRemoteReader(RemoteReader callback) {
518 this.callback = callback;
519 return this;
520 }
521
522
523
524
525
526
527
528
529
530
531
532 public RepoCommand setIncludedFileReader(IncludedFileReader reader) {
533 this.includedReader = reader;
534 return this;
535 }
536
537
538 @Override
539 public RevCommit call() throws GitAPIException {
540 checkCallable();
541 if (baseUri == null) {
542 baseUri = "";
543 }
544 if (inputStream == null) {
545 if (manifestPath == null || manifestPath.length() == 0)
546 throw new IllegalArgumentException(
547 JGitText.get().pathNotConfigured);
548 try {
549 inputStream = new FileInputStream(manifestPath);
550 } catch (IOException e) {
551 throw new IllegalArgumentException(
552 JGitText.get().pathNotConfigured, e);
553 }
554 }
555
556 List<RepoProject> filteredProjects;
557 try {
558 ManifestParser parser = new ManifestParser(includedReader,
559 manifestPath, branch, baseUri, groupsParam, repo);
560 parser.read(inputStream);
561 filteredProjects = parser.getFilteredProjects();
562 } catch (IOException e) {
563 throw new ManifestErrorException(e);
564 } finally {
565 try {
566 inputStream.close();
567 } catch (IOException e) {
568
569 }
570 }
571
572 if (repo.isBare()) {
573 if (author == null)
574 author = new PersonIdent(repo);
575 if (callback == null)
576 callback = new DefaultRemoteReader();
577 List<RepoProject> renamedProjects = renameProjects(filteredProjects);
578
579 DirCache index = DirCache.newInCore();
580 ObjectInserter inserter = repo.newObjectInserter();
581
582 try (RevWalk rw = new RevWalk(repo)) {
583 prepareIndex(renamedProjects, index, inserter);
584 ObjectId treeId = index.writeTree(inserter);
585 long prevDelay = 0;
586 for (int i = 0; i < LOCK_FAILURE_MAX_RETRIES - 1; i++) {
587 try {
588 return commitTreeOnCurrentTip(
589 inserter, rw, treeId);
590 } catch (ConcurrentRefUpdateException e) {
591 prevDelay = FileUtils.delay(prevDelay,
592 LOCK_FAILURE_MIN_RETRY_DELAY_MILLIS,
593 LOCK_FAILURE_MAX_RETRY_DELAY_MILLIS);
594 Thread.sleep(prevDelay);
595 repo.getRefDatabase().refresh();
596 }
597 }
598
599 return commitTreeOnCurrentTip(inserter, rw, treeId);
600 } catch (IOException | InterruptedException e) {
601 throw new ManifestErrorException(e);
602 }
603 }
604 try (Git git = new Git(repo)) {
605 for (RepoProject proj : filteredProjects) {
606 addSubmodule(proj.getName(), proj.getUrl(), proj.getPath(),
607 proj.getRevision(), proj.getCopyFiles(),
608 proj.getLinkFiles(), git);
609 }
610 return git.commit().setMessage(RepoText.get().repoCommitMessage)
611 .call();
612 } catch (IOException e) {
613 throw new ManifestErrorException(e);
614 }
615 }
616
617 private void prepareIndex(List<RepoProject> projects, DirCache index,
618 ObjectInserter inserter) throws IOException, GitAPIException {
619 Config cfg = new Config();
620 StringBuilder attributes = new StringBuilder();
621 DirCacheBuilder builder = index.builder();
622 for (RepoProject proj : projects) {
623 String name = proj.getName();
624 String path = proj.getPath();
625 String url = proj.getUrl();
626 ObjectId objectId;
627 if (ObjectId.isId(proj.getRevision())) {
628 objectId = ObjectId.fromString(proj.getRevision());
629 } else {
630 objectId = callback.sha1(url, proj.getRevision());
631 if (objectId == null && !ignoreRemoteFailures) {
632 throw new RemoteUnavailableException(url);
633 }
634 if (recordRemoteBranch) {
635
636
637 String field = proj.getRevision().startsWith(R_TAGS) ? "ref"
638 : "branch";
639 cfg.setString("submodule", name, field,
640 proj.getRevision());
641 }
642
643 if (recordShallowSubmodules
644 && proj.getRecommendShallow() != null) {
645
646
647
648
649
650 cfg.setBoolean("submodule", name, "shallow",
651 true);
652 }
653 }
654 if (recordSubmoduleLabels) {
655 StringBuilder rec = new StringBuilder();
656 rec.append("/");
657 rec.append(path);
658 for (String group : proj.getGroups()) {
659 rec.append(" ");
660 rec.append(group);
661 }
662 rec.append("\n");
663 attributes.append(rec.toString());
664 }
665
666 URI submodUrl = URI.create(url);
667 if (targetUri != null) {
668 submodUrl = relativize(targetUri, submodUrl);
669 }
670 cfg.setString("submodule", name, "path", path);
671 cfg.setString("submodule", name, "url",
672 submodUrl.toString());
673
674
675 if (objectId != null) {
676 DirCacheEntry dcEntry = new DirCacheEntry(path);
677 dcEntry.setObjectId(objectId);
678 dcEntry.setFileMode(FileMode.GITLINK);
679 builder.add(dcEntry);
680
681 for (CopyFile copyfile : proj.getCopyFiles()) {
682 RemoteFile rf = callback.readFileWithMode(url,
683 proj.getRevision(), copyfile.src);
684 objectId = inserter.insert(Constants.OBJ_BLOB,
685 rf.getContents());
686 dcEntry = new DirCacheEntry(copyfile.dest);
687 dcEntry.setObjectId(objectId);
688 dcEntry.setFileMode(rf.getFileMode());
689 builder.add(dcEntry);
690 }
691 for (LinkFile linkfile : proj.getLinkFiles()) {
692 String link;
693 if (linkfile.dest.contains("/")) {
694 link = FileUtils.relativizeGitPath(
695 linkfile.dest.substring(0,
696 linkfile.dest.lastIndexOf('/')),
697 proj.getPath() + "/" + linkfile.src);
698 } else {
699 link = proj.getPath() + "/" + linkfile.src;
700 }
701
702 objectId = inserter.insert(Constants.OBJ_BLOB,
703 link.getBytes(UTF_8));
704 dcEntry = new DirCacheEntry(linkfile.dest);
705 dcEntry.setObjectId(objectId);
706 dcEntry.setFileMode(FileMode.SYMLINK);
707 builder.add(dcEntry);
708 }
709 }
710 }
711 String content = cfg.toText();
712
713
714 DirCacheEntry dcEntry = new DirCacheEntry(
715 Constants.DOT_GIT_MODULES);
716 ObjectId objectId = inserter.insert(Constants.OBJ_BLOB,
717 content.getBytes(UTF_8));
718 dcEntry.setObjectId(objectId);
719 dcEntry.setFileMode(FileMode.REGULAR_FILE);
720 builder.add(dcEntry);
721
722 if (recordSubmoduleLabels) {
723
724 DirCacheEntry dcEntryAttr = new DirCacheEntry(
725 Constants.DOT_GIT_ATTRIBUTES);
726 ObjectId attrId = inserter.insert(Constants.OBJ_BLOB,
727 attributes.toString().getBytes(UTF_8));
728 dcEntryAttr.setObjectId(attrId);
729 dcEntryAttr.setFileMode(FileMode.REGULAR_FILE);
730 builder.add(dcEntryAttr);
731 }
732
733 builder.finish();
734 }
735
736 private RevCommit commitTreeOnCurrentTip(ObjectInserter inserter,
737 RevWalk rw, ObjectId treeId)
738 throws IOException, ConcurrentRefUpdateException {
739 ObjectId headId = repo.resolve(targetBranch + "^{commit}");
740 if (headId != null && rw.parseCommit(headId).getTree().getId().equals(treeId)) {
741
742 return rw.parseCommit(headId);
743 }
744
745 CommitBuilder commit = new CommitBuilder();
746 commit.setTreeId(treeId);
747 if (headId != null)
748 commit.setParentIds(headId);
749 commit.setAuthor(author);
750 commit.setCommitter(author);
751 commit.setMessage(RepoText.get().repoCommitMessage);
752
753 ObjectId commitId = inserter.insert(commit);
754 inserter.flush();
755
756 RefUpdate ru = repo.updateRef(targetBranch);
757 ru.setNewObjectId(commitId);
758 ru.setExpectedOldObjectId(headId != null ? headId : ObjectId.zeroId());
759 Result rc = ru.update(rw);
760 switch (rc) {
761 case NEW:
762 case FORCED:
763 case FAST_FORWARD:
764
765 break;
766 case REJECTED:
767 case LOCK_FAILURE:
768 throw new ConcurrentRefUpdateException(MessageFormat
769 .format(JGitText.get().cannotLock, targetBranch),
770 ru.getRef(), rc);
771 default:
772 throw new JGitInternalException(MessageFormat.format(
773 JGitText.get().updatingRefFailed,
774 targetBranch, commitId.name(), rc));
775 }
776
777 return rw.parseCommit(commitId);
778 }
779
780 private void addSubmodule(String name, String url, String path,
781 String revision, List<CopyFile> copyfiles, List<LinkFile> linkfiles,
782 Git git) throws GitAPIException, IOException {
783 assert (!repo.isBare());
784 assert (git != null);
785 if (!linkfiles.isEmpty()) {
786 throw new UnsupportedOperationException(
787 JGitText.get().nonBareLinkFilesNotSupported);
788 }
789
790 SubmoduleAddCommand add = git.submoduleAdd().setName(name).setPath(path)
791 .setURI(url);
792 if (monitor != null)
793 add.setProgressMonitor(monitor);
794
795 Repository subRepo = add.call();
796 if (revision != null) {
797 try (Git sub = new Git(subRepo)) {
798 sub.checkout().setName(findRef(revision, subRepo)).call();
799 }
800 subRepo.close();
801 git.add().addFilepattern(path).call();
802 }
803 for (CopyFile copyfile : copyfiles) {
804 copyfile.copy();
805 git.add().addFilepattern(copyfile.dest).call();
806 }
807 }
808
809
810
811
812
813
814
815
816 private List<RepoProject> renameProjects(List<RepoProject> projects) {
817 Map<String, List<RepoProject>> m = new TreeMap<>();
818 for (RepoProject proj : projects) {
819 List<RepoProject> l = m.get(proj.getName());
820 if (l == null) {
821 l = new ArrayList<>();
822 m.put(proj.getName(), l);
823 }
824 l.add(proj);
825 }
826
827 List<RepoProject> ret = new ArrayList<>();
828 for (List<RepoProject> ps : m.values()) {
829 boolean nameConflict = ps.size() != 1;
830 for (RepoProject proj : ps) {
831 String name = proj.getName();
832 if (nameConflict) {
833 name += SLASH + proj.getPath();
834 }
835 RepoProject p = new RepoProject(name,
836 proj.getPath(), proj.getRevision(), null,
837 proj.getGroups(), proj.getRecommendShallow());
838 p.setUrl(proj.getUrl());
839 p.addCopyFiles(proj.getCopyFiles());
840 p.addLinkFiles(proj.getLinkFiles());
841 ret.add(p);
842 }
843 }
844 return ret;
845 }
846
847
848
849
850
851
852 private static final String SLASH = "/";
853 static URI relativize(URI current, URI target) {
854 if (!Objects.equals(current.getHost(), target.getHost())) {
855 return target;
856 }
857
858 String cur = current.normalize().getPath();
859 String dest = target.normalize().getPath();
860
861
862 if (cur.startsWith(SLASH) != dest.startsWith(SLASH)) {
863 return target;
864 }
865
866 while (cur.startsWith(SLASH)) {
867 cur = cur.substring(1);
868 }
869 while (dest.startsWith(SLASH)) {
870 dest = dest.substring(1);
871 }
872
873 if (cur.indexOf('/') == -1 || dest.indexOf('/') == -1) {
874
875 String prefix = "prefix/";
876 cur = prefix + cur;
877 dest = prefix + dest;
878 }
879
880 if (!cur.endsWith(SLASH)) {
881
882 int lastSlash = cur.lastIndexOf('/');
883 cur = cur.substring(0, lastSlash);
884 }
885 String destFile = "";
886 if (!dest.endsWith(SLASH)) {
887
888 int lastSlash = dest.lastIndexOf('/');
889 destFile = dest.substring(lastSlash + 1, dest.length());
890 dest = dest.substring(0, dest.lastIndexOf('/'));
891 }
892
893 String[] cs = cur.split(SLASH);
894 String[] ds = dest.split(SLASH);
895
896 int common = 0;
897 while (common < cs.length && common < ds.length && cs[common].equals(ds[common])) {
898 common++;
899 }
900
901 StringJoiner j = new StringJoiner(SLASH);
902 for (int i = common; i < cs.length; i++) {
903 j.add("..");
904 }
905 for (int i = common; i < ds.length; i++) {
906 j.add(ds[i]);
907 }
908
909 j.add(destFile);
910 return URI.create(j.toString());
911 }
912
913 private static String findRef(String ref, Repository repo)
914 throws IOException {
915 if (!ObjectId.isId(ref)) {
916 Ref r = repo.exactRef(R_REMOTES + DEFAULT_REMOTE_NAME + "/" + ref);
917 if (r != null)
918 return r.getName();
919 }
920 return ref;
921 }
922 }