1
2
3
4
5
6
7
8
9
10 package org.eclipse.jgit.gitrepo;
11
12 import static java.nio.charset.StandardCharsets.UTF_8;
13 import static org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME;
14 import static org.eclipse.jgit.lib.Constants.R_REMOTES;
15 import static org.eclipse.jgit.lib.Constants.R_TAGS;
16
17 import java.io.File;
18 import java.io.FileInputStream;
19 import java.io.IOException;
20 import java.io.InputStream;
21 import java.net.URI;
22 import java.text.MessageFormat;
23 import java.util.ArrayList;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.Objects;
27 import java.util.StringJoiner;
28 import java.util.TreeMap;
29
30 import org.eclipse.jgit.annotations.NonNull;
31 import org.eclipse.jgit.annotations.Nullable;
32 import org.eclipse.jgit.api.Git;
33 import org.eclipse.jgit.api.GitCommand;
34 import org.eclipse.jgit.api.SubmoduleAddCommand;
35 import org.eclipse.jgit.api.errors.ConcurrentRefUpdateException;
36 import org.eclipse.jgit.api.errors.GitAPIException;
37 import org.eclipse.jgit.api.errors.InvalidRefNameException;
38 import org.eclipse.jgit.api.errors.JGitInternalException;
39 import org.eclipse.jgit.dircache.DirCache;
40 import org.eclipse.jgit.dircache.DirCacheBuilder;
41 import org.eclipse.jgit.dircache.DirCacheEntry;
42 import org.eclipse.jgit.gitrepo.ManifestParser.IncludedFileReader;
43 import org.eclipse.jgit.gitrepo.RepoProject.CopyFile;
44 import org.eclipse.jgit.gitrepo.RepoProject.LinkFile;
45 import org.eclipse.jgit.gitrepo.internal.RepoText;
46 import org.eclipse.jgit.internal.JGitText;
47 import org.eclipse.jgit.lib.CommitBuilder;
48 import org.eclipse.jgit.lib.Config;
49 import org.eclipse.jgit.lib.Constants;
50 import org.eclipse.jgit.lib.FileMode;
51 import org.eclipse.jgit.lib.ObjectId;
52 import org.eclipse.jgit.lib.ObjectInserter;
53 import org.eclipse.jgit.lib.PersonIdent;
54 import org.eclipse.jgit.lib.ProgressMonitor;
55 import org.eclipse.jgit.lib.Ref;
56 import org.eclipse.jgit.lib.RefDatabase;
57 import org.eclipse.jgit.lib.RefUpdate;
58 import org.eclipse.jgit.lib.RefUpdate.Result;
59 import org.eclipse.jgit.lib.Repository;
60 import org.eclipse.jgit.revwalk.RevCommit;
61 import org.eclipse.jgit.revwalk.RevWalk;
62 import org.eclipse.jgit.treewalk.TreeWalk;
63 import org.eclipse.jgit.util.FileUtils;
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82 public class RepoCommand extends GitCommand<RevCommit> {
83 private static final int LOCK_FAILURE_MAX_RETRIES = 5;
84
85
86 private static final int LOCK_FAILURE_MIN_RETRY_DELAY_MILLIS = 50;
87
88 private static final int LOCK_FAILURE_MAX_RETRY_DELAY_MILLIS = 5000;
89
90 private String manifestPath;
91 private String baseUri;
92 private URI targetUri;
93 private String groupsParam;
94 private String branch;
95 private String targetBranch = Constants.HEAD;
96 private boolean recordRemoteBranch = true;
97 private boolean recordSubmoduleLabels = true;
98 private boolean recordShallowSubmodules = true;
99 private PersonIdent author;
100 private RemoteReader callback;
101 private InputStream inputStream;
102 private IncludedFileReader includedReader;
103 private boolean ignoreRemoteFailures = false;
104
105 private ProgressMonitor monitor;
106
107
108
109
110
111
112
113
114
115
116
117 public interface RemoteReader {
118
119
120
121
122
123
124
125
126
127
128
129
130
131 @Nullable
132 public ObjectId sha1(String uri, String ref) throws GitAPIException;
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151 @Deprecated
152 public default byte[] readFile(String uri, String ref, String path)
153 throws GitAPIException, IOException {
154 return readFileWithMode(uri, ref, path).getContents();
155 }
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178 @NonNull
179 public RemoteFile readFileWithMode(String uri, String ref, String path)
180 throws GitAPIException, IOException;
181 }
182
183
184
185
186
187
188
189 public static final class RemoteFile {
190 @NonNull
191 private final byte[] contents;
192
193 @NonNull
194 private final FileMode fileMode;
195
196
197
198
199
200
201
202 public RemoteFile(@NonNull byte[] contents,
203 @NonNull FileMode fileMode) {
204 this.contents = Objects.requireNonNull(contents);
205 this.fileMode = Objects.requireNonNull(fileMode);
206 }
207
208
209
210
211
212
213
214
215
216 @NonNull
217 public byte[] getContents() {
218 return contents;
219 }
220
221
222
223
224 @NonNull
225 public FileMode getFileMode() {
226 return fileMode;
227 }
228
229 }
230
231
232 public static class DefaultRemoteReader implements RemoteReader {
233
234 @Override
235 public ObjectId sha1(String uri, String ref) throws GitAPIException {
236 Map<String, Ref> map = Git
237 .lsRemoteRepository()
238 .setRemote(uri)
239 .callAsMap();
240 Ref r = RefDatabase.findRef(map, ref);
241 return r != null ? r.getObjectId() : null;
242 }
243
244 @Override
245 public RemoteFile readFileWithMode(String uri, String ref, String path)
246 throws GitAPIException, IOException {
247 File dir = FileUtils.createTempDir("jgit_", ".git", null);
248 try (Git git = Git.cloneRepository().setBare(true).setDirectory(dir)
249 .setURI(uri).call()) {
250 Repository repo = git.getRepository();
251 ObjectId refCommitId = sha1(uri, ref);
252 if (refCommitId == null) {
253 throw new InvalidRefNameException(MessageFormat
254 .format(JGitText.get().refNotResolved, ref));
255 }
256 RevCommit commit = repo.parseCommit(refCommitId);
257 TreeWalk tw = TreeWalk.forPath(repo, path, commit.getTree());
258
259
260
261 return new RemoteFile(
262 tw.getObjectReader().open(tw.getObjectId(0))
263 .getCachedBytes(Integer.MAX_VALUE),
264 tw.getFileMode(0));
265 } finally {
266 FileUtils.delete(dir, FileUtils.RECURSIVE);
267 }
268 }
269 }
270
271 @SuppressWarnings("serial")
272 private static class ManifestErrorException extends GitAPIException {
273 ManifestErrorException(Throwable cause) {
274 super(RepoText.get().invalidManifest, cause);
275 }
276 }
277
278 @SuppressWarnings("serial")
279 private static class RemoteUnavailableException extends GitAPIException {
280 RemoteUnavailableException(String uri) {
281 super(MessageFormat.format(RepoText.get().errorRemoteUnavailable, uri));
282 }
283 }
284
285
286
287
288
289
290
291 public RepoCommand(Repository repo) {
292 super(repo);
293 }
294
295
296
297
298
299
300
301
302
303
304 public RepoCommand setPath(String path) {
305 this.manifestPath = path;
306 return this;
307 }
308
309
310
311
312
313
314
315
316
317
318
319 public RepoCommand setInputStream(InputStream inputStream) {
320 this.inputStream = inputStream;
321 return this;
322 }
323
324
325
326
327
328
329
330
331
332
333
334
335 public RepoCommand setURI(String uri) {
336 this.baseUri = uri;
337 return this;
338 }
339
340
341
342
343
344
345
346
347
348
349 public RepoCommand setTargetURI(String uri) {
350
351
352
353
354 this.targetUri = URI.create(uri + "/");
355 return this;
356 }
357
358
359
360
361
362
363
364 public RepoCommand setGroups(String groups) {
365 this.groupsParam = groups;
366 return this;
367 }
368
369
370
371
372
373
374
375
376
377
378
379
380 public RepoCommand setBranch(String branch) {
381 this.branch = branch;
382 return this;
383 }
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399 public RepoCommand setTargetBranch(String branch) {
400 this.targetBranch = Constants.R_HEADS + branch;
401 return this;
402 }
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423 public RepoCommand setRecordRemoteBranch(boolean enable) {
424 this.recordRemoteBranch = enable;
425 return this;
426 }
427
428
429
430
431
432
433
434
435
436
437
438 public RepoCommand setRecordSubmoduleLabels(boolean enable) {
439 this.recordSubmoduleLabels = enable;
440 return this;
441 }
442
443
444
445
446
447
448
449
450
451
452
453 public RepoCommand setRecommendShallow(boolean enable) {
454 this.recordShallowSubmodules = enable;
455 return this;
456 }
457
458
459
460
461
462
463
464
465
466
467 public RepoCommand setProgressMonitor(ProgressMonitor monitor) {
468 this.monitor = monitor;
469 return this;
470 }
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487 public RepoCommand setIgnoreRemoteFailures(boolean ignore) {
488 this.ignoreRemoteFailures = ignore;
489 return this;
490 }
491
492
493
494
495
496
497
498
499
500
501
502 public RepoCommand setAuthor(PersonIdent author) {
503 this.author = author;
504 return this;
505 }
506
507
508
509
510
511
512
513
514
515
516
517 public RepoCommand setRemoteReader(RemoteReader callback) {
518 this.callback = callback;
519 return this;
520 }
521
522
523
524
525
526
527
528
529
530
531
532 public RepoCommand setIncludedFileReader(IncludedFileReader reader) {
533 this.includedReader = reader;
534 return this;
535 }
536
537
538 @Override
539 public RevCommit call() throws GitAPIException {
540 checkCallable();
541 if (baseUri == null) {
542 baseUri = "";
543 }
544 if (inputStream == null) {
545 if (manifestPath == null || manifestPath.length() == 0)
546 throw new IllegalArgumentException(
547 JGitText.get().pathNotConfigured);
548 try {
549 inputStream = new FileInputStream(manifestPath);
550 } catch (IOException e) {
551 throw new IllegalArgumentException(
552 JGitText.get().pathNotConfigured, e);
553 }
554 }
555
556 List<RepoProject> filteredProjects;
557 try {
558 ManifestParser parser = new ManifestParser(includedReader,
559 manifestPath, branch, baseUri, groupsParam, repo);
560 parser.read(inputStream);
561 filteredProjects = parser.getFilteredProjects();
562 } catch (IOException e) {
563 throw new ManifestErrorException(e);
564 } finally {
565 try {
566 inputStream.close();
567 } catch (IOException e) {
568
569 }
570 }
571
572 if (repo.isBare()) {
573 if (author == null)
574 author = new PersonIdent(repo);
575 if (callback == null)
576 callback = new DefaultRemoteReader();
577 List<RepoProject> renamedProjects = renameProjects(filteredProjects);
578
579 DirCache index = DirCache.newInCore();
580 DirCacheBuilder builder = index.builder();
581 ObjectInserter inserter = repo.newObjectInserter();
582 try (RevWalkRevWalk.html#RevWalk">RevWalk rw = new RevWalk(repo)) {
583 Config cfg = new Config();
584 StringBuilder attributes = new StringBuilder();
585 for (RepoProject proj : renamedProjects) {
586 String name = proj.getName();
587 String path = proj.getPath();
588 String url = proj.getUrl();
589 ObjectId objectId;
590 if (ObjectId.isId(proj.getRevision())) {
591 objectId = ObjectId.fromString(proj.getRevision());
592 } else {
593 objectId = callback.sha1(url, proj.getRevision());
594 if (objectId == null && !ignoreRemoteFailures) {
595 throw new RemoteUnavailableException(url);
596 }
597 if (recordRemoteBranch) {
598
599
600 String field = proj.getRevision().startsWith(
601 R_TAGS) ? "ref" : "branch";
602 cfg.setString("submodule", name, field,
603 proj.getRevision());
604 }
605
606 if (recordShallowSubmodules && proj.getRecommendShallow() != null) {
607
608
609
610
611
612 cfg.setBoolean("submodule", name, "shallow",
613 true);
614 }
615 }
616 if (recordSubmoduleLabels) {
617 StringBuilder rec = new StringBuilder();
618 rec.append("/");
619 rec.append(path);
620 for (String group : proj.getGroups()) {
621 rec.append(" ");
622 rec.append(group);
623 }
624 rec.append("\n");
625 attributes.append(rec.toString());
626 }
627
628 URI submodUrl = URI.create(url);
629 if (targetUri != null) {
630 submodUrl = relativize(targetUri, submodUrl);
631 }
632 cfg.setString("submodule", name, "path", path);
633 cfg.setString("submodule", name, "url",
634 submodUrl.toString());
635
636
637 if (objectId != null) {
638 DirCacheEntry dcEntry = new DirCacheEntry(path);
639 dcEntry.setObjectId(objectId);
640 dcEntry.setFileMode(FileMode.GITLINK);
641 builder.add(dcEntry);
642
643 for (CopyFile copyfile : proj.getCopyFiles()) {
644 RemoteFile rf = callback.readFileWithMode(
645 url, proj.getRevision(), copyfile.src);
646 objectId = inserter.insert(Constants.OBJ_BLOB,
647 rf.getContents());
648 dcEntry = new DirCacheEntry(copyfile.dest);
649 dcEntry.setObjectId(objectId);
650 dcEntry.setFileMode(rf.getFileMode());
651 builder.add(dcEntry);
652 }
653 for (LinkFile linkfile : proj.getLinkFiles()) {
654 String link;
655 if (linkfile.dest.contains("/")) {
656 link = FileUtils.relativizeGitPath(
657 linkfile.dest.substring(0,
658 linkfile.dest.lastIndexOf('/')),
659 proj.getPath() + "/" + linkfile.src);
660 } else {
661 link = proj.getPath() + "/" + linkfile.src;
662 }
663
664 objectId = inserter.insert(Constants.OBJ_BLOB,
665 link.getBytes(UTF_8));
666 dcEntry = new DirCacheEntry(linkfile.dest);
667 dcEntry.setObjectId(objectId);
668 dcEntry.setFileMode(FileMode.SYMLINK);
669 builder.add(dcEntry);
670 }
671 }
672 }
673 String content = cfg.toText();
674
675
676 final DirCacheEntrytry.html#DirCacheEntry">DirCacheEntry dcEntry = new DirCacheEntry(Constants.DOT_GIT_MODULES);
677 ObjectId objectId = inserter.insert(Constants.OBJ_BLOB,
678 content.getBytes(UTF_8));
679 dcEntry.setObjectId(objectId);
680 dcEntry.setFileMode(FileMode.REGULAR_FILE);
681 builder.add(dcEntry);
682
683 if (recordSubmoduleLabels) {
684
685 final DirCacheEntryhtml#DirCacheEntry">DirCacheEntry dcEntryAttr = new DirCacheEntry(Constants.DOT_GIT_ATTRIBUTES);
686 ObjectId attrId = inserter.insert(Constants.OBJ_BLOB,
687 attributes.toString().getBytes(UTF_8));
688 dcEntryAttr.setObjectId(attrId);
689 dcEntryAttr.setFileMode(FileMode.REGULAR_FILE);
690 builder.add(dcEntryAttr);
691 }
692
693 builder.finish();
694 ObjectId treeId = index.writeTree(inserter);
695
696 long prevDelay = 0;
697 for (int i = 0; i < LOCK_FAILURE_MAX_RETRIES - 1; i++) {
698 try {
699 return commitTreeOnCurrentTip(
700 inserter, rw, treeId);
701 } catch (ConcurrentRefUpdateException e) {
702 prevDelay = FileUtils.delay(prevDelay,
703 LOCK_FAILURE_MIN_RETRY_DELAY_MILLIS,
704 LOCK_FAILURE_MAX_RETRY_DELAY_MILLIS);
705 Thread.sleep(prevDelay);
706 repo.getRefDatabase().refresh();
707 }
708 }
709
710 return commitTreeOnCurrentTip(inserter, rw, treeId);
711 } catch (GitAPIException | IOException | InterruptedException e) {
712 throw new ManifestErrorException(e);
713 }
714 }
715 try (Gitit.html#Git">Git git = new Git(repo)) {
716 for (RepoProject proj : filteredProjects) {
717 addSubmodule(proj.getName(), proj.getUrl(), proj.getPath(),
718 proj.getRevision(), proj.getCopyFiles(),
719 proj.getLinkFiles(), git);
720 }
721 return git.commit().setMessage(RepoText.get().repoCommitMessage)
722 .call();
723 } catch (GitAPIException | IOException e) {
724 throw new ManifestErrorException(e);
725 }
726 }
727
728
729 private RevCommit commitTreeOnCurrentTip(ObjectInserter inserter,
730 RevWalk rw, ObjectId treeId)
731 throws IOException, ConcurrentRefUpdateException {
732 ObjectId headId = repo.resolve(targetBranch + "^{commit}");
733 if (headId != null && rw.parseCommit(headId).getTree().getId().equals(treeId)) {
734
735 return rw.parseCommit(headId);
736 }
737
738 CommitBuilder commit = new CommitBuilder();
739 commit.setTreeId(treeId);
740 if (headId != null)
741 commit.setParentIds(headId);
742 commit.setAuthor(author);
743 commit.setCommitter(author);
744 commit.setMessage(RepoText.get().repoCommitMessage);
745
746 ObjectId commitId = inserter.insert(commit);
747 inserter.flush();
748
749 RefUpdate ru = repo.updateRef(targetBranch);
750 ru.setNewObjectId(commitId);
751 ru.setExpectedOldObjectId(headId != null ? headId : ObjectId.zeroId());
752 Result rc = ru.update(rw);
753 switch (rc) {
754 case NEW:
755 case FORCED:
756 case FAST_FORWARD:
757
758 break;
759 case REJECTED:
760 case LOCK_FAILURE:
761 throw new ConcurrentRefUpdateException(MessageFormat
762 .format(JGitText.get().cannotLock, targetBranch),
763 ru.getRef(), rc);
764 default:
765 throw new JGitInternalException(MessageFormat.format(
766 JGitText.get().updatingRefFailed,
767 targetBranch, commitId.name(), rc));
768 }
769
770 return rw.parseCommit(commitId);
771 }
772
773 private void addSubmodule(String name, String url, String path,
774 String revision, List<CopyFile> copyfiles, List<LinkFile> linkfiles,
775 Git git) throws GitAPIException, IOException {
776 assert (!repo.isBare());
777 assert (git != null);
778 if (!linkfiles.isEmpty()) {
779 throw new UnsupportedOperationException(
780 JGitText.get().nonBareLinkFilesNotSupported);
781 }
782
783 SubmoduleAddCommand add = git.submoduleAdd().setName(name).setPath(path)
784 .setURI(url);
785 if (monitor != null)
786 add.setProgressMonitor(monitor);
787
788 Repository subRepo = add.call();
789 if (revision != null) {
790 try (Gitit.html#Git">Git sub = new Git(subRepo)) {
791 sub.checkout().setName(findRef(revision, subRepo)).call();
792 }
793 subRepo.close();
794 git.add().addFilepattern(path).call();
795 }
796 for (CopyFile copyfile : copyfiles) {
797 copyfile.copy();
798 git.add().addFilepattern(copyfile.dest).call();
799 }
800 }
801
802
803
804
805
806
807
808
809 private List<RepoProject> renameProjects(List<RepoProject> projects) {
810 Map<String, List<RepoProject>> m = new TreeMap<>();
811 for (RepoProject proj : projects) {
812 List<RepoProject> l = m.get(proj.getName());
813 if (l == null) {
814 l = new ArrayList<>();
815 m.put(proj.getName(), l);
816 }
817 l.add(proj);
818 }
819
820 List<RepoProject> ret = new ArrayList<>();
821 for (List<RepoProject> ps : m.values()) {
822 boolean nameConflict = ps.size() != 1;
823 for (RepoProject proj : ps) {
824 String name = proj.getName();
825 if (nameConflict) {
826 name += SLASH + proj.getPath();
827 }
828 RepoProject p = new RepoProject(name,
829 proj.getPath(), proj.getRevision(), null,
830 proj.getGroups(), proj.getRecommendShallow());
831 p.setUrl(proj.getUrl());
832 p.addCopyFiles(proj.getCopyFiles());
833 p.addLinkFiles(proj.getLinkFiles());
834 ret.add(p);
835 }
836 }
837 return ret;
838 }
839
840
841
842
843
844
845 private static final String SLASH = "/";
846 static URI relativize(URI current, URI target) {
847 if (!Objects.equals(current.getHost(), target.getHost())) {
848 return target;
849 }
850
851 String cur = current.normalize().getPath();
852 String dest = target.normalize().getPath();
853
854
855 if (cur.startsWith(SLASH) != dest.startsWith(SLASH)) {
856 return target;
857 }
858
859 while (cur.startsWith(SLASH)) {
860 cur = cur.substring(1);
861 }
862 while (dest.startsWith(SLASH)) {
863 dest = dest.substring(1);
864 }
865
866 if (cur.indexOf('/') == -1 || dest.indexOf('/') == -1) {
867
868 String prefix = "prefix/";
869 cur = prefix + cur;
870 dest = prefix + dest;
871 }
872
873 if (!cur.endsWith(SLASH)) {
874
875 int lastSlash = cur.lastIndexOf('/');
876 cur = cur.substring(0, lastSlash);
877 }
878 String destFile = "";
879 if (!dest.endsWith(SLASH)) {
880
881 int lastSlash = dest.lastIndexOf('/');
882 destFile = dest.substring(lastSlash + 1, dest.length());
883 dest = dest.substring(0, dest.lastIndexOf('/'));
884 }
885
886 String[] cs = cur.split(SLASH);
887 String[] ds = dest.split(SLASH);
888
889 int common = 0;
890 while (common < cs.length && common < ds.length && cs[common].equals(ds[common])) {
891 common++;
892 }
893
894 StringJoiner j = new StringJoiner(SLASH);
895 for (int i = common; i < cs.length; i++) {
896 j.add("..");
897 }
898 for (int i = common; i < ds.length; i++) {
899 j.add(ds[i]);
900 }
901
902 j.add(destFile);
903 return URI.create(j.toString());
904 }
905
906 private static String findRef(String ref, Repository repo)
907 throws IOException {
908 if (!ObjectId.isId(ref)) {
909 Ref r = repo.exactRef(R_REMOTES + DEFAULT_REMOTE_NAME + "/" + ref);
910 if (r != null)
911 return r.getName();
912 }
913 return ref;
914 }
915 }