Mercurial > hg > mercurial-crew-with-dirclash
annotate contrib/convert-repo @ 2474:1e32e2fe8a67
Fix cold cache diff performance
cold cache diff performance has regressed in two ways. localrepo.changes
has optimizations for diffing against the working dir parent that expect
node1 to be None. commands.revpair() usage means that commands.dodiff()
never sends node1 == None. This is fixed in localrepo.changes by checking
against the dirstate parents.
In the non-dirstate parents case, localrepo.changes does a loop comparing
files without first sorting the file names, leading to random access
across the disk.
| author | Chris Mason <mason@suse.com> |
|---|---|
| date | Wed, 21 Jun 2006 09:28:48 -0700 |
| parents | 5cc414722587 |
| children | e6a7a6a33a62 |
| rev | line source |
|---|---|
| 316 | 1 #!/usr/bin/env python |
| 2 # | |
| 3 # This is a generalized framework for converting between SCM | |
| 4 # repository formats. | |
| 5 # | |
| 6 # In its current form, it's hardcoded to convert incrementally between | |
| 7 # git and Mercurial. | |
| 8 # | |
| 9 # To use, you must first import the first git version into Mercurial, | |
| 10 # and establish a mapping between the git commit hash and the hash in | |
| 11 # Mercurial for that version. This mapping is kept in a simple text | |
| 12 # file with lines like so: | |
| 13 # | |
| 14 # <git hash> <mercurial hash> | |
| 15 # | |
| 16 # To convert the rest of the repo, run: | |
| 17 # | |
| 18 # convert-repo <git-dir> <hg-dir> <mapfile> | |
| 19 # | |
| 20 # This updates the mapfile on each commit copied, so it can be | |
| 21 # interrupted and can be run repeatedly to copy new commits. | |
| 22 | |
| 694 | 23 import sys, os, zlib, sha, time |
|
1715
40346aa66b0f
Revert convert-repo changes
Matt Mackall <mpm@selenic.com>
parents:
1656
diff
changeset
|
24 from mercurial import hg, ui, util |
| 316 | 25 |
| 26 class convert_git: | |
| 27 def __init__(self, path): | |
| 28 self.path = path | |
| 29 | |
| 30 def getheads(self): | |
|
1335
bea6356b8bca
git -> hg conversion script
Florian La Roche <laroche@redhat.com>
parents:
1237
diff
changeset
|
31 return [file(self.path + "/HEAD").read()[:-1]] |
| 316 | 32 |
|
692
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
33 def catfile(self, rev, type): |
|
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
34 if rev == "0" * 40: raise IOError() |
|
1335
bea6356b8bca
git -> hg conversion script
Florian La Roche <laroche@redhat.com>
parents:
1237
diff
changeset
|
35 fh = os.popen("GIT_DIR=%s git-cat-file %s %s 2>/dev/null" % (self.path, type, rev)) |
|
692
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
36 return fh.read() |
|
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
37 |
| 316 | 38 def getfile(self, name, rev): |
|
692
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
39 return self.catfile(rev, "blob") |
| 316 | 40 |
| 41 def getchanges(self, version): | |
|
1335
bea6356b8bca
git -> hg conversion script
Florian La Roche <laroche@redhat.com>
parents:
1237
diff
changeset
|
42 fh = os.popen("GIT_DIR=%s git-diff-tree --root -m -r %s" % (self.path, version)) |
| 316 | 43 changes = [] |
| 44 for l in fh: | |
| 45 if "\t" not in l: continue | |
| 46 m, f = l[:-1].split("\t") | |
| 47 m = m.split() | |
| 48 h = m[3] | |
| 49 p = (m[1] == "100755") | |
| 50 changes.append((f, h, p)) | |
| 51 return changes | |
| 52 | |
| 53 def getcommit(self, version): | |
|
692
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
54 c = self.catfile(version, "commit") # read the commit hash |
| 316 | 55 end = c.find("\n\n") |
| 56 message = c[end+2:] | |
| 57 l = c[:end].splitlines() | |
| 58 manifest = l[0].split()[1] | |
| 59 parents = [] | |
| 60 for e in l[1:]: | |
| 61 n,v = e.split(" ", 1) | |
| 62 if n == "author": | |
| 63 p = v.split() | |
|
1385
adb3de56635b
convert-repo: Fix timezone handling
Matt Mackall <mpm@selenic.com>
parents:
1335
diff
changeset
|
64 tm, tz = p[-2:] |
| 316 | 65 author = " ".join(p[:-2]) |
| 66 if author[0] == "<": author = author[1:-1] | |
|
692
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
67 if n == "committer": |
| 431 | 68 p = v.split() |
|
1385
adb3de56635b
convert-repo: Fix timezone handling
Matt Mackall <mpm@selenic.com>
parents:
1335
diff
changeset
|
69 tm, tz = p[-2:] |
| 431 | 70 committer = " ".join(p[:-2]) |
| 71 if committer[0] == "<": committer = committer[1:-1] | |
|
1385
adb3de56635b
convert-repo: Fix timezone handling
Matt Mackall <mpm@selenic.com>
parents:
1335
diff
changeset
|
72 message += "\ncommitter: %s\n" % v |
| 316 | 73 if n == "parent": parents.append(v) |
|
1385
adb3de56635b
convert-repo: Fix timezone handling
Matt Mackall <mpm@selenic.com>
parents:
1335
diff
changeset
|
74 |
|
adb3de56635b
convert-repo: Fix timezone handling
Matt Mackall <mpm@selenic.com>
parents:
1335
diff
changeset
|
75 tzs, tzh, tzm = tz[-5:-4] + "1", tz[-4:-2], tz[-2:] |
|
2093
5cc414722587
convert-repo: fix reversed time zone offset
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
1715
diff
changeset
|
76 tz = -int(tzs) * (int(tzh) * 3600 + int(tzm)) |
|
1385
adb3de56635b
convert-repo: Fix timezone handling
Matt Mackall <mpm@selenic.com>
parents:
1335
diff
changeset
|
77 date = tm + " " + str(tz) |
| 316 | 78 return (parents, author, date, message) |
| 79 | |
| 694 | 80 def gettags(self): |
| 81 tags = {} | |
|
1335
bea6356b8bca
git -> hg conversion script
Florian La Roche <laroche@redhat.com>
parents:
1237
diff
changeset
|
82 for f in os.listdir(self.path + "/refs/tags"): |
| 694 | 83 try: |
|
1335
bea6356b8bca
git -> hg conversion script
Florian La Roche <laroche@redhat.com>
parents:
1237
diff
changeset
|
84 h = file(self.path + "/refs/tags/" + f).read().strip() |
|
1386
a1040345fdda
convert-repo: retrieve the commit hash from the tag object for tag import
Matt Mackall <mpm@selenic.com>
parents:
1385
diff
changeset
|
85 c = self.catfile(h, "tag") # read the commit hash |
|
a1040345fdda
convert-repo: retrieve the commit hash from the tag object for tag import
Matt Mackall <mpm@selenic.com>
parents:
1385
diff
changeset
|
86 h = c.splitlines()[0].split()[1] |
| 1237 | 87 tags[f] = h |
| 694 | 88 except: |
| 89 pass | |
| 90 return tags | |
| 91 | |
| 316 | 92 class convert_mercurial: |
| 93 def __init__(self, path): | |
| 94 self.path = path | |
| 95 u = ui.ui() | |
| 96 self.repo = hg.repository(u, path) | |
| 97 | |
| 98 def getheads(self): | |
| 99 h = self.repo.changelog.heads() | |
|
1335
bea6356b8bca
git -> hg conversion script
Florian La Roche <laroche@redhat.com>
parents:
1237
diff
changeset
|
100 return [ hg.hex(x) for x in h ] |
|
692
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
101 |
| 316 | 102 def putfile(self, f, e, data): |
| 103 self.repo.wfile(f, "w").write(data) | |
|
692
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
104 if self.repo.dirstate.state(f) == '?': |
|
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
105 self.repo.dirstate.update([f], "a") |
|
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
106 |
| 450 | 107 util.set_exec(self.repo.wjoin(f), e) |
| 316 | 108 |
| 109 def delfile(self, f): | |
| 110 try: | |
| 111 os.unlink(self.repo.wjoin(f)) | |
|
692
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
112 #self.repo.remove([f]) |
| 316 | 113 except: |
| 114 pass | |
| 115 | |
|
1715
40346aa66b0f
Revert convert-repo changes
Matt Mackall <mpm@selenic.com>
parents:
1656
diff
changeset
|
116 def putcommit(self, files, parents, author, dest, text): |
| 431 | 117 seen = {} |
| 118 pl = [] | |
| 119 for p in parents: | |
| 120 if p not in seen: | |
| 121 pl.append(p) | |
| 122 seen[p] = 1 | |
| 123 parents = pl | |
| 316 | 124 |
|
692
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
125 if len(parents) < 2: parents.append("0" * 40) |
|
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
126 if len(parents) < 2: parents.append("0" * 40) |
| 431 | 127 p2 = parents.pop(0) |
|
692
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
128 |
| 431 | 129 while parents: |
| 130 p1 = p2 | |
| 131 p2 = parents.pop(0) | |
|
1715
40346aa66b0f
Revert convert-repo changes
Matt Mackall <mpm@selenic.com>
parents:
1656
diff
changeset
|
132 self.repo.rawcommit(files, text, author, dest, |
|
40346aa66b0f
Revert convert-repo changes
Matt Mackall <mpm@selenic.com>
parents:
1656
diff
changeset
|
133 hg.bin(p1), hg.bin(p2)) |
| 431 | 134 text = "(octopus merge fixup)\n" |
|
1389
9b3ef6f3cef5
convert-repo: fix up octopus merge conversion
Matt Mackall <mpm@selenic.com>
parents:
1388
diff
changeset
|
135 p2 = hg.hex(self.repo.changelog.tip()) |
| 431 | 136 |
|
1389
9b3ef6f3cef5
convert-repo: fix up octopus merge conversion
Matt Mackall <mpm@selenic.com>
parents:
1388
diff
changeset
|
137 return p2 |
| 316 | 138 |
| 694 | 139 def puttags(self, tags): |
| 140 try: | |
| 141 old = self.repo.wfile(".hgtags").read() | |
| 142 oldlines = old.splitlines(1) | |
| 143 oldlines.sort() | |
| 144 except: | |
| 145 oldlines = [] | |
| 146 | |
| 147 k = tags.keys() | |
| 148 k.sort() | |
| 149 newlines = [] | |
| 150 for tag in k: | |
| 151 newlines.append("%s %s\n" % (tags[tag], tag)) | |
| 152 | |
| 153 newlines.sort() | |
| 154 | |
| 155 if newlines != oldlines: | |
|
1335
bea6356b8bca
git -> hg conversion script
Florian La Roche <laroche@redhat.com>
parents:
1237
diff
changeset
|
156 #print "updating tags" |
| 694 | 157 f = self.repo.wfile(".hgtags", "w") |
| 158 f.write("".join(newlines)) | |
| 159 f.close() | |
| 160 if not oldlines: self.repo.add([".hgtags"]) | |
|
1335
bea6356b8bca
git -> hg conversion script
Florian La Roche <laroche@redhat.com>
parents:
1237
diff
changeset
|
161 date = "%s 0" % int(time.mktime(time.gmtime())) |
| 694 | 162 self.repo.rawcommit([".hgtags"], "update tags", "convert-repo", |
| 163 date, self.repo.changelog.tip(), hg.nullid) | |
|
1387
0c7e8d345564
convert-repo: linearize the tag commit
Matt Mackall <mpm@selenic.com>
parents:
1386
diff
changeset
|
164 return hg.hex(self.repo.changelog.tip()) |
| 694 | 165 |
| 316 | 166 class convert: |
| 167 def __init__(self, source, dest, mapfile): | |
| 168 self.source = source | |
| 169 self.dest = dest | |
| 170 self.mapfile = mapfile | |
| 171 self.commitcache = {} | |
| 172 | |
| 173 self.map = {} | |
|
1655
7bfd4724932a
convert-repo: automatically create empty map file
Matt Mackall <mpm@selenic.com>
parents:
1389
diff
changeset
|
174 try: |
|
7bfd4724932a
convert-repo: automatically create empty map file
Matt Mackall <mpm@selenic.com>
parents:
1389
diff
changeset
|
175 for l in file(self.mapfile): |
|
7bfd4724932a
convert-repo: automatically create empty map file
Matt Mackall <mpm@selenic.com>
parents:
1389
diff
changeset
|
176 sv, dv = l[:-1].split() |
|
7bfd4724932a
convert-repo: automatically create empty map file
Matt Mackall <mpm@selenic.com>
parents:
1389
diff
changeset
|
177 self.map[sv] = dv |
|
7bfd4724932a
convert-repo: automatically create empty map file
Matt Mackall <mpm@selenic.com>
parents:
1389
diff
changeset
|
178 except IOError: |
|
7bfd4724932a
convert-repo: automatically create empty map file
Matt Mackall <mpm@selenic.com>
parents:
1389
diff
changeset
|
179 pass |
| 316 | 180 |
| 181 def walktree(self, heads): | |
| 182 visit = heads | |
| 183 known = {} | |
| 184 parents = {} | |
| 185 while visit: | |
| 186 n = visit.pop(0) | |
| 187 if n in known or n in self.map: continue | |
| 188 known[n] = 1 | |
| 189 self.commitcache[n] = self.source.getcommit(n) | |
| 190 cp = self.commitcache[n][0] | |
| 191 for p in cp: | |
| 192 parents.setdefault(n, []).append(p) | |
| 193 visit.append(p) | |
| 194 | |
| 195 return parents | |
| 196 | |
| 197 def toposort(self, parents): | |
| 198 visit = parents.keys() | |
| 199 seen = {} | |
| 200 children = {} | |
|
692
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
201 |
| 316 | 202 while visit: |
| 203 n = visit.pop(0) | |
| 204 if n in seen: continue | |
| 205 seen[n] = 1 | |
| 206 pc = 0 | |
| 207 if n in parents: | |
| 208 for p in parents[n]: | |
| 209 if p not in self.map: pc += 1 | |
| 210 visit.append(p) | |
| 211 children.setdefault(p, []).append(n) | |
| 212 if not pc: root = n | |
| 213 | |
| 214 s = [] | |
| 215 removed = {} | |
|
692
695dd9a491da
convert-repo: deal with packed git and other fixes
mpm@selenic.com
parents:
450
diff
changeset
|
216 visit = children.keys() |
| 316 | 217 while visit: |
| 218 n = visit.pop(0) | |
| 219 if n in removed: continue | |
| 220 dep = 0 | |
| 221 if n in parents: | |
| 222 for p in parents[n]: | |
| 223 if p in self.map: continue | |
| 224 if p not in removed: | |
| 225 # we're still dependent | |
| 226 visit.append(n) | |
| 227 dep = 1 | |
| 228 break | |
| 229 | |
| 230 if not dep: | |
| 231 # all n's parents are in the list | |
| 232 removed[n] = 1 | |
| 233 s.append(n) | |
| 234 if n in children: | |
| 235 for c in children[n]: | |
| 236 visit.insert(0, c) | |
| 237 | |
| 238 return s | |
| 239 | |
| 240 def copy(self, rev): | |
| 241 p, a, d, t = self.commitcache[rev] | |
| 242 files = self.source.getchanges(rev) | |
| 243 | |
| 244 for f,v,e in files: | |
| 245 try: | |
| 246 data = self.source.getfile(f, v) | |
| 247 except IOError, inst: | |
| 248 self.dest.delfile(f) | |
| 249 else: | |
| 250 self.dest.putfile(f, e, data) | |
| 251 | |
| 252 r = [self.map[v] for v in p] | |
| 253 f = [f for f,v,e in files] | |
| 254 self.map[rev] = self.dest.putcommit(f, r, a, d, t) | |
| 255 file(self.mapfile, "a").write("%s %s\n" % (rev, self.map[rev])) | |
| 256 | |
| 257 def convert(self): | |
| 258 heads = self.source.getheads() | |
| 259 parents = self.walktree(heads) | |
| 260 t = self.toposort(parents) | |
|
1388
5eb2d3c54165
convert-repo: change duplicate elimination
Matt Mackall <mpm@selenic.com>
parents:
1387
diff
changeset
|
261 t = [n for n in t if n not in self.map] |
| 316 | 262 num = len(t) |
|
1715
40346aa66b0f
Revert convert-repo changes
Matt Mackall <mpm@selenic.com>
parents:
1656
diff
changeset
|
263 c = None |
| 316 | 264 |
| 265 for c in t: | |
| 266 num -= 1 | |
| 267 desc = self.commitcache[c][3].splitlines()[0] | |
|
1335
bea6356b8bca
git -> hg conversion script
Florian La Roche <laroche@redhat.com>
parents:
1237
diff
changeset
|
268 #print num, desc |
| 316 | 269 self.copy(c) |
| 270 | |
| 694 | 271 tags = self.source.gettags() |
| 272 ctags = {} | |
| 273 for k in tags: | |
| 274 v = tags[k] | |
| 275 if v in self.map: | |
| 276 ctags[k] = self.map[v] | |
| 277 | |
|
1715
40346aa66b0f
Revert convert-repo changes
Matt Mackall <mpm@selenic.com>
parents:
1656
diff
changeset
|
278 if c and ctags: |
|
1387
0c7e8d345564
convert-repo: linearize the tag commit
Matt Mackall <mpm@selenic.com>
parents:
1386
diff
changeset
|
279 nrev = self.dest.puttags(ctags) |
|
0c7e8d345564
convert-repo: linearize the tag commit
Matt Mackall <mpm@selenic.com>
parents:
1386
diff
changeset
|
280 # write another hash correspondence to override the previous |
|
0c7e8d345564
convert-repo: linearize the tag commit
Matt Mackall <mpm@selenic.com>
parents:
1386
diff
changeset
|
281 # one so we don't end up with extra tag heads |
|
0c7e8d345564
convert-repo: linearize the tag commit
Matt Mackall <mpm@selenic.com>
parents:
1386
diff
changeset
|
282 file(self.mapfile, "a").write("%s %s\n" % (c, nrev)) |
| 694 | 283 |
| 316 | 284 gitpath, hgpath, mapfile = sys.argv[1:] |
|
1335
bea6356b8bca
git -> hg conversion script
Florian La Roche <laroche@redhat.com>
parents:
1237
diff
changeset
|
285 if os.path.isdir(gitpath + "/.git"): |
|
bea6356b8bca
git -> hg conversion script
Florian La Roche <laroche@redhat.com>
parents:
1237
diff
changeset
|
286 gitpath += "/.git" |
| 316 | 287 |
| 288 c = convert(convert_git(gitpath), convert_mercurial(hgpath), mapfile) | |
| 289 c.convert() |