Mercurial > hg > mercurial-crew-with-dirclash
annotate mercurial/streamclone.py @ 4661:a7e9b6b1adeb
Backed out changeset cfa477946181824f4c769580ebf59710090ba08a
Only printenv was changed, not the actual execution of hooks.
And not setting an empty value might cause problems on platforms
which can't always remove variables from the environment.
author | Thomas Arendsen Hein <thomas@intevation.de> |
---|---|
date | Thu, 21 Jun 2007 10:00:59 +0200 |
parents | 9dc64c8414ca |
children | 439e2f2fde42 |
rev | line source |
---|---|
2612
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
1 # streamclone.py - streaming clone server support for mercurial |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
2 # |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
3 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com> |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
4 # |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
5 # This software may be used and distributed according to the terms |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
6 # of the GNU General Public License, incorporated herein by reference. |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
7 |
3893 | 8 from i18n import _ |
3886
abaee83ce0a6
Replace demandload with new demandimport
Matt Mackall <mpm@selenic.com>
parents:
3856
diff
changeset
|
9 import os, stat, util, lock |
2612
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
10 |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
11 # if server supports streaming clone, it advertises "stream" |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
12 # capability with value that is version+flags of repo it is serving. |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
13 # client only streams if it can read that repo format. |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
14 |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
15 def walkrepo(root): |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
16 '''iterate over metadata files in repository. |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
17 walk in natural (sorted) order. |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
18 yields 2-tuples: name of .d or .i file, size of file.''' |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
19 |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
20 strip_count = len(root) + len(os.sep) |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
21 def walk(path, recurse): |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
22 ents = os.listdir(path) |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
23 ents.sort() |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
24 for e in ents: |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
25 pe = os.path.join(path, e) |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
26 st = os.lstat(pe) |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
27 if stat.S_ISDIR(st.st_mode): |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
28 if recurse: |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
29 for x in walk(pe, True): |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
30 yield x |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
31 else: |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
32 if not stat.S_ISREG(st.st_mode) or len(e) < 2: |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
33 continue |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
34 sfx = e[-2:] |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
35 if sfx in ('.d', '.i'): |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
36 yield pe[strip_count:], st.st_size |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
37 # write file data first |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
38 for x in walk(os.path.join(root, 'data'), True): |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
39 yield x |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
40 # write manifest before changelog |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
41 meta = list(walk(root, False)) |
2623
d1cbfe9e13cd
fix problem with uncompressed clone and python 2.3.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2622
diff
changeset
|
42 meta.sort() |
d1cbfe9e13cd
fix problem with uncompressed clone and python 2.3.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2622
diff
changeset
|
43 meta.reverse() |
2612
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
44 for x in meta: |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
45 yield x |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
46 |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
47 # stream file format is simple. |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
48 # |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
49 # server writes out line that says how many files, how many total |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
50 # bytes. separator is ascii space, byte counts are strings. |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
51 # |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
52 # then for each file: |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
53 # |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
54 # server writes out line that says file name, how many bytes in |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
55 # file. separator is ascii nul, byte count is string. |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
56 # |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
57 # server writes out raw file data. |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
58 |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
59 def stream_out(repo, fileobj): |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
60 '''stream out all metadata files in repository. |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
61 writes to file-like object, must support write() and optional flush().''' |
2621
5a5852a417b1
clone: disable stream support on server side by default.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2612
diff
changeset
|
62 |
2622
064aef9162cc
rename stream hgrc option to compressed.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2621
diff
changeset
|
63 if not repo.ui.configbool('server', 'uncompressed'): |
2621
5a5852a417b1
clone: disable stream support on server side by default.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2612
diff
changeset
|
64 fileobj.write('1\n') |
5a5852a417b1
clone: disable stream support on server side by default.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2612
diff
changeset
|
65 return |
5a5852a417b1
clone: disable stream support on server side by default.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
2612
diff
changeset
|
66 |
2612
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
67 # get consistent snapshot of repo. lock during scan so lock not |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
68 # needed while we stream, and commits can happen. |
3687
d5dd0a2a44bc
Handle locking exceptions if streaming clone can't lock the repo. (Issue324)
Thomas Arendsen Hein <thomas@intevation.de>
parents:
2623
diff
changeset
|
69 try: |
d5dd0a2a44bc
Handle locking exceptions if streaming clone can't lock the repo. (Issue324)
Thomas Arendsen Hein <thomas@intevation.de>
parents:
2623
diff
changeset
|
70 repolock = repo.lock() |
d5dd0a2a44bc
Handle locking exceptions if streaming clone can't lock the repo. (Issue324)
Thomas Arendsen Hein <thomas@intevation.de>
parents:
2623
diff
changeset
|
71 except (lock.LockHeld, lock.LockUnavailable), inst: |
d5dd0a2a44bc
Handle locking exceptions if streaming clone can't lock the repo. (Issue324)
Thomas Arendsen Hein <thomas@intevation.de>
parents:
2623
diff
changeset
|
72 repo.ui.warn('locking the repository failed: %s\n' % (inst,)) |
d5dd0a2a44bc
Handle locking exceptions if streaming clone can't lock the repo. (Issue324)
Thomas Arendsen Hein <thomas@intevation.de>
parents:
2623
diff
changeset
|
73 fileobj.write('2\n') |
d5dd0a2a44bc
Handle locking exceptions if streaming clone can't lock the repo. (Issue324)
Thomas Arendsen Hein <thomas@intevation.de>
parents:
2623
diff
changeset
|
74 return |
d5dd0a2a44bc
Handle locking exceptions if streaming clone can't lock the repo. (Issue324)
Thomas Arendsen Hein <thomas@intevation.de>
parents:
2623
diff
changeset
|
75 |
d5dd0a2a44bc
Handle locking exceptions if streaming clone can't lock the repo. (Issue324)
Thomas Arendsen Hein <thomas@intevation.de>
parents:
2623
diff
changeset
|
76 fileobj.write('0\n') |
2612
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
77 repo.ui.debug('scanning\n') |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
78 entries = [] |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
79 total_bytes = 0 |
3791
8643b9f90b51
introduce localrepo.spath for the store path, sopener fixes
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
3721
diff
changeset
|
80 for name, size in walkrepo(repo.spath): |
4127
a3a6f71edf2e
streamclone: stream_out tried to decodefilename non-normalized paths
Patrick Mezard <pmezard@gmail.com>
parents:
3856
diff
changeset
|
81 name = repo.decodefn(util.pconvert(name)) |
2612
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
82 entries.append((name, size)) |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
83 total_bytes += size |
3687
d5dd0a2a44bc
Handle locking exceptions if streaming clone can't lock the repo. (Issue324)
Thomas Arendsen Hein <thomas@intevation.de>
parents:
2623
diff
changeset
|
84 repolock.release() |
2612
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
85 |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
86 repo.ui.debug('%d files, %d bytes to transfer\n' % |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
87 (len(entries), total_bytes)) |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
88 fileobj.write('%d %d\n' % (len(entries), total_bytes)) |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
89 for name, size in entries: |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
90 repo.ui.debug('sending %s (%d bytes)\n' % (name, size)) |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
91 fileobj.write('%s\0%d\n' % (name, size)) |
3791
8643b9f90b51
introduce localrepo.spath for the store path, sopener fixes
Benoit Boissinot <benoit.boissinot@ens-lyon.org>
parents:
3721
diff
changeset
|
92 for chunk in util.filechunkiter(repo.sopener(name), limit=size): |
2612
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
93 fileobj.write(chunk) |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
94 flush = getattr(fileobj, 'flush', None) |
ffb895f16925
add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff
changeset
|
95 if flush: flush() |