Mercurial > hg > mercurial-crew-with-dirclash

annotate mercurial/streamclone.py @ 3569:a27d90c9336e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Stripping of query string (since 88b4755fa48f) stripped too much (issue327) rstrip(qs) not only strips qs from the right, but it continues stripping every char at the end of the URL that occurs on qs.
author Thomas Arendsen Hein <thomas@intevation.de>
date Sat, 28 Oct 2006 11:00:59 +0200
parents d1cbfe9e13cd
children d5dd0a2a44bc
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
2612
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
1 # streamclone.py - streaming clone server support for mercurial
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
2 #
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
3 # Copyright 2006 Vadim Gelfer <vadim.gelfer@gmail.com>
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
4 #
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
5 # This software may be used and distributed according to the terms
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
6 # of the GNU General Public License, incorporated herein by reference.
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
7
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
8 from demandload import demandload
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
9 from i18n import gettext as _
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
10 demandload(globals(), "os stat util")
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
11
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
12 # if server supports streaming clone, it advertises "stream"
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
13 # capability with value that is version+flags of repo it is serving.
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
14 # client only streams if it can read that repo format.
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
15
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
16 def walkrepo(root):
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
17 '''iterate over metadata files in repository.
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
18 walk in natural (sorted) order.
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
19 yields 2-tuples: name of .d or .i file, size of file.'''
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
20
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
21 strip_count = len(root) + len(os.sep)
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
22 def walk(path, recurse):
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
23 ents = os.listdir(path)
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
24 ents.sort()
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
25 for e in ents:
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
26 pe = os.path.join(path, e)
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
27 st = os.lstat(pe)
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
28 if stat.S_ISDIR(st.st_mode):
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
29 if recurse:
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
30 for x in walk(pe, True):
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
31 yield x
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
32 else:
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
33 if not stat.S_ISREG(st.st_mode) or len(e) < 2:
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
34 continue
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
35 sfx = e[-2:]
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
36 if sfx in ('.d', '.i'):
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
37 yield pe[strip_count:], st.st_size
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
38 # write file data first
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
39 for x in walk(os.path.join(root, 'data'), True):
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
40 yield x
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
41 # write manifest before changelog
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
42 meta = list(walk(root, False))
2623
d1cbfe9e13cd fix problem with uncompressed clone and python 2.3.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2622
diff changeset
43 meta.sort()
d1cbfe9e13cd fix problem with uncompressed clone and python 2.3.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2622
diff changeset
44 meta.reverse()
2612
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
45 for x in meta:
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
46 yield x
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
47
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
48 # stream file format is simple.
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
49 #
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
50 # server writes out line that says how many files, how many total
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
51 # bytes. separator is ascii space, byte counts are strings.
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
52 #
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
53 # then for each file:
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
54 #
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
55 # server writes out line that says file name, how many bytes in
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
56 # file. separator is ascii nul, byte count is string.
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
57 #
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
58 # server writes out raw file data.
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
59
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
60 def stream_out(repo, fileobj):
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
61 '''stream out all metadata files in repository.
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
62 writes to file-like object, must support write() and optional flush().'''
2621
5a5852a417b1 clone: disable stream support on server side by default.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2612
diff changeset
63
2622
064aef9162cc rename stream hgrc option to compressed.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2621
diff changeset
64 if not repo.ui.configbool('server', 'uncompressed'):
2621
5a5852a417b1 clone: disable stream support on server side by default.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2612
diff changeset
65 fileobj.write('1\n')
5a5852a417b1 clone: disable stream support on server side by default.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2612
diff changeset
66 return
5a5852a417b1 clone: disable stream support on server side by default.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2612
diff changeset
67
5a5852a417b1 clone: disable stream support on server side by default.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2612
diff changeset
68 fileobj.write('0\n')
5a5852a417b1 clone: disable stream support on server side by default.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2612
diff changeset
69
2612
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
70 # get consistent snapshot of repo. lock during scan so lock not
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
71 # needed while we stream, and commits can happen.
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
72 lock = repo.lock()
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
73 repo.ui.debug('scanning\n')
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
74 entries = []
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
75 total_bytes = 0
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
76 for name, size in walkrepo(repo.path):
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
77 entries.append((name, size))
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
78 total_bytes += size
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
79 lock.release()
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
80
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
81 repo.ui.debug('%d files, %d bytes to transfer\n' %
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
82 (len(entries), total_bytes))
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
83 fileobj.write('%d %d\n' % (len(entries), total_bytes))
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
84 for name, size in entries:
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
85 repo.ui.debug('sending %s (%d bytes)\n' % (name, size))
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
86 fileobj.write('%s\0%d\n' % (name, size))
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
87 for chunk in util.filechunkiter(repo.opener(name), limit=size):
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
88 fileobj.write(chunk)
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
89 flush = getattr(fileobj, 'flush', None)
ffb895f16925 add support for streaming clone.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents:
diff changeset
90 if flush: flush()