mercurial/byterange.py
author mason@suse.com
Tue, 04 Apr 2006 16:38:43 -0400
changeset 2073 1e6745f78989
parent 674 6513ba7d858a
child 3673 eb0b4a2d70a9
permissions -rw-r--r--
Implement data inlined with the index file This patch allows you to optionally inline data bytes with the revlog index file. It saves considerable space and checkout time by reducing the number of inodes, wasted partial blocks and system calls. To use the inline data add this to your .hgrc [revlog] # inline data only works with revlogng format=1 # inline is the only valid flag right now. flags=inline
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
     1
#   This library is free software; you can redistribute it and/or
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
     2
#   modify it under the terms of the GNU Lesser General Public
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
     3
#   License as published by the Free Software Foundation; either
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
     4
#   version 2.1 of the License, or (at your option) any later version.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
     5
#
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
     6
#   This library is distributed in the hope that it will be useful,
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
     7
#   but WITHOUT ANY WARRANTY; without even the implied warranty of
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
     8
#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
     9
#   Lesser General Public License for more details.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    10
#
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    11
#   You should have received a copy of the GNU Lesser General Public
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    12
#   License along with this library; if not, write to the
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    13
#      Free Software Foundation, Inc.,
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    14
#      59 Temple Place, Suite 330,
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    15
#      Boston, MA  02111-1307  USA
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    16
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    17
# This file is part of urlgrabber, a high-level cross-protocol url-grabber
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    18
# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    19
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    20
# $Id: byterange.py,v 1.9 2005/02/14 21:55:07 mstenner Exp $
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    21
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    22
import os
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    23
import stat
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    24
import urllib
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    25
import urllib2
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    26
import rfc822
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    27
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    28
try:
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    29
    from cStringIO import StringIO
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    30
except ImportError, msg:
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    31
    from StringIO import StringIO
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    32
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    33
class RangeError(IOError):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    34
    """Error raised when an unsatisfiable range is requested."""
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    35
    pass
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    36
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    37
class HTTPRangeHandler(urllib2.BaseHandler):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    38
    """Handler that enables HTTP Range headers.
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    39
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    40
    This was extremely simple. The Range header is a HTTP feature to
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    41
    begin with so all this class does is tell urllib2 that the
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    42
    "206 Partial Content" reponse from the HTTP server is what we
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    43
    expected.
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    44
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    45
    Example:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    46
        import urllib2
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    47
        import byterange
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    48
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    49
        range_handler = range.HTTPRangeHandler()
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    50
        opener = urllib2.build_opener(range_handler)
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    51
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    52
        # install it
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    53
        urllib2.install_opener(opener)
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    54
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    55
        # create Request and set Range header
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    56
        req = urllib2.Request('http://www.python.org/')
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    57
        req.header['Range'] = 'bytes=30-50'
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    58
        f = urllib2.urlopen(req)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    59
    """
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    60
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    61
    def http_error_206(self, req, fp, code, msg, hdrs):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    62
        # 206 Partial Content Response
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    63
        r = urllib.addinfourl(fp, hdrs, req.get_full_url())
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    64
        r.code = code
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    65
        r.msg = msg
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    66
        return r
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    67
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    68
    def http_error_416(self, req, fp, code, msg, hdrs):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    69
        # HTTP's Range Not Satisfiable error
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    70
        raise RangeError('Requested Range Not Satisfiable')
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    71
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    72
class RangeableFileObject:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    73
    """File object wrapper to enable raw range handling.
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    74
    This was implemented primarilary for handling range
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    75
    specifications for file:// urls. This object effectively makes
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    76
    a file object look like it consists only of a range of bytes in
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    77
    the stream.
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    78
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    79
    Examples:
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    80
        # expose 10 bytes, starting at byte position 20, from
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    81
        # /etc/aliases.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    82
        >>> fo = RangeableFileObject(file('/etc/passwd', 'r'), (20,30))
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    83
        # seek seeks within the range (to position 23 in this case)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    84
        >>> fo.seek(3)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    85
        # tell tells where your at _within the range_ (position 3 in
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    86
        # this case)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    87
        >>> fo.tell()
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    88
        # read EOFs if an attempt is made to read past the last
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    89
        # byte in the range. the following will return only 7 bytes.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    90
        >>> fo.read(30)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    91
    """
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    92
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    93
    def __init__(self, fo, rangetup):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    94
        """Create a RangeableFileObject.
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    95
        fo       -- a file like object. only the read() method need be
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
    96
                    supported but supporting an optimized seek() is
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    97
                    preferable.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    98
        rangetup -- a (firstbyte,lastbyte) tuple specifying the range
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
    99
                    to work over.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   100
        The file object provided is assumed to be at byte offset 0.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   101
        """
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   102
        self.fo = fo
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   103
        (self.firstbyte, self.lastbyte) = range_tuple_normalize(rangetup)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   104
        self.realpos = 0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   105
        self._do_seek(self.firstbyte)
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   106
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   107
    def __getattr__(self, name):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   108
        """This effectively allows us to wrap at the instance level.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   109
        Any attribute not found in _this_ object will be searched for
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   110
        in self.fo.  This includes methods."""
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   111
        if hasattr(self.fo, name):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   112
            return getattr(self.fo, name)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   113
        raise AttributeError, name
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   114
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   115
    def tell(self):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   116
        """Return the position within the range.
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   117
        This is different from fo.seek in that position 0 is the
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   118
        first byte position of the range tuple. For example, if
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   119
        this object was created with a range tuple of (500,899),
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   120
        tell() will return 0 when at byte position 500 of the file.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   121
        """
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   122
        return (self.realpos - self.firstbyte)
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   123
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   124
    def seek(self,offset,whence=0):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   125
        """Seek within the byte range.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   126
        Positioning is identical to that described under tell().
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   127
        """
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   128
        assert whence in (0, 1, 2)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   129
        if whence == 0:   # absolute seek
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   130
            realoffset = self.firstbyte + offset
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   131
        elif whence == 1: # relative seek
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   132
            realoffset = self.realpos + offset
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   133
        elif whence == 2: # absolute from end of file
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   134
            # XXX: are we raising the right Error here?
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   135
            raise IOError('seek from end of file not supported.')
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   136
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   137
        # do not allow seek past lastbyte in range
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   138
        if self.lastbyte and (realoffset >= self.lastbyte):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   139
            realoffset = self.lastbyte
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   140
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   141
        self._do_seek(realoffset - self.realpos)
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   142
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   143
    def read(self, size=-1):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   144
        """Read within the range.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   145
        This method will limit the size read based on the range.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   146
        """
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   147
        size = self._calc_read_size(size)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   148
        rslt = self.fo.read(size)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   149
        self.realpos += len(rslt)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   150
        return rslt
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   151
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   152
    def readline(self, size=-1):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   153
        """Read lines within the range.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   154
        This method will limit the size read based on the range.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   155
        """
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   156
        size = self._calc_read_size(size)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   157
        rslt = self.fo.readline(size)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   158
        self.realpos += len(rslt)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   159
        return rslt
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   160
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   161
    def _calc_read_size(self, size):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   162
        """Handles calculating the amount of data to read based on
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   163
        the range.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   164
        """
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   165
        if self.lastbyte:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   166
            if size > -1:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   167
                if ((self.realpos + size) >= self.lastbyte):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   168
                    size = (self.lastbyte - self.realpos)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   169
            else:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   170
                size = (self.lastbyte - self.realpos)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   171
        return size
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   172
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   173
    def _do_seek(self,offset):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   174
        """Seek based on whether wrapped object supports seek().
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   175
        offset is relative to the current position (self.realpos).
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   176
        """
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   177
        assert offset >= 0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   178
        if not hasattr(self.fo, 'seek'):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   179
            self._poor_mans_seek(offset)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   180
        else:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   181
            self.fo.seek(self.realpos + offset)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   182
        self.realpos+= offset
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   183
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   184
    def _poor_mans_seek(self,offset):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   185
        """Seek by calling the wrapped file objects read() method.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   186
        This is used for file like objects that do not have native
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   187
        seek support. The wrapped objects read() method is called
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   188
        to manually seek to the desired position.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   189
        offset -- read this number of bytes from the wrapped
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   190
                  file object.
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   191
        raise RangeError if we encounter EOF before reaching the
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   192
        specified offset.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   193
        """
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   194
        pos = 0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   195
        bufsize = 1024
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   196
        while pos < offset:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   197
            if (pos + bufsize) > offset:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   198
                bufsize = offset - pos
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   199
            buf = self.fo.read(bufsize)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   200
            if len(buf) != bufsize:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   201
                raise RangeError('Requested Range Not Satisfiable')
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   202
            pos+= bufsize
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   203
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   204
class FileRangeHandler(urllib2.FileHandler):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   205
    """FileHandler subclass that adds Range support.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   206
    This class handles Range headers exactly like an HTTP
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   207
    server would.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   208
    """
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   209
    def open_local_file(self, req):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   210
        import mimetypes
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   211
        import mimetools
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   212
        host = req.get_host()
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   213
        file = req.get_selector()
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   214
        localfile = urllib.url2pathname(file)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   215
        stats = os.stat(localfile)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   216
        size = stats[stat.ST_SIZE]
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   217
        modified = rfc822.formatdate(stats[stat.ST_MTIME])
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   218
        mtype = mimetypes.guess_type(file)[0]
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   219
        if host:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   220
            host, port = urllib.splitport(host)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   221
            if port or socket.gethostbyname(host) not in self.get_names():
667
31a9aa890016 A number of minor fixes to problems that pychecker found.
mark.williamson@cl.cam.ac.uk
parents: 575
diff changeset
   222
                raise urllib2.URLError('file not on local host')
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   223
        fo = open(localfile,'rb')
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   224
        brange = req.headers.get('Range',None)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   225
        brange = range_header_to_tuple(brange)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   226
        assert brange != ()
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   227
        if brange:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   228
            (fb,lb) = brange
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   229
            if lb == '': lb = size
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   230
            if fb < 0 or fb > size or lb > size:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   231
                raise RangeError('Requested Range Not Satisfiable')
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   232
            size = (lb - fb)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   233
            fo = RangeableFileObject(fo, (fb,lb))
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   234
        headers = mimetools.Message(StringIO(
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   235
            'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   236
            (mtype or 'text/plain', size, modified)))
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   237
        return urllib.addinfourl(fo, headers, 'file:'+file)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   238
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   239
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   240
# FTP Range Support
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   241
# Unfortunately, a large amount of base FTP code had to be copied
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   242
# from urllib and urllib2 in order to insert the FTP REST command.
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   243
# Code modifications for range support have been commented as
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   244
# follows:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   245
# -- range support modifications start/end here
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   246
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   247
from urllib import splitport, splituser, splitpasswd, splitattr, \
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   248
                   unquote, addclosehook, addinfourl
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   249
import ftplib
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   250
import socket
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   251
import sys
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   252
import mimetypes
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   253
import mimetools
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   254
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   255
class FTPRangeHandler(urllib2.FTPHandler):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   256
    def ftp_open(self, req):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   257
        host = req.get_host()
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   258
        if not host:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   259
            raise IOError, ('ftp error', 'no host given')
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   260
        host, port = splitport(host)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   261
        if port is None:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   262
            port = ftplib.FTP_PORT
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   263
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   264
        # username/password handling
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   265
        user, host = splituser(host)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   266
        if user:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   267
            user, passwd = splitpasswd(user)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   268
        else:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   269
            passwd = None
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   270
        host = unquote(host)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   271
        user = unquote(user or '')
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   272
        passwd = unquote(passwd or '')
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   273
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   274
        try:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   275
            host = socket.gethostbyname(host)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   276
        except socket.error, msg:
667
31a9aa890016 A number of minor fixes to problems that pychecker found.
mark.williamson@cl.cam.ac.uk
parents: 575
diff changeset
   277
            raise urllib2.URLError(msg)
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   278
        path, attrs = splitattr(req.get_selector())
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   279
        dirs = path.split('/')
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   280
        dirs = map(unquote, dirs)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   281
        dirs, file = dirs[:-1], dirs[-1]
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   282
        if dirs and not dirs[0]:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   283
            dirs = dirs[1:]
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   284
        try:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   285
            fw = self.connect_ftp(user, passwd, host, port, dirs)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   286
            type = file and 'I' or 'D'
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   287
            for attr in attrs:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   288
                attr, value = splitattr(attr)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   289
                if attr.lower() == 'type' and \
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   290
                   value in ('a', 'A', 'i', 'I', 'd', 'D'):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   291
                    type = value.upper()
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   292
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   293
            # -- range support modifications start here
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   294
            rest = None
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   295
            range_tup = range_header_to_tuple(req.headers.get('Range',None))
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   296
            assert range_tup != ()
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   297
            if range_tup:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   298
                (fb,lb) = range_tup
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   299
                if fb > 0: rest = fb
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   300
            # -- range support modifications end here
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   301
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   302
            fp, retrlen = fw.retrfile(file, type, rest)
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   303
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   304
            # -- range support modifications start here
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   305
            if range_tup:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   306
                (fb,lb) = range_tup
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   307
                if lb == '':
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   308
                    if retrlen is None or retrlen == 0:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   309
                        raise RangeError('Requested Range Not Satisfiable due to unobtainable file length.')
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   310
                    lb = retrlen
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   311
                    retrlen = lb - fb
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   312
                    if retrlen < 0:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   313
                        # beginning of range is larger than file
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   314
                        raise RangeError('Requested Range Not Satisfiable')
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   315
                else:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   316
                    retrlen = lb - fb
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   317
                    fp = RangeableFileObject(fp, (0,retrlen))
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   318
            # -- range support modifications end here
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   319
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   320
            headers = ""
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   321
            mtype = mimetypes.guess_type(req.get_full_url())[0]
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   322
            if mtype:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   323
                headers += "Content-Type: %s\n" % mtype
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   324
            if retrlen is not None and retrlen >= 0:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   325
                headers += "Content-Length: %d\n" % retrlen
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   326
            sf = StringIO(headers)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   327
            headers = mimetools.Message(sf)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   328
            return addinfourl(fp, headers, req.get_full_url())
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   329
        except ftplib.all_errors, msg:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   330
            raise IOError, ('ftp error', msg), sys.exc_info()[2]
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   331
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   332
    def connect_ftp(self, user, passwd, host, port, dirs):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   333
        fw = ftpwrapper(user, passwd, host, port, dirs)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   334
        return fw
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   335
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   336
class ftpwrapper(urllib.ftpwrapper):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   337
    # range support note:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   338
    # this ftpwrapper code is copied directly from
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   339
    # urllib. The only enhancement is to add the rest
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   340
    # argument and pass it on to ftp.ntransfercmd
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   341
    def retrfile(self, file, type, rest=None):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   342
        self.endtransfer()
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   343
        if type in ('d', 'D'): cmd = 'TYPE A'; isdir = 1
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   344
        else: cmd = 'TYPE ' + type; isdir = 0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   345
        try:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   346
            self.ftp.voidcmd(cmd)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   347
        except ftplib.all_errors:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   348
            self.init()
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   349
            self.ftp.voidcmd(cmd)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   350
        conn = None
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   351
        if file and not isdir:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   352
            # Use nlst to see if the file exists at all
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   353
            try:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   354
                self.ftp.nlst(file)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   355
            except ftplib.error_perm, reason:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   356
                raise IOError, ('ftp error', reason), sys.exc_info()[2]
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   357
            # Restore the transfer mode!
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   358
            self.ftp.voidcmd(cmd)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   359
            # Try to retrieve as a file
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   360
            try:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   361
                cmd = 'RETR ' + file
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   362
                conn = self.ftp.ntransfercmd(cmd, rest)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   363
            except ftplib.error_perm, reason:
674
6513ba7d858a Make consistent use of str.startswith() in conditionals.
chad.netzer@gmail.com
parents: 667
diff changeset
   364
                if str(reason).startswith('501'):
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   365
                    # workaround for REST not supported error
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   366
                    fp, retrlen = self.retrfile(file, type)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   367
                    fp = RangeableFileObject(fp, (rest,''))
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   368
                    return (fp, retrlen)
674
6513ba7d858a Make consistent use of str.startswith() in conditionals.
chad.netzer@gmail.com
parents: 667
diff changeset
   369
                elif not str(reason).startswith('550'):
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   370
                    raise IOError, ('ftp error', reason), sys.exc_info()[2]
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   371
        if not conn:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   372
            # Set transfer mode to ASCII!
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   373
            self.ftp.voidcmd('TYPE A')
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   374
            # Try a directory listing
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   375
            if file: cmd = 'LIST ' + file
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   376
            else: cmd = 'LIST'
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   377
            conn = self.ftp.ntransfercmd(cmd)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   378
        self.busy = 1
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   379
        # Pass back both a suitably decorated object and a retrieval length
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   380
        return (addclosehook(conn[0].makefile('rb'),
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   381
                            self.endtransfer), conn[1])
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   382
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   383
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   384
####################################################################
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   385
# Range Tuple Functions
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   386
# XXX: These range tuple functions might go better in a class.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   387
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   388
_rangere = None
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   389
def range_header_to_tuple(range_header):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   390
    """Get a (firstbyte,lastbyte) tuple from a Range header value.
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   391
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   392
    Range headers have the form "bytes=<firstbyte>-<lastbyte>". This
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   393
    function pulls the firstbyte and lastbyte values and returns
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   394
    a (firstbyte,lastbyte) tuple. If lastbyte is not specified in
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   395
    the header value, it is returned as an empty string in the
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   396
    tuple.
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   397
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   398
    Return None if range_header is None
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   399
    Return () if range_header does not conform to the range spec
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   400
    pattern.
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   401
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   402
    """
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   403
    global _rangere
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   404
    if range_header is None: return None
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   405
    if _rangere is None:
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   406
        import re
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   407
        _rangere = re.compile(r'^bytes=(\d{1,})-(\d*)')
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   408
    match = _rangere.match(range_header)
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   409
    if match:
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   410
        tup = range_tuple_normalize(match.group(1,2))
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   411
        if tup and tup[1]:
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   412
            tup = (tup[0],tup[1]+1)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   413
        return tup
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   414
    return ()
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   415
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   416
def range_tuple_to_header(range_tup):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   417
    """Convert a range tuple to a Range header value.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   418
    Return a string of the form "bytes=<firstbyte>-<lastbyte>" or None
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   419
    if no range is needed.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   420
    """
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   421
    if range_tup is None: return None
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   422
    range_tup = range_tuple_normalize(range_tup)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   423
    if range_tup:
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   424
        if range_tup[1]:
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   425
            range_tup = (range_tup[0],range_tup[1] - 1)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   426
        return 'bytes=%s-%s' % range_tup
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   427
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   428
def range_tuple_normalize(range_tup):
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   429
    """Normalize a (first_byte,last_byte) range tuple.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   430
    Return a tuple whose first element is guaranteed to be an int
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   431
    and whose second element will be '' (meaning: the last byte) or
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   432
    an int. Finally, return None if the normalized tuple == (0,'')
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   433
    as that is equivelant to retrieving the entire file.
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   434
    """
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   435
    if range_tup is None: return None
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   436
    # handle first byte
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   437
    fb = range_tup[0]
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   438
    if fb in (None,''): fb = 0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   439
    else: fb = int(fb)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   440
    # handle last byte
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   441
    try: lb = range_tup[1]
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   442
    except IndexError: lb = ''
575
7f5ce4bbdd7b More whitespace cleanups
mpm@selenic.com
parents: 0
diff changeset
   443
    else:
0
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   444
        if lb is None: lb = ''
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   445
        elif lb != '': lb = int(lb)
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   446
    # check if range is over the entire file
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   447
    if (fb,lb) == (0,''): return None
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   448
    # check that the range is valid
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   449
    if lb < fb: raise RangeError('Invalid byte range: %s-%s' % (fb,lb))
9117c6561b0b Add back links from file revisions to changeset revisions
mpm@selenic.com
parents:
diff changeset
   450
    return (fb,lb)