# HG changeset patch # User Vadim Gelfer # Date 1148337769 25200 # Node ID 3f24bc5dee81e35fd9bc7816bbf3b9af6daf7e9a # Parent f77edcffb83729c9644e6d15e25bf340fbc2e7d9 http: fix many problems with url parsing and auth. added proxy test. problems fixed: - https scheme handled properly for real and proxy urls. - url of form "http://user:password@host:port/path" now ok. - no-proxy check uses proper host names. diff --git a/mercurial/httprepo.py b/mercurial/httprepo.py --- a/mercurial/httprepo.py +++ b/mercurial/httprepo.py @@ -22,6 +22,9 @@ class passwordmgr(urllib2.HTTPPasswordMg if authinfo != (None, None): return authinfo + if not ui.interactive: + raise util.Abort(_('http authorization required')) + self.ui.write(_("http authorization required\n")) self.ui.status(_("realm: %s\n") % realm) user = self.ui.prompt(_("user:"), default=None) @@ -30,37 +33,95 @@ class passwordmgr(urllib2.HTTPPasswordMg self.add_password(realm, authuri, user, passwd) return (user, passwd) +def netlocsplit(netloc): + '''split [user[:passwd]@]host[:port] into 4-tuple.''' + + a = netloc.find('@') + if a == -1: + user, passwd = None, None + else: + userpass, netloc = netloc[:a], netloc[a+1:] + c = userpass.find(':') + if c == -1: + user, passwd = urllib.unquote(userpass), None + else: + user = urllib.unquote(userpass[:c]) + passwd = urllib.unquote(userpass[c+1:]) + c = netloc.find(':') + if c == -1: + host, port = netloc, None + else: + host, port = netloc[:c], netloc[c+1:] + return host, port, user, passwd + +def netlocunsplit(host, port, user=None, passwd=None): + '''turn host, port, user, passwd into [user[:passwd]@]host[:port].''' + if port: + hostport = host + ':' + port + else: + hostport = host + if user: + if passwd: + userpass = urllib.quote(user) + ':' + urllib.quote(passwd) + else: + userpass = urllib.quote(user) + return userpass + '@' + hostport + return hostport + class httprepository(remoterepository): def __init__(self, ui, path): - # fix missing / after hostname - s = urlparse.urlsplit(path) - partial = s[2] - if not partial: partial = "/" - self.url = urlparse.urlunsplit((s[0], s[1], partial, '', '')) + scheme, netloc, urlpath, query, frag = urlparse.urlsplit(path) + if query or frag: + raise util.Abort(_('unsupported URL component: "%s"') % + (query or frag)) + if not urlpath: urlpath = '/' + host, port, user, passwd = netlocsplit(netloc) + + # urllib cannot handle URLs with embedded user or passwd + self.url = urlparse.urlunsplit((scheme, netlocunsplit(host, port), + urlpath, '', '')) self.ui = ui - no_list = [ "localhost", "127.0.0.1" ] - host = ui.config("http_proxy", "host") - if host is None: - host = os.environ.get("http_proxy") - if host and host.startswith('http://'): - host = host[7:] - user = ui.config("http_proxy", "user") - passwd = ui.config("http_proxy", "passwd") - no = ui.config("http_proxy", "no") - if no is None: - no = os.environ.get("no_proxy") - if no: - no_list = no_list + no.split(",") + + proxyurl = ui.config("http_proxy", "host") or os.getenv('http_proxy') + proxyauthinfo = None + handler = urllib2.BaseHandler() + + if proxyurl: + # proxy can be proper url or host[:port] + if not (proxyurl.startswith('http:') or + proxyurl.startswith('https:')): + proxyurl = 'http://' + proxyurl + '/' + snpqf = urlparse.urlsplit(proxyurl) + proxyscheme, proxynetloc, proxypath, proxyquery, proxyfrag = snpqf + hpup = netlocsplit(proxynetloc) + + proxyhost, proxyport, proxyuser, proxypasswd = hpup + if not proxyuser: + proxyuser = ui.config("http_proxy", "user") + proxypasswd = ui.config("http_proxy", "passwd") - no_proxy = 0 - for h in no_list: - if (path.startswith("http://" + h + "/") or - path.startswith("http://" + h + ":") or - path == "http://" + h): - no_proxy = 1 + # see if we should use a proxy for this url + no_list = [ "localhost", "127.0.0.1" ] + no_list.extend([p.strip().lower() for + p in ui.config("http_proxy", "no", '').split(',') + if p.strip()]) + no_list.extend([p.strip().lower() for + p in os.getenv("no_proxy", '').split(',') + if p.strip()]) + # "http_proxy.always" config is for running tests on localhost + if (not ui.configbool("http_proxy", "always") and + host.lower() in no_list): + ui.debug(_('disabling proxy for %s\n') % host) + else: + proxyurl = urlparse.urlunsplit(( + proxyscheme, netlocunsplit(proxyhost, proxyport, + proxyuser, proxypasswd or ''), + proxypath, proxyquery, proxyfrag)) + handler = urllib2.ProxyHandler({scheme: proxyurl}) + ui.debug(_('proxying through %s\n') % proxyurl) - # Note: urllib2 takes proxy values from the environment and those will - # take precedence + # urllib2 takes proxy values from the environment and those + # will take precedence if found, so drop them for env in ["HTTP_PROXY", "http_proxy", "no_proxy"]: try: if os.environ.has_key(env): @@ -68,24 +129,15 @@ class httprepository(remoterepository): except OSError: pass - proxy_handler = urllib2.BaseHandler() - if host and not no_proxy: - proxy_handler = urllib2.ProxyHandler({"http" : "http://" + host}) + passmgr = passwordmgr(ui) + if user: + ui.debug(_('will use user %s for http auth\n') % user) + passmgr.add_password(None, host, user, passwd or '') - proxyauthinfo = None - if user and passwd: - passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm() - passmgr.add_password(None, host, user, passwd) - proxyauthinfo = urllib2.ProxyBasicAuthHandler(passmgr) - - if ui.interactive: - passmgr = passwordmgr(ui) - opener = urllib2.build_opener( - proxy_handler, proxyauthinfo, - urllib2.HTTPBasicAuthHandler(passmgr), - urllib2.HTTPDigestAuthHandler(passmgr)) - else: - opener = urllib2.build_opener(proxy_handler, proxyauthinfo) + opener = urllib2.build_opener( + handler, + urllib2.HTTPBasicAuthHandler(passmgr), + urllib2.HTTPDigestAuthHandler(passmgr)) # 1.0 here is the _protocol_ version opener.addheaders = [('User-agent', 'mercurial/proto-1.0')] diff --git a/tests/test-http-proxy b/tests/test-http-proxy new file mode 100755 --- /dev/null +++ b/tests/test-http-proxy @@ -0,0 +1,30 @@ +#!/bin/sh + +hg init a +cd a +echo a > a +hg ci -Ama -d '1123456789 0' +hg serve -p 20059 -d --pid-file=hg.pid + +cd .. +("$TESTDIR/tinyproxy.py" 20060 localhost >/dev/null 2>&1 proxy.pid) +sleep 2 + +echo %% url for proxy +http_proxy=http://localhost:20060/ hg --config http_proxy.always=True clone http://localhost:20059/ b + +echo %% host:port for proxy +http_proxy=localhost:20060 hg clone --config http_proxy.always=True http://localhost:20059/ c + +echo %% proxy url with user name and password +http_proxy=http://user:passwd@localhost:20060 hg clone --config http_proxy.always=True http://localhost:20059/ d + +echo %% url with user name and password +http_proxy=http://user:passwd@localhost:20060 hg clone --config http_proxy.always=True http://user:passwd@localhost:20059/ e + +echo %% bad host:port for proxy +http_proxy=localhost:20061 hg clone --config http_proxy.always=True http://localhost:20059/ f + +kill $(cat proxy.pid a/hg.pid) +exit 0 diff --git a/tests/test-http-proxy.out b/tests/test-http-proxy.out new file mode 100644 --- /dev/null +++ b/tests/test-http-proxy.out @@ -0,0 +1,31 @@ +adding a +%% url for proxy +requesting all changes +adding changesets +adding manifests +adding file changes +added 1 changesets with 1 changes to 1 files +1 files updated, 0 files merged, 0 files removed, 0 files unresolved +%% host:port for proxy +requesting all changes +adding changesets +adding manifests +adding file changes +added 1 changesets with 1 changes to 1 files +1 files updated, 0 files merged, 0 files removed, 0 files unresolved +%% proxy url with user name and password +requesting all changes +adding changesets +adding manifests +adding file changes +added 1 changesets with 1 changes to 1 files +1 files updated, 0 files merged, 0 files removed, 0 files unresolved +%% url with user name and password +requesting all changes +adding changesets +adding manifests +adding file changes +added 1 changesets with 1 changes to 1 files +1 files updated, 0 files merged, 0 files removed, 0 files unresolved +%% bad host:port for proxy +abort: error: Connection refused diff --git a/tests/tinyproxy.py b/tests/tinyproxy.py new file mode 100755 --- /dev/null +++ b/tests/tinyproxy.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python + +__doc__ = """Tiny HTTP Proxy. + +This module implements GET, HEAD, POST, PUT and DELETE methods +on BaseHTTPServer, and behaves as an HTTP proxy. The CONNECT +method is also implemented experimentally, but has not been +tested yet. + +Any help will be greatly appreciated. SUZUKI Hisao +""" + +__version__ = "0.2.1" + +import BaseHTTPServer, select, socket, SocketServer, urlparse + +class ProxyHandler (BaseHTTPServer.BaseHTTPRequestHandler): + __base = BaseHTTPServer.BaseHTTPRequestHandler + __base_handle = __base.handle + + server_version = "TinyHTTPProxy/" + __version__ + rbufsize = 0 # self.rfile Be unbuffered + + def handle(self): + (ip, port) = self.client_address + if hasattr(self, 'allowed_clients') and ip not in self.allowed_clients: + self.raw_requestline = self.rfile.readline() + if self.parse_request(): self.send_error(403) + else: + self.__base_handle() + + def _connect_to(self, netloc, soc): + i = netloc.find(':') + if i >= 0: + host_port = netloc[:i], int(netloc[i+1:]) + else: + host_port = netloc, 80 + print "\t" "connect to %s:%d" % host_port + try: soc.connect(host_port) + except socket.error, arg: + try: msg = arg[1] + except: msg = arg + self.send_error(404, msg) + return 0 + return 1 + + def do_CONNECT(self): + soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + if self._connect_to(self.path, soc): + self.log_request(200) + self.wfile.write(self.protocol_version + + " 200 Connection established\r\n") + self.wfile.write("Proxy-agent: %s\r\n" % self.version_string()) + self.wfile.write("\r\n") + self._read_write(soc, 300) + finally: + print "\t" "bye" + soc.close() + self.connection.close() + + def do_GET(self): + (scm, netloc, path, params, query, fragment) = urlparse.urlparse( + self.path, 'http') + if scm != 'http' or fragment or not netloc: + self.send_error(400, "bad url %s" % self.path) + return + soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + if self._connect_to(netloc, soc): + self.log_request() + soc.send("%s %s %s\r\n" % ( + self.command, + urlparse.urlunparse(('', '', path, params, query, '')), + self.request_version)) + self.headers['Connection'] = 'close' + del self.headers['Proxy-Connection'] + for key_val in self.headers.items(): + soc.send("%s: %s\r\n" % key_val) + soc.send("\r\n") + self._read_write(soc) + finally: + print "\t" "bye" + soc.close() + self.connection.close() + + def _read_write(self, soc, max_idling=20): + iw = [self.connection, soc] + ow = [] + count = 0 + while 1: + count += 1 + (ins, _, exs) = select.select(iw, ow, iw, 3) + if exs: break + if ins: + for i in ins: + if i is soc: + out = self.connection + else: + out = soc + data = i.recv(8192) + if data: + out.send(data) + count = 0 + else: + print "\t" "idle", count + if count == max_idling: break + + do_HEAD = do_GET + do_POST = do_GET + do_PUT = do_GET + do_DELETE=do_GET + +class ThreadingHTTPServer (SocketServer.ThreadingMixIn, + BaseHTTPServer.HTTPServer): pass + +if __name__ == '__main__': + from sys import argv + if argv[1:] and argv[1] in ('-h', '--help'): + print argv[0], "[port [allowed_client_name ...]]" + else: + if argv[2:]: + allowed = [] + for name in argv[2:]: + client = socket.gethostbyname(name) + allowed.append(client) + print "Accept: %s (%s)" % (client, name) + ProxyHandler.allowed_clients = allowed + del argv[2:] + else: + print "Any clients will be served..." + BaseHTTPServer.test(ProxyHandler, ThreadingHTTPServer)