mercurial/packagescan.py
author Vadim Gelfer <vadim.gelfer@gmail.com>
Fri, 14 Jul 2006 11:17:22 -0700
changeset 2612 ffb895f16925
parent 2600 c4325f0a9b91
child 2858 345bac2bc4ec
child 2981 0171a5432621
permissions -rw-r--r--
add support for streaming clone. existing clone code uses pull to get changes from remote repo. is very slow, uses lots of memory and cpu. new clone code has server write file data straight to client, client writes file data straight to disk. memory and cpu used are very low, clone is much faster over lan. new client can still clone with pull, can still clone from older servers. new server can still serve older clients.
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
1308
2073e5a71008 Cleanup of tabs and trailing spaces.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 1283
diff changeset
     1
# packagescan.py - Helper module for identifing used modules.
1283
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
     2
# Used for the py2exe distutil.
2323
c58a403aa830 setup.py: install packagescan before any mercurial modules is imported
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
     3
# This module must be the first mercurial module imported in setup.py
1283
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
     4
#
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
     5
# Copyright 2005 Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
     6
#
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
     7
# This software may be used and distributed according to the terms
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
     8
# of the GNU General Public License, incorporated herein by reference.
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
     9
import glob
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    10
import os
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    11
import sys
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    12
import ihooks
2322
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    13
import types
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    14
import string
1283
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    15
2323
c58a403aa830 setup.py: install packagescan before any mercurial modules is imported
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    16
# Install this module as fake demandload module
c58a403aa830 setup.py: install packagescan before any mercurial modules is imported
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    17
sys.modules['mercurial.demandload'] = sys.modules[__name__]
c58a403aa830 setup.py: install packagescan before any mercurial modules is imported
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    18
c58a403aa830 setup.py: install packagescan before any mercurial modules is imported
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    19
# Requiredmodules contains the modules imported by demandload.
2600
c4325f0a9b91 clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2497
diff changeset
    20
# Please note that demandload can be invoked before the
2323
c58a403aa830 setup.py: install packagescan before any mercurial modules is imported
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    21
# mercurial.packagescan.scan method is invoked in case a mercurial
c58a403aa830 setup.py: install packagescan before any mercurial modules is imported
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    22
# module is imported.
2600
c4325f0a9b91 clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2497
diff changeset
    23
requiredmodules = {}
1283
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    24
def demandload(scope, modules):
2600
c4325f0a9b91 clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2497
diff changeset
    25
    """ fake demandload function that collects the required modules
2322
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    26
        foo            import foo
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    27
        foo bar        import foo, bar
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    28
        foo.bar        import foo.bar
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    29
        foo:bar        from foo import bar
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    30
        foo:bar,quux   from foo import bar, quux
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    31
        foo.bar:quux   from foo.bar import quux"""
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    32
1283
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    33
    for m in modules.split():
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    34
        mod = None
1841
7f12a63568ae Fix py2exe packagescan problem with new demandload
Eung-ju Park <eungju@gmail.com>
parents: 1308
diff changeset
    35
        try:
2322
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    36
            module, fromlist = m.split(':')
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    37
            fromlist = fromlist.split(',')
1841
7f12a63568ae Fix py2exe packagescan problem with new demandload
Eung-ju Park <eungju@gmail.com>
parents: 1308
diff changeset
    38
        except:
7f12a63568ae Fix py2exe packagescan problem with new demandload
Eung-ju Park <eungju@gmail.com>
parents: 1308
diff changeset
    39
            module = m
2322
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    40
            fromlist = []
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    41
        mod = __import__(module, scope, scope, fromlist)
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    42
        if fromlist == []:
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    43
            # mod is only the top package, but we need all packages
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    44
            comp = module.split('.')
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    45
            i = 1
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    46
            mn = comp[0]
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    47
            while True:
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    48
                # mn and mod.__name__ might not be the same
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    49
                scope[mn] = mod
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    50
                requiredmodules[mod.__name__] = 1
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    51
                if len(comp) == i: break
2600
c4325f0a9b91 clean up trailing white space.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2497
diff changeset
    52
                mod = getattr(mod,comp[i])
2322
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    53
                mn = string.join(comp[:i+1],'.')
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    54
                i += 1
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    55
        else:
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    56
            # mod is the last package in the component list
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    57
            requiredmodules[mod.__name__] = 1
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    58
            for f in fromlist:
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    59
                scope[f] = getattr(mod,f)
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    60
                if type(scope[f]) == types.ModuleType:
685597676a13 packagescan: handle demandload module naming changes.
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    61
                    requiredmodules[scope[f].__name__] = 1
1283
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    62
2497
976b6b2a1613 do not try to package lsprof if not available.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2402
diff changeset
    63
class SkipPackage(Exception):
976b6b2a1613 do not try to package lsprof if not available.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2402
diff changeset
    64
    def __init__(self, reason):
976b6b2a1613 do not try to package lsprof if not available.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2402
diff changeset
    65
        self.reason = reason
976b6b2a1613 do not try to package lsprof if not available.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2402
diff changeset
    66
976b6b2a1613 do not try to package lsprof if not available.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2402
diff changeset
    67
scan_in_progress = False
976b6b2a1613 do not try to package lsprof if not available.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2402
diff changeset
    68
2323
c58a403aa830 setup.py: install packagescan before any mercurial modules is imported
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
    69
def scan(libpath,packagename):
1283
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    70
    """ helper for finding all required modules of package <packagename> """
2497
976b6b2a1613 do not try to package lsprof if not available.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2402
diff changeset
    71
    global scan_in_progress
976b6b2a1613 do not try to package lsprof if not available.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2402
diff changeset
    72
    scan_in_progress = True
1283
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    73
    # Use the package in the build directory
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    74
    libpath = os.path.abspath(libpath)
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    75
    sys.path.insert(0,libpath)
2402
82cef38fea56 packagescan can't scan nested packages
Eung-Ju PARK <eungju@gmail.com>
parents: 2325
diff changeset
    76
    packdir = os.path.join(libpath,packagename.replace('.', '/'))
1308
2073e5a71008 Cleanup of tabs and trailing spaces.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 1283
diff changeset
    77
    # A normal import would not find the package in
1283
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    78
    # the build directory. ihook is used to force the import.
1308
2073e5a71008 Cleanup of tabs and trailing spaces.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 1283
diff changeset
    79
    # After the package is imported the import scope for
1283
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    80
    # the following imports is settled.
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    81
    p = importfrom(packdir)
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    82
    globals()[packagename] = p
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    83
    sys.modules[packagename] = p
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    84
    # Fetch the python modules in the package
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    85
    cwd = os.getcwd()
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    86
    os.chdir(packdir)
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    87
    pymodulefiles = glob.glob('*.py')
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    88
    extmodulefiles = glob.glob('*.pyd')
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    89
    os.chdir(cwd)
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    90
    # Import all python modules and by that run the fake demandload
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    91
    for m in pymodulefiles:
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    92
        if m == '__init__.py': continue
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    93
        tmp = {}
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
    94
        mname,ext = os.path.splitext(m)
1308
2073e5a71008 Cleanup of tabs and trailing spaces.
Thomas Arendsen Hein <thomas@intevation.de>
parents: 1283
diff changeset
    95
        fullname = packagename+'.'+mname
2497
976b6b2a1613 do not try to package lsprof if not available.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2402
diff changeset
    96
        try:
976b6b2a1613 do not try to package lsprof if not available.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2402
diff changeset
    97
            __import__(fullname,tmp,tmp)
976b6b2a1613 do not try to package lsprof if not available.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2402
diff changeset
    98
        except SkipPackage, inst:
976b6b2a1613 do not try to package lsprof if not available.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2402
diff changeset
    99
            print >> sys.stderr, 'skipping %s: %s' % (fullname, inst.reason)
976b6b2a1613 do not try to package lsprof if not available.
Vadim Gelfer <vadim.gelfer@gmail.com>
parents: 2402
diff changeset
   100
            continue
1283
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   101
        requiredmodules[fullname] = 1
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   102
    # Import all extension modules and by that run the fake demandload
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   103
    for m in extmodulefiles:
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   104
        tmp = {}
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   105
        mname,ext = os.path.splitext(m)
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   106
        fullname = packagename+'.'+mname
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   107
        __import__(fullname,tmp,tmp)
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   108
        requiredmodules[fullname] = 1
2323
c58a403aa830 setup.py: install packagescan before any mercurial modules is imported
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
   109
c58a403aa830 setup.py: install packagescan before any mercurial modules is imported
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
   110
def getmodules():
c58a403aa830 setup.py: install packagescan before any mercurial modules is imported
Volker Kleinfeld <Volker.Kleinfeld@gmx.de>
parents: 1841
diff changeset
   111
    return requiredmodules.keys()
1283
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   112
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   113
def importfrom(filename):
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   114
    """
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   115
    import module/package from a named file and returns the module.
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   116
    It does not check on sys.modules or includes the module in the scope.
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   117
    """
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   118
    loader = ihooks.BasicModuleLoader()
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   119
    path, file = os.path.split(filename)
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   120
    name, ext  = os.path.splitext(file)
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   121
    m = loader.find_module_in_dir(name, path)
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   122
    if not m:
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   123
        raise ImportError, name
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   124
    m = loader.load_module(name, m)
f5faab34f32e Support for the distutils extention 'py2exe' added.
Volker.Kleinfeld@gmx.de
parents:
diff changeset
   125
    return m