comparison mercurial/localrepo.py @ 1998:65cc17ae9649

fix race in localrepo.addchangegroup. localrepo.addchangegroup writes to changelog, then manifest, then normal files. this breaks access ordering. if reader reads changelog while manifest is being written, can find pointers into places in manifest that are not yet written. same can happen for manifest and normal files. fix is to make almost no change to localrepo.addchangegroup. it must to write changelog and manifest data early because it has to read them while writing other files. instead, write changelog and manifest data to temp file that reader cannot see, then append temp data to manifest after all normal files written, finally append temp data to changelog. temp file code is in new appendfile module. can be used in other places with small changes. much smaller race still left. we write all new data in one write call, but reader can maybe see partial update because python or os or filesystem cannot always make write really atomic. file locking no help: slow, not portable, not reliable over nfs. only real safe other plan is write to temp file every time and rename, but performance bad when manifest or changelog is big.
author Vadim Gelfer <vadim.gelfer@gmail.com>
date Fri, 24 Mar 2006 09:08:12 -0800
parents 2da2d46862fb
children ff5c9a92f556 ced2d3620f95
comparison
equal deleted inserted replaced
1997:802e8a029d99 1998:65cc17ae9649
9 import filelog, manifest, changelog, dirstate, repo 9 import filelog, manifest, changelog, dirstate, repo
10 from node import * 10 from node import *
11 from i18n import gettext as _ 11 from i18n import gettext as _
12 from demandload import * 12 from demandload import *
13 demandload(globals(), "re lock transaction tempfile stat mdiff errno ui") 13 demandload(globals(), "re lock transaction tempfile stat mdiff errno ui")
14 demandload(globals(), "changegroup") 14 demandload(globals(), "appendfile changegroup")
15 15
16 class localrepository(object): 16 class localrepository(object):
17 def __del__(self): 17 def __del__(self):
18 self.transhandle = None 18 self.transhandle = None
19 def __init__(self, parentui, path=None, create=0): 19 def __init__(self, parentui, path=None, create=0):
1341 1341
1342 def addchangegroup(self, source): 1342 def addchangegroup(self, source):
1343 1343
1344 def csmap(x): 1344 def csmap(x):
1345 self.ui.debug(_("add changeset %s\n") % short(x)) 1345 self.ui.debug(_("add changeset %s\n") % short(x))
1346 return self.changelog.count() 1346 return cl.count()
1347 1347
1348 def revmap(x): 1348 def revmap(x):
1349 return self.changelog.rev(x) 1349 return cl.rev(x)
1350 1350
1351 if not source: 1351 if not source:
1352 return 1352 return
1353 1353
1354 self.hook('prechangegroup', throw=True) 1354 self.hook('prechangegroup', throw=True)
1355 1355
1356 changesets = files = revisions = 0 1356 changesets = files = revisions = 0
1357 1357
1358 tr = self.transaction() 1358 tr = self.transaction()
1359 1359
1360 oldheads = len(self.changelog.heads()) 1360 # write changelog and manifest data to temp files so
1361 # concurrent readers will not see inconsistent view
1362 cl = appendfile.appendchangelog(self.opener)
1363
1364 oldheads = len(cl.heads())
1361 1365
1362 # pull off the changeset group 1366 # pull off the changeset group
1363 self.ui.status(_("adding changesets\n")) 1367 self.ui.status(_("adding changesets\n"))
1364 co = self.changelog.tip() 1368 co = cl.tip()
1365 chunkiter = changegroup.chunkiter(source) 1369 chunkiter = changegroup.chunkiter(source)
1366 cn = self.changelog.addgroup(chunkiter, csmap, tr, 1) # unique 1370 cn = cl.addgroup(chunkiter, csmap, tr, 1) # unique
1367 cnr, cor = map(self.changelog.rev, (cn, co)) 1371 cnr, cor = map(cl.rev, (cn, co))
1368 if cn == nullid: 1372 if cn == nullid:
1369 cnr = cor 1373 cnr = cor
1370 changesets = cnr - cor 1374 changesets = cnr - cor
1371 1375
1376 mf = appendfile.appendmanifest(self.opener)
1377
1372 # pull off the manifest group 1378 # pull off the manifest group
1373 self.ui.status(_("adding manifests\n")) 1379 self.ui.status(_("adding manifests\n"))
1374 mm = self.manifest.tip() 1380 mm = mf.tip()
1375 chunkiter = changegroup.chunkiter(source) 1381 chunkiter = changegroup.chunkiter(source)
1376 mo = self.manifest.addgroup(chunkiter, revmap, tr) 1382 mo = mf.addgroup(chunkiter, revmap, tr)
1377 1383
1378 # process the files 1384 # process the files
1379 self.ui.status(_("adding file changes\n")) 1385 self.ui.status(_("adding file changes\n"))
1380 while 1: 1386 while 1:
1381 f = changegroup.getchunk(source) 1387 f = changegroup.getchunk(source)
1386 o = fl.count() 1392 o = fl.count()
1387 chunkiter = changegroup.chunkiter(source) 1393 chunkiter = changegroup.chunkiter(source)
1388 n = fl.addgroup(chunkiter, revmap, tr) 1394 n = fl.addgroup(chunkiter, revmap, tr)
1389 revisions += fl.count() - o 1395 revisions += fl.count() - o
1390 files += 1 1396 files += 1
1397
1398 # write order here is important so concurrent readers will see
1399 # consistent view of repo
1400 mf.writedata()
1401 cl.writedata()
1402
1403 # make changelog and manifest see real files again
1404 self.changelog = changelog.changelog(self.opener)
1405 self.manifest = manifest.manifest(self.opener)
1391 1406
1392 newheads = len(self.changelog.heads()) 1407 newheads = len(self.changelog.heads())
1393 heads = "" 1408 heads = ""
1394 if oldheads and newheads > oldheads: 1409 if oldheads and newheads > oldheads:
1395 heads = _(" (+%d heads)") % (newheads - oldheads) 1410 heads = _(" (+%d heads)") % (newheads - oldheads)