1397 iterator over chunks of arbitrary size.""" |
1397 iterator over chunks of arbitrary size.""" |
1398 |
1398 |
1399 def __init__(self, in_iter): |
1399 def __init__(self, in_iter): |
1400 """in_iter is the iterator that's iterating over the input chunks. |
1400 """in_iter is the iterator that's iterating over the input chunks. |
1401 targetsize is how big a buffer to try to maintain.""" |
1401 targetsize is how big a buffer to try to maintain.""" |
1402 self.in_iter = iter(in_iter) |
1402 self.iter = iter(in_iter) |
1403 self.buf = '' |
1403 self.buf = '' |
1404 self.targetsize = 2**16 |
1404 self.targetsize = 2**16 |
1405 self.iterempty = False |
|
1406 |
1405 |
1407 def read(self, l): |
1406 def read(self, l): |
1408 """Read L bytes of data from the iterator of chunks of data. |
1407 """Read L bytes of data from the iterator of chunks of data. |
1409 Returns less than L bytes if the iterator runs dry.""" |
1408 Returns less than L bytes if the iterator runs dry.""" |
1410 if l > len(self.buf) and not self.iterempty: |
1409 if l > len(self.buf) and self.iter: |
1411 # Clamp to a multiple of self.targetsize |
1410 # Clamp to a multiple of self.targetsize |
1412 targetsize = self.targetsize * ((l // self.targetsize) + 1) |
1411 targetsize = self.targetsize * ((l // self.targetsize) + 1) |
1413 collector = cStringIO.StringIO() |
1412 collector = cStringIO.StringIO() |
1414 collector.write(self.buf) |
1413 collector.write(self.buf) |
1415 collected = len(self.buf) |
1414 collected = len(self.buf) |
1416 for chunk in self.in_iter: |
1415 for chunk in self.iter: |
1417 collector.write(chunk) |
1416 collector.write(chunk) |
1418 collected += len(chunk) |
1417 collected += len(chunk) |
1419 if collected >= targetsize: |
1418 if collected >= targetsize: |
1420 break |
1419 break |
1421 if collected < targetsize: |
1420 if collected < targetsize: |
1422 self.iterempty = True |
1421 self.iter = False |
1423 self.buf = collector.getvalue() |
1422 self.buf = collector.getvalue() |
1424 s, self.buf = self.buf[:l], buffer(self.buf, l) |
1423 s, self.buf = self.buf[:l], buffer(self.buf, l) |
1425 return s |
1424 return s |
1426 |
1425 |
1427 def filechunkiter(f, size=65536, limit=None): |
1426 def filechunkiter(f, size=65536, limit=None): |