1394 |
1394 |
1395 class chunkbuffer(object): |
1395 class chunkbuffer(object): |
1396 """Allow arbitrary sized chunks of data to be efficiently read from an |
1396 """Allow arbitrary sized chunks of data to be efficiently read from an |
1397 iterator over chunks of arbitrary size.""" |
1397 iterator over chunks of arbitrary size.""" |
1398 |
1398 |
1399 def __init__(self, in_iter, targetsize = 2**16): |
1399 def __init__(self, in_iter): |
1400 """in_iter is the iterator that's iterating over the input chunks. |
1400 """in_iter is the iterator that's iterating over the input chunks. |
1401 targetsize is how big a buffer to try to maintain.""" |
1401 targetsize is how big a buffer to try to maintain.""" |
1402 self.in_iter = iter(in_iter) |
1402 self.in_iter = iter(in_iter) |
1403 self.buf = '' |
1403 self.buf = '' |
1404 self.targetsize = int(targetsize) |
1404 self.targetsize = 2**16 |
1405 if self.targetsize <= 0: |
|
1406 raise ValueError(_("targetsize must be greater than 0, was %d") % |
|
1407 targetsize) |
|
1408 self.iterempty = False |
1405 self.iterempty = False |
1409 |
|
1410 def fillbuf(self): |
|
1411 """Ignore target size; read every chunk from iterator until empty.""" |
|
1412 if not self.iterempty: |
|
1413 collector = cStringIO.StringIO() |
|
1414 collector.write(self.buf) |
|
1415 for ch in self.in_iter: |
|
1416 collector.write(ch) |
|
1417 self.buf = collector.getvalue() |
|
1418 self.iterempty = True |
|
1419 |
1406 |
1420 def read(self, l): |
1407 def read(self, l): |
1421 """Read L bytes of data from the iterator of chunks of data. |
1408 """Read L bytes of data from the iterator of chunks of data. |
1422 Returns less than L bytes if the iterator runs dry.""" |
1409 Returns less than L bytes if the iterator runs dry.""" |
1423 if l > len(self.buf) and not self.iterempty: |
1410 if l > len(self.buf) and not self.iterempty: |