Mercurial > hg > gitweb
comparison mercurial/revlog.py @ 2255:3f38e872f39a
Reduce ram used for very large inlined index files
During a clone, an inline index is not converted to a split index
file until the very end. When the conversion happens, the index
can be very large, and the inline index loading functions always load
the entire index file into ram.
This changes the revlog code to read the index in smaller chunks.
| author | mason@suse.com |
|---|---|
| date | Wed, 10 May 2006 15:26:56 -0700 |
| parents | 45aef5ddcdbe |
| children | 854954fd410a |
comparison
equal
deleted
inserted
replaced
| 2254:827fcfe88b14 | 2255:3f38e872f39a |
|---|---|
| 374 # big index, let's parse it on demand | 374 # big index, let's parse it on demand |
| 375 parser = lazyparser(f, st.st_size, self.indexformat, shaoffset) | 375 parser = lazyparser(f, st.st_size, self.indexformat, shaoffset) |
| 376 self.index = lazyindex(parser) | 376 self.index = lazyindex(parser) |
| 377 self.nodemap = lazymap(parser) | 377 self.nodemap = lazymap(parser) |
| 378 else: | 378 else: |
| 379 i = f.read() | 379 self.parseindex(f, st) |
| 380 self.parseindex(i) | |
| 381 if self.inlinedata(): | |
| 382 # we've already got the entire data file read in, save it | |
| 383 # in the chunk data | |
| 384 self.chunkcache = (0, i) | |
| 385 if self.version != REVLOGV0: | 380 if self.version != REVLOGV0: |
| 386 e = list(self.index[0]) | 381 e = list(self.index[0]) |
| 387 type = self.ngtype(e[0]) | 382 type = self.ngtype(e[0]) |
| 388 e[0] = self.offset_type(0, type) | 383 e[0] = self.offset_type(0, type) |
| 389 self.index[0] = e | 384 self.index[0] = e |
| 390 else: | 385 else: |
| 391 self.nodemap = { nullid: -1} | 386 self.nodemap = { nullid: -1} |
| 392 self.index = [] | 387 self.index = [] |
| 393 | 388 |
| 394 | 389 |
| 395 def parseindex(self, data): | 390 def parseindex(self, fp, st): |
| 396 s = struct.calcsize(self.indexformat) | 391 s = struct.calcsize(self.indexformat) |
| 397 l = len(data) | |
| 398 self.index = [] | 392 self.index = [] |
| 399 self.nodemap = {nullid: -1} | 393 self.nodemap = {nullid: -1} |
| 400 inline = self.inlinedata() | 394 inline = self.inlinedata() |
| 401 off = 0 | |
| 402 n = 0 | 395 n = 0 |
| 403 while off < l: | 396 leftover = None |
| 404 e = struct.unpack(self.indexformat, data[off:off + s]) | 397 while True: |
| 405 self.index.append(e) | 398 if st: |
| 406 self.nodemap[e[-1]] = n | 399 data = fp.read(65536) |
| 407 n += 1 | 400 else: |
| 408 off += s | 401 # hack for httprangereader, it doesn't do partial reads well |
| 409 if inline: | 402 data = fp.read() |
| 410 off += e[1] | 403 if not data: |
| 404 break | |
| 405 if n == 0 and self.inlinedata(): | |
| 406 # cache the first chunk | |
| 407 self.chunkcache = (0, data) | |
| 408 off = 0 | |
| 409 l = len(data) | |
| 410 while off < l: | |
| 411 if l - off < s: | |
| 412 leftover = data[off:] | |
| 413 break | |
| 414 if leftover: | |
| 415 cur = leftover + data[off:off + s - len(leftover)] | |
| 416 off += s - len(leftover) | |
| 417 leftover = None | |
| 418 else: | |
| 419 cur = data[off:off + s] | |
| 420 off += s | |
| 421 e = struct.unpack(self.indexformat, cur) | |
| 422 self.index.append(e) | |
| 423 self.nodemap[e[-1]] = n | |
| 424 n += 1 | |
| 425 if inline: | |
| 426 off += e[1] | |
| 427 if off > l: | |
| 428 # some things don't seek well, just read it | |
| 429 fp.read(off - l) | |
| 430 if not st: | |
| 431 break | |
| 432 | |
| 411 | 433 |
| 412 def ngoffset(self, q): | 434 def ngoffset(self, q): |
| 413 if q & 0xFFFF: | 435 if q & 0xFFFF: |
| 414 raise RevlogError(_('%s: incompatible revision flag %x') % | 436 raise RevlogError(_('%s: incompatible revision flag %x') % |
| 415 (self.indexfile, q)) | 437 (self.indexfile, q)) |