Mercurial > dovecot > original-hg > dovecot-1.2

changeset 6944:66b3e894041b HEAD

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Support mmap_disable=yes and some error handling improvements.
author Timo Sirainen <tss@iki.fi>
date Fri, 07 Dec 2007 16:10:19 +0200
parents 0a93c4e07776
children 5915aea5f070
files src/plugins/fts-squat/squat-trie-private.h src/plugins/fts-squat/squat-trie.c src/plugins/fts-squat/squat-uidlist.c
diffstat 3 files changed, 270 insertions(+), 90 deletions(-) [+]
line wrap: on
line diff
--- a/src/plugins/fts-squat/squat-trie-private.h	Fri Dec 07 00:46:31 2007 +0200
+++ b/src/plugins/fts-squat/squat-trie-private.h	Fri Dec 07 16:10:19 2007 +0200
@@ -118,13 +118,18 @@
 
 	char *path;
 	int fd;
+	struct file_cache *file_cache;
+
 	uoff_t locked_file_size;
+	const void *data;
+	size_t data_size;
 
 	void *mmap_base;
 	size_t mmap_size;
 
 	unsigned char default_normalize_map[256];
 
+	unsigned int mmap_disable:1;
 	unsigned int corrupted:1;
 };
 
--- a/src/plugins/fts-squat/squat-trie.c	Fri Dec 07 00:46:31 2007 +0200
+++ b/src/plugins/fts-squat/squat-trie.c	Fri Dec 07 16:10:19 2007 +0200
@@ -3,9 +3,11 @@
 #include "lib.h"
 #include "array.h"
 #include "str.h"
+#include "read-full.h"
 #include "istream.h"
 #include "ostream.h"
 #include "unichar.h"
+#include "file-cache.h"
 #include "seq-range-array.h"
 #include "squat-uidlist.h"
 #include "squat-trie-private.h"
@@ -13,9 +15,6 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
-#include <ctype.h>
-#include <fcntl.h>
-#include <time.h>
 #include <sys/mman.h>
 
 #define DEFAULT_NORMALIZE_MAP_CHARS \
@@ -26,6 +25,11 @@
 #define MAX_FAST_LEVEL 3
 #define SEQUENTIAL_COUNT 46
 
+#define TRIE_BYTES_LEFT(n) \
+	((n) * SQUAT_PACK_MAX_SIZE)
+#define TRIE_READAHEAD_SIZE \
+	I_MAX(4096, 1 + 256 + TRIE_BYTES_LEFT(256))
+
 struct squat_trie_build_context {
 	struct squat_trie *trie;
 	struct ostream *output;
@@ -130,6 +134,7 @@
 	trie->fd = -1;
 	trie->lock_method = lock_method;
 	trie->uidvalidity = uidvalidity;
+	trie->mmap_disable = mmap_disable;
 	squat_trie_normalize_map_build(trie);
 	return trie;
 }
@@ -141,6 +146,11 @@
 	memset(&trie->root, 0, sizeof(trie->root));
 	memset(&trie->hdr, 0, sizeof(trie->hdr));
 
+	trie->data = NULL;
+	trie->data_size = 0;
+
+	if (trie->file_cache != NULL)
+		file_cache_free(&trie->file_cache);
 	if (trie->mmap_size != 0) {
 		if (munmap(trie->mmap_base, trie->mmap_size) < 0)
 			i_error("munmap(%s) failed: %m", trie->path);
@@ -342,6 +352,20 @@
 }
 
 static int
+trie_file_cache_read(struct squat_trie *trie, size_t offset, size_t size)
+{
+	if (trie->file_cache == NULL)
+		return 0;
+
+	if (file_cache_read(trie->file_cache, offset, size) < 0) {
+		i_error("read(%s) failed: %m", trie->path);
+		return -1;
+	}
+	trie->data = file_cache_get_map(trie->file_cache, &trie->data_size);
+	return 0;
+}
+
+static int
 node_read_children(struct squat_trie *trie, struct squat_node *node, int level)
 {
 	const uint8_t *data, *end;
@@ -355,21 +379,24 @@
 	i_assert(node->children_not_mapped);
 	i_assert(!node->have_sequential);
 	i_assert(trie->unmapped_child_count > 0);
+	i_assert(trie->data_size <= trie->locked_file_size);
 
 	trie->unmapped_child_count--;
 	node_offset = node->children.offset;
 	node->children_not_mapped = FALSE;
 	node->children.data = NULL;
 
-	if (unlikely(node_offset >= trie->mmap_size)) {
+	if (trie_file_cache_read(trie, node_offset, TRIE_READAHEAD_SIZE) < 0)
+		return -1;
+	if (unlikely(node_offset >= trie->data_size)) {
 		squat_trie_set_corrupted(trie);
 		return -1;
 	}
 
-	data = CONST_PTR_OFFSET(trie->mmap_base, node_offset);
-	end = CONST_PTR_OFFSET(trie->mmap_base, trie->mmap_size);
+	data = CONST_PTR_OFFSET(trie->data, node_offset);
+	end = CONST_PTR_OFFSET(trie->data, trie->data_size);
 	child_count = *data++;
-	if (unlikely(node_offset + child_count >= trie->mmap_size)) {
+	if (unlikely(node_offset + child_count >= trie->data_size)) {
 		squat_trie_set_corrupted(trie);
 		return -1;
 	}
@@ -404,7 +431,7 @@
 			} else {
 				base_offset -= num >> 1;
 			}
-			if (base_offset >= trie->mmap_size) {
+			if (base_offset >= trie->locked_file_size) {
 				squat_trie_set_corrupted(trie);
 				return -1;
 			}
@@ -447,6 +474,24 @@
 				dest = child->children.leaf_string =
 					i_malloc(len);
 			}
+
+			if (trie->file_cache != NULL) {
+				/* the string may be long -
+				   recalculate the end pos */
+				size_t offset, size;
+
+				offset = (const char *)data -
+					(const char *)trie->data;
+				size = len + TRIE_BYTES_LEFT(child_count - i);
+
+				if (trie_file_cache_read(trie, offset,
+							 size) < 0)
+					return -1;
+				data = CONST_PTR_OFFSET(trie->data, offset);
+				end = CONST_PTR_OFFSET(trie->data,
+						       trie->data_size);
+			}
+
 			if (end - data < len) {
 				squat_trie_set_corrupted(trie);
 				return -1;
@@ -455,6 +500,11 @@
 			data += len;
 		}
 	}
+	if (unlikely(data == end)) {
+		/* we should never get this far */
+		squat_trie_set_corrupted(trie);
+		return -1;
+	}
 	return 0;
 }
 
@@ -1092,38 +1142,52 @@
 
 static int squat_trie_map_header(struct squat_trie *trie)
 {
+	int ret;
+
 	if (trie->locked_file_size == 0) {
 		/* newly created file */
-		squat_trie_header_init(trie);
-		return 0;
+		return 1;
 	}
 	i_assert(trie->fd != -1);
 
-	if (trie->mmap_size != 0) {
-		if (munmap(trie->mmap_base, trie->mmap_size) < 0)
-			i_error("munmap(%s) failed: %m", trie->path);
+	if (trie->mmap_disable) {
+		ret = pread_full(trie->fd, &trie->hdr, sizeof(trie->hdr), 0);
+		if (ret <= 0) {
+			if (ret < 0) {
+				i_error("pread(%s) failed: %m", trie->path);
+				return -1;
+			}
+			i_error("Corrupted %s: File too small", trie->path);
+			return 0;
+		}
+		trie->data = NULL;
+		trie->data_size = 0;
+	} else {
+		if (trie->locked_file_size < sizeof(trie->hdr)) {
+			i_error("Corrupted %s: File too small", trie->path);
+			return 0;
+		}
+		if (trie->mmap_size != 0) {
+			if (munmap(trie->mmap_base, trie->mmap_size) < 0)
+				i_error("munmap(%s) failed: %m", trie->path);
+		}
+
+		trie->mmap_size = trie->locked_file_size;
+		trie->mmap_base = mmap(NULL, trie->mmap_size,
+				       PROT_READ | PROT_WRITE,
+				       MAP_SHARED, trie->fd, 0);
+		if (trie->mmap_base == MAP_FAILED) {
+			trie->data = trie->mmap_base = NULL;
+			trie->data_size = trie->mmap_size = 0;
+			i_error("mmap(%s) failed: %m", trie->path);
+			return -1;
+		}
+		memcpy(&trie->hdr, trie->mmap_base, sizeof(trie->hdr));
+		trie->data = trie->mmap_base;
+		trie->data_size = trie->mmap_size;
 	}
 
-	trie->mmap_size = trie->locked_file_size;
-	trie->mmap_base = mmap(NULL, trie->mmap_size, PROT_READ | PROT_WRITE,
-			       MAP_SHARED, trie->fd, 0);
-	if (trie->mmap_base == MAP_FAILED) {
-		trie->mmap_base = NULL;
-		trie->mmap_size = 0;
-		i_error("mmap(%s) failed: %m", trie->path);
-		return -1;
-	}
-	memcpy(&trie->hdr, trie->mmap_base, sizeof(trie->hdr));
-
-	if (trie->hdr.root_offset == 0)
-		return 0;
-	if (!squat_trie_check_header(trie)) {
-		squat_trie_delete(trie);
-		squat_trie_close(trie);
-		squat_trie_header_init(trie);
-		return 0;
-	}
-	return 0;
+	return squat_trie_check_header(trie) ? 1 : 0;
 }
 
 static int squat_trie_map(struct squat_trie *trie, bool building)
@@ -1135,9 +1199,17 @@
 	if (trie->fd != -1) {
 		if (squat_trie_lock(trie, F_RDLCK, &file_lock) <= 0)
 			return -1;
+		if (trie->mmap_disable && trie->file_cache == NULL)
+			trie->file_cache = file_cache_new(trie->fd);
 	}
 
 	ret = squat_trie_map_header(trie);
+	if (ret == 0) {
+		file_lock_free(&file_lock);
+		squat_trie_delete(trie);
+		squat_trie_close(trie);
+		squat_trie_header_init(trie);
+	}
 	changed = trie->root.children.offset != trie->hdr.root_offset;
 
 	if (changed || trie->hdr.root_offset == 0) {
@@ -1157,7 +1229,7 @@
 		}
 	}
 
-	if (ret == 0 && !building) {
+	if (ret >= 0 && !building) {
 		/* do this while we're still locked */
 		ret = squat_uidlist_refresh(trie->uidlist);
 	}
@@ -1267,6 +1339,10 @@
 	ret = squat_write_nodes(ctx);
 	ctx->output = NULL;
 
+	/* write 1 byte guard at the end of file, so that we can verify broken
+	   squat_unpack_num() input by checking if data==end */
+	o_stream_send(output, "", 1);
+
 	if (trie->corrupted)
 		ret = -1;
 	if (ret == 0)
--- a/src/plugins/fts-squat/squat-uidlist.c	Fri Dec 07 00:46:31 2007 +0200
+++ b/src/plugins/fts-squat/squat-uidlist.c	Fri Dec 07 16:10:19 2007 +0200
@@ -3,9 +3,11 @@
 #include "lib.h"
 #include "array.h"
 #include "bsearch-insert-pos.h"
+#include "file-cache.h"
 #include "file-lock.h"
+#include "read-full.h"
+#include "write-full.h"
 #include "ostream.h"
-#include "write-full.h"
 #include "squat-trie-private.h"
 #include "squat-uidlist.h"
 
@@ -34,6 +36,7 @@
 
 	char *path;
 	int fd;
+	struct file_cache *file_cache;
 
 	struct file_lock *file_lock;
 	uoff_t locked_file_size;
@@ -42,6 +45,9 @@
 	size_t mmap_size;
 	struct squat_uidlist_file_header hdr;
 
+	const void *data;
+	size_t data_size;
+
 	unsigned int cur_block_count;
 	const uint32_t *cur_block_offsets;
 	const uint32_t *cur_block_end_indexes;
@@ -263,32 +269,72 @@
 	return ret;
 }
 
-static int node_uidlist_map_blocks(struct squat_uidlist *uidlist)
+static void squat_uidlist_map_blocks_set_pointers(struct squat_uidlist *uidlist)
+{
+	const void *base;
+	size_t end_index_size, end_size;
+
+	base = CONST_PTR_OFFSET(uidlist->data, uidlist->hdr.block_list_offset +
+				sizeof(uint32_t));
+
+	end_index_size = uidlist->cur_block_count * sizeof(uint32_t);
+	end_size = end_index_size + uidlist->cur_block_count * sizeof(uint32_t);
+	if (end_size <= uidlist->data_size) {
+		uidlist->cur_block_end_indexes = base;
+		uidlist->cur_block_offsets =
+			CONST_PTR_OFFSET(base, end_index_size);
+	} else {
+		uidlist->cur_block_end_indexes = NULL;
+		uidlist->cur_block_offsets = NULL;
+	}
+}
+
+static int uidlist_file_cache_read(struct squat_uidlist *uidlist,
+				   size_t offset, size_t size)
+{
+	if (uidlist->file_cache == NULL)
+		return 0;
+
+	if (file_cache_read(uidlist->file_cache, offset, size) < 0) {
+		i_error("read(%s) failed: %m", uidlist->path);
+		return -1;
+	}
+	uidlist->data = file_cache_get_map(uidlist->file_cache,
+					   &uidlist->data_size);
+	squat_uidlist_map_blocks_set_pointers(uidlist);
+	return 0;
+}
+
+static int squat_uidlist_map_blocks(struct squat_uidlist *uidlist)
 {
 	const struct squat_uidlist_file_header *hdr = &uidlist->hdr;
 	const void *base;
-	uint32_t block_count, block_end_offset, i, verify_count;
+	uint32_t block_count, blocks_offset, blocks_size, i, verify_count;
 
-	block_end_offset = hdr->block_list_offset + sizeof(block_count);
-	if (block_end_offset > uidlist->mmap_size) {
+	/* get number of blocks */
+	if (uidlist_file_cache_read(uidlist, hdr->block_list_offset,
+				    sizeof(block_count)) < 0)
+		return -1;
+	blocks_offset = hdr->block_list_offset + sizeof(block_count);
+	if (blocks_offset > uidlist->data_size) {
 		squat_uidlist_set_corrupted(uidlist, "block list outside file");
 		return -1;
 	}
 
-	base = CONST_PTR_OFFSET(uidlist->mmap_base, hdr->block_list_offset);
+	base = CONST_PTR_OFFSET(uidlist->data, hdr->block_list_offset);
 	memcpy(&block_count, base, sizeof(block_count));
-	base = CONST_PTR_OFFSET(base, sizeof(block_count));
 
-	block_end_offset += block_count * sizeof(uint32_t)*2;
-	if (block_end_offset > uidlist->mmap_size) {
+	/* map the blocks */
+	blocks_size = block_count * sizeof(uint32_t)*2;
+	if (uidlist_file_cache_read(uidlist, blocks_offset, blocks_size) < 0)
+		return -1;
+	if (blocks_offset + blocks_size > uidlist->data_size) {
 		squat_uidlist_set_corrupted(uidlist, "block list outside file");
 		return -1;
 	}
 
 	uidlist->cur_block_count = block_count;
-	uidlist->cur_block_end_indexes = base;
-	uidlist->cur_block_offsets =
-		CONST_PTR_OFFSET(base, block_count * sizeof(uint32_t));
+	squat_uidlist_map_blocks_set_pointers(uidlist);
 
 	/* verify just a couple of the end indexes to make sure they
 	   look correct */
@@ -319,13 +365,25 @@
 		return -1;
 	}
 	if (uidlist->hdr.used_file_size < sizeof(uidlist->hdr) ||
-	    uidlist->hdr.used_file_size > uidlist->mmap_size) {
+	    (uidlist->hdr.used_file_size > uidlist->mmap_size &&
+	     uidlist->mmap_base != NULL)) {
 		squat_uidlist_set_corrupted(uidlist, "broken used_file_size");
 		return -1;
 	}
-	if (node_uidlist_map_blocks(uidlist) < 0)
-		return -1;
-	return 0;
+	return squat_uidlist_map_blocks(uidlist);
+}
+
+static void squat_uidlist_unmap(struct squat_uidlist *uidlist)
+{
+	if (uidlist->mmap_size != 0) {
+		if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0)
+			i_error("munmap(%s) failed: %m", uidlist->path);
+		uidlist->mmap_base = NULL;
+		uidlist->mmap_size = 0;
+	}
+	uidlist->cur_block_count = 0;
+	uidlist->cur_block_end_indexes = NULL;
+	uidlist->cur_block_offsets = NULL;
 }
 
 static int squat_uidlist_mmap(struct squat_uidlist *uidlist)
@@ -343,26 +401,26 @@
 		return -1;
 	}
 
-	if (uidlist->mmap_size != 0) {
-		if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0)
-			i_error("munmap(%s) failed: %m", uidlist->path);
-	}
+	squat_uidlist_unmap(uidlist);
 	uidlist->mmap_size = st.st_size;
 	uidlist->mmap_base = mmap(NULL, uidlist->mmap_size,
 				  PROT_READ | PROT_WRITE,
 				  MAP_SHARED, uidlist->fd, 0);
 	if (uidlist->mmap_base == MAP_FAILED) {
-		uidlist->mmap_base = NULL;
-		uidlist->mmap_size = 0;
+		uidlist->data = uidlist->mmap_base = NULL;
+		uidlist->data_size = uidlist->mmap_size = 0;
 		i_error("mmap(%s) failed: %m", uidlist->path);
 		return -1;
 	}
+	uidlist->data = uidlist->mmap_base;
+	uidlist->data_size = uidlist->mmap_size;
 	return 0;
 }
 
 static int squat_uidlist_map(struct squat_uidlist *uidlist)
 {
 	const struct squat_uidlist_file_header *mmap_hdr = uidlist->mmap_base;
+	int ret;
 
 	if (mmap_hdr != NULL && !uidlist->building &&
 	    uidlist->hdr.block_list_offset == mmap_hdr->block_list_offset) {
@@ -370,15 +428,36 @@
 		return 0;
 	}
 
-	if (mmap_hdr == NULL || uidlist->building ||
-	    uidlist->mmap_size < mmap_hdr->used_file_size) {
-		if (squat_uidlist_mmap(uidlist) < 0)
-			return -1;
+	if (!uidlist->trie->mmap_disable) {
+		if (mmap_hdr == NULL || uidlist->building ||
+		    uidlist->mmap_size < mmap_hdr->used_file_size) {
+			if (squat_uidlist_mmap(uidlist) < 0)
+				return -1;
+		}
+
+		if (!uidlist->building) {
+			memcpy(&uidlist->hdr, uidlist->mmap_base,
+			       sizeof(uidlist->hdr));
+		}
+	} else if (uidlist->building) {
+		/* we want to update blocks mapping, but using the header
+		   in memory */
+	} else {
+		ret = pread_full(uidlist->fd, &uidlist->hdr,
+				 sizeof(uidlist->hdr), 0);
+		if (ret <= 0) {
+			if (ret < 0) {
+				i_error("pread(%s) failed: %m", uidlist->path);
+				return -1;
+			}
+			i_error("Corrupted %s: File too small", uidlist->path);
+			return 0;
+		}
+		uidlist->data = NULL;
+		uidlist->data_size = 0;
 	}
-
-	if (!uidlist->building)
-		memcpy(&uidlist->hdr, uidlist->mmap_base, sizeof(uidlist->hdr));
-
+	if (uidlist->file_cache == NULL && uidlist->trie->mmap_disable)
+		uidlist->file_cache = file_cache_new(uidlist->fd);
 	return squat_uidlist_map_header(uidlist);
 }
 
@@ -422,26 +501,23 @@
 {
 	i_assert(!uidlist->building);
 
+	squat_uidlist_unmap(uidlist);
+	if (uidlist->file_cache != NULL)
+		file_cache_free(&uidlist->file_cache);
 	if (uidlist->file_lock != NULL)
 		file_lock_free(&uidlist->file_lock);
-	if (uidlist->mmap_size != 0) {
-		if (munmap(uidlist->mmap_base, uidlist->mmap_size) < 0)
-			i_error("munmap(%s) failed: %m", uidlist->path);
-		uidlist->mmap_base = NULL;
-		uidlist->mmap_size = 0;
-	}
 	if (uidlist->fd != -1) {
 		if (close(uidlist->fd) < 0)
 			i_error("close(%s) failed: %m", uidlist->path);
 		uidlist->fd = -1;
 	}
-	uidlist->cur_block_end_indexes = NULL;
-	uidlist->cur_block_offsets = NULL;
 	uidlist->corrupted = FALSE;
 }
 
 int squat_uidlist_refresh(struct squat_uidlist *uidlist)
 {
+	/* we assume here that trie is locked, so that we don't need to worry
+	   about it when reading the header */
 	if (uidlist->fd == -1 ||
 	    uidlist->hdr.indexid != uidlist->trie->hdr.indexid) {
 		if (squat_uidlist_open(uidlist) < 0)
@@ -1080,26 +1156,38 @@
 }
 
 static int
-node_uidlist_get_at_offset(struct squat_uidlist *uidlist, uoff_t offset,
-			   uint32_t num, ARRAY_TYPE(uint32_t) *uids)
+squat_uidlist_get_at_offset(struct squat_uidlist *uidlist, uoff_t offset,
+			    uint32_t num, ARRAY_TYPE(uint32_t) *uids)
 {
 	const uint8_t *p, *end;
 	uint32_t size, base_uid;
+	uoff_t uidlist_data_offset;
 	unsigned int i, j, extra = 0;
 
-	p = CONST_PTR_OFFSET(uidlist->mmap_base, offset);
-	end = CONST_PTR_OFFSET(uidlist->mmap_base, uidlist->mmap_size);
+	if (num != 0)
+		uidlist_data_offset = offset;
+	else {
+		/* not given, read it */
+		if (uidlist_file_cache_read(uidlist, offset,
+					    SQUAT_PACK_MAX_SIZE) < 0)
+			return -1;
 
-	if (num == 0) {
-		/* not given, read it */
+		p = CONST_PTR_OFFSET(uidlist->data, offset);
+		end = CONST_PTR_OFFSET(uidlist->data, uidlist->data_size);
 		num = squat_unpack_num(&p, end);
+		uidlist_data_offset = p - (const uint8_t *)uidlist->data;
 	}
 	size = num >> 2;
-	if (p + size > end) {
+
+	if (uidlist_file_cache_read(uidlist, uidlist_data_offset, size) < 0)
+		return -1;
+	if (uidlist_data_offset + size > uidlist->data_size) {
 		squat_uidlist_set_corrupted(uidlist,
 					    "size points outside file");
 		return -1;
 	}
+
+	p = CONST_PTR_OFFSET(uidlist->data, uidlist_data_offset);
 	end = p + size;
 
 	if ((num & UIDLIST_PACKED_FLAG_BEGINS_WITH_POINTER) != 0) {
@@ -1113,8 +1201,8 @@
 				return -1;
 		} else {
 			prev = offset - (prev >> 1);
-			if (node_uidlist_get_at_offset(uidlist, prev,
-						       0, uids) < 0)
+			if (squat_uidlist_get_at_offset(uidlist, prev,
+							0, uids) < 0)
 				return -1;
 		}
 	}
@@ -1160,12 +1248,13 @@
 }
 
 static int
-node_uidlist_get_offset(struct squat_uidlist *uidlist, uint32_t uid_list_idx,
-			uint32_t *offset_r, uint32_t *num_r)
+squat_uidlist_get_offset(struct squat_uidlist *uidlist, uint32_t uid_list_idx,
+			 uint32_t *offset_r, uint32_t *num_r)
 {
 	const uint8_t *p, *end;
 	unsigned int idx;
 	uint32_t num, skip_bytes, uidlists_offset;
+	size_t max_map_size;
 
 	if (uidlist->fd == -1) {
 		squat_uidlist_set_corrupted(uidlist, "no uidlists");
@@ -1186,14 +1275,19 @@
 		return -1;
 	}
 
+	/* make sure everything is mapped */
+	uid_list_idx -= idx == 0 ? 0 : uidlist->cur_block_end_indexes[idx-1];
+	max_map_size = SQUAT_PACK_MAX_SIZE * (1+uid_list_idx);
+	if (uidlist_file_cache_read(uidlist, uidlist->cur_block_offsets[idx],
+				    max_map_size) < 0)
+		return -1;
+
 	/* find the uidlist inside the block */
-	p = CONST_PTR_OFFSET(uidlist->mmap_base,
-			     uidlist->cur_block_offsets[idx]);
-	end = CONST_PTR_OFFSET(uidlist->mmap_base, uidlist->mmap_size);
+	p = CONST_PTR_OFFSET(uidlist->data, uidlist->cur_block_offsets[idx]);
+	end = CONST_PTR_OFFSET(uidlist->data, uidlist->data_size);
 
 	uidlists_offset = uidlist->cur_block_offsets[idx] -
 		squat_unpack_num(&p, end);
-	uid_list_idx -= idx == 0 ? 0 : uidlist->cur_block_end_indexes[idx-1];
 	for (skip_bytes = 0; uid_list_idx > 0; uid_list_idx--) {
 		num = squat_unpack_num(&p, end);
 		skip_bytes += num >> 2;
@@ -1201,7 +1295,12 @@
 	*offset_r = uidlists_offset + skip_bytes;
 	*num_r = squat_unpack_num(&p, end);
 
-	if (unlikely(*offset_r > uidlist->mmap_size)) {
+	if (unlikely(p == end)) {
+		squat_uidlist_set_corrupted(uidlist, "broken file");
+		return -1;
+	}
+	if (unlikely(*offset_r > uidlist->mmap_size &&
+		     uidlist->mmap_base != NULL)) {
 		squat_uidlist_set_corrupted(uidlist, "broken offset");
 		return -1;
 	}
@@ -1229,9 +1328,9 @@
 	}
 
 	uid_list_idx = (uid_list_idx >> 1) - 0x100;
-	if (node_uidlist_get_offset(uidlist, uid_list_idx, &offset, &num) < 0)
+	if (squat_uidlist_get_offset(uidlist, uid_list_idx, &offset, &num) < 0)
 		return -1;
-	return node_uidlist_get_at_offset(uidlist, offset, num, uids);
+	return squat_uidlist_get_at_offset(uidlist, offset, num, uids);
 }
 
 uint32_t squat_uidlist_singleton_last_uid(uint32_t uid_list_idx)