# HG changeset patch
# User Timo Sirainen <tss@iki.fi>
# Date 1195279983 -7200
# Node ID 85385079b0447ff3056675de20af00e019029818
# Parent  b8c66f0255835146a3fa2a959eec3ff65d05fe84
Use likely() and unlikely() macros to make commonly checked error handling
paths unlikely.

diff -r b8c66f025583 -r 85385079b044 src/lib/base64.c
--- a/src/lib/base64.c	Sat Nov 17 02:07:40 2007 +0200
+++ b/src/lib/base64.c	Sat Nov 17 08:13:03 2007 +0200
@@ -93,16 +93,16 @@
 	for (src_pos = 0; src_pos+3 < src_size; ) {
 		input[0] = b64dec[src_c[src_pos]];
 		if (input[0] == 0xff) {
-			if (IS_EMPTY(src_c[src_pos])) {
-				src_pos++;
-				continue;
+			if (unlikely(!IS_EMPTY(src_c[src_pos]))) {
+				ret = -1;
+				break;
 			}
-			ret = -1;
-			break;
+			src_pos++;
+			continue;
 		}
 
 		input[1] = b64dec[src_c[src_pos+1]];
-		if (input[1] == 0xff) {
+		if (unlikely(input[1] == 0xff)) {
 			ret = -1;
 			break;
 		}
@@ -110,8 +110,8 @@
 
 		input[2] = b64dec[src_c[src_pos+2]];
 		if (input[2] == 0xff) {
-			if (src_c[src_pos+2] != '=' ||
-			    src_c[src_pos+3] != '=') {
+			if (unlikely(src_c[src_pos+2] != '=' ||
+				     src_c[src_pos+3] != '=')) {
 				ret = -1;
 				break;
 			}
@@ -124,7 +124,7 @@
 		output[1] = (input[1] << 4) | (input[2] >> 2);
 		input[3] = b64dec[src_c[src_pos+3]];
 		if (input[3] == 0xff) {
-			if (src_c[src_pos+3] != '=') {
+			if (unlikely(src_c[src_pos+3] != '=')) {
 				ret = -1;
 				break;
 			}
diff -r b8c66f025583 -r 85385079b044 src/lib/buffer.c
--- a/src/lib/buffer.c	Sat Nov 17 02:07:40 2007 +0200
+++ b/src/lib/buffer.c	Sat Nov 17 08:13:03 2007 +0200
@@ -41,14 +41,14 @@
 {
 	size_t new_size;
 
-	if ((size_t)-1 - pos < data_size) {
+	if (unlikely((size_t)-1 - pos < data_size)) {
 		i_panic("Buffer write out of range (%"PRIuSIZE_T
 			" + %"PRIuSIZE_T")", pos, data_size);
 	}
 	new_size = pos + data_size;
 
 	if (new_size > buf->alloc) {
-		if (!buf->dynamic) {
+		if (unlikely(!buf->dynamic)) {
 			i_panic("Buffer full (%"PRIuSIZE_T" > %"PRIuSIZE_T", "
 				"pool %s)", pos + data_size, buf->alloc,
 				pool_get_name(buf->pool));
diff -r b8c66f025583 -r 85385079b044 src/lib/data-stack.c
--- a/src/lib/data-stack.c	Sat Nov 17 02:07:40 2007 +0200
+++ b/src/lib/data-stack.c	Sat Nov 17 08:13:03 2007 +0200
@@ -152,7 +152,7 @@
 	struct stack_frame_block *frame_block;
 	int popped_frame_pos;
 
-	if (frame_pos < 0)
+	if (unlikely(frame_pos < 0))
 		i_panic("t_pop() called with empty stack");
 
 	/* update the current block */
@@ -199,7 +199,7 @@
 #else
 	block = GC_malloc(SIZEOF_MEMBLOCK + alloc_size);
 #endif
-	if (block == NULL) {
+	if (unlikely(block == NULL)) {
 		i_panic("data stack: Out of memory when allocating %"
 			PRIuSIZE_T" bytes", alloc_size + SIZEOF_MEMBLOCK);
 	}
@@ -217,10 +217,10 @@
 	bool warn = FALSE;
 #endif
 
-	if (size == 0 || size > SSIZE_T_MAX)
+	if (unlikely(size == 0 || size > SSIZE_T_MAX))
 		i_panic("Trying to allocate %"PRIuSIZE_T" bytes", size);
 
-	if (data_stack_frame == 0) {
+	if (unlikely(data_stack_frame == 0)) {
 		/* kludgy, but allow this before initialization */
 		data_stack_init();
 	}
@@ -295,7 +295,7 @@
 {
 	size_t last_alloc_size;
 
-	if (size == 0 || size > SSIZE_T_MAX)
+	if (unlikely(size == 0 || size > SSIZE_T_MAX))
 		i_panic("Trying to allocate %"PRIuSIZE_T" bytes", size);
 
 	last_alloc_size = current_frame_block->last_alloc_size[frame_pos];
diff -r b8c66f025583 -r 85385079b044 src/lib/hash.c
--- a/src/lib/hash.c	Sat Nov 17 02:07:40 2007 +0200
+++ b/src/lib/hash.c	Sat Nov 17 08:13:03 2007 +0200
@@ -311,7 +311,7 @@
 	hash = table->hash_cb(key);
 
 	node = hash_lookup_node(table, key, hash);
-	if (node == NULL)
+	if (unlikely(node == NULL))
 		i_panic("key not found from hash");
 
 	node->key = NULL;
diff -r b8c66f025583 -r 85385079b044 src/lib/istream-file.c
--- a/src/lib/istream-file.c	Sat Nov 17 02:07:40 2007 +0200
+++ b/src/lib/istream-file.c	Sat Nov 17 08:13:03 2007 +0200
@@ -58,7 +58,8 @@
 			ret = read(stream->fd, stream->w_buffer + stream->pos,
 				   size);
 		}
-	} while (ret < 0 && errno == EINTR && stream->istream.blocking);
+	} while (unlikely(ret < 0 && errno == EINTR &&
+			  stream->istream.blocking));
 
 	if (ret == 0) {
 		/* EOF */
@@ -66,7 +67,7 @@
 		return -1;
 	}
 
-	if (ret < 0) {
+	if (unlikely(ret < 0)) {
 		if (errno == EINTR || errno == EAGAIN) {
 			i_assert(!stream->istream.blocking);
 			ret = 0;
diff -r b8c66f025583 -r 85385079b044 src/lib/istream.c
--- a/src/lib/istream.c	Sat Nov 17 02:07:40 2007 +0200
+++ b/src/lib/istream.c	Sat Nov 17 08:13:03 2007 +0200
@@ -50,7 +50,7 @@
 {
 	struct istream_private *_stream = stream->real_stream;
 
-	if (stream->closed)
+	if (unlikely(stream->closed))
 		return -1;
 
 	stream->eof = FALSE;
@@ -75,7 +75,7 @@
 	_stream->skip = _stream->pos;
 	stream->v_offset += data_size;
 
-	if (stream->closed)
+	if (unlikely(stream->closed))
 		return;
 
 	_stream->seek(_stream, stream->v_offset + count, FALSE);
@@ -90,7 +90,7 @@
 		return;
 	}
 
-	if (stream->closed)
+	if (unlikely(stream->closed))
 		return;
 
 	stream->eof = FALSE;
@@ -101,7 +101,7 @@
 {
 	struct istream_private *_stream = stream->real_stream;
 
-	if (stream->closed)
+	if (unlikely(stream->closed))
 		return;
 
 	stream->eof = FALSE;
@@ -112,7 +112,10 @@
 {
 	struct istream_private *_stream = stream->real_stream;
 
-	if (!stream->closed && _stream->sync != NULL)
+	if (unlikely(stream->closed))
+		return;
+
+	if (_stream->sync != NULL)
 		_stream->sync(_stream);
 }
 
@@ -120,7 +123,7 @@
 {
 	struct istream_private *_stream = stream->real_stream;
 
-	if (stream->closed)
+	if (unlikely(stream->closed))
 		return NULL;
 
 	return _stream->stat(_stream, exact);
@@ -174,7 +177,7 @@
 		return NULL;
 	}
 
-	if (_stream->w_buffer == NULL) {
+	if (unlikely(_stream->w_buffer == NULL)) {
 		i_error("i_stream_next_line() called for unmodifiable stream");
 		return NULL;
 	}
diff -r b8c66f025583 -r 85385079b044 src/lib/mempool-alloconly.c
--- a/src/lib/mempool-alloconly.c	Sat Nov 17 02:07:40 2007 +0200
+++ b/src/lib/mempool-alloconly.c	Sat Nov 17 08:13:03 2007 +0200
@@ -218,7 +218,7 @@
 #else
 	block = GC_malloc(size);
 #endif
-	if (block == NULL)
+	if (unlikely(block == NULL))
 		i_fatal_status(FATAL_OUTOFMEM, "block_alloc(): Out of memory");
 	block->prev = apool->block;
 	apool->block = block;
@@ -232,7 +232,7 @@
 	struct alloconly_pool *apool = (struct alloconly_pool *)pool;
 	void *mem;
 
-	if (size == 0 || size > SSIZE_T_MAX)
+	if (unlikely(size == 0 || size > SSIZE_T_MAX))
 		i_panic("Trying to allocate %"PRIuSIZE_T" bytes", size);
 
 	size = MEM_ALIGN(size);
@@ -289,7 +289,7 @@
 	struct alloconly_pool *apool = (struct alloconly_pool *)pool;
 	unsigned char *new_mem;
 
-	if (new_size == 0 || new_size > SSIZE_T_MAX)
+	if (unlikely(new_size == 0 || new_size > SSIZE_T_MAX))
 		i_panic("Trying to allocate %"PRIuSIZE_T" bytes", new_size);
 
 	if (mem == NULL)
diff -r b8c66f025583 -r 85385079b044 src/lib/mempool-datastack.c
--- a/src/lib/mempool-datastack.c	Sat Nov 17 02:07:40 2007 +0200
+++ b/src/lib/mempool-datastack.c	Sat Nov 17 08:13:03 2007 +0200
@@ -64,7 +64,7 @@
 {
 	struct datastack_pool *dpool = (struct datastack_pool *) pool;
 
-	if (dpool->data_stack_frame != data_stack_frame)
+	if (unlikely(dpool->data_stack_frame != data_stack_frame))
 		i_panic("pool_data_stack_ref(): stack frame changed");
 
 	dpool->refcount++;
@@ -74,7 +74,7 @@
 {
 	struct datastack_pool *dpool = (struct datastack_pool *)*pool;
 
-	if (dpool->data_stack_frame != data_stack_frame)
+	if (unlikely(dpool->data_stack_frame != data_stack_frame))
 		i_panic("pool_data_stack_unref(): stack frame changed");
 
 	dpool->refcount--;
@@ -87,10 +87,10 @@
 {
 	struct datastack_pool *dpool = (struct datastack_pool *) pool;
 
-	if (size == 0 || size > SSIZE_T_MAX)
+	if (unlikely(size == 0 || size > SSIZE_T_MAX))
 		i_panic("Trying to allocate %"PRIuSIZE_T" bytes", size);
 
-	if (dpool->data_stack_frame != data_stack_frame)
+	if (unlikely(dpool->data_stack_frame != data_stack_frame))
 		i_panic("pool_data_stack_malloc(): stack frame changed");
 
 	return t_malloc0(size);
@@ -100,7 +100,7 @@
 {
 	struct datastack_pool *dpool = (struct datastack_pool *) pool;
 
-	if (dpool->data_stack_frame != data_stack_frame)
+	if (unlikely(dpool->data_stack_frame != data_stack_frame))
 		i_panic("pool_data_stack_free(): stack frame changed");
 }
 
@@ -111,10 +111,10 @@
 	void *new_mem;
 
 	/* @UNSAFE */
-	if (new_size == 0 || new_size > SSIZE_T_MAX)
+	if (unlikely(new_size == 0 || new_size > SSIZE_T_MAX))
 		i_panic("Trying to allocate %"PRIuSIZE_T" bytes", new_size);
 
-	if (dpool->data_stack_frame != data_stack_frame)
+	if (unlikely(dpool->data_stack_frame != data_stack_frame))
 		i_panic("pool_data_stack_realloc(): stack frame changed");
 
 	if (mem == NULL)
diff -r b8c66f025583 -r 85385079b044 src/lib/mempool-system-clean.c
--- a/src/lib/mempool-system-clean.c	Sat Nov 17 02:07:40 2007 +0200
+++ b/src/lib/mempool-system-clean.c	Sat Nov 17 08:13:03 2007 +0200
@@ -88,7 +88,7 @@
 {
 	void *mem;
 
-	if (size == 0 || size > SSIZE_T_MAX)
+	if (unlikely(size == 0 || size > SSIZE_T_MAX))
 		i_panic("Trying to allocate %"PRIuSIZE_T" bytes", size);
 
 #ifdef USE_GC
@@ -99,7 +99,7 @@
 #endif
 	mem = calloc(size, 1);
 #endif
-	if (mem == NULL) {
+	if (unlikely(mem == NULL)) {
 		i_fatal_status(FATAL_OUTOFMEM,
 			       "pool_system_clean_malloc(): Out of memory");
 	}
@@ -132,7 +132,7 @@
 {
 	void *new_mem;
 
-	if (new_size == 0 || new_size > SSIZE_T_MAX)
+	if (unlikely(new_size == 0 || new_size > SSIZE_T_MAX))
 		i_panic("Trying to allocate %"PRIuSIZE_T" bytes", new_size);
 
 	new_mem = pool_system_clean_malloc(pool, new_size);
diff -r b8c66f025583 -r 85385079b044 src/lib/mempool-system.c
--- a/src/lib/mempool-system.c	Sat Nov 17 02:07:40 2007 +0200
+++ b/src/lib/mempool-system.c	Sat Nov 17 08:13:03 2007 +0200
@@ -67,7 +67,7 @@
 {
 	void *mem;
 
-	if (size == 0 || size > SSIZE_T_MAX)
+	if (unlikely(size == 0 || size > SSIZE_T_MAX))
 		i_panic("Trying to allocate %"PRIuSIZE_T" bytes", size);
 
 #ifndef USE_GC
@@ -75,7 +75,7 @@
 #else
 	mem = GC_malloc(size);
 #endif
-	if (mem == NULL) {
+	if (unlikely(mem == NULL)) {
 		i_fatal_status(FATAL_OUTOFMEM,
 			       "pool_system_malloc(): Out of memory");
 	}
@@ -94,7 +94,7 @@
 static void *pool_system_realloc(pool_t pool ATTR_UNUSED, void *mem,
 				 size_t old_size, size_t new_size)
 {
-	if (new_size == 0 || new_size > SSIZE_T_MAX)
+	if (unlikely(new_size == 0 || new_size > SSIZE_T_MAX))
 		i_panic("Trying to allocate %"PRIuSIZE_T" bytes", new_size);
 
 #if !defined(USE_GC) && defined(HAVE_MALLOC_USABLE_SIZE)
@@ -107,7 +107,7 @@
 #else
 	mem = GC_realloc(mem, new_size);
 #endif
-	if (mem == NULL) {
+	if (unlikely(mem == NULL)) {
 		i_fatal_status(FATAL_OUTOFMEM,
 			       "pool_system_realloc(): Out of memory");
 	}
diff -r b8c66f025583 -r 85385079b044 src/lib/mempool-unsafe-datastack.c
--- a/src/lib/mempool-unsafe-datastack.c	Sat Nov 17 02:07:40 2007 +0200
+++ b/src/lib/mempool-unsafe-datastack.c	Sat Nov 17 08:13:03 2007 +0200
@@ -55,7 +55,7 @@
 static void *pool_unsafe_data_stack_malloc(pool_t pool ATTR_UNUSED,
 					   size_t size)
 {
-	if (size == 0 || size > SSIZE_T_MAX)
+	if (unlikely(size == 0 || size > SSIZE_T_MAX))
 		i_panic("Trying to allocate %"PRIuSIZE_T" bytes", size);
 
 	return t_malloc0(size);
diff -r b8c66f025583 -r 85385079b044 src/lib/network.c
--- a/src/lib/network.c	Sat Nov 17 02:07:40 2007 +0200
+++ b/src/lib/network.c	Sat Nov 17 08:13:03 2007 +0200
@@ -402,7 +402,7 @@
 		return -2;
 	}
 
-	if (ret < 0) {
+	if (unlikely(ret < 0)) {
 		if (errno == EINTR || errno == EAGAIN)
 			return 0;
 
@@ -423,10 +423,10 @@
 	i_assert(len <= SSIZE_T_MAX);
 
 	ret = send(fd, data, len, 0);
-	if (ret == -1 && (errno == EINTR || errno == EAGAIN))
+	if (unlikely(ret == -1 && (errno == EINTR || errno == EAGAIN)))
 		return 0;
 
-	if (errno == EPIPE)
+	if (unlikely(errno == EPIPE))
 		return -2;
 
         return ret;
diff -r b8c66f025583 -r 85385079b044 src/lib/nfs-workarounds.c
--- a/src/lib/nfs-workarounds.c	Sat Nov 17 02:07:40 2007 +0200
+++ b/src/lib/nfs-workarounds.c	Sat Nov 17 08:13:03 2007 +0200
@@ -169,7 +169,7 @@
 	uid_t uid;
 
 	if (fstat(fd, &st) < 0) {
-		if (errno == ESTALE) {
+		if (likely(errno == ESTALE)) {
 			/* ESTALE causes the OS to flush the attr cache */
 			return FALSE;
 		}
@@ -182,10 +182,9 @@
 	uid = st.st_uid;
 #endif
 	if (fchown(fd, uid, (gid_t)-1) < 0) {
-		if (errno == ESTALE) {
+		if (errno == ESTALE)
 			return FALSE;
-		}
-		if (errno == EACCES || errno == EPERM) {
+		if (likely(errno == EACCES || errno == EPERM)) {
 			/* attr cache is flushed */
 			return TRUE;
 		}
@@ -212,7 +211,7 @@
 			/* ESTALE causes the OS to flush the attr cache */
 			return;
 		}
-		if (errno != ENOENT) {
+		if (unlikely(errno != ENOENT)) {
 			i_error("nfs_flush_chown_uid: stat(%s) failed: %m",
 				path);
 			return;
@@ -226,8 +225,8 @@
 	}
 #endif
 	if (chown(path, uid, (gid_t)-1) < 0) {
-		if (errno == ESTALE || errno == EACCES ||
-		    errno == EPERM || errno == ENOENT) {
+		if (likely(errno == ESTALE || errno == EACCES ||
+			   errno == EPERM || errno == ENOENT)) {
 			/* attr cache is flushed */
 			return;
 		}
@@ -255,7 +254,7 @@
 	ret = fcntl(fd, F_SETLKW, &fl);
 	alarm(0);
 
-	if (ret < 0) {
+	if (unlikely(ret < 0)) {
 		i_error("nfs_flush_fcntl: fcntl(%s, F_RDLCK) failed: %m", path);
 		return;
 	}
@@ -272,7 +271,7 @@
 #ifdef __FreeBSD__
 	/* Unfortunately rmdir() seems to be the only way to flush a
 	   directory's attribute cache. */
-	if (rmdir(path) == 0) {
+	if (unlikely(rmdir(path) == 0)) {
 		if (mkdir(path, 0600) == 0) {
 			i_warning("nfs_flush_dir: rmdir(%s) unexpectedly "
 				  "removed the dir. recreated.", path);
@@ -280,7 +279,8 @@
 			i_error("nfs_flush_dir: rmdir(%s) unexpectedly "
 				"removed the dir. mkdir() failed: %m", path);
 		}
-	} else if (errno == ESTALE || errno == ENOENT || errno == ENOTEMPTY) {
+	} else if (likely(errno == ESTALE || errno == ENOENT ||
+			  errno == ENOTEMPTY)) {
 		/* expected failures */
 	} else {
 		i_error("nfs_flush_dir: rmdir(%s) failed: %m", path);
diff -r b8c66f025583 -r 85385079b044 src/lib/ostream.c
--- a/src/lib/ostream.c	Sat Nov 17 02:07:40 2007 +0200
+++ b/src/lib/ostream.c	Sat Nov 17 08:13:03 2007 +0200
@@ -55,7 +55,7 @@
 {
 	struct ostream_private *_stream = stream->real_stream;
 
-	if (stream->closed)
+	if (unlikely(stream->closed))
 		return;
 
 	_stream->cork(_stream, TRUE);
@@ -65,7 +65,7 @@
 {
 	struct ostream_private *_stream = stream->real_stream;
 
-	if (stream->closed)
+	if (unlikely(stream->closed))
 		return;
 
 	_stream->cork(_stream, FALSE);
@@ -75,7 +75,7 @@
 {
 	struct ostream_private *_stream = stream->real_stream;
 
-	if (stream->closed)
+	if (unlikely(stream->closed))
 		return -1;
 
 	return _stream->flush(_stream);
@@ -85,8 +85,10 @@
 {
 	struct ostream_private *_stream = stream->real_stream;
 
-	if (!stream->closed)
-		_stream->flush_pending(_stream, set);
+	if (unlikely(stream->closed))
+		return;
+
+	_stream->flush_pending(_stream, set);
 }
 
 size_t o_stream_get_buffer_used_size(struct ostream *stream)
@@ -100,7 +102,7 @@
 {
 	struct ostream_private *_stream = stream->real_stream;
 
-	if (stream->closed)
+	if (unlikely(stream->closed))
 		return -1;
 
 	return _stream->seek(_stream, offset);
@@ -124,14 +126,14 @@
 	size_t total_size;
 	ssize_t ret;
 
-	if (stream->closed)
+	if (unlikely(stream->closed))
 		return -1;
 
 	for (i = 0, total_size = 0; i < iov_count; i++)
 		total_size += iov[i].iov_len;
 
 	ret = _stream->sendv(_stream, iov, iov_count);
-	if (ret != (ssize_t)total_size)
+	if (unlikely(ret != (ssize_t)total_size))
 		stream->overflow = TRUE;
 	return ret;
 }
@@ -147,11 +149,11 @@
 	struct ostream_private *_outstream = outstream->real_stream;
 	off_t ret;
 
-	if (outstream->closed || instream->closed)
+	if (unlikely(outstream->closed || instream->closed))
 		return -1;
 
 	ret = _outstream->send_istream(_outstream, instream);
-	if (ret < 0)
+	if (unlikely(ret < 0))
 		errno = outstream->stream_errno;
 	return ret;
 }
diff -r b8c66f025583 -r 85385079b044 src/lib/randgen.c
--- a/src/lib/randgen.c	Sat Nov 17 02:07:40 2007 +0200
+++ b/src/lib/randgen.c	Sat Nov 17 08:13:03 2007 +0200
@@ -24,7 +24,7 @@
 
 	for (pos = 0; pos < size; pos += ret) {
 		ret = read(urandom_fd, (char *) buf + pos, size - pos);
-		if (ret < 0 && errno != EINTR)
+		if (unlikely(ret < 0 && errno != EINTR))
 			i_fatal("Error reading from /dev/urandom: %m");
 	}
 }
diff -r b8c66f025583 -r 85385079b044 src/lib/write-full.c
--- a/src/lib/write-full.c	Sat Nov 17 02:07:40 2007 +0200
+++ b/src/lib/write-full.c	Sat Nov 17 08:13:03 2007 +0200
@@ -11,10 +11,10 @@
 
 	while (size > 0) {
 		ret = write(fd, data, size < SSIZE_T_MAX ? size : SSIZE_T_MAX);
-		if (ret < 0)
+		if (unlikely(ret < 0))
 			return -1;
 
-		if (ret == 0) {
+		if (unlikely(ret == 0)) {
 			/* nothing was written, only reason for this should
 			   be out of disk space */
 			errno = ENOSPC;
@@ -35,10 +35,10 @@
 	while (size > 0) {
 		ret = pwrite(fd, data, size < SSIZE_T_MAX ?
 			     size : SSIZE_T_MAX, offset);
-		if (ret < 0)
+		if (unlikely(ret < 0))
 			return -1;
 
-		if (ret == 0) {
+		if (unlikely(ret == 0)) {
 			/* nothing was written, only reason for this should
 			   be out of disk space */
 			errno = ENOSPC;