# HG changeset patch
# User Timo Sirainen <tss@iki.fi>
# Date 1287509250 -3600
# Node ID 21c70ff6b344bb7b6607c63a32886e47035a61d0
# Parent  e3fe87b855efd27316a62cf55c75e4fd36e2e0ff
doveadm: Added commands for managing SiS directories.

diff -r e3fe87b855ef -r 21c70ff6b344 src/doveadm/Makefile.am
--- a/src/doveadm/Makefile.am	Tue Oct 19 18:16:10 2010 +0100
+++ b/src/doveadm/Makefile.am	Tue Oct 19 18:27:30 2010 +0100
@@ -9,6 +9,7 @@
 	-I$(top_srcdir)/src/lib-settings \
 	-I$(top_srcdir)/src/lib-auth \
 	-I$(top_srcdir)/src/lib-dict \
+	-I$(top_srcdir)/src/lib-fs \
 	-I$(top_srcdir)/src/lib-master \
 	-I$(top_srcdir)/src/lib-mail \
 	-I$(top_srcdir)/src/lib-imap \
@@ -92,6 +93,7 @@
 	doveadm-print-tab.c \
 	doveadm-print-table.c \
 	doveadm-pw.c \
+	doveadm-sis.c \
 	doveadm-who.c \
 	server-connection.c
 
diff -r e3fe87b855ef -r 21c70ff6b344 src/doveadm/doveadm-sis.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/doveadm/doveadm-sis.c	Tue Oct 19 18:27:30 2010 +0100
@@ -0,0 +1,321 @@
+/* Copyright (c) 2009-2010 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "hex-binary.h"
+#include "hostpid.h"
+#include "randgen.h"
+#include "read-full.h"
+#include "fs-sis-common.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <sys/stat.h>
+
+/* Files are in <rootdir>/ha/sh/<hash>-<guid>
+   They may be hard linked to hashes/<hash>
+*/
+
+static const char *sis_get_dir(const char *rootdir, const char *hash)
+{
+	if (strlen(hash) < 4 || strchr(hash, '/') != NULL)
+		i_fatal("Invalid hash in filename: %s", hash);
+	return t_strdup_printf("%s/%c%c/%c%c", rootdir,
+			       hash[0], hash[1], hash[2], hash[3]);
+}
+
+static int
+file_contents_equal(const char *path1, const char *path2, ino_t *path2_inode_r)
+{
+	struct stat st1, st2;
+	int fd1, fd2, ret = -1;
+
+	/* do a byte-by-byte comparison for the files to find out if they're
+	   the same or if this is a hash collision */
+	fd1 = open(path1, O_RDONLY);
+	if (fd1 == -1) {
+		if (errno != ENOENT)
+			i_error("open(%s) failed: %m", path1);
+		return -1;
+	}
+	fd2 = open(path2, O_RDONLY);
+	if (fd1 == -1) {
+		if (errno != ENOENT)
+			i_error("open(%s) failed: %m", path2);
+		(void)close(fd1);
+		return -1;
+	}
+
+	if (fstat(fd1, &st1) < 0)
+		i_error("fstat(%s) failed: %m", path1);
+	else if (fstat(fd2, &st2) < 0)
+		i_error("fstat(%s) failed: %m", path1);
+	else if (st1.st_size != st2.st_size)
+		ret = 0;
+	else {
+		/* @UNSAFE: sizes match. compare. */
+		unsigned char buf1[IO_BLOCK_SIZE], buf2[IO_BLOCK_SIZE];
+		ssize_t ret1;
+		int ret2;
+
+		while ((ret1 = read(fd1, buf1, sizeof(buf1))) > 0) {
+			if ((ret2 = read_full(fd2, buf2, ret1)) <= 0) {
+				if (ret2 < 0)
+					i_error("read(%s) failed: %m", path2);
+				else
+					ret = 0;
+				break;
+			}
+			if (memcmp(buf1, buf2, ret1) != 0) {
+				ret = 0;
+				break;
+			}
+		}
+		if (ret1 < 0)
+			i_error("read(%s) failed: %m", path1);
+		else if (ret1 == 0)
+			ret = 1;
+	}
+
+	if (close(fd1) < 0)
+		i_error("close(%s) failed: %m", path1);
+	if (close(fd2) < 0)
+		i_error("close(%s) failed: %m", path2);
+
+	*path2_inode_r = st2.st_ino;
+	return ret;
+}
+
+static int
+hardlink_replace(const char *src, const char *dest, ino_t src_inode)
+{
+	const char *p, *destdir, *tmppath;
+	unsigned char randbuf[8];
+	struct stat st;
+
+	p = strrchr(dest, '/');
+	i_assert(p != NULL);
+	destdir = t_strdup_until(dest, p);
+
+	random_fill_weak(randbuf, sizeof(randbuf));
+	tmppath = t_strdup_printf("%s/temp.%s.%s.%s",
+				  destdir, my_hostname, my_pid,
+				  binary_to_hex(randbuf, sizeof(randbuf)));
+	if (link(src, tmppath) < 0) {
+		if (errno == EMLINK)
+			return 0;
+		i_error("link(%s, %s) failed: %m", src, tmppath);
+		return -1;
+	}
+	if (stat(tmppath, &st) < 0) {
+		i_error("stat(%s) failed: %m", tmppath);
+		return -1;
+	}
+	if (st.st_ino != src_inode) {
+		if (unlink(tmppath) < 0)
+			i_error("unlink(%s) failed: %m", tmppath);
+		return 0;
+	}
+	if (rename(tmppath, dest) < 0) {
+		i_error("rename(%s, %s) failed: %m", src, tmppath);
+		if (unlink(tmppath) < 0)
+			i_error("unlink(%s) failed: %m", tmppath);
+		return -1;
+	}
+	return 1;
+}
+
+static int sis_try_deduplicate(const char *rootdir, const char *fname)
+{
+	const char *p, *hash, *hashdir, *path, *hashes_dir, *hashes_path;
+	struct stat st;
+	ino_t inode;
+	int ret;
+
+	/* fname should be in <hash>-<guid> format */
+	p = strchr(fname, '-');
+	i_assert(p != NULL);
+
+	hash = t_strdup_until(fname, p);
+	hashdir = sis_get_dir(rootdir, hash);
+	path = t_strdup_printf("%s/%s", hashdir, fname);
+
+	hashes_dir = t_strconcat(hashdir, "/", HASH_DIR_NAME, NULL);
+	hashes_path = t_strconcat(hashes_dir, "/", hash, NULL);
+	if (link(path, hashes_path) == 0) {
+		/* first file with this hash. we're done */
+		return 0;
+	}
+	if (errno == ENOENT) {
+		/* either path was already deleted or hashes dir
+		   doesn't exist */
+		if (mkdir(hashes_dir, 0700) < 0) {
+			if (errno == EEXIST)
+				return 0;
+			i_error("mkdir(%s) failed: %m", hashes_dir);
+			return -1;
+		}
+		/* try again */
+		if (link(path, hashes_path) == 0 || errno == ENOENT)
+			return 0;
+	}
+	if (errno != EEXIST) {
+		i_error("link(%s, %s) failed: %m", path, hashes_path);
+		return -1;
+	}
+
+	/* need to do a byte-by-byte comparison. but check first if someone
+	   else already had deduplicated the file. */
+	if (stat(path, &st) < 0) {
+		if (errno == ENOENT) {
+			/* just got deleted */
+			return 0;
+		}
+		i_error("stat(%s) failed: %m", path);
+		return -1;
+	}
+	if (st.st_nlink > 1) {
+		/* already deduplicated */
+		return 0;
+	}
+
+	ret = file_contents_equal(path, hashes_path, &inode);
+	if (ret < 0) {
+		if (errno == ENOENT) {
+			/* either path or hashes_path was deleted. */
+			return sis_try_deduplicate(rootdir, fname);
+		}
+		return -1;
+	}
+	if (ret > 0) {
+		/* equal, replace with hard link */
+		ret = hardlink_replace(hashes_path, path, inode);
+		if (ret > 0)
+			return 0;
+		else if (ret < 0)
+			return -1;
+		/* too many hard links or inode changed */
+	}
+
+	/* replace hashes link with this  */
+	return hardlink_replace(path, hashes_path, st.st_ino) < 0 ? -1 : 0;
+}
+
+static void cmd_sis_deduplicate(int argc, char *argv[])
+{
+	const char *rootdir, *queuedir;
+	DIR *dir;
+	struct dirent *d;
+	struct stat st, first_st;
+	string_t *path;
+	unsigned int dir_len;
+	int ret;
+
+	if (argc < 3)
+		help(&doveadm_cmd_sis_deduplicate);
+
+	/* go through the filenames in the queue dir and see if
+	   we can deduplicate them. */
+	rootdir = argv[1];
+	queuedir = argv[2];
+
+	if (stat(rootdir, &st) < 0)
+		i_fatal("stat(%s) failed: %m", rootdir);
+
+	path = t_str_new(256);
+	str_append(path, queuedir);
+	str_append_c(path, '/');
+	dir_len = str_len(path);
+
+	dir = opendir(queuedir);
+	if (dir == NULL)
+		i_fatal("opendir(%s) failed: %m", queuedir);
+
+	first_st.st_size = -1;
+	while ((d = readdir(dir)) != NULL) {
+		if (d->d_name[0] == '.')
+			continue;
+
+		str_truncate(path, dir_len);
+		str_append(path, d->d_name);
+
+		if (first_st.st_size < 0) {
+			if (stat(str_c(path), &first_st) < 0)
+				i_fatal("stat(%s) failed: %m", str_c(path));
+		}
+		if (strchr(d->d_name, '-') == NULL || first_st.st_size != 0) {
+			i_fatal("%s is not a valid sis-queue file, "
+				"is the queue directory correct?",
+				str_c(path));
+		}
+
+		T_BEGIN {
+			ret = sis_try_deduplicate(rootdir, d->d_name);
+		} T_END;
+		if (ret == 0) {
+			if (unlink(str_c(path)) < 0)
+				i_error("unlink(%s) failed: %m", str_c(path));
+		}
+	}
+	if (closedir(dir) < 0)
+		i_error("closedir(%s) failed: %m", queuedir);
+}
+
+static void cmd_sis_find(int argc, char *argv[])
+{
+	const char *rootdir, *path, *hash;
+	DIR *dir;
+	struct dirent *d;
+	struct stat st;
+	string_t *str;
+	unsigned int dir_len, hash_len;
+
+	if (argc < 3 || strlen(argv[2]) < 4)
+		help(&doveadm_cmd_sis_find);
+
+	rootdir = argv[1];
+	if (stat(rootdir, &st) < 0) {
+		if (errno == ENOENT)
+			i_fatal("Attachment dir doesn't exist: %s", rootdir);
+		i_fatal("stat(%s) failed: %m", rootdir);
+	}
+	hash = argv[2];
+	hash_len = strlen(hash);
+
+	path = sis_get_dir(rootdir, hash);
+	str = t_str_new(256);
+	str_append(str, path);
+	str_append_c(str, '/');
+	dir_len = str_len(str);
+
+	dir = opendir(path);
+	if (dir == NULL) {
+		if (errno == ENOENT)
+			return;
+		i_fatal("opendir(%s) failed: %m", path);
+	}
+
+	doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW);
+	doveadm_print_header("path", "path",
+			     DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+	while ((d = readdir(dir)) != NULL) {
+		if (strncmp(d->d_name, hash, hash_len) == 0) {
+			str_truncate(str, dir_len);
+			str_append(str, d->d_name);
+			doveadm_print(str_c(str));
+		}
+	}
+	if (closedir(dir) < 0)
+		i_error("closedir(%s) failed: %m", path);
+}
+
+struct doveadm_cmd doveadm_cmd_sis_deduplicate = {
+	cmd_sis_deduplicate, "sis deduplicate", "<root dir> <queue dir>"
+};
+struct doveadm_cmd doveadm_cmd_sis_find = {
+	cmd_sis_find, "sis find", "<root dir> <hash>"
+};
diff -r e3fe87b855ef -r 21c70ff6b344 src/doveadm/doveadm.c
--- a/src/doveadm/doveadm.c	Tue Oct 19 18:16:10 2010 +0100
+++ b/src/doveadm/doveadm.c	Tue Oct 19 18:27:30 2010 +0100
@@ -255,7 +255,9 @@
 	&doveadm_cmd_who,
 	&doveadm_cmd_penalty,
 	&doveadm_cmd_kick,
-	&doveadm_cmd_mailbox_mutf7
+	&doveadm_cmd_mailbox_mutf7,
+	&doveadm_cmd_sis_deduplicate,
+	&doveadm_cmd_sis_find
 };
 
 int main(int argc, char *argv[])
diff -r e3fe87b855ef -r 21c70ff6b344 src/doveadm/doveadm.h
--- a/src/doveadm/doveadm.h	Tue Oct 19 18:16:10 2010 +0100
+++ b/src/doveadm/doveadm.h	Tue Oct 19 18:27:30 2010 +0100
@@ -24,6 +24,8 @@
 extern struct doveadm_cmd doveadm_cmd_penalty;
 extern struct doveadm_cmd doveadm_cmd_kick;
 extern struct doveadm_cmd doveadm_cmd_mailbox_mutf7;
+extern struct doveadm_cmd doveadm_cmd_sis_deduplicate;
+extern struct doveadm_cmd doveadm_cmd_sis_find;
 
 void doveadm_register_cmd(const struct doveadm_cmd *cmd);