Mercurial > dovecot > core-2.2

comparison src/lib-storage/index/index-sort.c @ 21604:fb8ef6e2c2fe

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
lib-storage: Add mail_sort_max_read_count setting. This controls how many slow mail accesses sorting can perform before it fails: a NO [LIMIT] Requested sort would have taken too long The SORT reply is still returned, but it's likely not correct.
author Timo Sirainen <timo.sirainen@dovecot.fi>
date Fri, 17 Feb 2017 19:07:53 +0200
parents a90f51ba2e9c
children bcd286fcdbad
comparison
equal deleted inserted replaced
21603:6d50f63cfa67 21604:fb8ef6e2c2fe
38 38
39 static void 39 static void
40 index_sort_program_set_mail_failed(struct mail_search_sort_program *program, 40 index_sort_program_set_mail_failed(struct mail_search_sort_program *program,
41 struct mail *mail) 41 struct mail *mail)
42 { 42 {
43 if (mailbox_get_last_mail_error(mail->box) != MAIL_ERROR_EXPUNGED) 43 switch (mailbox_get_last_mail_error(mail->box)) {
44 case MAIL_ERROR_EXPUNGED:
45 break;
46 case MAIL_ERROR_NOTPOSSIBLE:
47 /* just change the error message */
48 mail_storage_set_error(program->t->box->storage, MAIL_ERROR_LIMIT,
49 "Requested sort would have taken too long.");
50 /* fall through */
51 default:
44 program->failed = TRUE; 52 program->failed = TRUE;
53 break;
54 }
55 }
56
57 static time_t
58 index_sort_program_set_date_failed(struct mail_search_sort_program *program,
59 struct mail *mail)
60 {
61 index_sort_program_set_mail_failed(program, mail);
62
63 if (mailbox_get_last_mail_error(mail->box) == MAIL_ERROR_LIMIT) {
64 /* limit reached - sort the rest of the mails at the end of
65 the list by their UIDs */
66 return LONG_MAX;
67 } else {
68 /* expunged / some other error - sort in the beginning */
69 return 0;
70 }
45 } 71 }
46 72
47 static void 73 static void
48 index_sort_list_add_arrival(struct mail_search_sort_program *program, 74 index_sort_list_add_arrival(struct mail_search_sort_program *program,
49 struct mail *mail) 75 struct mail *mail)
51 ARRAY_TYPE(mail_sort_node_date) *nodes = program->context; 77 ARRAY_TYPE(mail_sort_node_date) *nodes = program->context;
52 struct mail_sort_node_date *node; 78 struct mail_sort_node_date *node;
53 79
54 node = array_append_space(nodes); 80 node = array_append_space(nodes);
55 node->seq = mail->seq; 81 node->seq = mail->seq;
56 if (mail_get_received_date(mail, &node->date) < 0) { 82 if (mail_get_received_date(mail, &node->date) < 0)
57 index_sort_program_set_mail_failed(program, mail); 83 node->date = index_sort_program_set_date_failed(program, mail);
58 node->date = 0;
59 }
60 } 84 }
61 85
62 static void 86 static void
63 index_sort_list_add_date(struct mail_search_sort_program *program, 87 index_sort_list_add_date(struct mail_search_sort_program *program,
64 struct mail *mail) 88 struct mail *mail)
67 struct mail_sort_node_date *node; 91 struct mail_sort_node_date *node;
68 int tz; 92 int tz;
69 93
70 node = array_append_space(nodes); 94 node = array_append_space(nodes);
71 node->seq = mail->seq; 95 node->seq = mail->seq;
72 if (mail_get_date(mail, &node->date, &tz) < 0) { 96 if (mail_get_date(mail, &node->date, &tz) < 0)
73 index_sort_program_set_mail_failed(program, mail); 97 node->date = index_sort_program_set_date_failed(program, mail);
74 node->date = 0; 98 else if (node->date == 0) {
75 } else if (node->date == 0) { 99 if (mail_get_received_date(mail, &node->date) < 0)
76 if (mail_get_received_date(mail, &node->date) < 0) { 100 node->date = index_sort_program_set_date_failed(program, mail);
77 index_sort_program_set_mail_failed(program, mail);
78 node->date = 0;
79 }
80 } 101 }
81 } 102 }
82 103
83 static void 104 static void
84 index_sort_list_add_size(struct mail_search_sort_program *program, 105 index_sort_list_add_size(struct mail_search_sort_program *program,
146 } 167 }
147 168
148 void index_sort_list_add(struct mail_search_sort_program *program, 169 void index_sort_list_add(struct mail_search_sort_program *program,
149 struct mail *mail) 170 struct mail *mail)
150 { 171 {
172 enum mail_lookup_abort orig_abort = mail->lookup_abort;
173 bool prev_slow = mail->mail_stream_opened ||
174 mail->mail_metadata_accessed;
175
151 i_assert(mail->transaction == program->t); 176 i_assert(mail->transaction == program->t);
152 177
178 if (program->slow_mails_left == 0)
179 mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
153 T_BEGIN { 180 T_BEGIN {
154 program->sort_list_add(program, mail); 181 program->sort_list_add(program, mail);
155 } T_END; 182 } T_END;
183 if (!prev_slow && (mail->mail_stream_opened ||
184 mail->mail_metadata_accessed)) {
185 i_assert(program->slow_mails_left > 0);
186 program->slow_mails_left--;
187 }
188 mail->lookup_abort = orig_abort;
156 } 189 }
157 190
158 static int sort_node_date_cmp(const struct mail_sort_node_date *n1, 191 static int sort_node_date_cmp(const struct mail_sort_node_date *n1,
159 const struct mail_sort_node_date *n2) 192 const struct mail_sort_node_date *n2)
160 { 193 {
273 306
274 /* we support internal sorting by the primary condition */ 307 /* we support internal sorting by the primary condition */
275 program = i_new(struct mail_search_sort_program, 1); 308 program = i_new(struct mail_search_sort_program, 1);
276 program->t = t; 309 program->t = t;
277 program->temp_mail = mail_alloc(t, 0, NULL); 310 program->temp_mail = mail_alloc(t, 0, NULL);
311
312 program->slow_mails_left =
313 program->t->box->storage->set->mail_sort_max_read_count;
314 if (program->slow_mails_left == 0)
315 program->slow_mails_left = UINT_MAX;
278 316
279 for (i = 0; i < MAX_SORT_PROGRAM_SIZE; i++) { 317 for (i = 0; i < MAX_SORT_PROGRAM_SIZE; i++) {
280 program->sort_program[i] = sort_program[i]; 318 program->sort_program[i] = sort_program[i];
281 if (sort_program[i] == MAIL_SORT_END) 319 if (sort_program[i] == MAIL_SORT_END)
282 break; 320 break;
423 else if (addr->mailbox != NULL) 461 else if (addr->mailbox != NULL)
424 *name_r = addr->mailbox; 462 *name_r = addr->mailbox;
425 return 0; 463 return 0;
426 } 464 }
427 465
428 int index_sort_header_get(struct mail *mail, uint32_t seq, 466 static void
467 index_sort_set_seq(struct mail_search_sort_program *program,
468 struct mail *mail, uint32_t seq)
469 {
470 if ((mail->mail_stream_opened || mail->mail_metadata_accessed) &&
471 program->slow_mails_left > 0)
472 program->slow_mails_left--;
473 mail_set_seq(mail, seq);
474 if (program->slow_mails_left == 0) {
475 /* too many slow lookups - just return the rest of the results
476 in whatever order. */
477 mail->lookup_abort = MAIL_LOOKUP_ABORT_NOT_IN_CACHE;
478 }
479 }
480
481 int index_sort_header_get(struct mail_search_sort_program *program, uint32_t seq,
429 enum mail_sort_type sort_type, string_t *dest) 482 enum mail_sort_type sort_type, string_t *dest)
430 { 483 {
484 struct mail *mail = program->temp_mail;
431 const char *str; 485 const char *str;
432 int ret; 486 int ret;
433 bool reply_or_fw; 487 bool reply_or_fw;
434 488
435 mail_set_seq(mail, seq); 489 index_sort_set_seq(program, mail, seq);
436 str_truncate(dest, 0); 490 str_truncate(dest, 0);
437 491
438 switch (sort_type & MAIL_SORT_MASK) { 492 switch (sort_type & MAIL_SORT_MASK) {
439 case MAIL_SORT_SUBJECT: 493 case MAIL_SORT_SUBJECT:
440 ret = mail_get_first_header(mail, "Subject", &str); 494 ret = mail_get_first_header(mail, "Subject", &str);
498 T_BEGIN { 552 T_BEGIN {
499 string_t *str1, *str2; 553 string_t *str1, *str2;
500 554
501 str1 = t_str_new(256); 555 str1 = t_str_new(256);
502 str2 = t_str_new(256); 556 str2 = t_str_new(256);
503 if (index_sort_header_get(mail, seq1, sort_type, str1) < 0) 557 if (index_sort_header_get(program, seq1, sort_type, str1) < 0)
504 index_sort_program_set_mail_failed(program, mail); 558 index_sort_program_set_mail_failed(program, mail);
505 if (index_sort_header_get(mail, seq2, sort_type, str2) < 0) 559 if (index_sort_header_get(program, seq2, sort_type, str2) < 0)
506 index_sort_program_set_mail_failed(program, mail); 560 index_sort_program_set_mail_failed(program, mail);
507 561
508 ret = strcmp(str_c(str1), str_c(str2)); 562 ret = strcmp(str_c(str1), str_c(str2));
509 } T_END; 563 } T_END;
510 break; 564 break;
511 case MAIL_SORT_ARRIVAL: 565 case MAIL_SORT_ARRIVAL:
512 mail_set_seq(mail, seq1); 566 index_sort_set_seq(program, mail, seq1);
513 if (mail_get_received_date(mail, &time1) < 0) { 567 if (mail_get_received_date(mail, &time1) < 0)
514 index_sort_program_set_mail_failed(program, mail); 568 time1 = index_sort_program_set_date_failed(program, mail);
515 time1 = 0; 569
516 } 570 index_sort_set_seq(program, mail, seq2);
517 571 if (mail_get_received_date(mail, &time2) < 0)
518 mail_set_seq(mail, seq2); 572 time2 = index_sort_program_set_date_failed(program, mail);
519 if (mail_get_received_date(mail, &time2) < 0) {
520 index_sort_program_set_mail_failed(program, mail);
521 time2 = 0;
522 }
523 573
524 ret = time1 < time2 ? -1 : 574 ret = time1 < time2 ? -1 :
525 (time1 > time2 ? 1 : 0); 575 (time1 > time2 ? 1 : 0);
526 break; 576 break;
527 case MAIL_SORT_DATE: 577 case MAIL_SORT_DATE:
528 mail_set_seq(mail, seq1); 578 index_sort_set_seq(program, mail, seq1);
529 if (mail_get_date(mail, &time1, &tz) < 0) { 579 if (mail_get_date(mail, &time1, &tz) < 0)
530 index_sort_program_set_mail_failed(program, mail); 580 time1 = index_sort_program_set_date_failed(program, mail);
531 time1 = 0; 581 else if (time1 == 0) {
532 } else if (time1 == 0) { 582 if (mail_get_received_date(mail, &time1) < 0)
533 if (mail_get_received_date(mail, &time1) < 0) { 583 time1 = index_sort_program_set_date_failed(program, mail);
534 index_sort_program_set_mail_failed(program, mail);
535 time1 = 0;
536 }
537 } 584 }
538 585
539 mail_set_seq(mail, seq2); 586 index_sort_set_seq(program, mail, seq2);
540 if (mail_get_date(mail, &time2, &tz) < 0) { 587 if (mail_get_date(mail, &time2, &tz) < 0)
541 index_sort_program_set_mail_failed(program, mail); 588 time2 = index_sort_program_set_date_failed(program, mail);
542 time2 = 0; 589 else if (time2 == 0) {
543 } else if (time2 == 0) { 590 if (mail_get_received_date(mail, &time2) < 0)
544 if (mail_get_received_date(mail, &time2) < 0) { 591 time2 = index_sort_program_set_date_failed(program, mail);
545 index_sort_program_set_mail_failed(program, mail);
546 time2 = 0;
547 }
548 } 592 }
549 593
550 ret = time1 < time2 ? -1 : 594 ret = time1 < time2 ? -1 :
551 (time1 > time2 ? 1 : 0); 595 (time1 > time2 ? 1 : 0);
552 break; 596 break;
553 case MAIL_SORT_SIZE: 597 case MAIL_SORT_SIZE:
554 mail_set_seq(mail, seq1); 598 index_sort_set_seq(program, mail, seq1);
555 if (mail_get_virtual_size(mail, &size1) < 0) { 599 if (mail_get_virtual_size(mail, &size1) < 0) {
556 index_sort_program_set_mail_failed(program, mail); 600 index_sort_program_set_mail_failed(program, mail);
557 size1 = 0; 601 size1 = 0;
558 } 602 }
559 603
560 mail_set_seq(mail, seq2); 604 index_sort_set_seq(program, mail, seq2);
561 if (mail_get_virtual_size(mail, &size2) < 0) { 605 if (mail_get_virtual_size(mail, &size2) < 0) {
562 index_sort_program_set_mail_failed(program, mail); 606 index_sort_program_set_mail_failed(program, mail);
563 size2 = 0; 607 size2 = 0;
564 } 608 }
565 609
566 ret = size1 < size2 ? -1 : 610 ret = size1 < size2 ? -1 :
567 (size1 > size2 ? 1 : 0); 611 (size1 > size2 ? 1 : 0);
568 break; 612 break;
569 case MAIL_SORT_RELEVANCY: 613 case MAIL_SORT_RELEVANCY:
570 mail_set_seq(mail, seq1); 614 index_sort_set_seq(program, mail, seq1);
571 if (index_sort_get_relevancy(mail, &float1) < 0) 615 if (index_sort_get_relevancy(mail, &float1) < 0)
572 index_sort_program_set_mail_failed(program, mail); 616 index_sort_program_set_mail_failed(program, mail);
573 mail_set_seq(mail, seq2); 617 index_sort_set_seq(program, mail, seq2);
574 if (index_sort_get_relevancy(mail, &float2) < 0) 618 if (index_sort_get_relevancy(mail, &float2) < 0)
575 index_sort_program_set_mail_failed(program, mail); 619 index_sort_program_set_mail_failed(program, mail);
576 620
577 /* NOTE: higher relevancy is returned first, unlike with all 621 /* NOTE: higher relevancy is returned first, unlike with all
578 other number based sort keys */ 622 other number based sort keys */
580 (float1 > float2 ? -1 : 0); 624 (float1 > float2 ? -1 : 0);
581 break; 625 break;
582 case MAIL_SORT_POP3_ORDER: 626 case MAIL_SORT_POP3_ORDER:
583 /* 32bit numbers would be enough, but since there is already 627 /* 32bit numbers would be enough, but since there is already
584 existing code for uoff_t in sizes, just use them. */ 628 existing code for uoff_t in sizes, just use them. */
585 mail_set_seq(mail, seq1); 629 index_sort_set_seq(program, mail, seq1);
586 if (index_sort_get_pop3_order(mail, &size1) < 0) 630 if (index_sort_get_pop3_order(mail, &size1) < 0)
587 index_sort_program_set_mail_failed(program, mail); 631 index_sort_program_set_mail_failed(program, mail);
588 mail_set_seq(mail, seq2); 632 index_sort_set_seq(program, mail, seq2);
589 if (index_sort_get_pop3_order(mail, &size2) < 0) 633 if (index_sort_get_pop3_order(mail, &size2) < 0)
590 index_sort_program_set_mail_failed(program, mail); 634 index_sort_program_set_mail_failed(program, mail);
591 635
592 ret = size1 < size2 ? -1 : 636 ret = size1 < size2 ? -1 :
593 (size1 > size2 ? 1 : 0); 637 (size1 > size2 ? 1 : 0);