Mercurial > illumos > illumos-gate
changeset 10777:66a375946c31
6868616 Using Intel CRC32 instruction to accelerate CRC32c algorithm
Contributed by Xinping Huang <xinping.huang@intel.com>
author | Jack Meng <Jack.Meng@Sun.COM> |
---|---|
date | Tue, 13 Oct 2009 11:35:11 +0800 |
parents | 0c765af76a3a |
children | 48cb25c7b1aa |
files | usr/src/cmd/iscsi/Makefile.iscsi usr/src/cmd/iscsi/iscsitgtd/Makefile usr/src/cmd/iscsi/iscsitgtd/Makefile.com usr/src/cmd/iscsi/iscsitgtd/iscsi_crc.c usr/src/common/hdcrc/hd_crc.h usr/src/uts/common/io/idm/idm_impl.c usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_crc.c usr/src/uts/intel/idm/Makefile usr/src/uts/intel/iscsi/Makefile usr/src/uts/sparc/idm/Makefile usr/src/uts/sparc/iscsi/Makefile |
diffstat | 11 files changed, 376 insertions(+), 11 deletions(-) [+] |
line wrap: on
line diff
--- a/usr/src/cmd/iscsi/Makefile.iscsi Mon Oct 12 20:17:24 2009 -0700 +++ b/usr/src/cmd/iscsi/Makefile.iscsi Tue Oct 13 11:35:11 2009 +0800 @@ -19,13 +19,12 @@ # CDDL HEADER END # # -# Copyright 2006 Sun Microsystems, Inc. All rights reserved. +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # -# ident "%Z%%M% %I% %E% SMI" -# CPPFLAGS += -D_FILE_OFFSET_BITS=64 -I/usr/include/libxml2 ISCSISRC = $(SRC)/cmd/iscsi ISCSICOMMONDIR = $(ISCSISRC)/common +HDCRCCOMMONDIR = $(SRC)/common/hdcrc
--- a/usr/src/cmd/iscsi/iscsitgtd/Makefile Mon Oct 12 20:17:24 2009 -0700 +++ b/usr/src/cmd/iscsi/iscsitgtd/Makefile Tue Oct 13 11:35:11 2009 +0800 @@ -52,7 +52,7 @@ ROOTDLIBS = $(DLIBSRCS:%=$(ROOTDLIBDIR)/%) $(ROOTDLIBS) := FILEMODE = 0644 -CPPFLAGS += -D_FILE_OFFSET_BITS=64 -I${ISCSICOMMONDIR} +CPPFLAGS += -D_FILE_OFFSET_BITS=64 -I${ISCSICOMMONDIR} -I${HDCRCCOMMONDIR} all := TARGET = all install := TARGET = install
--- a/usr/src/cmd/iscsi/iscsitgtd/Makefile.com Mon Oct 12 20:17:24 2009 -0700 +++ b/usr/src/cmd/iscsi/iscsitgtd/Makefile.com Tue Oct 13 11:35:11 2009 +0800 @@ -47,7 +47,7 @@ NATIVE_CFLAGS += $(CTF_FLAGS) CFLAGS += $(CCVERBOSE) -CPPFLAGS += -D_LARGEFILE64_SOURCE=1 -I/usr/include/libxml2 +CPPFLAGS += -D_LARGEFILE64_SOURCE=1 -I/usr/include/libxml2 -I${HDCRCCOMMONDIR} CFLAGS64 += $(CCVERBOSE) CLEANFILES += $(OBJS) ../$(DTRACE_HEADER)
--- a/usr/src/cmd/iscsi/iscsitgtd/iscsi_crc.c Mon Oct 12 20:17:24 2009 -0700 +++ b/usr/src/cmd/iscsi/iscsitgtd/iscsi_crc.c Tue Oct 13 11:35:11 2009 +0800 @@ -20,16 +20,16 @@ */ /* * Copyright 2000 by Cisco Systems, Inc. All rights reserved. - * Copyright 2006 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * * This file contains CRC-32C code use to verify * iSCSI HeaderDigests and DataDigests. */ -#pragma ident "%Z%%M% %I% %E% SMI" #include <sys/types.h> /* standard types */ #include <sys/iscsi_protocol.h> /* contains prototypes */ +#include <hd_crc.h> /* * This is the CRC-32C table @@ -109,6 +109,13 @@ }; /* + * -1 - uninitialized + * 0 - applicable + * others - NA + */ +static int iscsi_crc32_hd = -1; + +/* * iscsi_crc32c - Steps through buffer one byte at at time, calculates * reflected crc using table. */ @@ -117,10 +124,20 @@ { uint8_t *buffer = address; uint32_t crc = 0xffffffff, result; -#ifdef _BIG_ENDIAN +#ifdef _BIG_ENDIAN uint8_t byte0, byte1, byte2, byte3; #endif + if (iscsi_crc32_hd == -1) { + if (hd_crc32_avail((uint32_t *)iscsi_crc32c_table) == B_TRUE) { + iscsi_crc32_hd = 0; + } else { + iscsi_crc32_hd = 1; + } + } + if (iscsi_crc32_hd == 0) + return (HW_CRC32(buffer, length, crc)); + while (length--) { crc = iscsi_crc32c_table[(crc ^ *buffer++) & 0xFFL] ^ (crc >> 8); @@ -152,6 +169,16 @@ uint8_t byte0, byte1, byte2, byte3; #endif + if (iscsi_crc32_hd == -1) { + if (hd_crc32_avail((uint32_t *)iscsi_crc32c_table) == B_TRUE) { + iscsi_crc32_hd = 0; + } else { + iscsi_crc32_hd = 1; + } + } + if (iscsi_crc32_hd == 0) + return (HW_CRC32_CONT(buffer, length, crc)); + #ifdef _BIG_ENDIAN byte0 = (uint8_t)((crc >> 24) & 0xFF); byte1 = (uint8_t)((crc >> 16) & 0xFF);
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/usr/src/common/hdcrc/hd_crc.h Tue Oct 13 11:35:11 2009 +0800 @@ -0,0 +1,275 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2009, Intel Corporation. + * All rights reserved. + */ + +/* + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + +#ifndef _CRC_HD_H +#define _CRC_HD_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include <sys/types.h> + +#ifdef __cplusplus +#ifndef INLINE +#define INLINE inline +#endif +#else +#ifndef INLINE +#define INLINE +#endif +#endif + +#if defined(__i386) || defined(__amd_64) || defined(__x86_64) + +#define INTEL_CPU_FEATURE_SSE42 0x0100000 + +static INLINE unsigned int +#ifdef _KERNEL +/*LINTED:E_FUNC_ARG_UNUSED*/ +cpuid_ecx(unsigned int op) +#else +cpuid_ecx(unsigned int op) +#endif /* _KERNEL */ +{ + /*LINTED:E_FUNC_VAR_UNUSED*/ + unsigned int eax, ecx; + __asm__ __volatile__("cpuid" + : "=a" (eax), "=c" (ecx) + : "0" (op) + : "bx", "dx"); + /*LINTED:E_VAR_USED_BEFORE_SET*/ + return (ecx); +} + +static INLINE uint32_t +#ifdef _KERNEL +/*LINTED:E_FUNC_ARG_UNUSED*/ +mm_crc32_u8(uint32_t crc, uint8_t *data) +#else +mm_crc32_u8(uint32_t crc, uint8_t *data) +#endif /* _KERNEL */ +{ + __asm__ __volatile__( + /* "crc32 r32, r/m8" */ + ".byte 0xF2, 0x0F, 0x38, 0xF0, 0xF1" + : "=S" (crc) + : "0" (crc), "c" (*data)); + + return (crc); +} + +#if defined(__amd64) || defined(__x86_64) + +static INLINE uint32_t +#ifdef _KERNEL +/*LINTED:E_FUNC_ARG_UNUSED*/ +mm_crc32_u64(uint32_t crc, uint64_t *data) +#else +mm_crc32_u64(uint32_t crc, uint64_t *data) +#endif /* _KERNEL */ +{ + __asm__ __volatile__( + /* "crc32 r32, r/m64" */ + ".byte 0xF2, 0x48, 0x0F, 0x38, 0xF1, 0xF1" + : "=S" (crc) + : "0" (crc), "c" (*data)); + + return (crc); +} + +#define INTEL_CRC_ALIGN_MASK 0x7 +#define INTEL_CRC_SIZE 8 +#define INTEL_CRC_DATA_TYPE uint64_t +#define INTEL_CRC_FUNC mm_crc32_u64 + +#else + +static INLINE uint32_t +#ifdef _KERNEL +/*LINTED:E_FUNC_ARG_UNUSED*/ +mm_crc32_u32(uint32_t crc, uint32_t *data) +#else +mm_crc32_u32(uint32_t crc, uint32_t *data) +#endif /* _KERNEL */ +{ + __asm__ __volatile__( + /* "crc32 r32, r/m32" */ + ".byte 0xF2, 0x0F, 0x38, 0xF1, 0xF1" + : "=S" (crc) + : "0" (crc), "c" (*data)); + + return (crc); +} + +#define INTEL_CRC_ALIGN_MASK 0x3 +#define INTEL_CRC_SIZE 4 +#define INTEL_CRC_DATA_TYPE uint32_t +#define INTEL_CRC_FUNC mm_crc32_u32 + +#endif + +static INLINE uint32_t +intel_crc32c(uint8_t *address, unsigned long length, uint32_t crc) +{ + uint32_t i = (uintptr_t)address & INTEL_CRC_ALIGN_MASK; + uint8_t *data = address; + + /* Process unaligned header data */ + while ((length > 0) && (i > 0)) { + crc = mm_crc32_u8(crc, data); + ++ data; + -- length; + i ++; + i &= INTEL_CRC_ALIGN_MASK; + } + + /* aligned data part */ + while (length >= INTEL_CRC_SIZE) { + /*LINTED:E_BAD_PTR_CAST_ALIGN*/ + crc = INTEL_CRC_FUNC(crc, (INTEL_CRC_DATA_TYPE *)data); + data += INTEL_CRC_SIZE; + length -= INTEL_CRC_SIZE; + } + + /* unaligned tail data */ + while (length > 0) { + crc = mm_crc32_u8(crc, data); + ++ data; + -- length; + } + + return (crc ^ 0xFFFFFFFF); +} + +#define HW_CRC32(buffer, length, crc) (intel_crc32c((buffer), (length), (crc))) +#define HW_CRC32_CONT(buffer, length, crc) \ + (intel_crc32c((buffer), (length), (crc) ^ 0xFFFFFFFF)) +#else +#define HW_CRC32(buffer, length, crc) 0 +#define HW_CRC32_CONT(buffer, length, crc) 0 +#endif + +static INLINE boolean_t +#if defined(_KERNEL) && !defined(__i386) && !defined(__amd_64) &&\ + !defined(__x86_64) +/*LINTED:E_FUNC_ARG_UNUSED*/ +hd_crc32_avail(uint32_t *crc32_table) +#else +hd_crc32_avail(uint32_t *crc32_table) +#endif +{ +#if defined(__i386) || defined(__amd_64) || defined(__x86_64) + int i; + /* poly = 0x1EDC6F41 */ + static const uint32_t _intel_crc32_hd_table[256] = { + 0x00000000, 0xF26B8303, 0xE13B70F7, 0x1350F3F4, + 0xC79A971F, 0x35F1141C, 0x26A1E7E8, 0xD4CA64EB, + 0x8AD958CF, 0x78B2DBCC, 0x6BE22838, 0x9989AB3B, + 0x4D43CFD0, 0xBF284CD3, 0xAC78BF27, 0x5E133C24, + 0x105EC76F, 0xE235446C, 0xF165B798, 0x030E349B, + 0xD7C45070, 0x25AFD373, 0x36FF2087, 0xC494A384, + 0x9A879FA0, 0x68EC1CA3, 0x7BBCEF57, 0x89D76C54, + 0x5D1D08BF, 0xAF768BBC, 0xBC267848, 0x4E4DFB4B, + 0x20BD8EDE, 0xD2D60DDD, 0xC186FE29, 0x33ED7D2A, + 0xE72719C1, 0x154C9AC2, 0x061C6936, 0xF477EA35, + 0xAA64D611, 0x580F5512, 0x4B5FA6E6, 0xB93425E5, + 0x6DFE410E, 0x9F95C20D, 0x8CC531F9, 0x7EAEB2FA, + 0x30E349B1, 0xC288CAB2, 0xD1D83946, 0x23B3BA45, + 0xF779DEAE, 0x05125DAD, 0x1642AE59, 0xE4292D5A, + 0xBA3A117E, 0x4851927D, 0x5B016189, 0xA96AE28A, + 0x7DA08661, 0x8FCB0562, 0x9C9BF696, 0x6EF07595, + 0x417B1DBC, 0xB3109EBF, 0xA0406D4B, 0x522BEE48, + 0x86E18AA3, 0x748A09A0, 0x67DAFA54, 0x95B17957, + 0xCBA24573, 0x39C9C670, 0x2A993584, 0xD8F2B687, + 0x0C38D26C, 0xFE53516F, 0xED03A29B, 0x1F682198, + 0x5125DAD3, 0xA34E59D0, 0xB01EAA24, 0x42752927, + 0x96BF4DCC, 0x64D4CECF, 0x77843D3B, 0x85EFBE38, + 0xDBFC821C, 0x2997011F, 0x3AC7F2EB, 0xC8AC71E8, + 0x1C661503, 0xEE0D9600, 0xFD5D65F4, 0x0F36E6F7, + 0x61C69362, 0x93AD1061, 0x80FDE395, 0x72966096, + 0xA65C047D, 0x5437877E, 0x4767748A, 0xB50CF789, + 0xEB1FCBAD, 0x197448AE, 0x0A24BB5A, 0xF84F3859, + 0x2C855CB2, 0xDEEEDFB1, 0xCDBE2C45, 0x3FD5AF46, + 0x7198540D, 0x83F3D70E, 0x90A324FA, 0x62C8A7F9, + 0xB602C312, 0x44694011, 0x5739B3E5, 0xA55230E6, + 0xFB410CC2, 0x092A8FC1, 0x1A7A7C35, 0xE811FF36, + 0x3CDB9BDD, 0xCEB018DE, 0xDDE0EB2A, 0x2F8B6829, + 0x82F63B78, 0x709DB87B, 0x63CD4B8F, 0x91A6C88C, + 0x456CAC67, 0xB7072F64, 0xA457DC90, 0x563C5F93, + 0x082F63B7, 0xFA44E0B4, 0xE9141340, 0x1B7F9043, + 0xCFB5F4A8, 0x3DDE77AB, 0x2E8E845F, 0xDCE5075C, + 0x92A8FC17, 0x60C37F14, 0x73938CE0, 0x81F80FE3, + 0x55326B08, 0xA759E80B, 0xB4091BFF, 0x466298FC, + 0x1871A4D8, 0xEA1A27DB, 0xF94AD42F, 0x0B21572C, + 0xDFEB33C7, 0x2D80B0C4, 0x3ED04330, 0xCCBBC033, + 0xA24BB5A6, 0x502036A5, 0x4370C551, 0xB11B4652, + 0x65D122B9, 0x97BAA1BA, 0x84EA524E, 0x7681D14D, + 0x2892ED69, 0xDAF96E6A, 0xC9A99D9E, 0x3BC21E9D, + 0xEF087A76, 0x1D63F975, 0x0E330A81, 0xFC588982, + 0xB21572C9, 0x407EF1CA, 0x532E023E, 0xA145813D, + 0x758FE5D6, 0x87E466D5, 0x94B49521, 0x66DF1622, + 0x38CC2A06, 0xCAA7A905, 0xD9F75AF1, 0x2B9CD9F2, + 0xFF56BD19, 0x0D3D3E1A, 0x1E6DCDEE, 0xEC064EED, + 0xC38D26C4, 0x31E6A5C7, 0x22B65633, 0xD0DDD530, + 0x0417B1DB, 0xF67C32D8, 0xE52CC12C, 0x1747422F, + 0x49547E0B, 0xBB3FFD08, 0xA86F0EFC, 0x5A048DFF, + 0x8ECEE914, 0x7CA56A17, 0x6FF599E3, 0x9D9E1AE0, + 0xD3D3E1AB, 0x21B862A8, 0x32E8915C, 0xC083125F, + 0x144976B4, 0xE622F5B7, 0xF5720643, 0x07198540, + 0x590AB964, 0xAB613A67, 0xB831C993, 0x4A5A4A90, + 0x9E902E7B, 0x6CFBAD78, 0x7FAB5E8C, 0x8DC0DD8F, + 0xE330A81A, 0x115B2B19, 0x020BD8ED, 0xF0605BEE, + 0x24AA3F05, 0xD6C1BC06, 0xC5914FF2, 0x37FACCF1, + 0x69E9F0D5, 0x9B8273D6, 0x88D28022, 0x7AB90321, + 0xAE7367CA, 0x5C18E4C9, 0x4F48173D, 0xBD23943E, + 0xF36E6F75, 0x0105EC76, 0x12551F82, 0xE03E9C81, + 0x34F4F86A, 0xC69F7B69, 0xD5CF889D, 0x27A40B9E, + 0x79B737BA, 0x8BDCB4B9, 0x988C474D, 0x6AE7C44E, + 0xBE2DA0A5, 0x4C4623A6, 0x5F16D052, 0xAD7D5351 + }; + + if ((cpuid_ecx(1) & INTEL_CPU_FEATURE_SSE42) != 0) { + for (i = 0; i < 256; i++) { + if (crc32_table[i] != _intel_crc32_hd_table[i]) + return (B_FALSE); + } + return (B_TRUE); + } +#endif + return (B_FALSE); +} + +#ifdef __cplusplus +} +#endif + +#endif /* _CRC_HD_H */
--- a/usr/src/uts/common/io/idm/idm_impl.c Mon Oct 12 20:17:24 2009 -0700 +++ b/usr/src/uts/common/io/idm/idm_impl.c Tue Oct 13 11:35:11 2009 +0800 @@ -38,8 +38,15 @@ #include <sys/idm/idm.h> #include <sys/idm/idm_so.h> +#include <hd_crc.h> extern idm_transport_t idm_transport_list[]; +/* + * -1 - uninitialized + * 0 - applicable + * others - NA + */ +static int iscsi_crc32_hd = -1; void idm_pdu_rx(idm_conn_t *ic, idm_pdu_t *pdu) @@ -894,6 +901,16 @@ ASSERT(address != NULL); + if (iscsi_crc32_hd == -1) { + if (hd_crc32_avail((uint32_t *)idm_crc32c_table) == B_TRUE) { + iscsi_crc32_hd = 0; + } else { + iscsi_crc32_hd = 1; + } + } + if (iscsi_crc32_hd == 0) + return (HW_CRC32(buffer, length, crc)); + while (length--) { crc = idm_crc32c_table[(crc ^ *buffer++) & 0xFFL] ^ (crc >> 8); @@ -927,6 +944,17 @@ ASSERT(address != NULL); + if (iscsi_crc32_hd == -1) { + if (hd_crc32_avail((uint32_t *)idm_crc32c_table) == B_TRUE) { + iscsi_crc32_hd = 0; + } else { + iscsi_crc32_hd = 1; + } + } + if (iscsi_crc32_hd == 0) + return (HW_CRC32_CONT(buffer, length, crc)); + + #ifdef _BIG_ENDIAN byte0 = (uint8_t)((crc >> 24) & 0xFF); byte1 = (uint8_t)((crc >> 16) & 0xFF);
--- a/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_crc.c Mon Oct 12 20:17:24 2009 -0700 +++ b/usr/src/uts/common/io/scsi/adapters/iscsi/iscsi_crc.c Tue Oct 13 11:35:11 2009 +0800 @@ -20,7 +20,7 @@ */ /* * Copyright 2000 by Cisco Systems, Inc. All rights reserved. - * Copyright 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright 2009 Sun Microsystems, Inc. All rights reserved. * Use is subject to license terms. * * This file contains CRC-32C code use to verify @@ -29,6 +29,7 @@ #include <sys/types.h> /* standard types */ #include "iscsi.h" /* contains prototypes */ +#include <hd_crc.h> /* * This is the CRC-32C table @@ -108,6 +109,13 @@ }; /* + * -1 - uninitialized + * 0 - applicable + * others - NA + */ +static int iscsi_crc32_hd = -1; + +/* * iscsi_crc32c - Steps through buffer one byte at at time, calculates * reflected crc using table. */ @@ -120,6 +128,16 @@ uint8_t byte0, byte1, byte2, byte3; #endif + if (iscsi_crc32_hd == -1) { + if (hd_crc32_avail((uint32_t *)iscsi_crc32c_table) == B_TRUE) { + iscsi_crc32_hd = 0; + } else { + iscsi_crc32_hd = 1; + } + } + if (iscsi_crc32_hd == 0) + return (HW_CRC32(buffer, length, crc)); + ASSERT(address != NULL); while (length--) { @@ -153,6 +171,16 @@ uint8_t byte0, byte1, byte2, byte3; #endif + if (iscsi_crc32_hd == -1) { + if (hd_crc32_avail((uint32_t *)iscsi_crc32c_table) == B_TRUE) { + iscsi_crc32_hd = 0; + } else { + iscsi_crc32_hd = 1; + } + } + if (iscsi_crc32_hd == 0) + return (HW_CRC32_CONT(buffer, length, crc)); + ASSERT(address != NULL); #ifdef _BIG_ENDIAN
--- a/usr/src/uts/intel/idm/Makefile Mon Oct 12 20:17:24 2009 -0700 +++ b/usr/src/uts/intel/idm/Makefile Tue Oct 13 11:35:11 2009 +0800 @@ -21,7 +21,7 @@ # # uts/intel/io/idm/Makefile # -# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # @@ -62,6 +62,9 @@ DEBUG_DEFS += $(DEBUG_FLGS) LDFLAGS += -dy -Nfs/sockfs -Nmisc/ksocket +# Include +INC_PATH += -I$(SRC)/common/hdcrc + # # Default build targets. #
--- a/usr/src/uts/intel/iscsi/Makefile Mon Oct 12 20:17:24 2009 -0700 +++ b/usr/src/uts/intel/iscsi/Makefile Tue Oct 13 11:35:11 2009 +0800 @@ -57,6 +57,7 @@ # includes INC_PATH += -I$(UTSBASE)/common/io/scsi/adapters/iscsi +INC_PATH += -I$(SRC)/common/hdcrc # # Note dependancy on misc/scsi.
--- a/usr/src/uts/sparc/idm/Makefile Mon Oct 12 20:17:24 2009 -0700 +++ b/usr/src/uts/sparc/idm/Makefile Tue Oct 13 11:35:11 2009 +0800 @@ -21,7 +21,7 @@ # # uts/intel/io/idm/Makefile # -# Copyright 2008 Sun Microsystems, Inc. All rights reserved. +# Copyright 2009 Sun Microsystems, Inc. All rights reserved. # Use is subject to license terms. # # @@ -60,6 +60,9 @@ DEBUG_DEFS += $(DEBUG_FLGS) LDFLAGS += -dy -Nfs/sockfs -Nmisc/ksocket +# Include +INC_PATH += -I$(SRC)/common/hdcrc + # # Default build targets. #